From 0ac8f15e0eca5d601b5ee2f2e62907a369cb0271 Mon Sep 17 00:00:00 2001 From: rozetko Date: Wed, 28 Dec 2022 14:43:30 +0300 Subject: [PATCH 1/9] move all `/api` endpoints to a separate router --- src/index.ts | 13 +++---------- src/routes/api.ts | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 src/routes/api.ts diff --git a/src/index.ts b/src/index.ts index 73841bf..64fd506 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,24 +1,17 @@ -import { EXPORTED_PATH } from './config'; -import { router as tasksRouter } from './routes/tasks'; -import { router as statusRouter } from './routes/status'; -import { router as deleteRouter } from './routes/delete'; +import { router as apiRouter } from './routes/api'; import { port } from './config'; import * as express from 'express'; import * as bodyParser from 'body-parser'; + const app = express(); app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: true })); -// TODO: move everything with /api prefix to an apiRouter -app.use('/api/status', statusRouter); - -app.use('/api/tasks', tasksRouter); -app.use('/api/delete', deleteRouter); +app.use('/api', apiRouter); -app.use('/api/static', express.static(EXPORTED_PATH)); app.use('/', (req, res) => { res.send('Grafana-data-exporter server works') }); app.listen(port, () => { diff --git a/src/routes/api.ts b/src/routes/api.ts new file mode 100644 index 0000000..8e55309 --- /dev/null +++ b/src/routes/api.ts @@ -0,0 +1,16 @@ +import { EXPORTED_PATH } from '../config'; +import { router as tasksRouter } from '../routes/tasks'; +import { router as statusRouter } from '../routes/status'; +import { router as deleteRouter } from '../routes/delete'; + +import * as express from 'express'; + + +export const router = express.Router(); + +router.use('/status', statusRouter); + +router.use('/tasks', tasksRouter); +router.use('/delete', deleteRouter); + +router.use('/static', express.static(EXPORTED_PATH)); From ee162490040e3167547eba82a1807d68a83d7b3a Mon Sep 17 00:00:00 2001 From: rozetko Date: Wed, 28 Dec 2022 14:45:27 +0300 Subject: [PATCH 2/9] deleteRouter -> `DELETE /api/tasks` --- src/routes/api.ts | 2 -- src/routes/delete.ts | 25 ------------------------- src/routes/tasks.ts | 27 ++++++++++++++++++++++++--- 3 files changed, 24 insertions(+), 30 deletions(-) delete mode 100644 src/routes/delete.ts diff --git a/src/routes/api.ts b/src/routes/api.ts index 8e55309..9c808d8 100644 --- a/src/routes/api.ts +++ b/src/routes/api.ts @@ -1,7 +1,6 @@ import { EXPORTED_PATH } from '../config'; import { router as tasksRouter } from '../routes/tasks'; import { router as statusRouter } from '../routes/status'; -import { router as deleteRouter } from '../routes/delete'; import * as express from 'express'; @@ -11,6 +10,5 @@ export const router = express.Router(); router.use('/status', statusRouter); router.use('/tasks', tasksRouter); -router.use('/delete', deleteRouter); router.use('/static', express.static(EXPORTED_PATH)); diff --git a/src/routes/delete.ts b/src/routes/delete.ts deleted file mode 100644 index 3079222..0000000 --- a/src/routes/delete.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { EXPORTED_PATH } from '../config' - -import * as express from 'express' -import * as fs from 'fs' -import * as path from 'path' - - -async function deleteTask(req, res) { - let filename = req.query.filename; - let csvFilePath = path.join(EXPORTED_PATH, `${filename}.csv`); - let jsonFilePath = path.join(EXPORTED_PATH, `${filename}.json`); - - if(fs.existsSync(csvFilePath)) { - fs.unlink(csvFilePath, err => console.error(err)); - } - if(fs.existsSync(jsonFilePath)) { - fs.unlink(jsonFilePath, err => console.error(err)); - } - - res.status(200).send({ status: 'OK' }); -} - -export const router = express.Router(); - -router.get('/', deleteTask); diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts index 27fbed5..b4359d7 100644 --- a/src/routes/tasks.ts +++ b/src/routes/tasks.ts @@ -1,8 +1,13 @@ -import { Target } from '../types/target' +import { Target } from '../types/target'; +import { exporterFactory } from '../services/exporter.factory'; +import { EXPORTED_PATH } from '../config'; -import * as express from 'express' import { Datasource } from '@corpglory/tsdb-kit'; -import { exporterFactory } from '../services/exporter.factory'; + +import * as express from 'express'; + +import * as path from 'path'; +import * as fs from 'fs'; type TRequest = { body: { @@ -60,7 +65,23 @@ async function addTask(req: TRequest, res) { } } +async function deleteTask(req, res) { + let filename = req.query.filename; + let csvFilePath = path.join(EXPORTED_PATH, `${filename}.csv`); + let jsonFilePath = path.join(EXPORTED_PATH, `${filename}.json`); + + if(fs.existsSync(csvFilePath)) { + fs.unlink(csvFilePath, err => console.error(err)); + } + if(fs.existsSync(jsonFilePath)) { + fs.unlink(jsonFilePath, err => console.error(err)); + } + + res.status(200).send({ status: 'OK' }); +} + export const router = express.Router(); router.get('/', getTasks); router.post('/', addTask); +router.delete('/', deleteTask); From 2eb23da9cdcf6f9c099f63fe8a02794cfd74d1f7 Mon Sep 17 00:00:00 2001 From: rozetko Date: Wed, 28 Dec 2022 14:45:42 +0300 Subject: [PATCH 3/9] `/api/tasks` -> `/api/task` --- src/routes/api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/api.ts b/src/routes/api.ts index 9c808d8..49a57b3 100644 --- a/src/routes/api.ts +++ b/src/routes/api.ts @@ -9,6 +9,6 @@ export const router = express.Router(); router.use('/status', statusRouter); -router.use('/tasks', tasksRouter); +router.use('/task', tasksRouter); router.use('/static', express.static(EXPORTED_PATH)); From bb22cf887f20c8f0599695305de74e64c9b984b3 Mon Sep 17 00:00:00 2001 From: rozetko Date: Wed, 28 Dec 2022 19:05:34 +0300 Subject: [PATCH 4/9] upd tsdb-kit to the latest version --- package.json | 2 +- src/services/exporter.ts | 8 +++++--- yarn.lock | 8 ++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 9fa4d9b..4b4ddf2 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "license": "ISC", "dependencies": {}, "devDependencies": { - "@corpglory/tsdb-kit": "^1.1.1", + "@corpglory/tsdb-kit": "^2.0.1", "axios": "^1.2.1", "express": "^4.18.2", "fast-csv": "^4.3.6", diff --git a/src/services/exporter.ts b/src/services/exporter.ts index 820520f..ab0b766 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -4,7 +4,9 @@ import { apiKeys } from '../config'; import { promisify } from '../utils'; import { ExportStatus } from '../types/export-status'; -import { Metric, queryByMetric } from '@corpglory/tsdb-kit'; +import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit'; +// TODO: export QueryType directly from @corpglory/tsdb-kit +import { QueryType } from '@corpglory/tsdb-kit/lib/connectors'; import * as moment from 'moment'; import * as csv from 'fast-csv'; @@ -60,7 +62,7 @@ export class Exporter { this.validateTargets(data); const targets = data.map(target => ({ ...target, - metric: new Metric(target.datasource, target.targets) + metric: new QueryConfig(QueryType.GRAFANA, target.datasource, target.targets) })); this.datasource = data.length === 1 ? data[0].datasourceName : 'all'; @@ -82,7 +84,7 @@ export class Exporter { const host = new URL(target.panelUrl).origin; const apiKey = apiKeys[host]; - const datasourceMetrics = await queryByMetric(target.metric, target.panelUrl, from, to, apiKey); + const datasourceMetrics = await queryByConfig(target.metric, target.panelUrl, from, to, apiKey); const column = `${target.panelId}` + `-${target.panelTitle.replace(' ', '-')}-${datasourceMetrics.columns[1]}`; diff --git a/yarn.lock b/yarn.lock index b8a5f34..138d2f9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,10 +2,10 @@ # yarn lockfile v1 -"@corpglory/tsdb-kit@^1.1.1": - version "1.1.1" - resolved "https://registry.yarnpkg.com/@corpglory/tsdb-kit/-/tsdb-kit-1.1.1.tgz#ad2c85a4c05748db56c2a65138f2f0d8fd48cdd8" - integrity sha512-OmJdgeFavbbKpXsQ8Aq1Sb8NvaMgPhdXXPArBjnzciizi5WwLs/O91S2TOztZWljnTi+mH+TpWKy9ryH3AuGaw== +"@corpglory/tsdb-kit@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@corpglory/tsdb-kit/-/tsdb-kit-2.0.1.tgz#13435a69b2bb8c9890b838a06038daf76e96ec39" + integrity sha512-qIoCy0DXjPFkAE/G9URNVJ56vfU9SPyPoNR2wJSRut6vA6eLP8kljFf4GQE/7/yFpvulHI9+RT9UrlhImWbEZA== dependencies: axios "^0.18.0" moment "^2.22.2" From 45f5da66a332e5811ba285fa8955d6bd10482dba Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 30 Dec 2022 03:31:53 +0300 Subject: [PATCH 5/9] seems to work now --- package.json | 2 +- src/routes/tasks.ts | 83 +++++++++++++++++++++------------- src/services/exporter.ts | 53 ++++++++++++++-------- src/types/index.ts | 97 ++++++++++++++++++++++++++++++++++++++++ src/types/target.ts | 10 ++--- yarn.lock | 8 ++-- 6 files changed, 192 insertions(+), 61 deletions(-) create mode 100644 src/types/index.ts diff --git a/package.json b/package.json index 4b4ddf2..41827b4 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "license": "ISC", "dependencies": {}, "devDependencies": { - "@corpglory/tsdb-kit": "^2.0.1", + "@corpglory/tsdb-kit": "^2.0.2", "axios": "^1.2.1", "express": "^4.18.2", "fast-csv": "^4.3.6", diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts index b4359d7..9c5f646 100644 --- a/src/routes/tasks.ts +++ b/src/routes/tasks.ts @@ -2,7 +2,7 @@ import { Target } from '../types/target'; import { exporterFactory } from '../services/exporter.factory'; import { EXPORTED_PATH } from '../config'; -import { Datasource } from '@corpglory/tsdb-kit'; +import { Task, TaskTableRowConfig } from '../types'; import * as express from 'express'; @@ -13,55 +13,76 @@ type TRequest = { body: { from: string, to: string, - data: Array<{ - panelUrl: string, - panelTitle: string, - panelId: number, - datasourceRequest: Datasource, - datasourceName: string, - target: object, - }>, - user: string, - } + username: string, + tasks: Task[], + url: string, + }, }; async function getTasks(req, res) { - res.status(200).send([{ - timestamp: 12343567, - user: 'admin', - datasource: 'postgres', - rowsCount: 2345, - progress: 100, - status: 'Success', - }]); + const resp: TaskTableRowConfig[] = []; + fs.readdir(EXPORTED_PATH, (err, items) => { + if(err) { + console.error(err); + res.status(500).send('Something went wrong'); + } else { + for(let item of items) { + let file = path.parse(item); + if(file.ext !== '.json') { + continue; + } + // TODO: read async + let data = fs.readFileSync(path.join(EXPORTED_PATH, item), 'utf8'); + let status = JSON.parse(data); + + let requestedUrl = `http://${req.headers.host}`; + let downloadLink = ''; + let deleteLink = ''; + if(status.status === 'finished') { + downloadLink = ``; + } + resp.push({ + timestamp: status.time, + user: status.user, + datasourceRef: status.datasourceRef, + rowsCount: status.exportedRows, + progress: status.progress, + status: status.status, + downloadLink, + }); + } + + res.status(200).send(resp); + } + }); } async function addTask(req: TRequest, res) { + const body = req.body; + + const clientUrl = body.url; const from = parseInt(body.from); const to = parseInt(body.to); - const data = body.data; - const user = body.user; + const username = body.username; + const tasks = body.tasks; + + const datasourceUrl = `${new URL(clientUrl).origin}/api/ds/query`; if(isNaN(from) || isNaN(to)) { res.status(400).send('Range error: please fill both "from" and "to" fields'); } else if(from >= to) { res.status(400).send('Range error: "from" should be less than "to"'); } else { - const names = data.map(item => item.datasourceName).join(', '); + const names = tasks.map(item => item.datasource.name).join(', '); res.status(200).send(`Exporting ${names} data from ${new Date(from).toLocaleString()} to ${new Date(to).toLocaleString()}`); - const targets = data.map(item => new Target( - item.panelUrl, - item.panelTitle, - item.panelId, - item.datasourceRequest, - [item.target], - item.datasourceName, + const targets = tasks.map((task: Task) => new Target( + task.panel, + task.datasource, )); - const exporter = exporterFactory.getExporter(); - exporter.export(targets, user, from, to); + exporter.export(targets, datasourceUrl, username, from, to); } } diff --git a/src/services/exporter.ts b/src/services/exporter.ts index ab0b766..e7899b2 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -3,6 +3,7 @@ import { URL } from 'url'; import { apiKeys } from '../config'; import { promisify } from '../utils'; import { ExportStatus } from '../types/export-status'; +import { DataSourceRef } from '../types'; import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit'; // TODO: export QueryType directly from @corpglory/tsdb-kit @@ -21,11 +22,12 @@ export class Exporter { private exportedRows = 0; private createdTimestamp: number; private user: string; - private datasource: string; + private datasourceRef: DataSourceRef; private initCsvStream() { - // @ts-ignore - const csvStream = csv.createWriteStream({ headers: true }); + const csvStream = csv.parse({ headers: true }) + .on('error', error => console.error(error)); + const writableStream = fs.createWriteStream(this.getFilePath('csv')); csvStream.pipe(writableStream); @@ -46,7 +48,7 @@ export class Exporter { exportedRows: this.exportedRows, progress: progress.toLocaleString('en', { style: 'percent' }), status, - datasourceName: this.datasource, + datasourceRef: this.datasourceRef, }; await promisify(fs.writeFile, this.getFilePath('json'), JSON.stringify(data), 'utf8') @@ -56,16 +58,31 @@ export class Exporter { } } - public async export(data: Target[], user: string, from: number, to: number) { + public async export(data: Target[], datasourceUrl: string, user: string, from: number, to: number) { this.user = user; - this.validateTargets(data); - const targets = data.map(target => ({ - ...target, - metric: new QueryConfig(QueryType.GRAFANA, target.datasource, target.targets) - })); + this.validateTargets(datasourceUrl, data); + + // console.log('ds', data[0].datasource) + const targets = data.map(target => { + console.log({ + ...target.datasource, + url: datasourceUrl + }) + return { + ...target, + metric: new QueryConfig( + QueryType.GRAFANA, + { + ...target.datasource, + url: datasourceUrl + }, + target.panel.targets + ) + } + }); - this.datasource = data.length === 1 ? data[0].datasourceName : 'all'; + this.datasourceRef = data.length === 1 ? data[0].datasource : { uid: 'all', type: 'all' }; const stream = this.initCsvStream(); const days = Math.ceil((to - from) / MS_IN_DAY); @@ -81,13 +98,13 @@ export class Exporter { const values = {}; for(const [index, target] of targets.entries()) { - const host = new URL(target.panelUrl).origin; + const host = new URL(datasourceUrl).origin; const apiKey = apiKeys[host]; - const datasourceMetrics = await queryByConfig(target.metric, target.panelUrl, from, to, apiKey); + const datasourceMetrics = await queryByConfig(target.metric, datasourceUrl, from, to, apiKey); - const column = `${target.panelId}` + - `-${target.panelTitle.replace(' ', '-')}-${datasourceMetrics.columns[1]}`; + const column = `${target.panel.id}` + + `-${target.panel.title.replace(' ', '-')}-${datasourceMetrics.columns[1]}`; columns.push(column); @@ -120,13 +137,13 @@ export class Exporter { stream.end(); } - private validateTargets(targets: Target[]) { + private validateTargets(datasourceUrl, targets: Target[]) { if(!targets || !Array.isArray(targets)) { throw new Error('Incorrect targets format'); } for(const target of targets) { - const host = new URL(target.panelUrl).origin; + const host = new URL(datasourceUrl).origin; const apiKey = apiKeys[host]; if(apiKey === undefined || apiKey === '') { @@ -156,7 +173,7 @@ export class Exporter { if(this.createdTimestamp === undefined) { this.createdTimestamp = moment().valueOf(); } - return `${this.createdTimestamp}.${this.datasource}.${extension}`; + return `${this.createdTimestamp}.${this.datasourceRef.uid}.${extension}`; } private getFilePath(extension) { diff --git a/src/types/index.ts b/src/types/index.ts new file mode 100644 index 0000000..df63d77 --- /dev/null +++ b/src/types/index.ts @@ -0,0 +1,97 @@ +import { DatasourceType } from '@corpglory/tsdb-kit'; + +export interface DataSourceRef { + /** The plugin type-id */ + type?: string; + /** Specific datasource instance */ + uid?: string; +} + +export interface DataQuery { + /** + * A - Z + */ + refId: string; + /** + * true if query is disabled (ie should not be returned to the dashboard) + */ + hide?: boolean; + /** + * Unique, guid like, string used in explore mode + */ + key?: string; + /** + * Specify the query flavor + */ + queryType?: string; + /** + * For mixed data sources the selected datasource is on the query level. + * For non mixed scenarios this is undefined. + */ + datasource?: DataSourceRef | null; +} + +/** + * Data Source instance edit model. This is returned from: + * /api/datasources + */ +export interface DataSourceSettings { + id: number; + uid: string; + orgId: number; + name: string; + typeLogoUrl: string; + type: DatasourceType; + typeName: string; + access: string; + url: string; + user: string; + database: string; + basicAuth: boolean; + basicAuthUser: string; + isDefault: boolean; + jsonData: any; + secureJsonData?: any; + secureJsonFields: any; + readOnly: boolean; + withCredentials: boolean; + version?: number; + accessControl?: any; +} + +export interface PanelModel { + /** ID of the panel within the current dashboard */ + id: number; + /** Panel title */ + title?: string; + /** Description */ + description?: string; + /** Panel options */ + options: any; + /** Field options configuration */ + fieldConfig: any; + /** Version of the panel plugin */ + pluginVersion?: string; + /** The datasource used in all targets */ + datasource?: DataSourceRef | null; + /** The queries in a panel */ + targets?: DataQuery[]; + /** alerting v1 object */ + alert?: any; +} + +export type Task = DataQuery & { + selected: boolean; + panel: PanelModel; + datasource: DataSourceSettings; +}; + +export type TaskTableRowConfig = { + timestamp: number; + user: string; + datasourceRef: DataSourceRef; + rowsCount: number; + progress: number; + status: string; + downloadLink: string; +}; diff --git a/src/types/target.ts b/src/types/target.ts index 4ad80ae..d6639b3 100644 --- a/src/types/target.ts +++ b/src/types/target.ts @@ -1,12 +1,8 @@ -import { Datasource } from '@corpglory/tsdb-kit'; +import { DataSourceSettings, PanelModel } from '.'; export class Target { constructor( - public panelUrl: string, - public panelTitle: string, - public panelId: number, - public datasource: Datasource, - public targets: Array, - public datasourceName: string, + public panel: PanelModel, + public datasource: DataSourceSettings, ) {} } diff --git a/yarn.lock b/yarn.lock index 138d2f9..1b1ca59 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,10 +2,10 @@ # yarn lockfile v1 -"@corpglory/tsdb-kit@^2.0.1": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@corpglory/tsdb-kit/-/tsdb-kit-2.0.1.tgz#13435a69b2bb8c9890b838a06038daf76e96ec39" - integrity sha512-qIoCy0DXjPFkAE/G9URNVJ56vfU9SPyPoNR2wJSRut6vA6eLP8kljFf4GQE/7/yFpvulHI9+RT9UrlhImWbEZA== +"@corpglory/tsdb-kit@^2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@corpglory/tsdb-kit/-/tsdb-kit-2.0.2.tgz#0bbba2b344b651e374a1f89967abee7ef3172f30" + integrity sha512-USXpz9kXcHavJ0kZAuf64wE+zhmHfI9EKplyXtAAEL9FYZ3YErR2Awz7mAma6juKrkDELMtW/V0tx1e//0aRaw== dependencies: axios "^0.18.0" moment "^2.22.2" From dd25cd1f60b0b33726b1fde0db5aa82019536564 Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 30 Dec 2022 14:29:06 +0300 Subject: [PATCH 6/9] hotfix --- src/routes/tasks.ts | 2 +- src/services/exporter.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts index 9c5f646..37ba9ca 100644 --- a/src/routes/tasks.ts +++ b/src/routes/tasks.ts @@ -75,7 +75,6 @@ async function addTask(req: TRequest, res) { res.status(400).send('Range error: "from" should be less than "to"'); } else { const names = tasks.map(item => item.datasource.name).join(', '); - res.status(200).send(`Exporting ${names} data from ${new Date(from).toLocaleString()} to ${new Date(to).toLocaleString()}`); const targets = tasks.map((task: Task) => new Target( task.panel, @@ -83,6 +82,7 @@ async function addTask(req: TRequest, res) { )); const exporter = exporterFactory.getExporter(); exporter.export(targets, datasourceUrl, username, from, to); + res.status(200).send(`Exporting ${names} data from ${new Date(from).toLocaleString()} to ${new Date(to).toLocaleString()}`); } } diff --git a/src/services/exporter.ts b/src/services/exporter.ts index e7899b2..1bda0d6 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -84,6 +84,8 @@ export class Exporter { this.datasourceRef = data.length === 1 ? data[0].datasource : { uid: 'all', type: 'all' }; + await this.updateStatus(ExportStatus.EXPORTING, 0); + const stream = this.initCsvStream(); const days = Math.ceil((to - from) / MS_IN_DAY); From 4ef537eab8541a356d764a8338076bc97168d131 Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 30 Dec 2022 14:41:02 +0300 Subject: [PATCH 7/9] it works --- package.json | 2 +- src/services/exporter.ts | 3 +- yarn.lock | 158 ++++++++++++++++++++------------------- 3 files changed, 84 insertions(+), 79 deletions(-) diff --git a/package.json b/package.json index 41827b4..82be763 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "@corpglory/tsdb-kit": "^2.0.2", "axios": "^1.2.1", "express": "^4.18.2", - "fast-csv": "^4.3.6", + "fast-csv": "^2.5.0", "lodash": "^4.17.21", "moment": "^2.29.4", "nodemon": "^2.0.20", diff --git a/src/services/exporter.ts b/src/services/exporter.ts index 1bda0d6..8c94182 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -25,7 +25,7 @@ export class Exporter { private datasourceRef: DataSourceRef; private initCsvStream() { - const csvStream = csv.parse({ headers: true }) + const csvStream = csv.createWriteStream({ headers: true }) .on('error', error => console.error(error)); const writableStream = fs.createWriteStream(this.getFilePath('csv')); @@ -127,6 +127,7 @@ export class Exporter { }); if(metricsValues.length > 0) { + console.log(metricsValues) this.writeCsv(stream, { columns, values: metricsValues, diff --git a/yarn.lock b/yarn.lock index 1b1ca59..5251a87 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16,31 +16,6 @@ resolved "https://registry.yarnpkg.com/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz#1d572bfbbe14b7704e0ba0f39b74815b84870d70" integrity sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw== -"@fast-csv/format@4.3.5": - version "4.3.5" - resolved "https://registry.yarnpkg.com/@fast-csv/format/-/format-4.3.5.tgz#90d83d1b47b6aaf67be70d6118f84f3e12ee1ff3" - integrity sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A== - dependencies: - "@types/node" "^14.0.1" - lodash.escaperegexp "^4.1.2" - lodash.isboolean "^3.0.3" - lodash.isequal "^4.5.0" - lodash.isfunction "^3.0.9" - lodash.isnil "^4.0.0" - -"@fast-csv/parse@4.3.6": - version "4.3.6" - resolved "https://registry.yarnpkg.com/@fast-csv/parse/-/parse-4.3.6.tgz#ee47d0640ca0291034c7aa94039a744cfb019264" - integrity sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA== - dependencies: - "@types/node" "^14.0.1" - lodash.escaperegexp "^4.1.2" - lodash.groupby "^4.6.0" - lodash.isfunction "^3.0.9" - lodash.isnil "^4.0.0" - lodash.isundefined "^3.0.1" - lodash.uniq "^4.5.0" - "@jridgewell/gen-mapping@^0.3.0": version "0.3.2" resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz#c1aedc61e853f2bb9f5dfe6d4442d3b565b253b9" @@ -117,11 +92,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.17.tgz#5c009e1d9c38f4a2a9d45c0b0c493fe6cdb4bcb5" integrity sha512-HJSUJmni4BeDHhfzn6nF0sVmd1SMezP7/4F0Lq+aXzmp2xm9O7WXrUtHW/CHlYVtZUbByEvWidHqRtcJXGF2Ng== -"@types/node@^14.0.1": - version "14.18.35" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.35.tgz#879c4659cb7b3fe515844f029c75079c941bb65c" - integrity sha512-2ATO8pfhG1kDvw4Lc4C0GXIMSQFFJBCo/R1fSgTwmUlq5oy95LXyjDQinsRVgQY6gp6ghh3H91wk9ES5/5C+Tw== - "@webassemblyjs/ast@1.11.1": version "1.11.1" resolved "https://registry.yarnpkg.com/@webassemblyjs/ast/-/ast-1.11.1.tgz#2bfd767eae1a6996f432ff7e8d7fc75679c0b6a7" @@ -321,6 +291,23 @@ anymatch@~3.1.2: normalize-path "^3.0.0" picomatch "^2.0.4" +arguments-extended@~0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/arguments-extended/-/arguments-extended-0.0.3.tgz#6107e4917d0eb6f0a4dd66320fc15afc72ef4946" + integrity sha512-MNYdPKgCiywbgHAmNsYr1tSNLtfbSdwE1akZV+33hU9A8RG0lO5HAK9oMnw7y7bjYUhc04dJpcIBMUaPPYYtXg== + dependencies: + extended "~0.0.3" + is-extended "~0.0.8" + +array-extended@~0.0.3, array-extended@~0.0.4, array-extended@~0.0.5: + version "0.0.11" + resolved "https://registry.yarnpkg.com/array-extended/-/array-extended-0.0.11.tgz#d7144ae748de93ca726f121009dbff1626d164bd" + integrity sha512-Fe4Ti2YgM1onQgrcCD8dUhFuZgHQxzqylSl1C5IDJVVVqY5D07h8RghIXL9sZ6COZ0e+oTL5IusTv5eXABJ9Kw== + dependencies: + arguments-extended "~0.0.3" + extended "~0.0.3" + is-extended "~0.0.3" + array-flatten@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" @@ -531,6 +518,15 @@ cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" +date-extended@~0.0.3: + version "0.0.6" + resolved "https://registry.yarnpkg.com/date-extended/-/date-extended-0.0.6.tgz#23802d57dd1bf7818813fe0c32e851a86da267c9" + integrity sha512-v9a2QLTVn1GQGXf02TQaSvNfeXA/V1FL2Tr0OQYqjI5+L9T5jEtCpLYG01sxFk+m1OtwMxydkKa8NKcflANAoQ== + dependencies: + array-extended "~0.0.3" + extended "~0.0.3" + is-extended "~0.0.3" + debug@2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -552,6 +548,11 @@ debug@^3.2.7: dependencies: ms "^2.1.1" +declare.js@~0.0.4: + version "0.0.8" + resolved "https://registry.yarnpkg.com/declare.js/-/declare.js-0.0.8.tgz#0478adff9564c004f51df73d8bc134019d28dcde" + integrity sha512-O659hy1gcHef7JnwtqdQlrj2c5DAEgtxm8pgFXofW7eUE1L4FjsSLlziovWcrOJAOFlEPaOJshY+0hBWCG/AnA== + delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" @@ -682,13 +683,30 @@ express@^4.18.2: utils-merge "1.0.1" vary "~1.1.2" -fast-csv@^4.3.6: - version "4.3.6" - resolved "https://registry.yarnpkg.com/fast-csv/-/fast-csv-4.3.6.tgz#70349bdd8fe4d66b1130d8c91820b64a21bc4a63" - integrity sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw== +extended@0.0.6, extended@~0.0.3: + version "0.0.6" + resolved "https://registry.yarnpkg.com/extended/-/extended-0.0.6.tgz#7fb8bf7b9dae397586e48570acfd642c78e50669" + integrity sha512-rvAV3BDGsV1SYGzUOu7aO0k82quhfl0QAyZudYhAcTeIr1rPbBnyOhOlkCLwLpDfP7HyKAWAPNSjRb9p7lE3rg== + dependencies: + extender "~0.0.5" + +extender@~0.0.5: + version "0.0.10" + resolved "https://registry.yarnpkg.com/extender/-/extender-0.0.10.tgz#589c07482be61a1460b6d81f9c24aa67e8f324cd" + integrity sha512-iPLUHZJaNW6RuOShQX33ZpewEUIlijFBcsXnKWyiYERKWPsFxfKgx8J0xRz29hKQWPFFPACgBW6cHM7Ke1pfaA== + dependencies: + declare.js "~0.0.4" + +fast-csv@^2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/fast-csv/-/fast-csv-2.5.0.tgz#5332dfede3f59340cb8e9f46b2e6dff1e7612005" + integrity sha512-M/9ezLU9/uDwvDZTt9sNFJa0iLDUsbhYJwPtnE0D9MjeuB6DY9wRCyUPZta9iI6cSz5wBWGaUPL61QH8h92cNA== dependencies: - "@fast-csv/format" "4.3.5" - "@fast-csv/parse" "4.3.6" + extended "0.0.6" + is-extended "0.0.10" + object-extended "0.0.7" + safer-buffer "^2.1.2" + string-extended "0.0.8" fast-deep-equal@^3.1.1: version "3.1.3" @@ -887,6 +905,13 @@ is-core-module@^2.9.0: dependencies: has "^1.0.3" +is-extended@0.0.10, is-extended@~0.0.3, is-extended@~0.0.8: + version "0.0.10" + resolved "https://registry.yarnpkg.com/is-extended/-/is-extended-0.0.10.tgz#244e140df75bb1c9a3106f412ff182fb534a6d62" + integrity sha512-qp+HR+L9QXbgFurvqiVgD+JiGyUboRgICNzCXmbiLtZBFVSNFbxRsI4q7Be9mCWTO5PoO1IxoWp5sl+j5b83FA== + dependencies: + extended "~0.0.3" + is-extglob@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" @@ -957,46 +982,6 @@ locate-path@^5.0.0: dependencies: p-locate "^4.1.0" -lodash.escaperegexp@^4.1.2: - version "4.1.2" - resolved "https://registry.yarnpkg.com/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz#64762c48618082518ac3df4ccf5d5886dae20347" - integrity sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw== - -lodash.groupby@^4.6.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/lodash.groupby/-/lodash.groupby-4.6.0.tgz#0b08a1dcf68397c397855c3239783832df7403d1" - integrity sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw== - -lodash.isboolean@^3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6" - integrity sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg== - -lodash.isequal@^4.5.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0" - integrity sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ== - -lodash.isfunction@^3.0.9: - version "3.0.9" - resolved "https://registry.yarnpkg.com/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz#06de25df4db327ac931981d1bdb067e5af68d051" - integrity sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw== - -lodash.isnil@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/lodash.isnil/-/lodash.isnil-4.0.0.tgz#49e28cd559013458c814c5479d3c663a21bfaa6c" - integrity sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng== - -lodash.isundefined@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz#23ef3d9535565203a66cefd5b830f848911afb48" - integrity sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA== - -lodash.uniq@^4.5.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" - integrity sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ== - lodash@^4.17.21: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" @@ -1119,6 +1104,15 @@ normalize-path@^3.0.0, normalize-path@~3.0.0: resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== +object-extended@0.0.7: + version "0.0.7" + resolved "https://registry.yarnpkg.com/object-extended/-/object-extended-0.0.7.tgz#84fd23f56b15582aeb3e88b05cb55d2432d68a33" + integrity sha512-2LJYIacEXoZ1glGkAZuvA/4pfJM4Y1ShReAo9jWpBSuz89TiUCdiPqhGJJ6m97F3WjhCSRwrbgaxYEAm9dRYBw== + dependencies: + array-extended "~0.0.4" + extended "~0.0.3" + is-extended "~0.0.3" + object-inspect@^1.9.0: version "1.12.2" resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea" @@ -1294,7 +1288,7 @@ safe-buffer@5.2.1, safe-buffer@^5.1.0: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== -"safer-buffer@>= 2.1.2 < 3": +"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== @@ -1419,6 +1413,16 @@ statuses@2.0.1: resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== +string-extended@0.0.8: + version "0.0.8" + resolved "https://registry.yarnpkg.com/string-extended/-/string-extended-0.0.8.tgz#741957dff487b0272a79eec5a44f239ee6f17ccd" + integrity sha512-CK46p3AxBvBhJbBi6WrF9bCcaWH20E4NwlLSzpooG2nXWvcP2gy2YR8VN6fSwZyrbcvL4S4zoNKbR0QG52X4rw== + dependencies: + array-extended "~0.0.5" + date-extended "~0.0.3" + extended "~0.0.3" + is-extended "~0.0.3" + supports-color@^5.5.0: version "5.5.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" From cbd7e67161d34fede0b8169206aef2e2e1ff5d14 Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 30 Dec 2022 14:57:54 +0300 Subject: [PATCH 8/9] it works --- src/routes/tasks.ts | 10 ++++------ src/services/exporter.ts | 19 ++++++++++--------- src/types/index.ts | 7 ++++--- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts index 37ba9ca..2061a6e 100644 --- a/src/routes/tasks.ts +++ b/src/routes/tasks.ts @@ -35,20 +35,18 @@ async function getTasks(req, res) { let data = fs.readFileSync(path.join(EXPORTED_PATH, item), 'utf8'); let status = JSON.parse(data); - let requestedUrl = `http://${req.headers.host}`; - let downloadLink = ''; - let deleteLink = ''; + let filename = ''; if(status.status === 'finished') { - downloadLink = ``; + filename = file.name; } resp.push({ timestamp: status.time, - user: status.user, + username: status.user, datasourceRef: status.datasourceRef, rowsCount: status.exportedRows, progress: status.progress, status: status.status, - downloadLink, + filename, }); } diff --git a/src/services/exporter.ts b/src/services/exporter.ts index 8c94182..4f5ba90 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -21,7 +21,7 @@ const TIMESTAMP_COLUMN = 'timestamp'; export class Exporter { private exportedRows = 0; private createdTimestamp: number; - private user: string; + private username: string; private datasourceRef: DataSourceRef; private initCsvStream() { @@ -44,9 +44,9 @@ export class Exporter { let time = moment().valueOf(); let data = { time, - user: this.user, + username: this.username, exportedRows: this.exportedRows, - progress: progress.toLocaleString('en', { style: 'percent' }), + progress: progress, status, datasourceRef: this.datasourceRef, }; @@ -58,11 +58,11 @@ export class Exporter { } } - public async export(data: Target[], datasourceUrl: string, user: string, from: number, to: number) { - this.user = user; + public async export(data: Target[], datasourceUrl: string, username: string, from: number, to: number) { + this.username = username; this.validateTargets(datasourceUrl, data); - + // console.log('ds', data[0].datasource) const targets = data.map(target => { console.log({ @@ -72,17 +72,18 @@ export class Exporter { return { ...target, metric: new QueryConfig( - QueryType.GRAFANA, + QueryType.GRAFANA, { ...target.datasource, url: datasourceUrl - }, + }, target.panel.targets ) } }); - this.datasourceRef = data.length === 1 ? data[0].datasource : { uid: 'all', type: 'all' }; + const datasource = data[0].datasource; + this.datasourceRef = data.length === 1 ? { uid: datasource.uid, type: datasource.type } : { uid: 'all', type: 'all' }; await this.updateStatus(ExportStatus.EXPORTING, 0); diff --git a/src/types/index.ts b/src/types/index.ts index df63d77..ca9a1b5 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -80,7 +80,8 @@ export interface PanelModel { alert?: any; } -export type Task = DataQuery & { +// TODO: rename to query +export type Task = Omit & { selected: boolean; panel: PanelModel; datasource: DataSourceSettings; @@ -88,10 +89,10 @@ export type Task = DataQuery & { export type TaskTableRowConfig = { timestamp: number; - user: string; + username: string; datasourceRef: DataSourceRef; rowsCount: number; progress: number; status: string; - downloadLink: string; + filename?: string; }; From f3b52778c4dfc69f90a86c0440b20bd65edfe4cc Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 30 Dec 2022 15:29:00 +0300 Subject: [PATCH 9/9] fix username --- src/routes/tasks.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts index 2061a6e..c8d782b 100644 --- a/src/routes/tasks.ts +++ b/src/routes/tasks.ts @@ -34,19 +34,14 @@ async function getTasks(req, res) { // TODO: read async let data = fs.readFileSync(path.join(EXPORTED_PATH, item), 'utf8'); let status = JSON.parse(data); - - let filename = ''; - if(status.status === 'finished') { - filename = file.name; - } resp.push({ timestamp: status.time, - username: status.user, + username: status.username, datasourceRef: status.datasourceRef, rowsCount: status.exportedRows, progress: status.progress, status: status.status, - filename, + filename: file.name, }); } @@ -85,7 +80,7 @@ async function addTask(req: TRequest, res) { } async function deleteTask(req, res) { - let filename = req.query.filename; + let filename = req.body.filename; let csvFilePath = path.join(EXPORTED_PATH, `${filename}.csv`); let jsonFilePath = path.join(EXPORTED_PATH, `${filename}.json`);