From 45f5da66a332e5811ba285fa8955d6bd10482dba Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 30 Dec 2022 03:31:53 +0300 Subject: [PATCH] seems to work now --- package.json | 2 +- src/routes/tasks.ts | 83 +++++++++++++++++++++------------- src/services/exporter.ts | 53 ++++++++++++++-------- src/types/index.ts | 97 ++++++++++++++++++++++++++++++++++++++++ src/types/target.ts | 10 ++--- yarn.lock | 8 ++-- 6 files changed, 192 insertions(+), 61 deletions(-) create mode 100644 src/types/index.ts diff --git a/package.json b/package.json index 4b4ddf2..41827b4 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "license": "ISC", "dependencies": {}, "devDependencies": { - "@corpglory/tsdb-kit": "^2.0.1", + "@corpglory/tsdb-kit": "^2.0.2", "axios": "^1.2.1", "express": "^4.18.2", "fast-csv": "^4.3.6", diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts index b4359d7..9c5f646 100644 --- a/src/routes/tasks.ts +++ b/src/routes/tasks.ts @@ -2,7 +2,7 @@ import { Target } from '../types/target'; import { exporterFactory } from '../services/exporter.factory'; import { EXPORTED_PATH } from '../config'; -import { Datasource } from '@corpglory/tsdb-kit'; +import { Task, TaskTableRowConfig } from '../types'; import * as express from 'express'; @@ -13,55 +13,76 @@ type TRequest = { body: { from: string, to: string, - data: Array<{ - panelUrl: string, - panelTitle: string, - panelId: number, - datasourceRequest: Datasource, - datasourceName: string, - target: object, - }>, - user: string, - } + username: string, + tasks: Task[], + url: string, + }, }; async function getTasks(req, res) { - res.status(200).send([{ - timestamp: 12343567, - user: 'admin', - datasource: 'postgres', - rowsCount: 2345, - progress: 100, - status: 'Success', - }]); + const resp: TaskTableRowConfig[] = []; + fs.readdir(EXPORTED_PATH, (err, items) => { + if(err) { + console.error(err); + res.status(500).send('Something went wrong'); + } else { + for(let item of items) { + let file = path.parse(item); + if(file.ext !== '.json') { + continue; + } + // TODO: read async + let data = fs.readFileSync(path.join(EXPORTED_PATH, item), 'utf8'); + let status = JSON.parse(data); + + let requestedUrl = `http://${req.headers.host}`; + let downloadLink = ''; + let deleteLink = ''; + if(status.status === 'finished') { + downloadLink = ``; + } + resp.push({ + timestamp: status.time, + user: status.user, + datasourceRef: status.datasourceRef, + rowsCount: status.exportedRows, + progress: status.progress, + status: status.status, + downloadLink, + }); + } + + res.status(200).send(resp); + } + }); } async function addTask(req: TRequest, res) { + const body = req.body; + + const clientUrl = body.url; const from = parseInt(body.from); const to = parseInt(body.to); - const data = body.data; - const user = body.user; + const username = body.username; + const tasks = body.tasks; + + const datasourceUrl = `${new URL(clientUrl).origin}/api/ds/query`; if(isNaN(from) || isNaN(to)) { res.status(400).send('Range error: please fill both "from" and "to" fields'); } else if(from >= to) { res.status(400).send('Range error: "from" should be less than "to"'); } else { - const names = data.map(item => item.datasourceName).join(', '); + const names = tasks.map(item => item.datasource.name).join(', '); res.status(200).send(`Exporting ${names} data from ${new Date(from).toLocaleString()} to ${new Date(to).toLocaleString()}`); - const targets = data.map(item => new Target( - item.panelUrl, - item.panelTitle, - item.panelId, - item.datasourceRequest, - [item.target], - item.datasourceName, + const targets = tasks.map((task: Task) => new Target( + task.panel, + task.datasource, )); - const exporter = exporterFactory.getExporter(); - exporter.export(targets, user, from, to); + exporter.export(targets, datasourceUrl, username, from, to); } } diff --git a/src/services/exporter.ts b/src/services/exporter.ts index ab0b766..e7899b2 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -3,6 +3,7 @@ import { URL } from 'url'; import { apiKeys } from '../config'; import { promisify } from '../utils'; import { ExportStatus } from '../types/export-status'; +import { DataSourceRef } from '../types'; import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit'; // TODO: export QueryType directly from @corpglory/tsdb-kit @@ -21,11 +22,12 @@ export class Exporter { private exportedRows = 0; private createdTimestamp: number; private user: string; - private datasource: string; + private datasourceRef: DataSourceRef; private initCsvStream() { - // @ts-ignore - const csvStream = csv.createWriteStream({ headers: true }); + const csvStream = csv.parse({ headers: true }) + .on('error', error => console.error(error)); + const writableStream = fs.createWriteStream(this.getFilePath('csv')); csvStream.pipe(writableStream); @@ -46,7 +48,7 @@ export class Exporter { exportedRows: this.exportedRows, progress: progress.toLocaleString('en', { style: 'percent' }), status, - datasourceName: this.datasource, + datasourceRef: this.datasourceRef, }; await promisify(fs.writeFile, this.getFilePath('json'), JSON.stringify(data), 'utf8') @@ -56,16 +58,31 @@ export class Exporter { } } - public async export(data: Target[], user: string, from: number, to: number) { + public async export(data: Target[], datasourceUrl: string, user: string, from: number, to: number) { this.user = user; - this.validateTargets(data); - const targets = data.map(target => ({ - ...target, - metric: new QueryConfig(QueryType.GRAFANA, target.datasource, target.targets) - })); + this.validateTargets(datasourceUrl, data); + + // console.log('ds', data[0].datasource) + const targets = data.map(target => { + console.log({ + ...target.datasource, + url: datasourceUrl + }) + return { + ...target, + metric: new QueryConfig( + QueryType.GRAFANA, + { + ...target.datasource, + url: datasourceUrl + }, + target.panel.targets + ) + } + }); - this.datasource = data.length === 1 ? data[0].datasourceName : 'all'; + this.datasourceRef = data.length === 1 ? data[0].datasource : { uid: 'all', type: 'all' }; const stream = this.initCsvStream(); const days = Math.ceil((to - from) / MS_IN_DAY); @@ -81,13 +98,13 @@ export class Exporter { const values = {}; for(const [index, target] of targets.entries()) { - const host = new URL(target.panelUrl).origin; + const host = new URL(datasourceUrl).origin; const apiKey = apiKeys[host]; - const datasourceMetrics = await queryByConfig(target.metric, target.panelUrl, from, to, apiKey); + const datasourceMetrics = await queryByConfig(target.metric, datasourceUrl, from, to, apiKey); - const column = `${target.panelId}` + - `-${target.panelTitle.replace(' ', '-')}-${datasourceMetrics.columns[1]}`; + const column = `${target.panel.id}` + + `-${target.panel.title.replace(' ', '-')}-${datasourceMetrics.columns[1]}`; columns.push(column); @@ -120,13 +137,13 @@ export class Exporter { stream.end(); } - private validateTargets(targets: Target[]) { + private validateTargets(datasourceUrl, targets: Target[]) { if(!targets || !Array.isArray(targets)) { throw new Error('Incorrect targets format'); } for(const target of targets) { - const host = new URL(target.panelUrl).origin; + const host = new URL(datasourceUrl).origin; const apiKey = apiKeys[host]; if(apiKey === undefined || apiKey === '') { @@ -156,7 +173,7 @@ export class Exporter { if(this.createdTimestamp === undefined) { this.createdTimestamp = moment().valueOf(); } - return `${this.createdTimestamp}.${this.datasource}.${extension}`; + return `${this.createdTimestamp}.${this.datasourceRef.uid}.${extension}`; } private getFilePath(extension) { diff --git a/src/types/index.ts b/src/types/index.ts new file mode 100644 index 0000000..df63d77 --- /dev/null +++ b/src/types/index.ts @@ -0,0 +1,97 @@ +import { DatasourceType } from '@corpglory/tsdb-kit'; + +export interface DataSourceRef { + /** The plugin type-id */ + type?: string; + /** Specific datasource instance */ + uid?: string; +} + +export interface DataQuery { + /** + * A - Z + */ + refId: string; + /** + * true if query is disabled (ie should not be returned to the dashboard) + */ + hide?: boolean; + /** + * Unique, guid like, string used in explore mode + */ + key?: string; + /** + * Specify the query flavor + */ + queryType?: string; + /** + * For mixed data sources the selected datasource is on the query level. + * For non mixed scenarios this is undefined. + */ + datasource?: DataSourceRef | null; +} + +/** + * Data Source instance edit model. This is returned from: + * /api/datasources + */ +export interface DataSourceSettings { + id: number; + uid: string; + orgId: number; + name: string; + typeLogoUrl: string; + type: DatasourceType; + typeName: string; + access: string; + url: string; + user: string; + database: string; + basicAuth: boolean; + basicAuthUser: string; + isDefault: boolean; + jsonData: any; + secureJsonData?: any; + secureJsonFields: any; + readOnly: boolean; + withCredentials: boolean; + version?: number; + accessControl?: any; +} + +export interface PanelModel { + /** ID of the panel within the current dashboard */ + id: number; + /** Panel title */ + title?: string; + /** Description */ + description?: string; + /** Panel options */ + options: any; + /** Field options configuration */ + fieldConfig: any; + /** Version of the panel plugin */ + pluginVersion?: string; + /** The datasource used in all targets */ + datasource?: DataSourceRef | null; + /** The queries in a panel */ + targets?: DataQuery[]; + /** alerting v1 object */ + alert?: any; +} + +export type Task = DataQuery & { + selected: boolean; + panel: PanelModel; + datasource: DataSourceSettings; +}; + +export type TaskTableRowConfig = { + timestamp: number; + user: string; + datasourceRef: DataSourceRef; + rowsCount: number; + progress: number; + status: string; + downloadLink: string; +}; diff --git a/src/types/target.ts b/src/types/target.ts index 4ad80ae..d6639b3 100644 --- a/src/types/target.ts +++ b/src/types/target.ts @@ -1,12 +1,8 @@ -import { Datasource } from '@corpglory/tsdb-kit'; +import { DataSourceSettings, PanelModel } from '.'; export class Target { constructor( - public panelUrl: string, - public panelTitle: string, - public panelId: number, - public datasource: Datasource, - public targets: Array, - public datasourceName: string, + public panel: PanelModel, + public datasource: DataSourceSettings, ) {} } diff --git a/yarn.lock b/yarn.lock index 138d2f9..1b1ca59 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,10 +2,10 @@ # yarn lockfile v1 -"@corpglory/tsdb-kit@^2.0.1": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@corpglory/tsdb-kit/-/tsdb-kit-2.0.1.tgz#13435a69b2bb8c9890b838a06038daf76e96ec39" - integrity sha512-qIoCy0DXjPFkAE/G9URNVJ56vfU9SPyPoNR2wJSRut6vA6eLP8kljFf4GQE/7/yFpvulHI9+RT9UrlhImWbEZA== +"@corpglory/tsdb-kit@^2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@corpglory/tsdb-kit/-/tsdb-kit-2.0.2.tgz#0bbba2b344b651e374a1f89967abee7ef3172f30" + integrity sha512-USXpz9kXcHavJ0kZAuf64wE+zhmHfI9EKplyXtAAEL9FYZ3YErR2Awz7mAma6juKrkDELMtW/V0tx1e//0aRaw== dependencies: axios "^0.18.0" moment "^2.22.2"