From 5c88c5adbeed79f019fcc8aeb3231671acee9716 Mon Sep 17 00:00:00 2001 From: rozetko Date: Wed, 18 Jan 2023 18:27:42 +0300 Subject: [PATCH 1/7] add all errors to the progress --- src/services/exporter.ts | 66 ++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/src/services/exporter.ts b/src/services/exporter.ts index 7930739..5711ad3 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -28,39 +28,39 @@ export class Exporter { private _task: ExportTask; public async export(task: ExportTask, datasourceUrl: string) { - this._task = _.cloneDeep(task); - this._task.id = uuidv4(); - this._task.progress = _.cloneDeep(DEFAULT_PROGRESS); - - const targets = task.queries.map((query: DashboardQuery) => new Target( - query.panel, - query.datasource, - )); - - this._validateTargets(datasourceUrl, targets); - - await this._updateProgress(); - - const queryConfigs = targets.map( - target => - new QueryConfig( - QueryType.GRAFANA, - { - ...target.datasource, - url: datasourceUrl - }, - target.panel.targets - ) - ); - - let from = +this._task.timeRange.from; - let to = +this._task.timeRange.to; - const days = Math.ceil((to - from) / MS_IN_DAY); - - console.log(`Total days: ${days}`); - - const stream = this._initCsvStream(); try { + this._task = _.cloneDeep(task); + this._task.id = uuidv4(); + this._task.progress = _.cloneDeep(DEFAULT_PROGRESS); + + const targets = task.queries.map((query: DashboardQuery) => new Target( + query.panel, + query.datasource, + )); + + this._validateTargets(datasourceUrl, targets); + + await this._updateProgress(); + + const queryConfigs = targets.map( + target => + new QueryConfig( + QueryType.GRAFANA, + { + ...target.datasource, + url: datasourceUrl + }, + target.panel.targets + ) + ); + + let from = +this._task.timeRange.from; + let to = +this._task.timeRange.to; + const days = Math.ceil((to - from) / MS_IN_DAY); + + console.log(`Total days: ${days}`); + + const stream = this._initCsvStream(); for(let day = 0; day < days; day++) { to = from + MS_IN_DAY; @@ -86,10 +86,10 @@ export class Exporter { from += MS_IN_DAY; } + stream.end(); } catch (e) { await this._updateProgress({ status: ExportStatus.ERROR, errorMessage: e.message }); } - stream.end(); } private _initCsvStream() { From d1d63e88231d5c76d1fd000a437471152475e460 Mon Sep 17 00:00:00 2001 From: rozetko Date: Wed, 18 Jan 2023 18:38:09 +0300 Subject: [PATCH 2/7] fix queries sometimes failing --- src/routes/tasks.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts index c03ef20..fa0d902 100644 --- a/src/routes/tasks.ts +++ b/src/routes/tasks.ts @@ -31,8 +31,12 @@ async function getTasks(req, res) { } // TODO: read async let data = fs.readFileSync(path.join(EXPORTED_PATH, item), 'utf8'); - let status = JSON.parse(data); - resp.push(status); + try { + let status = JSON.parse(data); + resp.push(status); + } catch(e) { + console.log(`Cannot read file /exporter/${item}. if this error doesn't repeat, maybe the file is being updated at the moment`); + } } res.status(200).send(resp); From e8f6a53d925352dcab0f22d9ac6ee846408bc247 Mon Sep 17 00:00:00 2001 From: rozetko Date: Wed, 18 Jan 2023 18:38:36 +0300 Subject: [PATCH 3/7] write empty values to the csv --- src/services/exporter.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/services/exporter.ts b/src/services/exporter.ts index 5711ad3..2654f8d 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -146,18 +146,12 @@ export class Exporter { private _writeCsv(stream, series) { for(let row of series.values) { - const isEmpty = _.every( - _.slice(row, 1), - val => val === null - ); - if(!isEmpty) { - let csvRow = {}; - for(let col in series.columns) { - csvRow[series.columns[col]] = row[col]; - } - stream.write(csvRow); - this._task.progress.exportedRowsCount++; + let csvRow = {}; + for(let col in series.columns) { + csvRow[series.columns[col]] = row[col]; } + stream.write(csvRow); + this._task.progress.exportedRowsCount++; } } From 3b0b8162c72e386f6a84ef91394fab70a200a02d Mon Sep 17 00:00:00 2001 From: rozetko Date: Wed, 18 Jan 2023 18:45:42 +0300 Subject: [PATCH 4/7] minor fix --- src/routes/tasks.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts index fa0d902..1d3c702 100644 --- a/src/routes/tasks.ts +++ b/src/routes/tasks.ts @@ -22,7 +22,7 @@ async function getTasks(req, res) { fs.readdir(EXPORTED_PATH, (err, items) => { if(err) { console.error(err); - res.status(500).send('Something went wrong'); + res.status(500).send(err.message); } else { for(let item of items) { let file = path.parse(item); From c813d21151e01d9c4813862dd2962cdd138396db Mon Sep 17 00:00:00 2001 From: rozetko Date: Wed, 18 Jan 2023 18:51:45 +0300 Subject: [PATCH 5/7] better task validation --- src/services/exporter.ts | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/services/exporter.ts b/src/services/exporter.ts index 2654f8d..a1fe347 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -33,13 +33,14 @@ export class Exporter { this._task.id = uuidv4(); this._task.progress = _.cloneDeep(DEFAULT_PROGRESS); + this._validateQueries(task.queries); + this._validateDatasourceUrl(datasourceUrl); + const targets = task.queries.map((query: DashboardQuery) => new Target( query.panel, query.datasource, )); - this._validateTargets(datasourceUrl, targets); - await this._updateProgress(); const queryConfigs = targets.map( @@ -127,15 +128,22 @@ export class Exporter { } } - private _validateTargets(datasourceUrl: string, targets: Target[]) { - if(!targets || !Array.isArray(targets)) { - throw new Error('Incorrect targets format'); + private _validateQueries(queries: DashboardQuery[]) { + if(!queries || !Array.isArray(queries)) { + throw new Error('`queries` field is required and should be an array'); } - if(targets.length > 1) { - throw new Error(`Multiple queries are not supported yet`); + for(const query of queries) { + if(!_.isEmpty(query.datasource)) { + throw new Error('all queries should have a `datasource` field'); + } + if(!_.isEmpty(query.panel.targets)) { + throw new Error('all queries should have a `panel` field with non-empty `targets` field'); + } } + } + private _validateDatasourceUrl(datasourceUrl: string) { const host = new URL(datasourceUrl).origin; const apiKey = apiKeys[host]; From 5d6e7a6178f0c3662b897bd7f58821bd7ecf5dc4 Mon Sep 17 00:00:00 2001 From: rozetko Date: Wed, 18 Jan 2023 19:00:47 +0300 Subject: [PATCH 6/7] hotfix --- src/services/exporter.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/exporter.ts b/src/services/exporter.ts index a1fe347..e35bbb3 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -129,15 +129,15 @@ export class Exporter { } private _validateQueries(queries: DashboardQuery[]) { - if(!queries || !Array.isArray(queries)) { + if(!_.isArray(queries)) { throw new Error('`queries` field is required and should be an array'); } for(const query of queries) { - if(!_.isEmpty(query.datasource)) { + if(_.isEmpty(query.datasource)) { throw new Error('all queries should have a `datasource` field'); } - if(!_.isEmpty(query.panel.targets)) { + if(_.isEmpty(query.panel.targets)) { throw new Error('all queries should have a `panel` field with non-empty `targets` field'); } } From 66b84718484e589477966110c17ae4d14c590d7c Mon Sep 17 00:00:00 2001 From: rozetko Date: Thu, 19 Jan 2023 14:40:13 +0300 Subject: [PATCH 7/7] convert timestamps to dates --- package.json | 1 + src/models/target.ts | 8 ------- src/routes/tasks.ts | 3 ++- src/services/exporter.ts | 49 +++++++++++++++++++++++++--------------- src/types.ts | 3 ++- src/utils.ts | 6 +++++ yarn.lock | 9 +++++++- 7 files changed, 50 insertions(+), 29 deletions(-) delete mode 100644 src/models/target.ts diff --git a/package.json b/package.json index d44c232..828f7cd 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "fast-csv": "^2.5.0", "lodash": "^4.17.21", "moment": "^2.29.4", + "moment-timezone": "^0.5.40", "nodemon": "^2.0.20", "ts-loader": "^9.4.2", "typescript": "^4.9.4", diff --git a/src/models/target.ts b/src/models/target.ts deleted file mode 100644 index ca24bdf..0000000 --- a/src/models/target.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { DataSourceSettings, PanelModel } from '../types'; - -export class Target { - constructor( - public panel: PanelModel, - public datasource: DataSourceSettings, - ) {} -} diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts index 1d3c702..ecf19be 100644 --- a/src/routes/tasks.ts +++ b/src/routes/tasks.ts @@ -14,6 +14,7 @@ type TRequest = { body: { task: ExportTask, url: string, + timeZoneName: string, }, }; @@ -67,7 +68,7 @@ async function addTask(req: TRequest, res) { } const exporter = exporterFactory.getExporter(); - exporter.export(task, datasourceUrl); + exporter.export(task, datasourceUrl, body.timeZoneName); res.status(200).send(`Export process started`); } diff --git a/src/services/exporter.ts b/src/services/exporter.ts index e35bbb3..8d63013 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -1,7 +1,6 @@ -import { Target } from '../models/target'; import { URL } from 'url'; import { apiKeys } from '../config'; -import { promisify } from '../utils'; +import { promisify, toIsoString } from '../utils'; import { DashboardQuery, ExportProgress, ExportStatus, ExportTask } from '../types'; import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit'; @@ -27,7 +26,7 @@ const DEFAULT_PROGRESS = { export class Exporter { private _task: ExportTask; - public async export(task: ExportTask, datasourceUrl: string) { + public async export(task: ExportTask, datasourceUrl: string, timeZoneName: string) { try { this._task = _.cloneDeep(task); this._task.id = uuidv4(); @@ -36,22 +35,17 @@ export class Exporter { this._validateQueries(task.queries); this._validateDatasourceUrl(datasourceUrl); - const targets = task.queries.map((query: DashboardQuery) => new Target( - query.panel, - query.datasource, - )); - await this._updateProgress(); - const queryConfigs = targets.map( - target => + const queryConfigs = task.queries.map( + query => new QueryConfig( QueryType.GRAFANA, { - ...target.datasource, - url: datasourceUrl + ...query.datasource, + url: datasourceUrl, }, - target.panel.targets + [query.target] ) ); @@ -76,12 +70,31 @@ export class Exporter { const datasourceMetrics = await queryByConfig(queryConfig, datasourceUrl, from, to, apiKey); - columns = datasourceMetrics.columns; - values = datasourceMetrics.values; + if(_.isEmpty(columns)) { + columns = datasourceMetrics.columns; + } else { + columns = _.concat(columns, datasourceMetrics.columns.slice(1)); + } + + if(_.isEmpty(values)) { + values = datasourceMetrics.values; + } else { + if(values.length !== datasourceMetrics.values.length) { + throw new Error(`All queries should return rows of the same lengths`); + } + for(const rowIdx in values) { + if(datasourceMetrics.values[rowIdx][0] !== values[rowIdx][0]) { + throw new Error('Queries should return the same timestamps'); + } + values[rowIdx] = _.concat(values[rowIdx], datasourceMetrics.values[rowIdx].slice(1)); + } + } } + values = values.map((row: number[]) => [toIsoString(row[0], timeZoneName), ...row.slice(1)]); + if(columns.length > 0) { - this._writeCsv(stream, { columns, values, }); + this._writeCsv(stream, { columns, values }); } await this._updateProgress({ status: ExportStatus.EXPORTING, progress: (day + 1) / days }); @@ -137,8 +150,8 @@ export class Exporter { if(_.isEmpty(query.datasource)) { throw new Error('all queries should have a `datasource` field'); } - if(_.isEmpty(query.panel.targets)) { - throw new Error('all queries should have a `panel` field with non-empty `targets` field'); + if(_.isEmpty(query.target)) { + throw new Error('all queries should have a `target` field'); } } } diff --git a/src/types.ts b/src/types.ts index 05aa249..16ae109 100644 --- a/src/types.ts +++ b/src/types.ts @@ -105,8 +105,9 @@ export type ExportTask = { id?: string; }; -export type DashboardQuery = DataQuery & { +export type DashboardQuery = { selected: boolean; + target: DataQuery; panel: PanelModel; datasource: DataSourceSettings; }; diff --git a/src/utils.ts b/src/utils.ts index 094d51c..a1fcc7f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,3 +1,5 @@ +import * as moment from 'moment-timezone'; + export async function promisify(method: (...params: any[]) => Promise | void, ...params: any[]) { return new Promise((resolve, reject) => { method(...params, (err, result) => { @@ -9,3 +11,7 @@ export async function promisify(method: (...params: any[]) => Promise | voi }) }); } + +export function toIsoString(msTimestamp: number, timeZone: string): string { + return moment.tz(msTimestamp, timeZone).format('YYYY-MM-DD HH:mm:ssZ').replace(/:00$/, ''); +} diff --git a/yarn.lock b/yarn.lock index 5462164..9d920c4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1046,7 +1046,14 @@ minimatch@^3.1.2: dependencies: brace-expansion "^1.1.7" -moment@^2.22.2, moment@^2.29.4: +moment-timezone@^0.5.40: + version "0.5.40" + resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.40.tgz#c148f5149fd91dd3e29bf481abc8830ecba16b89" + integrity sha512-tWfmNkRYmBkPJz5mr9GVDn9vRlVZOTe6yqY92rFxiOdWXbjaR0+9LwQnZGGuNR63X456NqmEkbskte8tWL5ePg== + dependencies: + moment ">= 2.9.0" + +"moment@>= 2.9.0", moment@^2.22.2, moment@^2.29.4: version "2.29.4" resolved "https://registry.yarnpkg.com/moment/-/moment-2.29.4.tgz#3dbe052889fe7c1b2ed966fcb3a77328964ef108" integrity sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==