From 45b3439f0830ce5945aa9ca94af6c3f19b76a3b9 Mon Sep 17 00:00:00 2001 From: vargburz Date: Fri, 28 Apr 2023 18:04:33 +0300 Subject: [PATCH 1/4] minor refactoring && renaming --- src/services/exporter.ts | 58 ++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/src/services/exporter.ts b/src/services/exporter.ts index 7c4c2e3..3a80f15 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -5,7 +5,7 @@ import { CSV_PATH } from '../config'; import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit'; // TODO: export QueryType directly from @corpglory/tsdb-kit -import { QueryType } from '@corpglory/tsdb-kit/lib/connectors'; +import { DataTable, QueryType } from '@corpglory/tsdb-kit/lib/connectors'; import { v4 as uuidv4 } from 'uuid'; @@ -63,42 +63,27 @@ export class Exporter { console.log(`Total days: ${days}`); const stream = this._initCsvStream(); + console.log("queryConfigs", queryConfigs); for(let day = 0; day < days; day++) { to = from + MS_IN_DAY; console.log(`${day} day: ${from}ms -> ${to}ms`); let columns = []; - let values = []; + let rows = []; for(const queryConfig of queryConfigs) { - const datasourceMetrics = await queryByConfig(queryConfig, datasourceUrl, from, to, apiKey); - - if(_.isEmpty(columns)) { - columns = datasourceMetrics.columns; - } else { - columns = _.concat(columns, datasourceMetrics.columns.slice(1)); - } - - if(_.isEmpty(values)) { - values = datasourceMetrics.values; - } else { - if(values.length !== datasourceMetrics.values.length) { - throw new Error(`All queries should return rows of the same lengths`); - } - for(const rowIdx in values) { - if(datasourceMetrics.values[rowIdx][0] !== values[rowIdx][0]) { - throw new Error('Queries should return the same timestamps'); - } - values[rowIdx] = _.concat(values[rowIdx], datasourceMetrics.values[rowIdx].slice(1)); - } - } + const datasourceMetrics: DataTable = await queryByConfig(queryConfig, datasourceUrl, from, to, apiKey); + columns = this._updateColumns(columns, datasourceMetrics.columns); + console.log("columns", columns) + rows = this._updateRows(rows, datasourceMetrics.values); + console.log("rows", rows.slice(0, 10)); } - values = values.map((row: number[]) => [toIsoString(row[0], timeZoneName), ...row.slice(1)]); + rows = rows.map((row: number[]) => [toIsoString(row[0], timeZoneName), ...row.slice(1)]); if(columns.length > 0) { - this._writeCsv(stream, { columns, values }); + this._writeCsv(stream, { columns, rows }); } await this._updateProgress({ status: ExportStatus.EXPORTING, progress: (day + 1) / days }); @@ -110,6 +95,27 @@ export class Exporter { } } + private _updateColumns(columnsToUpdate: string[], queriedColumns: string[]): string[] { + // slice(1) to avoid time fields + return _.concat(columnsToUpdate, queriedColumns.slice(1)); + } + + private _updateRows(rowsToUpdate: (number | null)[][], queriedRows: (number | null)[][]): (number | null)[][] { + if(_.isEmpty(rowsToUpdate)) { + return queriedRows; + } + if(rowsToUpdate.length !== queriedRows.length) { + throw new Error(`All queries should return rows of the same lengths`); + } + for(const rowIdx in rowsToUpdate) { + if(queriedRows[rowIdx][0] !== rowsToUpdate[rowIdx][0]) { + throw new Error('Queries should return the same timestamps'); + } + rowsToUpdate[rowIdx] = _.concat(rowsToUpdate[rowIdx], queriedRows[rowIdx].slice(1)); + } + return rowsToUpdate; + } + private _initCsvStream() { const csvStream = csv.createWriteStream({ headers: true, delimiter: this._task.csvDelimiter }) .on('error', async e => await this._updateProgress({ status: ExportStatus.ERROR, errorMessage: e.message })); @@ -161,7 +167,7 @@ export class Exporter { } private _writeCsv(stream, series) { - for(let row of series.values) { + for(let row of series.rows) { let csvRow = {}; for(let col in series.columns) { csvRow[series.columns[col]] = row[col]; From 65813cd00689d14991da4abbc7cc8fb5db183bfe Mon Sep 17 00:00:00 2001 From: vargburz Date: Fri, 28 Apr 2023 20:11:35 +0300 Subject: [PATCH 2/4] concat rows in correct way --- src/services/exporter.ts | 43 ++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/services/exporter.ts b/src/services/exporter.ts index 3a80f15..1908852 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -70,17 +70,15 @@ export class Exporter { console.log(`${day} day: ${from}ms -> ${to}ms`); let columns = []; - let rows = []; + let rowsDict = {}; for(const queryConfig of queryConfigs) { const datasourceMetrics: DataTable = await queryByConfig(queryConfig, datasourceUrl, from, to, apiKey); columns = this._updateColumns(columns, datasourceMetrics.columns); - console.log("columns", columns) - rows = this._updateRows(rows, datasourceMetrics.values); - console.log("rows", rows.slice(0, 10)); + rowsDict = this._updateRows(rowsDict, datasourceMetrics.values, datasourceMetrics.columns); } - - rows = rows.map((row: number[]) => [toIsoString(row[0], timeZoneName), ...row.slice(1)]); + const rows = this._getRowsFromDict(rowsDict, columns); + // const formattedRows = rows.map((row: number[]) => [toIsoString(row[0], timeZoneName), ...row.slice(1)]); if(columns.length > 0) { this._writeCsv(stream, { columns, rows }); @@ -100,20 +98,31 @@ export class Exporter { return _.concat(columnsToUpdate, queriedColumns.slice(1)); } - private _updateRows(rowsToUpdate: (number | null)[][], queriedRows: (number | null)[][]): (number | null)[][] { - if(_.isEmpty(rowsToUpdate)) { - return queriedRows; - } - if(rowsToUpdate.length !== queriedRows.length) { - throw new Error(`All queries should return rows of the same lengths`); + private _updateRows(dict, queriedRows: (number | null)[][], queriedColumns: string[]): any { + const columns = queriedColumns.slice(1); + for(const rowIdx in queriedRows) { + const key = queriedRows[rowIdx][0]; + const values = queriedRows[rowIdx].slice(1); + dict[key] = dict[key] || {}; + for(const valueIdx in values) { + dict[key][columns[valueIdx]] = values[valueIdx]; + } } - for(const rowIdx in rowsToUpdate) { - if(queriedRows[rowIdx][0] !== rowsToUpdate[rowIdx][0]) { - throw new Error('Queries should return the same timestamps'); + return dict; + } + + private _getRowsFromDict(dict: any, columns: string[]): (number | null)[][] { + let keyList = _.orderBy(_.keys(dict)); + let rows = []; + for(const keyIdx in keyList) { + const key = keyList[keyIdx]; + rows[keyIdx] = [key]; + for(const column of columns) { + const value = dict[key][column] || null; + rows[keyIdx].push(value) } - rowsToUpdate[rowIdx] = _.concat(rowsToUpdate[rowIdx], queriedRows[rowIdx].slice(1)); } - return rowsToUpdate; + return rows; } private _initCsvStream() { From 322620952d1e1ca77817c99b72d246fba2103bf6 Mon Sep 17 00:00:00 2001 From: vargburz Date: Wed, 3 May 2023 12:28:49 +0300 Subject: [PATCH 3/4] fix missing column --- src/services/exporter.ts | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/services/exporter.ts b/src/services/exporter.ts index 1908852..3333280 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -63,25 +63,29 @@ export class Exporter { console.log(`Total days: ${days}`); const stream = this._initCsvStream(); - console.log("queryConfigs", queryConfigs); + for(let day = 0; day < days; day++) { to = from + MS_IN_DAY; console.log(`${day} day: ${from}ms -> ${to}ms`); - let columns = []; + let metricColumns = []; + let keyColumn; let rowsDict = {}; - for(const queryConfig of queryConfigs) { const datasourceMetrics: DataTable = await queryByConfig(queryConfig, datasourceUrl, from, to, apiKey); - columns = this._updateColumns(columns, datasourceMetrics.columns); + if(keyColumn == undefined) { + keyColumn = datasourceMetrics.columns[0]; + } + metricColumns = this._updateColumns(metricColumns, datasourceMetrics.columns); rowsDict = this._updateRows(rowsDict, datasourceMetrics.values, datasourceMetrics.columns); } - const rows = this._getRowsFromDict(rowsDict, columns); - // const formattedRows = rows.map((row: number[]) => [toIsoString(row[0], timeZoneName), ...row.slice(1)]); + const rows = this._getRowsFromDict(rowsDict, metricColumns); + const columns = _.concat(keyColumn, metricColumns); + const formattedRows = rows.map((row: number[]) => [toIsoString(row[0], timeZoneName), ...row.slice(1)]); - if(columns.length > 0) { - this._writeCsv(stream, { columns, rows }); + if(metricColumns.length > 0) { + this._writeCsv(stream, { columns, rows: formattedRows }); } await this._updateProgress({ status: ExportStatus.EXPORTING, progress: (day + 1) / days }); @@ -98,7 +102,7 @@ export class Exporter { return _.concat(columnsToUpdate, queriedColumns.slice(1)); } - private _updateRows(dict, queriedRows: (number | null)[][], queriedColumns: string[]): any { + private _updateRows(dict, queriedRows: (number | null)[][], queriedColumns: string[]): { [key: number]: { [column: string]: number} } { const columns = queriedColumns.slice(1); for(const rowIdx in queriedRows) { const key = queriedRows[rowIdx][0]; @@ -115,7 +119,7 @@ export class Exporter { let keyList = _.orderBy(_.keys(dict)); let rows = []; for(const keyIdx in keyList) { - const key = keyList[keyIdx]; + const key = _.toNumber(keyList[keyIdx]); rows[keyIdx] = [key]; for(const column of columns) { const value = dict[key][column] || null; From ccbf72d17f3555592f19e236502c605ea46691d7 Mon Sep 17 00:00:00 2001 From: vargburz Date: Wed, 3 May 2023 14:56:38 +0300 Subject: [PATCH 4/4] fix ordering --- src/services/exporter.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/services/exporter.ts b/src/services/exporter.ts index 3333280..0778385 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -98,7 +98,7 @@ export class Exporter { } private _updateColumns(columnsToUpdate: string[], queriedColumns: string[]): string[] { - // slice(1) to avoid time fields + // slice(1) to avoid key fields return _.concat(columnsToUpdate, queriedColumns.slice(1)); } @@ -116,10 +116,11 @@ export class Exporter { } private _getRowsFromDict(dict: any, columns: string[]): (number | null)[][] { - let keyList = _.orderBy(_.keys(dict)); + const keyList = _.map(_.keys(dict), value => _.toNumber(value)); + const orderedKeyList = _.orderBy(keyList); let rows = []; - for(const keyIdx in keyList) { - const key = _.toNumber(keyList[keyIdx]); + for(const keyIdx in orderedKeyList) { + const key = orderedKeyList[keyIdx]; rows[keyIdx] = [key]; for(const column of columns) { const value = dict[key][column] || null;