|
|
|
@ -1,7 +1,6 @@
|
|
|
|
|
import { Target } from '../models/target'; |
|
|
|
|
import { URL } from 'url'; |
|
|
|
|
import { apiKeys } from '../config'; |
|
|
|
|
import { promisify } from '../utils'; |
|
|
|
|
import { promisify, toIsoString } from '../utils'; |
|
|
|
|
import { DashboardQuery, ExportProgress, ExportStatus, ExportTask } from '../types'; |
|
|
|
|
|
|
|
|
|
import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit'; |
|
|
|
@ -27,40 +26,36 @@ const DEFAULT_PROGRESS = {
|
|
|
|
|
export class Exporter { |
|
|
|
|
private _task: ExportTask; |
|
|
|
|
|
|
|
|
|
public async export(task: ExportTask, datasourceUrl: string) { |
|
|
|
|
this._task = _.cloneDeep(task); |
|
|
|
|
this._task.id = uuidv4(); |
|
|
|
|
this._task.progress = _.cloneDeep(DEFAULT_PROGRESS); |
|
|
|
|
|
|
|
|
|
const targets = task.queries.map((query: DashboardQuery) => new Target( |
|
|
|
|
query.panel, |
|
|
|
|
query.datasource, |
|
|
|
|
)); |
|
|
|
|
|
|
|
|
|
this._validateTargets(datasourceUrl, targets); |
|
|
|
|
|
|
|
|
|
await this._updateProgress(); |
|
|
|
|
|
|
|
|
|
const queryConfigs = targets.map( |
|
|
|
|
target => |
|
|
|
|
new QueryConfig( |
|
|
|
|
QueryType.GRAFANA, |
|
|
|
|
{ |
|
|
|
|
...target.datasource, |
|
|
|
|
url: datasourceUrl |
|
|
|
|
}, |
|
|
|
|
target.panel.targets |
|
|
|
|
) |
|
|
|
|
); |
|
|
|
|
public async export(task: ExportTask, datasourceUrl: string, timeZoneName: string) { |
|
|
|
|
try { |
|
|
|
|
this._task = _.cloneDeep(task); |
|
|
|
|
this._task.id = uuidv4(); |
|
|
|
|
this._task.progress = _.cloneDeep(DEFAULT_PROGRESS); |
|
|
|
|
|
|
|
|
|
this._validateQueries(task.queries); |
|
|
|
|
this._validateDatasourceUrl(datasourceUrl); |
|
|
|
|
|
|
|
|
|
await this._updateProgress(); |
|
|
|
|
|
|
|
|
|
const queryConfigs = task.queries.map( |
|
|
|
|
query => |
|
|
|
|
new QueryConfig( |
|
|
|
|
QueryType.GRAFANA, |
|
|
|
|
{ |
|
|
|
|
...query.datasource, |
|
|
|
|
url: datasourceUrl, |
|
|
|
|
}, |
|
|
|
|
[query.target] |
|
|
|
|
) |
|
|
|
|
); |
|
|
|
|
|
|
|
|
|
let from = +this._task.timeRange.from; |
|
|
|
|
let to = +this._task.timeRange.to; |
|
|
|
|
const days = Math.ceil((to - from) / MS_IN_DAY); |
|
|
|
|
let from = +this._task.timeRange.from; |
|
|
|
|
let to = +this._task.timeRange.to; |
|
|
|
|
const days = Math.ceil((to - from) / MS_IN_DAY); |
|
|
|
|
|
|
|
|
|
console.log(`Total days: ${days}`); |
|
|
|
|
console.log(`Total days: ${days}`); |
|
|
|
|
|
|
|
|
|
const stream = this._initCsvStream(); |
|
|
|
|
try { |
|
|
|
|
const stream = this._initCsvStream(); |
|
|
|
|
for(let day = 0; day < days; day++) { |
|
|
|
|
to = from + MS_IN_DAY; |
|
|
|
|
|
|
|
|
@ -75,21 +70,40 @@ export class Exporter {
|
|
|
|
|
|
|
|
|
|
const datasourceMetrics = await queryByConfig(queryConfig, datasourceUrl, from, to, apiKey); |
|
|
|
|
|
|
|
|
|
columns = datasourceMetrics.columns; |
|
|
|
|
values = datasourceMetrics.values; |
|
|
|
|
if(_.isEmpty(columns)) { |
|
|
|
|
columns = datasourceMetrics.columns; |
|
|
|
|
} else { |
|
|
|
|
columns = _.concat(columns, datasourceMetrics.columns.slice(1)); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if(_.isEmpty(values)) { |
|
|
|
|
values = datasourceMetrics.values; |
|
|
|
|
} else { |
|
|
|
|
if(values.length !== datasourceMetrics.values.length) { |
|
|
|
|
throw new Error(`All queries should return rows of the same lengths`); |
|
|
|
|
} |
|
|
|
|
for(const rowIdx in values) { |
|
|
|
|
if(datasourceMetrics.values[rowIdx][0] !== values[rowIdx][0]) { |
|
|
|
|
throw new Error('Queries should return the same timestamps'); |
|
|
|
|
} |
|
|
|
|
values[rowIdx] = _.concat(values[rowIdx], datasourceMetrics.values[rowIdx].slice(1)); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
values = values.map((row: number[]) => [toIsoString(row[0], timeZoneName), ...row.slice(1)]); |
|
|
|
|
|
|
|
|
|
if(columns.length > 0) { |
|
|
|
|
this._writeCsv(stream, { columns, values, }); |
|
|
|
|
this._writeCsv(stream, { columns, values }); |
|
|
|
|
} |
|
|
|
|
await this._updateProgress({ status: ExportStatus.EXPORTING, progress: (day + 1) / days }); |
|
|
|
|
|
|
|
|
|
from += MS_IN_DAY; |
|
|
|
|
} |
|
|
|
|
stream.end(); |
|
|
|
|
} catch (e) { |
|
|
|
|
await this._updateProgress({ status: ExportStatus.ERROR, errorMessage: e.message }); |
|
|
|
|
} |
|
|
|
|
stream.end(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private _initCsvStream() { |
|
|
|
@ -127,15 +141,22 @@ export class Exporter {
|
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private _validateTargets(datasourceUrl: string, targets: Target[]) { |
|
|
|
|
if(!targets || !Array.isArray(targets)) { |
|
|
|
|
throw new Error('Incorrect targets format'); |
|
|
|
|
private _validateQueries(queries: DashboardQuery[]) { |
|
|
|
|
if(!_.isArray(queries)) { |
|
|
|
|
throw new Error('`queries` field is required and should be an array'); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if(targets.length > 1) { |
|
|
|
|
throw new Error(`Multiple queries are not supported yet`); |
|
|
|
|
for(const query of queries) { |
|
|
|
|
if(_.isEmpty(query.datasource)) { |
|
|
|
|
throw new Error('all queries should have a `datasource` field'); |
|
|
|
|
} |
|
|
|
|
if(_.isEmpty(query.target)) { |
|
|
|
|
throw new Error('all queries should have a `target` field'); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private _validateDatasourceUrl(datasourceUrl: string) { |
|
|
|
|
const host = new URL(datasourceUrl).origin; |
|
|
|
|
const apiKey = apiKeys[host]; |
|
|
|
|
|
|
|
|
@ -146,18 +167,12 @@ export class Exporter {
|
|
|
|
|
|
|
|
|
|
private _writeCsv(stream, series) { |
|
|
|
|
for(let row of series.values) { |
|
|
|
|
const isEmpty = _.every( |
|
|
|
|
_.slice(row, 1), |
|
|
|
|
val => val === null |
|
|
|
|
); |
|
|
|
|
if(!isEmpty) { |
|
|
|
|
let csvRow = {}; |
|
|
|
|
for(let col in series.columns) { |
|
|
|
|
csvRow[series.columns[col]] = row[col]; |
|
|
|
|
} |
|
|
|
|
stream.write(csvRow); |
|
|
|
|
this._task.progress.exportedRowsCount++; |
|
|
|
|
let csvRow = {}; |
|
|
|
|
for(let col in series.columns) { |
|
|
|
|
csvRow[series.columns[col]] = row[col]; |
|
|
|
|
} |
|
|
|
|
stream.write(csvRow); |
|
|
|
|
this._task.progress.exportedRowsCount++; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|