|
|
|
@ -1,14 +1,15 @@
|
|
|
|
|
import { Target } from '../types/target'; |
|
|
|
|
import { Target } from '../models/target'; |
|
|
|
|
import { URL } from 'url'; |
|
|
|
|
import { apiKeys } from '../config'; |
|
|
|
|
import { promisify } from '../utils'; |
|
|
|
|
import { ExportStatus } from '../types/export-status'; |
|
|
|
|
import { DataSourceRef } from '../types'; |
|
|
|
|
import { DashboardQuery, ExportProgress, ExportStatus, ExportTask } from '../types'; |
|
|
|
|
|
|
|
|
|
import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit'; |
|
|
|
|
// TODO: export QueryType directly from @corpglory/tsdb-kit
|
|
|
|
|
import { QueryType } from '@corpglory/tsdb-kit/lib/connectors'; |
|
|
|
|
|
|
|
|
|
import { v4 as uuidv4 } from 'uuid'; |
|
|
|
|
|
|
|
|
|
import * as moment from 'moment'; |
|
|
|
|
import * as csv from 'fast-csv'; |
|
|
|
|
import * as fs from 'fs'; |
|
|
|
@ -17,60 +18,32 @@ import * as _ from 'lodash';
|
|
|
|
|
|
|
|
|
|
const MS_IN_DAY = 24 * 60 * 60 * 1000; |
|
|
|
|
|
|
|
|
|
const DEFAULT_PROGRESS = { |
|
|
|
|
exportedRowsCount: 0, |
|
|
|
|
progress: 0, |
|
|
|
|
status: ExportStatus.EXPORTING, |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
export class Exporter { |
|
|
|
|
private exportedRows = 0; |
|
|
|
|
private createdTimestamp: number; |
|
|
|
|
private username: string; |
|
|
|
|
private datasourceRef: DataSourceRef; |
|
|
|
|
private _task: ExportTask; |
|
|
|
|
|
|
|
|
|
private initCsvStream() { |
|
|
|
|
const csvStream = csv.createWriteStream({ headers: true }) |
|
|
|
|
.on('error', error => console.error(error)); |
|
|
|
|
public async export(task: ExportTask, datasourceUrl: string) { |
|
|
|
|
this._task = _.cloneDeep(task); |
|
|
|
|
this._task.id = uuidv4(); |
|
|
|
|
this._task.progress = _.cloneDeep(DEFAULT_PROGRESS); |
|
|
|
|
|
|
|
|
|
const writableStream = fs.createWriteStream(this.getFilePath('csv')); |
|
|
|
|
const targets = task.queries.map((query: DashboardQuery) => new Target( |
|
|
|
|
query.panel, |
|
|
|
|
query.datasource, |
|
|
|
|
)); |
|
|
|
|
|
|
|
|
|
csvStream.pipe(writableStream); |
|
|
|
|
writableStream.on('finish', async () => { |
|
|
|
|
console.log(`Everything is written to ${this.getFilename('csv')}`); |
|
|
|
|
await this.updateStatus(ExportStatus.FINISHED, 1); |
|
|
|
|
}) |
|
|
|
|
this._validateTargets(datasourceUrl, targets); |
|
|
|
|
|
|
|
|
|
return csvStream; |
|
|
|
|
} |
|
|
|
|
await this._updateProgress(); |
|
|
|
|
|
|
|
|
|
public async updateStatus(status: string, progress: number) { |
|
|
|
|
try { |
|
|
|
|
let time = moment().valueOf(); |
|
|
|
|
let data = { |
|
|
|
|
time, |
|
|
|
|
username: this.username, |
|
|
|
|
exportedRows: this.exportedRows, |
|
|
|
|
progress: progress, |
|
|
|
|
status, |
|
|
|
|
datasourceRef: this.datasourceRef, |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
await promisify(fs.writeFile, this.getFilePath('json'), JSON.stringify(data), 'utf8') |
|
|
|
|
} catch(err) { |
|
|
|
|
console.error(err); |
|
|
|
|
throw new Error('Can`t write file'); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// TODO: rename `data` to `targets` or `queries`
|
|
|
|
|
public async export(data: Target[], datasourceUrl: string, username: string, from: number, to: number) { |
|
|
|
|
this.username = username; |
|
|
|
|
|
|
|
|
|
this.validateTargets(datasourceUrl, data); |
|
|
|
|
|
|
|
|
|
const targets = data.map(target => { |
|
|
|
|
console.log('target', { |
|
|
|
|
...target.datasource, |
|
|
|
|
url: datasourceUrl |
|
|
|
|
}); |
|
|
|
|
return { |
|
|
|
|
...target, |
|
|
|
|
metric: new QueryConfig( |
|
|
|
|
const queryConfigs = targets.map( |
|
|
|
|
target => |
|
|
|
|
new QueryConfig( |
|
|
|
|
QueryType.GRAFANA, |
|
|
|
|
{ |
|
|
|
|
...target.datasource, |
|
|
|
@ -78,52 +51,83 @@ export class Exporter {
|
|
|
|
|
}, |
|
|
|
|
target.panel.targets |
|
|
|
|
) |
|
|
|
|
} |
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
const datasource = data[0].datasource; |
|
|
|
|
this.datasourceRef = { uid: datasource.uid, type: datasource.type }; |
|
|
|
|
); |
|
|
|
|
|
|
|
|
|
await this.updateStatus(ExportStatus.EXPORTING, 0); |
|
|
|
|
|
|
|
|
|
const stream = this.initCsvStream(); |
|
|
|
|
let from = +this._task.timeRange.from; |
|
|
|
|
let to = +this._task.timeRange.to; |
|
|
|
|
const days = Math.ceil((to - from) / MS_IN_DAY); |
|
|
|
|
|
|
|
|
|
console.log(`Total days: ${days}`); |
|
|
|
|
|
|
|
|
|
for(let day = 0; day < days; day++) { |
|
|
|
|
to = from + MS_IN_DAY; |
|
|
|
|
const stream = this._initCsvStream(); |
|
|
|
|
try { |
|
|
|
|
for(let day = 0; day < days; day++) { |
|
|
|
|
to = from + MS_IN_DAY; |
|
|
|
|
|
|
|
|
|
console.log(`${day} day: ${from}ms -> ${to}ms`); |
|
|
|
|
|
|
|
|
|
console.log(`${day} day: ${from}ms -> ${to}ms`); |
|
|
|
|
let columns = []; |
|
|
|
|
let values = []; |
|
|
|
|
|
|
|
|
|
let columns = []; |
|
|
|
|
let values = []; |
|
|
|
|
for(const queryConfig of queryConfigs) { |
|
|
|
|
const host = new URL(datasourceUrl).origin; |
|
|
|
|
const apiKey = apiKeys[host]; |
|
|
|
|
|
|
|
|
|
for(const [index, target] of targets.entries()) { |
|
|
|
|
const host = new URL(datasourceUrl).origin; |
|
|
|
|
const apiKey = apiKeys[host]; |
|
|
|
|
const datasourceMetrics = await queryByConfig(queryConfig, datasourceUrl, from, to, apiKey); |
|
|
|
|
|
|
|
|
|
const datasourceMetrics = await queryByConfig(target.metric, datasourceUrl, from, to, apiKey); |
|
|
|
|
columns = datasourceMetrics.columns; |
|
|
|
|
values = datasourceMetrics.values; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if(columns.length > 0) { |
|
|
|
|
this._writeCsv(stream, { columns, values, }); |
|
|
|
|
} |
|
|
|
|
await this._updateProgress({ status: ExportStatus.EXPORTING, progress: (day + 1) / days }); |
|
|
|
|
|
|
|
|
|
columns = datasourceMetrics.columns; |
|
|
|
|
values = datasourceMetrics.values; |
|
|
|
|
from += MS_IN_DAY; |
|
|
|
|
} |
|
|
|
|
} catch (e) { |
|
|
|
|
await this._updateProgress({ status: ExportStatus.ERROR, errorMessage: e.message }); |
|
|
|
|
} |
|
|
|
|
stream.end(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if(columns.length > 0) { |
|
|
|
|
console.log('values', values); |
|
|
|
|
this.writeCsv(stream, { |
|
|
|
|
columns, |
|
|
|
|
values, |
|
|
|
|
}); |
|
|
|
|
private _initCsvStream() { |
|
|
|
|
const csvStream = csv.createWriteStream({ headers: true }) |
|
|
|
|
.on('error', async e => await this._updateProgress({ status: ExportStatus.ERROR, errorMessage: e.message })); |
|
|
|
|
|
|
|
|
|
const writableStream = fs.createWriteStream(this._getFilePath('csv')); |
|
|
|
|
|
|
|
|
|
csvStream.pipe(writableStream); |
|
|
|
|
writableStream.on('finish', async () => { |
|
|
|
|
if(this._task.progress.status !== ExportStatus.ERROR) { |
|
|
|
|
console.log(`Everything is written to ${this._getFilename('csv')}`); |
|
|
|
|
await this._updateProgress({ status: ExportStatus.FINISHED, progress: 1 }); |
|
|
|
|
} else { |
|
|
|
|
console.log(`${this._getFilename('csv')} export is finished with error`); |
|
|
|
|
} |
|
|
|
|
await this.updateStatus(ExportStatus.EXPORTING, (day + 1) / days); |
|
|
|
|
}); |
|
|
|
|
|
|
|
|
|
from += MS_IN_DAY; |
|
|
|
|
return csvStream; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private async _updateProgress(progress?: Partial<ExportProgress>) { |
|
|
|
|
try { |
|
|
|
|
let time = moment().valueOf(); |
|
|
|
|
|
|
|
|
|
const data = { |
|
|
|
|
...this._task, |
|
|
|
|
progress: _.assign(this._task.progress, progress, { time }), |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
await promisify(fs.writeFile, this._getFilePath('json'), JSON.stringify(data), 'utf8'); |
|
|
|
|
} catch(err) { |
|
|
|
|
console.error(err); |
|
|
|
|
throw new Error('Can`t write file'); |
|
|
|
|
} |
|
|
|
|
stream.end(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private validateTargets(datasourceUrl: string, targets: Target[]) { |
|
|
|
|
private _validateTargets(datasourceUrl: string, targets: Target[]) { |
|
|
|
|
if(!targets || !Array.isArray(targets)) { |
|
|
|
|
throw new Error('Incorrect targets format'); |
|
|
|
|
} |
|
|
|
@ -140,7 +144,7 @@ export class Exporter {
|
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private writeCsv(stream, series) { |
|
|
|
|
private _writeCsv(stream, series) { |
|
|
|
|
for(let row of series.values) { |
|
|
|
|
const isEmpty = _.every( |
|
|
|
|
_.slice(row, 1), |
|
|
|
@ -152,20 +156,17 @@ export class Exporter {
|
|
|
|
|
csvRow[series.columns[col]] = row[col]; |
|
|
|
|
} |
|
|
|
|
stream.write(csvRow); |
|
|
|
|
this.exportedRows++; |
|
|
|
|
this._task.progress.exportedRowsCount++; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private getFilename(extension) { |
|
|
|
|
if(this.createdTimestamp === undefined) { |
|
|
|
|
this.createdTimestamp = moment().valueOf(); |
|
|
|
|
} |
|
|
|
|
return `${this.createdTimestamp}.${this.datasourceRef.uid}.${extension}`; |
|
|
|
|
private _getFilename(extension: string): string { |
|
|
|
|
return `${this._task.id}.${extension}`; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private getFilePath(extension) { |
|
|
|
|
let filename = this.getFilename(extension); |
|
|
|
|
private _getFilePath(extension: string): string { |
|
|
|
|
let filename = this._getFilename(extension); |
|
|
|
|
return path.join(__dirname, `../exported/${filename}`); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|