|
|
|
import { Target } from '../models/target';
|
|
|
|
import { URL } from 'url';
|
|
|
|
import { apiKeys } from '../config';
|
|
|
|
import { promisify } from '../utils';
|
|
|
|
import { DashboardQuery, ExportProgress, ExportStatus, ExportTask } from '../types';
|
|
|
|
|
|
|
|
import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit';
|
|
|
|
// TODO: export QueryType directly from @corpglory/tsdb-kit
|
|
|
|
import { QueryType } from '@corpglory/tsdb-kit/lib/connectors';
|
|
|
|
|
|
|
|
import { v4 as uuidv4 } from 'uuid';
|
|
|
|
|
|
|
|
import * as moment from 'moment';
|
|
|
|
import * as csv from 'fast-csv';
|
|
|
|
import * as fs from 'fs';
|
|
|
|
import * as path from 'path';
|
|
|
|
import * as _ from 'lodash';
|
|
|
|
|
|
|
|
const MS_IN_DAY = 24 * 60 * 60 * 1000;
|
|
|
|
|
|
|
|
const DEFAULT_PROGRESS = {
|
|
|
|
exportedRowsCount: 0,
|
|
|
|
progress: 0,
|
|
|
|
status: ExportStatus.EXPORTING,
|
|
|
|
};
|
|
|
|
|
|
|
|
export class Exporter {
|
|
|
|
private _task: ExportTask;
|
|
|
|
|
|
|
|
public async export(task: ExportTask, datasourceUrl: string) {
|
|
|
|
try {
|
|
|
|
this._task = _.cloneDeep(task);
|
|
|
|
this._task.id = uuidv4();
|
|
|
|
this._task.progress = _.cloneDeep(DEFAULT_PROGRESS);
|
|
|
|
|
|
|
|
const targets = task.queries.map((query: DashboardQuery) => new Target(
|
|
|
|
query.panel,
|
|
|
|
query.datasource,
|
|
|
|
));
|
|
|
|
|
|
|
|
this._validateTargets(datasourceUrl, targets);
|
|
|
|
|
|
|
|
await this._updateProgress();
|
|
|
|
|
|
|
|
const queryConfigs = targets.map(
|
|
|
|
target =>
|
|
|
|
new QueryConfig(
|
|
|
|
QueryType.GRAFANA,
|
|
|
|
{
|
|
|
|
...target.datasource,
|
|
|
|
url: datasourceUrl
|
|
|
|
},
|
|
|
|
target.panel.targets
|
|
|
|
)
|
|
|
|
);
|
|
|
|
|
|
|
|
let from = +this._task.timeRange.from;
|
|
|
|
let to = +this._task.timeRange.to;
|
|
|
|
const days = Math.ceil((to - from) / MS_IN_DAY);
|
|
|
|
|
|
|
|
console.log(`Total days: ${days}`);
|
|
|
|
|
|
|
|
const stream = this._initCsvStream();
|
|
|
|
for(let day = 0; day < days; day++) {
|
|
|
|
to = from + MS_IN_DAY;
|
|
|
|
|
|
|
|
console.log(`${day} day: ${from}ms -> ${to}ms`);
|
|
|
|
|
|
|
|
let columns = [];
|
|
|
|
let values = [];
|
|
|
|
|
|
|
|
for(const queryConfig of queryConfigs) {
|
|
|
|
const host = new URL(datasourceUrl).origin;
|
|
|
|
const apiKey = apiKeys[host];
|
|
|
|
|
|
|
|
const datasourceMetrics = await queryByConfig(queryConfig, datasourceUrl, from, to, apiKey);
|
|
|
|
|
|
|
|
columns = datasourceMetrics.columns;
|
|
|
|
values = datasourceMetrics.values;
|
|
|
|
}
|
|
|
|
|
|
|
|
if(columns.length > 0) {
|
|
|
|
this._writeCsv(stream, { columns, values, });
|
|
|
|
}
|
|
|
|
await this._updateProgress({ status: ExportStatus.EXPORTING, progress: (day + 1) / days });
|
|
|
|
|
|
|
|
from += MS_IN_DAY;
|
|
|
|
}
|
|
|
|
stream.end();
|
|
|
|
} catch (e) {
|
|
|
|
await this._updateProgress({ status: ExportStatus.ERROR, errorMessage: e.message });
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private _initCsvStream() {
|
|
|
|
const csvStream = csv.createWriteStream({ headers: true })
|
|
|
|
.on('error', async e => await this._updateProgress({ status: ExportStatus.ERROR, errorMessage: e.message }));
|
|
|
|
|
|
|
|
const writableStream = fs.createWriteStream(this._getFilePath('csv'));
|
|
|
|
|
|
|
|
csvStream.pipe(writableStream);
|
|
|
|
writableStream.on('finish', async () => {
|
|
|
|
if(this._task.progress.status !== ExportStatus.ERROR) {
|
|
|
|
console.log(`Everything is written to ${this._getFilename('csv')}`);
|
|
|
|
await this._updateProgress({ status: ExportStatus.FINISHED, progress: 1 });
|
|
|
|
} else {
|
|
|
|
console.log(`${this._getFilename('csv')} export is finished with error`);
|
|
|
|
}
|
|
|
|
});
|
|
|
|
|
|
|
|
return csvStream;
|
|
|
|
}
|
|
|
|
|
|
|
|
private async _updateProgress(progress?: Partial<ExportProgress>) {
|
|
|
|
try {
|
|
|
|
let time = moment().valueOf();
|
|
|
|
|
|
|
|
const data = {
|
|
|
|
...this._task,
|
|
|
|
progress: _.assign(this._task.progress, progress, { time }),
|
|
|
|
};
|
|
|
|
|
|
|
|
await promisify(fs.writeFile, this._getFilePath('json'), JSON.stringify(data), 'utf8');
|
|
|
|
} catch(err) {
|
|
|
|
console.error(err);
|
|
|
|
throw new Error('Can`t write file');
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private _validateTargets(datasourceUrl: string, targets: Target[]) {
|
|
|
|
if(!targets || !Array.isArray(targets)) {
|
|
|
|
throw new Error('Incorrect targets format');
|
|
|
|
}
|
|
|
|
|
|
|
|
if(targets.length > 1) {
|
|
|
|
throw new Error(`Multiple queries are not supported yet`);
|
|
|
|
}
|
|
|
|
|
|
|
|
const host = new URL(datasourceUrl).origin;
|
|
|
|
const apiKey = apiKeys[host];
|
|
|
|
|
|
|
|
if(apiKey === undefined || apiKey === '') {
|
|
|
|
throw new Error(`Please configure API key for ${host}`);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private _writeCsv(stream, series) {
|
|
|
|
for(let row of series.values) {
|
|
|
|
let csvRow = {};
|
|
|
|
for(let col in series.columns) {
|
|
|
|
csvRow[series.columns[col]] = row[col];
|
|
|
|
}
|
|
|
|
stream.write(csvRow);
|
|
|
|
this._task.progress.exportedRowsCount++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private _getFilename(extension: string): string {
|
|
|
|
return `${this._task.id}.${extension}`;
|
|
|
|
}
|
|
|
|
|
|
|
|
private _getFilePath(extension: string): string {
|
|
|
|
let filename = this._getFilename(extension);
|
|
|
|
return path.join(__dirname, `../exported/${filename}`);
|
|
|
|
}
|
|
|
|
}
|