|
|
|
import { Target } from '../types/target';
|
|
|
|
import { URL } from 'url';
|
|
|
|
import { apiKeys } from '../config';
|
|
|
|
import { promisify } from '../utils';
|
|
|
|
import { ExportStatus } from '../types/export-status';
|
|
|
|
import { DataSourceRef } from '../types';
|
|
|
|
|
|
|
|
import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit';
|
|
|
|
// TODO: export QueryType directly from @corpglory/tsdb-kit
|
|
|
|
import { QueryType } from '@corpglory/tsdb-kit/lib/connectors';
|
|
|
|
|
|
|
|
import * as moment from 'moment';
|
|
|
|
import * as csv from 'fast-csv';
|
|
|
|
import * as fs from 'fs';
|
|
|
|
import * as path from 'path';
|
|
|
|
import * as _ from 'lodash';
|
|
|
|
|
|
|
|
const MS_IN_DAY = 24 * 60 * 60 * 1000;
|
|
|
|
const TIMESTAMP_COLUMN = 'timestamp';
|
|
|
|
|
|
|
|
export class Exporter {
|
|
|
|
private exportedRows = 0;
|
|
|
|
private createdTimestamp: number;
|
|
|
|
private user: string;
|
|
|
|
private datasourceRef: DataSourceRef;
|
|
|
|
|
|
|
|
private initCsvStream() {
|
|
|
|
const csvStream = csv.parse({ headers: true })
|
|
|
|
.on('error', error => console.error(error));
|
|
|
|
|
|
|
|
const writableStream = fs.createWriteStream(this.getFilePath('csv'));
|
|
|
|
|
|
|
|
csvStream.pipe(writableStream);
|
|
|
|
writableStream.on('finish', async () => {
|
|
|
|
console.log(`Everything is written to ${this.getFilename('csv')}`);
|
|
|
|
await this.updateStatus(ExportStatus.FINISHED, 1);
|
|
|
|
})
|
|
|
|
|
|
|
|
return csvStream;
|
|
|
|
}
|
|
|
|
|
|
|
|
public async updateStatus(status: string, progress: number) {
|
|
|
|
try {
|
|
|
|
let time = moment().valueOf();
|
|
|
|
let data = {
|
|
|
|
time,
|
|
|
|
user: this.user,
|
|
|
|
exportedRows: this.exportedRows,
|
|
|
|
progress: progress.toLocaleString('en', { style: 'percent' }),
|
|
|
|
status,
|
|
|
|
datasourceRef: this.datasourceRef,
|
|
|
|
};
|
|
|
|
|
|
|
|
await promisify(fs.writeFile, this.getFilePath('json'), JSON.stringify(data), 'utf8')
|
|
|
|
} catch(err) {
|
|
|
|
console.error(err);
|
|
|
|
throw new Error('Can`t write file');
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
public async export(data: Target[], datasourceUrl: string, user: string, from: number, to: number) {
|
|
|
|
this.user = user;
|
|
|
|
|
|
|
|
this.validateTargets(datasourceUrl, data);
|
|
|
|
|
|
|
|
// console.log('ds', data[0].datasource)
|
|
|
|
const targets = data.map(target => {
|
|
|
|
console.log({
|
|
|
|
...target.datasource,
|
|
|
|
url: datasourceUrl
|
|
|
|
})
|
|
|
|
return {
|
|
|
|
...target,
|
|
|
|
metric: new QueryConfig(
|
|
|
|
QueryType.GRAFANA,
|
|
|
|
{
|
|
|
|
...target.datasource,
|
|
|
|
url: datasourceUrl
|
|
|
|
},
|
|
|
|
target.panel.targets
|
|
|
|
)
|
|
|
|
}
|
|
|
|
});
|
|
|
|
|
|
|
|
this.datasourceRef = data.length === 1 ? data[0].datasource : { uid: 'all', type: 'all' };
|
|
|
|
|
|
|
|
const stream = this.initCsvStream();
|
|
|
|
const days = Math.ceil((to - from) / MS_IN_DAY);
|
|
|
|
|
|
|
|
console.log(`Total days: ${days}`);
|
|
|
|
|
|
|
|
for(let day = 0; day < days; day++) {
|
|
|
|
to = from + MS_IN_DAY;
|
|
|
|
|
|
|
|
console.log(`${day} day: ${from}ms -> ${to}ms`);
|
|
|
|
|
|
|
|
const columns = [TIMESTAMP_COLUMN];
|
|
|
|
const values = {};
|
|
|
|
|
|
|
|
for(const [index, target] of targets.entries()) {
|
|
|
|
const host = new URL(datasourceUrl).origin;
|
|
|
|
const apiKey = apiKeys[host];
|
|
|
|
|
|
|
|
const datasourceMetrics = await queryByConfig(target.metric, datasourceUrl, from, to, apiKey);
|
|
|
|
|
|
|
|
const column = `${target.panel.id}` +
|
|
|
|
`-${target.panel.title.replace(' ', '-')}-${datasourceMetrics.columns[1]}`;
|
|
|
|
|
|
|
|
columns.push(column);
|
|
|
|
|
|
|
|
for(const row of datasourceMetrics.values) {
|
|
|
|
const [timestamp, value] = row;
|
|
|
|
|
|
|
|
if(values[timestamp] === undefined) {
|
|
|
|
values[timestamp] = new Array(targets.length);
|
|
|
|
}
|
|
|
|
values[timestamp][index] = value;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
const metricsValues = [];
|
|
|
|
|
|
|
|
Object.keys(values).forEach(timestamp => {
|
|
|
|
metricsValues.push([timestamp, ...values[timestamp]]);
|
|
|
|
});
|
|
|
|
|
|
|
|
if(metricsValues.length > 0) {
|
|
|
|
this.writeCsv(stream, {
|
|
|
|
columns,
|
|
|
|
values: metricsValues,
|
|
|
|
});
|
|
|
|
}
|
|
|
|
await this.updateStatus(ExportStatus.EXPORTING, (day + 1) / days);
|
|
|
|
|
|
|
|
from += MS_IN_DAY;
|
|
|
|
}
|
|
|
|
stream.end();
|
|
|
|
}
|
|
|
|
|
|
|
|
private validateTargets(datasourceUrl, targets: Target[]) {
|
|
|
|
if(!targets || !Array.isArray(targets)) {
|
|
|
|
throw new Error('Incorrect targets format');
|
|
|
|
}
|
|
|
|
|
|
|
|
for(const target of targets) {
|
|
|
|
const host = new URL(datasourceUrl).origin;
|
|
|
|
const apiKey = apiKeys[host];
|
|
|
|
|
|
|
|
if(apiKey === undefined || apiKey === '') {
|
|
|
|
throw new Error(`Please configure API key for ${host}`);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private writeCsv(stream, series) {
|
|
|
|
for(let row of series.values) {
|
|
|
|
const isEmpty = _.every(
|
|
|
|
_.slice(row, 1),
|
|
|
|
val => val === null
|
|
|
|
);
|
|
|
|
if(!isEmpty) {
|
|
|
|
let csvRow = {};
|
|
|
|
for(let col in series.columns) {
|
|
|
|
csvRow[series.columns[col]] = row[col];
|
|
|
|
}
|
|
|
|
stream.write(csvRow);
|
|
|
|
this.exportedRows++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private getFilename(extension) {
|
|
|
|
if(this.createdTimestamp === undefined) {
|
|
|
|
this.createdTimestamp = moment().valueOf();
|
|
|
|
}
|
|
|
|
return `${this.createdTimestamp}.${this.datasourceRef.uid}.${extension}`;
|
|
|
|
}
|
|
|
|
|
|
|
|
private getFilePath(extension) {
|
|
|
|
let filename = this.getFilename(extension);
|
|
|
|
return path.join(__dirname, `../exported/${filename}`);
|
|
|
|
}
|
|
|
|
}
|