// TODO: import promisify from 'util' import { getApiKey } from './api_keys'; import { promisify, toIsoString } from '../utils'; import { DashboardQuery, ExportProgress, ExportStatus, ExportTask } from '../types'; import { CSV_PATH } from '../config'; import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit'; // TODO: export QueryType directly from @corpglory/tsdb-kit import { QueryType } from '@corpglory/tsdb-kit/lib/connectors'; import { v4 as uuidv4 } from 'uuid'; import * as moment from 'moment'; import * as csv from 'fast-csv'; import * as fs from 'fs'; import * as path from 'path'; import { URL } from 'url'; import * as _ from 'lodash'; const MS_IN_DAY = 24 * 60 * 60 * 1000; const DEFAULT_PROGRESS = { exportedRowsCount: 0, progress: 0, status: ExportStatus.EXPORTING, }; export class Exporter { private _task: ExportTask; public async export(task: ExportTask, datasourceUrl: string, timeZoneName: string) { try { this._task = _.cloneDeep(task); this._task.id = uuidv4(); this._task.progress = _.cloneDeep(DEFAULT_PROGRESS); this._validateQueries(task.queries); await this._updateProgress(); const queryConfigs = task.queries.map( query => new QueryConfig( QueryType.GRAFANA, { ...query.datasource, url: datasourceUrl, }, [query.target] ) ); let from = +this._task.timeRange.from; let to = +this._task.timeRange.to; const days = Math.ceil((to - from) / MS_IN_DAY); console.log(`Total days: ${days}`); const stream = this._initCsvStream(); for(let day = 0; day < days; day++) { to = from + MS_IN_DAY; console.log(`${day} day: ${from}ms -> ${to}ms`); let columns = []; let values = []; for(const queryConfig of queryConfigs) { const host = new URL(datasourceUrl).origin; const apiKey = getApiKey(host); const datasourceMetrics = await queryByConfig(queryConfig, datasourceUrl, from, to, apiKey); if(_.isEmpty(columns)) { columns = datasourceMetrics.columns; } else { columns = _.concat(columns, datasourceMetrics.columns.slice(1)); } if(_.isEmpty(values)) { values = datasourceMetrics.values; } else { if(values.length !== datasourceMetrics.values.length) { throw new Error(`All queries should return rows of the same lengths`); } for(const rowIdx in values) { if(datasourceMetrics.values[rowIdx][0] !== values[rowIdx][0]) { throw new Error('Queries should return the same timestamps'); } values[rowIdx] = _.concat(values[rowIdx], datasourceMetrics.values[rowIdx].slice(1)); } } } values = values.map((row: number[]) => [toIsoString(row[0], timeZoneName), ...row.slice(1)]); if(columns.length > 0) { this._writeCsv(stream, { columns, values }); } await this._updateProgress({ status: ExportStatus.EXPORTING, progress: (day + 1) / days }); from += MS_IN_DAY; } stream.end(); } catch (e) { await this._updateProgress({ status: ExportStatus.ERROR, errorMessage: e.message }); } } private _initCsvStream() { const csvStream = csv.createWriteStream({ headers: true }) .on('error', async e => await this._updateProgress({ status: ExportStatus.ERROR, errorMessage: e.message })); const writableStream = fs.createWriteStream(this._getFilePath('csv')); csvStream.pipe(writableStream); writableStream.on('finish', async () => { if(this._task.progress.status !== ExportStatus.ERROR) { console.log(`Everything is written to ${this._getFilename('csv')}`); await this._updateProgress({ status: ExportStatus.FINISHED, progress: 1 }); } else { console.log(`${this._getFilename('csv')} export is finished with error`); } }); return csvStream; } private async _updateProgress(progress?: Partial) { try { let time = moment().valueOf(); const data = { ...this._task, progress: _.assign(this._task.progress, progress, { time }), }; await promisify(fs.writeFile, this._getFilePath('json'), JSON.stringify(data), 'utf8'); } catch(err) { console.error(err); throw new Error('Can`t write file'); } } private _validateQueries(queries: DashboardQuery[]) { if(!_.isArray(queries)) { throw new Error('`queries` field is required and should be an array'); } for(const query of queries) { if(_.isEmpty(query.datasource)) { throw new Error('all queries should have a `datasource` field'); } if(_.isEmpty(query.target)) { throw new Error('all queries should have a `target` field'); } } } private _writeCsv(stream, series) { for(let row of series.values) { let csvRow = {}; for(let col in series.columns) { csvRow[series.columns[col]] = row[col]; } stream.write(csvRow); this._task.progress.exportedRowsCount++; } } private _getFilename(extension: string): string { return `${this._task.id}.${extension}`; } private _getFilePath(extension: string): string { let filename = this._getFilename(extension); return path.join(CSV_PATH, filename); } }