import { Target } from '../types/target'; import { URL } from 'url'; import { apiKeys } from '../config'; import { promisify } from '../utils'; import { ExportStatus } from '../types/export-status'; import { DataSourceRef } from '../types'; import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit'; // TODO: export QueryType directly from @corpglory/tsdb-kit import { QueryType } from '@corpglory/tsdb-kit/lib/connectors'; import * as moment from 'moment'; import * as csv from 'fast-csv'; import * as fs from 'fs'; import * as path from 'path'; import * as _ from 'lodash'; const MS_IN_DAY = 24 * 60 * 60 * 1000; export class Exporter { private exportedRows = 0; private createdTimestamp: number; private username: string; private datasourceRef: DataSourceRef; private initCsvStream() { const csvStream = csv.createWriteStream({ headers: true }) .on('error', error => console.error(error)); const writableStream = fs.createWriteStream(this.getFilePath('csv')); csvStream.pipe(writableStream); writableStream.on('finish', async () => { console.log(`Everything is written to ${this.getFilename('csv')}`); await this.updateStatus(ExportStatus.FINISHED, 1); }) return csvStream; } public async updateStatus(status: string, progress: number) { try { let time = moment().valueOf(); let data = { time, username: this.username, exportedRows: this.exportedRows, progress: progress, status, datasourceRef: this.datasourceRef, }; await promisify(fs.writeFile, this.getFilePath('json'), JSON.stringify(data), 'utf8') } catch(err) { console.error(err); throw new Error('Can`t write file'); } } // TODO: rename `data` to `targets` or `queries` public async export(data: Target[], datasourceUrl: string, username: string, from: number, to: number) { this.username = username; this.validateTargets(datasourceUrl, data); const targets = data.map(target => { console.log('target', { ...target.datasource, url: datasourceUrl }); return { ...target, metric: new QueryConfig( QueryType.GRAFANA, { ...target.datasource, url: datasourceUrl }, target.panel.targets ) } }); const datasource = data[0].datasource; this.datasourceRef = { uid: datasource.uid, type: datasource.type }; await this.updateStatus(ExportStatus.EXPORTING, 0); const stream = this.initCsvStream(); const days = Math.ceil((to - from) / MS_IN_DAY); console.log(`Total days: ${days}`); for(let day = 0; day < days; day++) { to = from + MS_IN_DAY; console.log(`${day} day: ${from}ms -> ${to}ms`); let columns = []; let values = []; for(const [index, target] of targets.entries()) { const host = new URL(datasourceUrl).origin; const apiKey = apiKeys[host]; const datasourceMetrics = await queryByConfig(target.metric, datasourceUrl, from, to, apiKey); columns = datasourceMetrics.columns; values = datasourceMetrics.values; } if(columns.length > 0) { console.log('values', values); this.writeCsv(stream, { columns, values, }); } await this.updateStatus(ExportStatus.EXPORTING, (day + 1) / days); from += MS_IN_DAY; } stream.end(); } private validateTargets(datasourceUrl: string, targets: Target[]) { if(!targets || !Array.isArray(targets)) { throw new Error('Incorrect targets format'); } if(targets.length > 1) { throw new Error(`Multiple queries are not supported yet`); } const host = new URL(datasourceUrl).origin; const apiKey = apiKeys[host]; if(apiKey === undefined || apiKey === '') { throw new Error(`Please configure API key for ${host}`); } } private writeCsv(stream, series) { for(let row of series.values) { const isEmpty = _.every( _.slice(row, 1), val => val === null ); if(!isEmpty) { let csvRow = {}; for(let col in series.columns) { csvRow[series.columns[col]] = row[col]; } stream.write(csvRow); this.exportedRows++; } } } private getFilename(extension) { if(this.createdTimestamp === undefined) { this.createdTimestamp = moment().valueOf(); } return `${this.createdTimestamp}.${this.datasourceRef.uid}.${extension}`; } private getFilePath(extension) { let filename = this.getFilename(extension); return path.join(__dirname, `../exported/${filename}`); } }