grafana-data-exporter/src/services/exporter.ts

import { Target } from '../models/target';
import { URL } from 'url';
import { apiKeys } from '../config';
import { promisify } from '../utils';
import { DashboardQuery, ExportProgress, ExportStatus, ExportTask } from '../types';

import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit';
// TODO: export QueryType directly from @corpglory/tsdb-kit
import { QueryType } from '@corpglory/tsdb-kit/lib/connectors';

import { v4 as uuidv4 } from 'uuid';

import * as moment from 'moment';
import * as csv from 'fast-csv';
import * as fs from 'fs';
import * as path from 'path';
import * as _ from 'lodash';

const MS_IN_DAY = 24 * 60 * 60 * 1000;

const DEFAULT_PROGRESS = {
  exportedRowsCount: 0,
  progress: 0,
  status: ExportStatus.EXPORTING,
};

export class Exporter {
  private _task: ExportTask;

  public async export(task: ExportTask, datasourceUrl: string) {
    try {
      this._task = _.cloneDeep(task);
      this._task.id = uuidv4();
      this._task.progress = _.cloneDeep(DEFAULT_PROGRESS);

      const targets = task.queries.map((query: DashboardQuery) => new Target(
        query.panel,
        query.datasource,
      ));

      this._validateTargets(datasourceUrl, targets);

      await this._updateProgress();

      const queryConfigs = targets.map(
        target =>
          new QueryConfig(
            QueryType.GRAFANA,
            {
              ...target.datasource,
              url: datasourceUrl
            },
            target.panel.targets
          )
      );

      let from = +this._task.timeRange.from;
      let to = +this._task.timeRange.to;
      const days = Math.ceil((to - from) / MS_IN_DAY);

      console.log(`Total days: ${days}`);

      const stream = this._initCsvStream();
      for(let day = 0; day < days; day++) {
        to = from + MS_IN_DAY;

        console.log(`${day} day: ${from}ms -> ${to}ms`);

        let columns = [];
        let values = [];

        for(const queryConfig of queryConfigs) {
          const host = new URL(datasourceUrl).origin;
          const apiKey = apiKeys[host];

          const datasourceMetrics = await queryByConfig(queryConfig, datasourceUrl, from, to, apiKey);

          columns = datasourceMetrics.columns;
          values = datasourceMetrics.values;
        }

        if(columns.length > 0) {
          this._writeCsv(stream, { columns, values, });
        }
        await this._updateProgress({ status: ExportStatus.EXPORTING, progress: (day + 1) / days });

        from += MS_IN_DAY;
      }
      stream.end();
    } catch (e) {
      await this._updateProgress({ status: ExportStatus.ERROR, errorMessage: e.message });
    }
  }

  private _initCsvStream() {
    const csvStream = csv.createWriteStream({ headers: true })
      .on('error', async e => await this._updateProgress({ status: ExportStatus.ERROR, errorMessage: e.message }));

    const writableStream = fs.createWriteStream(this._getFilePath('csv'));

    csvStream.pipe(writableStream);
    writableStream.on('finish', async () => {
      if(this._task.progress.status !== ExportStatus.ERROR) {
        console.log(`Everything is written to ${this._getFilename('csv')}`);
        await this._updateProgress({ status: ExportStatus.FINISHED, progress: 1 });
      } else {
        console.log(`${this._getFilename('csv')} export is finished with error`);
      }
    });

    return csvStream;
  }

  private async _updateProgress(progress?: Partial<ExportProgress>) {
    try {
      let time = moment().valueOf();

      const data = {
        ...this._task,
        progress: _.assign(this._task.progress, progress, { time }),
      };

      await promisify(fs.writeFile, this._getFilePath('json'), JSON.stringify(data), 'utf8');
    } catch(err) {
      console.error(err);
      throw new Error('Can`t write file');
    }
  }

  private _validateTargets(datasourceUrl: string, targets: Target[]) {
    if(!targets || !Array.isArray(targets)) {
      throw new Error('Incorrect targets format');
    }

    if(targets.length > 1) {
      throw new Error(`Multiple queries are not supported yet`);
    }

    const host = new URL(datasourceUrl).origin;
    const apiKey = apiKeys[host];

    if(apiKey === undefined || apiKey === '') {
      throw new Error(`Please configure API key for ${host}`);
    }
  }

  private _writeCsv(stream, series) {
    for(let row of series.values) {
      const isEmpty = _.every(
        _.slice(row, 1),
        val => val === null
      );
      if(!isEmpty) {
        let csvRow = {};
        for(let col in series.columns) {
          csvRow[series.columns[col]] = row[col];
        }
        stream.write(csvRow);
        this._task.progress.exportedRowsCount++;
      }
    }
  }

  private _getFilename(extension: string): string {
    return `${this._task.id}.${extension}`;
  }

  private _getFilePath(extension: string): string {
    let filename = this._getFilename(extension);
    return path.join(__dirname, `../exported/${filename}`);
  }
}