grafana-data-exporter/src/services/exporter.ts

import { Target } from '../types/target';
import { URL } from 'url';
import { apiKeys } from '../config';
import { promisify } from '../utils';
import { ExportStatus } from '../types/export-status';
import { DataSourceRef } from '../types';

import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit';
// TODO: export QueryType directly from @corpglory/tsdb-kit
import { QueryType } from '@corpglory/tsdb-kit/lib/connectors';

import * as moment from 'moment';
import * as csv from 'fast-csv';
import * as fs from 'fs';
import * as path from 'path';
import * as _ from 'lodash';

const MS_IN_DAY = 24 * 60 * 60 * 1000;
const TIMESTAMP_COLUMN = 'timestamp';

export class Exporter {
  private exportedRows = 0;
  private createdTimestamp: number;
  private username: string;
  private datasourceRef: DataSourceRef;

  private initCsvStream() {
    const csvStream = csv.createWriteStream({ headers: true })
      .on('error', error => console.error(error));

    const writableStream = fs.createWriteStream(this.getFilePath('csv'));

    csvStream.pipe(writableStream);
    writableStream.on('finish', async () => {
      console.log(`Everything is written to ${this.getFilename('csv')}`);
      await this.updateStatus(ExportStatus.FINISHED, 1);
    })

    return csvStream;
  }

  public async updateStatus(status: string, progress: number) {
    try {
      let time = moment().valueOf();
      let data = {
        time,
        username: this.username,
        exportedRows: this.exportedRows,
        progress: progress,
        status,
        datasourceRef: this.datasourceRef,
      };

      await promisify(fs.writeFile, this.getFilePath('json'), JSON.stringify(data), 'utf8')
    } catch(err) {
      console.error(err);
      throw new Error('Can`t write file');
    }
  }

  public async export(data: Target[], datasourceUrl: string, username: string, from: number, to: number) {
    this.username = username;

    this.validateTargets(datasourceUrl, data);

    // console.log('ds', data[0].datasource)
    const targets = data.map(target => {
      console.log({
        ...target.datasource,
        url: datasourceUrl
      })
      return {
        ...target,
        metric: new QueryConfig(
          QueryType.GRAFANA,
          {
            ...target.datasource,
            url: datasourceUrl
          },
          target.panel.targets
        )
      }
    });

    const datasource = data[0].datasource;
    this.datasourceRef = data.length === 1 ? { uid: datasource.uid, type: datasource.type } : { uid: 'all', type: 'all' };

    await this.updateStatus(ExportStatus.EXPORTING, 0);

    const stream = this.initCsvStream();
    const days = Math.ceil((to - from) / MS_IN_DAY);

    console.log(`Total days: ${days}`);

    for(let day = 0; day < days; day++) {
      to = from + MS_IN_DAY;

      console.log(`${day} day: ${from}ms -> ${to}ms`);

      const columns = [TIMESTAMP_COLUMN];
      const values = {};

      for(const [index, target] of targets.entries()) {
        const host = new URL(datasourceUrl).origin;
        const apiKey = apiKeys[host];

        const datasourceMetrics = await queryByConfig(target.metric, datasourceUrl, from, to, apiKey);

        const column = `${target.panel.id}` +
          `-${target.panel.title.replace(' ', '-')}-${datasourceMetrics.columns[1]}`;

        columns.push(column);

        for(const row of datasourceMetrics.values) {
          const [timestamp, value] = row;

          if(values[timestamp] === undefined) {
            values[timestamp] = new Array(targets.length);
          }
          values[timestamp][index] = value;
        }
      }

      const metricsValues = [];

      Object.keys(values).forEach(timestamp => {
        metricsValues.push([timestamp, ...values[timestamp]]);
      });

      if(metricsValues.length > 0) {
        console.log(metricsValues)
        this.writeCsv(stream, {
          columns,
          values: metricsValues,
        });
      }
      await this.updateStatus(ExportStatus.EXPORTING, (day + 1) / days);

      from += MS_IN_DAY;
    }
    stream.end();
  }

  private validateTargets(datasourceUrl, targets: Target[]) {
    if(!targets || !Array.isArray(targets)) {
      throw new Error('Incorrect targets format');
    }

    for(const target of targets) {
      const host = new URL(datasourceUrl).origin;
      const apiKey = apiKeys[host];

      if(apiKey === undefined || apiKey === '') {
        throw new Error(`Please configure API key for ${host}`);
      }
    }
  }

  private writeCsv(stream, series) {
    for(let row of series.values) {
      const isEmpty = _.every(
        _.slice(row, 1),
        val => val === null
      );
      if(!isEmpty) {
        let csvRow = {};
        for(let col in series.columns) {
          csvRow[series.columns[col]] = row[col];
        }
        stream.write(csvRow);
        this.exportedRows++;
      }
    }
  }

  private getFilename(extension) {
    if(this.createdTimestamp === undefined) {
      this.createdTimestamp = moment().valueOf();
    }
    return `${this.createdTimestamp}.${this.datasourceRef.uid}.${extension}`;
  }

  private getFilePath(extension) {
    let filename = this.getFilename(extension);
    return path.join(__dirname, `../exported/${filename}`);
  }
}