"Export all" to one file (#34)

src/routes/tasks.ts

@@ -1,7 +1,8 @@
-import { Target } from '../target'
+import { Target } from '../types/target'
 
 import * as express from 'express'
 import { Datasource } from '@corpglory/tsdb-kit';
+import { exporterFactory } from '../services/exporter.factory';
 
 type TRequest = {
   body: {
@@ -9,6 +10,8 @@ type TRequest = {
     to: string,
     data: Array<{
       panelUrl: string,
+      panelTitle: string,
+      panelId: number,
       datasourceRequest: Datasource,
       datasourceName: string,
       target: object,
@@ -32,10 +35,17 @@ async function addTask(req: TRequest, res) {
     const names = data.map(item => item.datasourceName).join(', ');
     res.status(200).send(`Exporting ${names} data from ${new Date(from).toLocaleString()} to ${new Date(to).toLocaleString()}`);
 
-    data.forEach(request => {
+    const targets = data.map(item => new Target(
-      const target = new Target(request.panelUrl, user, request.datasourceRequest, [request.target], from, to, request.datasourceName);
+      item.panelUrl,
-      target.export();
+      item.panelTitle,
-    });
+      item.panelId,
+      item.datasourceRequest,
+      [item.target],
+      item.datasourceName,
+    ));
+
+    const exporter = exporterFactory.getExporter();
+    exporter.export(targets, user, from, to);
   }
 }
 

src/services/exporter.factory.ts

@@ -0,0 +1,9 @@
+import { Exporter } from './exporter';
+
+class ExporterFactory {
+  getExporter() {
+    return new Exporter();
+  }
+}
+
+export const exporterFactory = new ExporterFactory();

src/services/exporter.ts

@@ -0,0 +1,163 @@
+import { Target } from '../types/target';
+import { URL } from 'url';
+import { apiKeys } from '../config';
+import { promisify } from '../utils';
+import { ExportStatus } from '../types/export-status';
+
+import { Metric, queryByMetric } from '@corpglory/tsdb-kit';
+
+import * as moment from 'moment';
+import * as csv from 'fast-csv';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as _ from 'lodash';
+
+const MS_IN_DAY = 24 * 60 * 60 * 1000;
+const TIMESTAMP_COLUMN = 'timestamp';
+
+export class Exporter {
+  private exportedRows = 0;
+  private createdTimestamp: number;
+  private user: string;
+  private datasource: string;
+
+  private initCsvStream() {
+    const csvStream = csv.createWriteStream({ headers: true });
+    const writableStream = fs.createWriteStream(this.getFilePath('csv'));
+
+    csvStream.pipe(writableStream);
+    writableStream.on('finish', async () => {
+      console.log(`Everything is written to ${this.getFilename('csv')}`);
+      await this.updateStatus(ExportStatus.FINISHED, 1);
+    })
+
+    return csvStream;
+  }
+
+  public async updateStatus(status: string, progress: number) {
+    try {
+      let time = moment().valueOf();
+      let data = {
+        time,
+        user: this.user,
+        exportedRows: this.exportedRows,
+        progress: progress.toLocaleString('en', { style: 'percent' }),
+        status,
+        datasourceName: this.datasource,
+      };
+
+      await promisify(fs.writeFile, this.getFilePath('json'), JSON.stringify(data), 'utf8')
+    } catch(err) {
+      console.error(err);
+      throw new Error('Can`t write file');
+    }
+  }
+
+  public async export(data: Target[], user: string, from: number, to: number) {
+    this.user = user;
+
+    this.validateTargets(data);
+    const targets = data.map(target => ({
+      ...target,
+      metric: new Metric(target.datasource, target.targets)
+    }));
+
+    this.datasource = data.length === 1 ? data[0].datasourceName : 'all';
+
+    const stream = this.initCsvStream();
+    const days = Math.ceil((to - from) / MS_IN_DAY);
+
+    console.log(`Total days: ${days}`);
+
+    for(let day = 0; day < days; day++) {
+      to = from + MS_IN_DAY;
+
+      console.log(`${day} day: ${from}ms -> ${to}ms`);
+
+      const columns = [TIMESTAMP_COLUMN];
+      const values = {};
+
+      for(const [index, target] of targets.entries()) {
+        const host = new URL(target.panelUrl).origin;
+        const apiKey = apiKeys[host];
+
+        const datasourceMetrics = await queryByMetric(target.metric, target.panelUrl, from, to, apiKey);
+
+        const column = `${target.panelId}` +
+          `-${target.panelTitle.replace(' ', '-')}-${datasourceMetrics.columns[1]}`;
+
+        columns.push(column);
+
+        for(const row of datasourceMetrics.values) {
+          const [timestamp, value] = row;
+
+          if(values[timestamp] === undefined) {
+            values[timestamp] = new Array(targets.length);
+          }
+          values[timestamp][index] = value;
+        }
+      }
+
+      const metricsValues = [];
+
+      Object.keys(values).forEach(timestamp => {
+        metricsValues.push([timestamp, ...values[timestamp]]);
+      });
+
+      if(metricsValues.length > 0) {
+        this.writeCsv(stream, {
+          columns,
+          values: metricsValues,
+        });
+      }
+      await this.updateStatus(ExportStatus.EXPORTING, (day + 1) / days);
+
+      from += MS_IN_DAY;
+    }
+    stream.end();
+  }
+
+  private validateTargets(targets: Target[]) {
+    if(!targets || !Array.isArray(targets)) {
+      throw new Error('Incorrect targets format');
+    }
+
+    for(const target of targets) {
+      const host = new URL(target.panelUrl).origin;
+      const apiKey = apiKeys[host];
+
+      if(apiKey === undefined || apiKey === '') {
+        throw new Error(`Please configure API key for ${host}`);
+      }
+    }
+  }
+
+  private writeCsv(stream, series) {
+    for(let row of series.values) {
+      const isEmpty = _.every(
+        _.slice(row, 1),
+        val => val === null
+      );
+      if(!isEmpty) {
+        let csvRow = {};
+        for(let col in series.columns) {
+          csvRow[series.columns[col]] = row[col];
+        }
+        stream.write(csvRow);
+        this.exportedRows++;
+      }
+    }
+  }
+
+  private getFilename(extension) {
+    if(this.createdTimestamp === undefined) {
+      this.createdTimestamp = moment().valueOf();
+    }
+    return `${this.createdTimestamp}.${this.datasource}.${extension}`;
+  }
+
+  private getFilePath(extension) {
+    let filename = this.getFilename(extension);
+    return path.join(__dirname, `../exported/${filename}`);
+  }
+}

src/target.ts

@@ -1,128 +0,0 @@
-import { apiKeys } from './config';
-
-import { queryByMetric, Datasource, Metric } from '@corpglory/tsdb-kit';
-
-import * as csv from 'fast-csv';
-import * as path from 'path';
-import * as fs from 'fs';
-import * as moment from 'moment';
-import { URL } from 'url';
-
-const MS_IN_DAY = 24 * 60 * 60 * 1000;
-
-export class Target {
-  private exportedRows: number;
-  private days: number;
-  private day: number;
-  private csvStream: any;
-  private metric: Metric;
-  private createdTimestamp: number;
-
-  constructor(
-    private panelUrl: string,
-    private user: string,
-    datasource: Datasource,
-    targets: Array<Object>,
-    private from: number,
-    private to: number,
-    private datasourceName: string,
-  ) {
-    this.metric = new Metric(datasource, targets);
-  }
-
-  public updateStatus(status) {
-    let time = moment().valueOf();
-    let data = {
-      time,
-      user: this.user,
-      exportedRows: this.exportedRows,
-      progress: (this.day / this.days).toLocaleString('en', { style: 'percent' }),
-      status,
-      datasourceName: this.datasourceName
-    };
-    return new Promise((resolve, reject) => {
-      fs.writeFile(this.getFilePath('json'), JSON.stringify(data), 'utf8', err => {
-        if(err) {
-          console.error(err);
-          reject('Can`t write file');
-        } else {
-          resolve();
-        }
-      });
-    });
-  }
-
-  public async export() {
-    this.exportedRows = 0;
-    this.days = Math.ceil((this.to - this.from) / MS_IN_DAY);
-    this.day = 0;
-    this.initCsvStream();
-
-    let to = this.to;
-    let from = this.from;
-
-    console.log(`Total days: ${this.days}`);
-    while(this.day < this.days) {
-      this.day++;
-      to = from + MS_IN_DAY;
-
-      console.log(`${this.day} day: ${from}ms -> ${to}ms`);
-
-      let host = new URL(this.panelUrl).origin;
-      let apiKey = apiKeys[host];
-
-      if(apiKey === undefined || apiKey === '') {
-        throw new Error(`Please configure API key for ${host}`);
-      }
-      let metrics = await queryByMetric(this.metric, this.panelUrl, from, to, apiKey);
-
-      if(metrics.values.length > 0) {
-        if(metrics !== undefined) {
-          this.writeCsv(metrics);
-        }
-      }
-      await this.updateStatus('exporting');
-
-      from += MS_IN_DAY;
-    }
-    this.csvStream.end();
-  }
-
-  // TODO: move csv-related stuff to a service
-  private initCsvStream() {
-    this.csvStream = csv.createWriteStream({ headers: true });
-    let writableStream = fs.createWriteStream(this.getFilePath('csv'));
-
-    this.csvStream.pipe(writableStream);
-    writableStream.on('finish', async () => {
-      console.log(`Everything is written to ${this.getFilename('csv')}`);
-      await this.updateStatus('finished');
-    })
-  }
-
-  private writeCsv(series) {
-    for(let val of series.values) {
-      if(val[1] !== null) {
-        let row = {};
-        for(let col in series.columns) {
-          row[series.columns[col]] = val[col];
-        }
-        this.csvStream.write(row);
-        this.exportedRows++;
-      }
-    }
-  }
-
-  private getFilename(extension) {
-    if(this.createdTimestamp === undefined) {
-      this.createdTimestamp = moment().valueOf();
-    }
-    return `${this.createdTimestamp}.${this.datasourceName}.${extension}`;
-  }
-
-  private getFilePath(extension) {
-    let filename = this.getFilename(extension);
-    return path.join(__dirname, `../exported/${filename}`);
-  }
-
-}

src/types/export-status.ts

@@ -0,0 +1,4 @@
+export enum ExportStatus {
+  EXPORTING = 'exporting',
+  FINISHED = 'finished',
+}

src/types/target.ts

@@ -0,0 +1,12 @@
+import { Datasource } from '@corpglory/tsdb-kit';
+
+export class Target {
+    constructor(
+      public panelUrl: string,
+      public panelTitle: string,
+      public panelId: number,
+      public datasource: Datasource,
+      public targets: Array<object>,
+      public datasourceName: string,
+    ) {}
+}

src/utils.ts

@@ -0,0 +1,11 @@
+export async function promisify(method: (...params: any[]) => Promise<any> | void, ...params: any[]) {
+  return new Promise((resolve, reject) => {
+    method(...params, (err, result) => {
+      if(err) {
+        reject(err);
+      } else {
+        resolve(result);
+      }
+    })
+  });
+}