From f93aaf2b103f91eb81816df070392ae89a9ed209 Mon Sep 17 00:00:00 2001 From: Dmitry Nalobin Date: Tue, 6 Oct 2020 17:39:28 +0400 Subject: [PATCH] "Export all" to one file (#34) --- src/routes/tasks.ts | 20 +++- src/services/exporter.factory.ts | 9 ++ src/services/exporter.ts | 163 +++++++++++++++++++++++++++++++ src/target.ts | 128 ------------------------ src/types/export-status.ts | 4 + src/types/target.ts | 12 +++ src/utils.ts | 11 +++ 7 files changed, 214 insertions(+), 133 deletions(-) create mode 100644 src/services/exporter.factory.ts create mode 100644 src/services/exporter.ts delete mode 100644 src/target.ts create mode 100644 src/types/export-status.ts create mode 100644 src/types/target.ts create mode 100644 src/utils.ts diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts index a16ab80..1884076 100644 --- a/src/routes/tasks.ts +++ b/src/routes/tasks.ts @@ -1,7 +1,8 @@ -import { Target } from '../target' +import { Target } from '../types/target' import * as express from 'express' import { Datasource } from '@corpglory/tsdb-kit'; +import { exporterFactory } from '../services/exporter.factory'; type TRequest = { body: { @@ -9,6 +10,8 @@ type TRequest = { to: string, data: Array<{ panelUrl: string, + panelTitle: string, + panelId: number, datasourceRequest: Datasource, datasourceName: string, target: object, @@ -32,10 +35,17 @@ async function addTask(req: TRequest, res) { const names = data.map(item => item.datasourceName).join(', '); res.status(200).send(`Exporting ${names} data from ${new Date(from).toLocaleString()} to ${new Date(to).toLocaleString()}`); - data.forEach(request => { - const target = new Target(request.panelUrl, user, request.datasourceRequest, [request.target], from, to, request.datasourceName); - target.export(); - }); + const targets = data.map(item => new Target( + item.panelUrl, + item.panelTitle, + item.panelId, + item.datasourceRequest, + [item.target], + item.datasourceName, + )); + + const exporter = exporterFactory.getExporter(); + exporter.export(targets, user, from, to); } } diff --git a/src/services/exporter.factory.ts b/src/services/exporter.factory.ts new file mode 100644 index 0000000..d641c13 --- /dev/null +++ b/src/services/exporter.factory.ts @@ -0,0 +1,9 @@ +import { Exporter } from './exporter'; + +class ExporterFactory { + getExporter() { + return new Exporter(); + } +} + +export const exporterFactory = new ExporterFactory(); diff --git a/src/services/exporter.ts b/src/services/exporter.ts new file mode 100644 index 0000000..ba999e8 --- /dev/null +++ b/src/services/exporter.ts @@ -0,0 +1,163 @@ +import { Target } from '../types/target'; +import { URL } from 'url'; +import { apiKeys } from '../config'; +import { promisify } from '../utils'; +import { ExportStatus } from '../types/export-status'; + +import { Metric, queryByMetric } from '@corpglory/tsdb-kit'; + +import * as moment from 'moment'; +import * as csv from 'fast-csv'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as _ from 'lodash'; + +const MS_IN_DAY = 24 * 60 * 60 * 1000; +const TIMESTAMP_COLUMN = 'timestamp'; + +export class Exporter { + private exportedRows = 0; + private createdTimestamp: number; + private user: string; + private datasource: string; + + private initCsvStream() { + const csvStream = csv.createWriteStream({ headers: true }); + const writableStream = fs.createWriteStream(this.getFilePath('csv')); + + csvStream.pipe(writableStream); + writableStream.on('finish', async () => { + console.log(`Everything is written to ${this.getFilename('csv')}`); + await this.updateStatus(ExportStatus.FINISHED, 1); + }) + + return csvStream; + } + + public async updateStatus(status: string, progress: number) { + try { + let time = moment().valueOf(); + let data = { + time, + user: this.user, + exportedRows: this.exportedRows, + progress: progress.toLocaleString('en', { style: 'percent' }), + status, + datasourceName: this.datasource, + }; + + await promisify(fs.writeFile, this.getFilePath('json'), JSON.stringify(data), 'utf8') + } catch(err) { + console.error(err); + throw new Error('Can`t write file'); + } + } + + public async export(data: Target[], user: string, from: number, to: number) { + this.user = user; + + this.validateTargets(data); + const targets = data.map(target => ({ + ...target, + metric: new Metric(target.datasource, target.targets) + })); + + this.datasource = data.length === 1 ? data[0].datasourceName : 'all'; + + const stream = this.initCsvStream(); + const days = Math.ceil((to - from) / MS_IN_DAY); + + console.log(`Total days: ${days}`); + + for(let day = 0; day < days; day++) { + to = from + MS_IN_DAY; + + console.log(`${day} day: ${from}ms -> ${to}ms`); + + const columns = [TIMESTAMP_COLUMN]; + const values = {}; + + for(const [index, target] of targets.entries()) { + const host = new URL(target.panelUrl).origin; + const apiKey = apiKeys[host]; + + const datasourceMetrics = await queryByMetric(target.metric, target.panelUrl, from, to, apiKey); + + const column = `${target.panelId}` + + `-${target.panelTitle.replace(' ', '-')}-${datasourceMetrics.columns[1]}`; + + columns.push(column); + + for(const row of datasourceMetrics.values) { + const [timestamp, value] = row; + + if(values[timestamp] === undefined) { + values[timestamp] = new Array(targets.length); + } + values[timestamp][index] = value; + } + } + + const metricsValues = []; + + Object.keys(values).forEach(timestamp => { + metricsValues.push([timestamp, ...values[timestamp]]); + }); + + if(metricsValues.length > 0) { + this.writeCsv(stream, { + columns, + values: metricsValues, + }); + } + await this.updateStatus(ExportStatus.EXPORTING, (day + 1) / days); + + from += MS_IN_DAY; + } + stream.end(); + } + + private validateTargets(targets: Target[]) { + if(!targets || !Array.isArray(targets)) { + throw new Error('Incorrect targets format'); + } + + for(const target of targets) { + const host = new URL(target.panelUrl).origin; + const apiKey = apiKeys[host]; + + if(apiKey === undefined || apiKey === '') { + throw new Error(`Please configure API key for ${host}`); + } + } + } + + private writeCsv(stream, series) { + for(let row of series.values) { + const isEmpty = _.every( + _.slice(row, 1), + val => val === null + ); + if(!isEmpty) { + let csvRow = {}; + for(let col in series.columns) { + csvRow[series.columns[col]] = row[col]; + } + stream.write(csvRow); + this.exportedRows++; + } + } + } + + private getFilename(extension) { + if(this.createdTimestamp === undefined) { + this.createdTimestamp = moment().valueOf(); + } + return `${this.createdTimestamp}.${this.datasource}.${extension}`; + } + + private getFilePath(extension) { + let filename = this.getFilename(extension); + return path.join(__dirname, `../exported/${filename}`); + } +} diff --git a/src/target.ts b/src/target.ts deleted file mode 100644 index b824231..0000000 --- a/src/target.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { apiKeys } from './config'; - -import { queryByMetric, Datasource, Metric } from '@corpglory/tsdb-kit'; - -import * as csv from 'fast-csv'; -import * as path from 'path'; -import * as fs from 'fs'; -import * as moment from 'moment'; -import { URL } from 'url'; - -const MS_IN_DAY = 24 * 60 * 60 * 1000; - -export class Target { - private exportedRows: number; - private days: number; - private day: number; - private csvStream: any; - private metric: Metric; - private createdTimestamp: number; - - constructor( - private panelUrl: string, - private user: string, - datasource: Datasource, - targets: Array, - private from: number, - private to: number, - private datasourceName: string, - ) { - this.metric = new Metric(datasource, targets); - } - - public updateStatus(status) { - let time = moment().valueOf(); - let data = { - time, - user: this.user, - exportedRows: this.exportedRows, - progress: (this.day / this.days).toLocaleString('en', { style: 'percent' }), - status, - datasourceName: this.datasourceName - }; - return new Promise((resolve, reject) => { - fs.writeFile(this.getFilePath('json'), JSON.stringify(data), 'utf8', err => { - if(err) { - console.error(err); - reject('Can`t write file'); - } else { - resolve(); - } - }); - }); - } - - public async export() { - this.exportedRows = 0; - this.days = Math.ceil((this.to - this.from) / MS_IN_DAY); - this.day = 0; - this.initCsvStream(); - - let to = this.to; - let from = this.from; - - console.log(`Total days: ${this.days}`); - while(this.day < this.days) { - this.day++; - to = from + MS_IN_DAY; - - console.log(`${this.day} day: ${from}ms -> ${to}ms`); - - let host = new URL(this.panelUrl).origin; - let apiKey = apiKeys[host]; - - if(apiKey === undefined || apiKey === '') { - throw new Error(`Please configure API key for ${host}`); - } - let metrics = await queryByMetric(this.metric, this.panelUrl, from, to, apiKey); - - if(metrics.values.length > 0) { - if(metrics !== undefined) { - this.writeCsv(metrics); - } - } - await this.updateStatus('exporting'); - - from += MS_IN_DAY; - } - this.csvStream.end(); - } - - // TODO: move csv-related stuff to a service - private initCsvStream() { - this.csvStream = csv.createWriteStream({ headers: true }); - let writableStream = fs.createWriteStream(this.getFilePath('csv')); - - this.csvStream.pipe(writableStream); - writableStream.on('finish', async () => { - console.log(`Everything is written to ${this.getFilename('csv')}`); - await this.updateStatus('finished'); - }) - } - - private writeCsv(series) { - for(let val of series.values) { - if(val[1] !== null) { - let row = {}; - for(let col in series.columns) { - row[series.columns[col]] = val[col]; - } - this.csvStream.write(row); - this.exportedRows++; - } - } - } - - private getFilename(extension) { - if(this.createdTimestamp === undefined) { - this.createdTimestamp = moment().valueOf(); - } - return `${this.createdTimestamp}.${this.datasourceName}.${extension}`; - } - - private getFilePath(extension) { - let filename = this.getFilename(extension); - return path.join(__dirname, `../exported/${filename}`); - } - -} diff --git a/src/types/export-status.ts b/src/types/export-status.ts new file mode 100644 index 0000000..c209d54 --- /dev/null +++ b/src/types/export-status.ts @@ -0,0 +1,4 @@ +export enum ExportStatus { + EXPORTING = 'exporting', + FINISHED = 'finished', +} diff --git a/src/types/target.ts b/src/types/target.ts new file mode 100644 index 0000000..4ad80ae --- /dev/null +++ b/src/types/target.ts @@ -0,0 +1,12 @@ +import { Datasource } from '@corpglory/tsdb-kit'; + +export class Target { + constructor( + public panelUrl: string, + public panelTitle: string, + public panelId: number, + public datasource: Datasource, + public targets: Array, + public datasourceName: string, + ) {} +} diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 0000000..094d51c --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,11 @@ +export async function promisify(method: (...params: any[]) => Promise | void, ...params: any[]) { + return new Promise((resolve, reject) => { + method(...params, (err, result) => { + if(err) { + reject(err); + } else { + resolve(result); + } + }) + }); +}