Browse Source

"Export all" to one file (#34)

pull/1/head
Dmitry Nalobin 4 years ago committed by GitHub
parent
commit
f93aaf2b10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 20
      src/routes/tasks.ts
  2. 9
      src/services/exporter.factory.ts
  3. 163
      src/services/exporter.ts
  4. 128
      src/target.ts
  5. 4
      src/types/export-status.ts
  6. 12
      src/types/target.ts
  7. 11
      src/utils.ts

20
src/routes/tasks.ts

@ -1,7 +1,8 @@
import { Target } from '../target' import { Target } from '../types/target'
import * as express from 'express' import * as express from 'express'
import { Datasource } from '@corpglory/tsdb-kit'; import { Datasource } from '@corpglory/tsdb-kit';
import { exporterFactory } from '../services/exporter.factory';
type TRequest = { type TRequest = {
body: { body: {
@ -9,6 +10,8 @@ type TRequest = {
to: string, to: string,
data: Array<{ data: Array<{
panelUrl: string, panelUrl: string,
panelTitle: string,
panelId: number,
datasourceRequest: Datasource, datasourceRequest: Datasource,
datasourceName: string, datasourceName: string,
target: object, target: object,
@ -32,10 +35,17 @@ async function addTask(req: TRequest, res) {
const names = data.map(item => item.datasourceName).join(', '); const names = data.map(item => item.datasourceName).join(', ');
res.status(200).send(`Exporting ${names} data from ${new Date(from).toLocaleString()} to ${new Date(to).toLocaleString()}`); res.status(200).send(`Exporting ${names} data from ${new Date(from).toLocaleString()} to ${new Date(to).toLocaleString()}`);
data.forEach(request => { const targets = data.map(item => new Target(
const target = new Target(request.panelUrl, user, request.datasourceRequest, [request.target], from, to, request.datasourceName); item.panelUrl,
target.export(); item.panelTitle,
}); item.panelId,
item.datasourceRequest,
[item.target],
item.datasourceName,
));
const exporter = exporterFactory.getExporter();
exporter.export(targets, user, from, to);
} }
} }

9
src/services/exporter.factory.ts

@ -0,0 +1,9 @@
import { Exporter } from './exporter';
class ExporterFactory {
getExporter() {
return new Exporter();
}
}
export const exporterFactory = new ExporterFactory();

163
src/services/exporter.ts

@ -0,0 +1,163 @@
import { Target } from '../types/target';
import { URL } from 'url';
import { apiKeys } from '../config';
import { promisify } from '../utils';
import { ExportStatus } from '../types/export-status';
import { Metric, queryByMetric } from '@corpglory/tsdb-kit';
import * as moment from 'moment';
import * as csv from 'fast-csv';
import * as fs from 'fs';
import * as path from 'path';
import * as _ from 'lodash';
const MS_IN_DAY = 24 * 60 * 60 * 1000;
const TIMESTAMP_COLUMN = 'timestamp';
export class Exporter {
private exportedRows = 0;
private createdTimestamp: number;
private user: string;
private datasource: string;
private initCsvStream() {
const csvStream = csv.createWriteStream({ headers: true });
const writableStream = fs.createWriteStream(this.getFilePath('csv'));
csvStream.pipe(writableStream);
writableStream.on('finish', async () => {
console.log(`Everything is written to ${this.getFilename('csv')}`);
await this.updateStatus(ExportStatus.FINISHED, 1);
})
return csvStream;
}
public async updateStatus(status: string, progress: number) {
try {
let time = moment().valueOf();
let data = {
time,
user: this.user,
exportedRows: this.exportedRows,
progress: progress.toLocaleString('en', { style: 'percent' }),
status,
datasourceName: this.datasource,
};
await promisify(fs.writeFile, this.getFilePath('json'), JSON.stringify(data), 'utf8')
} catch(err) {
console.error(err);
throw new Error('Can`t write file');
}
}
public async export(data: Target[], user: string, from: number, to: number) {
this.user = user;
this.validateTargets(data);
const targets = data.map(target => ({
...target,
metric: new Metric(target.datasource, target.targets)
}));
this.datasource = data.length === 1 ? data[0].datasourceName : 'all';
const stream = this.initCsvStream();
const days = Math.ceil((to - from) / MS_IN_DAY);
console.log(`Total days: ${days}`);
for(let day = 0; day < days; day++) {
to = from + MS_IN_DAY;
console.log(`${day} day: ${from}ms -> ${to}ms`);
const columns = [TIMESTAMP_COLUMN];
const values = {};
for(const [index, target] of targets.entries()) {
const host = new URL(target.panelUrl).origin;
const apiKey = apiKeys[host];
const datasourceMetrics = await queryByMetric(target.metric, target.panelUrl, from, to, apiKey);
const column = `${target.panelId}` +
`-${target.panelTitle.replace(' ', '-')}-${datasourceMetrics.columns[1]}`;
columns.push(column);
for(const row of datasourceMetrics.values) {
const [timestamp, value] = row;
if(values[timestamp] === undefined) {
values[timestamp] = new Array(targets.length);
}
values[timestamp][index] = value;
}
}
const metricsValues = [];
Object.keys(values).forEach(timestamp => {
metricsValues.push([timestamp, ...values[timestamp]]);
});
if(metricsValues.length > 0) {
this.writeCsv(stream, {
columns,
values: metricsValues,
});
}
await this.updateStatus(ExportStatus.EXPORTING, (day + 1) / days);
from += MS_IN_DAY;
}
stream.end();
}
private validateTargets(targets: Target[]) {
if(!targets || !Array.isArray(targets)) {
throw new Error('Incorrect targets format');
}
for(const target of targets) {
const host = new URL(target.panelUrl).origin;
const apiKey = apiKeys[host];
if(apiKey === undefined || apiKey === '') {
throw new Error(`Please configure API key for ${host}`);
}
}
}
private writeCsv(stream, series) {
for(let row of series.values) {
const isEmpty = _.every(
_.slice(row, 1),
val => val === null
);
if(!isEmpty) {
let csvRow = {};
for(let col in series.columns) {
csvRow[series.columns[col]] = row[col];
}
stream.write(csvRow);
this.exportedRows++;
}
}
}
private getFilename(extension) {
if(this.createdTimestamp === undefined) {
this.createdTimestamp = moment().valueOf();
}
return `${this.createdTimestamp}.${this.datasource}.${extension}`;
}
private getFilePath(extension) {
let filename = this.getFilename(extension);
return path.join(__dirname, `../exported/${filename}`);
}
}

128
src/target.ts

@ -1,128 +0,0 @@
import { apiKeys } from './config';
import { queryByMetric, Datasource, Metric } from '@corpglory/tsdb-kit';
import * as csv from 'fast-csv';
import * as path from 'path';
import * as fs from 'fs';
import * as moment from 'moment';
import { URL } from 'url';
const MS_IN_DAY = 24 * 60 * 60 * 1000;
export class Target {
private exportedRows: number;
private days: number;
private day: number;
private csvStream: any;
private metric: Metric;
private createdTimestamp: number;
constructor(
private panelUrl: string,
private user: string,
datasource: Datasource,
targets: Array<Object>,
private from: number,
private to: number,
private datasourceName: string,
) {
this.metric = new Metric(datasource, targets);
}
public updateStatus(status) {
let time = moment().valueOf();
let data = {
time,
user: this.user,
exportedRows: this.exportedRows,
progress: (this.day / this.days).toLocaleString('en', { style: 'percent' }),
status,
datasourceName: this.datasourceName
};
return new Promise((resolve, reject) => {
fs.writeFile(this.getFilePath('json'), JSON.stringify(data), 'utf8', err => {
if(err) {
console.error(err);
reject('Can`t write file');
} else {
resolve();
}
});
});
}
public async export() {
this.exportedRows = 0;
this.days = Math.ceil((this.to - this.from) / MS_IN_DAY);
this.day = 0;
this.initCsvStream();
let to = this.to;
let from = this.from;
console.log(`Total days: ${this.days}`);
while(this.day < this.days) {
this.day++;
to = from + MS_IN_DAY;
console.log(`${this.day} day: ${from}ms -> ${to}ms`);
let host = new URL(this.panelUrl).origin;
let apiKey = apiKeys[host];
if(apiKey === undefined || apiKey === '') {
throw new Error(`Please configure API key for ${host}`);
}
let metrics = await queryByMetric(this.metric, this.panelUrl, from, to, apiKey);
if(metrics.values.length > 0) {
if(metrics !== undefined) {
this.writeCsv(metrics);
}
}
await this.updateStatus('exporting');
from += MS_IN_DAY;
}
this.csvStream.end();
}
// TODO: move csv-related stuff to a service
private initCsvStream() {
this.csvStream = csv.createWriteStream({ headers: true });
let writableStream = fs.createWriteStream(this.getFilePath('csv'));
this.csvStream.pipe(writableStream);
writableStream.on('finish', async () => {
console.log(`Everything is written to ${this.getFilename('csv')}`);
await this.updateStatus('finished');
})
}
private writeCsv(series) {
for(let val of series.values) {
if(val[1] !== null) {
let row = {};
for(let col in series.columns) {
row[series.columns[col]] = val[col];
}
this.csvStream.write(row);
this.exportedRows++;
}
}
}
private getFilename(extension) {
if(this.createdTimestamp === undefined) {
this.createdTimestamp = moment().valueOf();
}
return `${this.createdTimestamp}.${this.datasourceName}.${extension}`;
}
private getFilePath(extension) {
let filename = this.getFilename(extension);
return path.join(__dirname, `../exported/${filename}`);
}
}

4
src/types/export-status.ts

@ -0,0 +1,4 @@
export enum ExportStatus {
EXPORTING = 'exporting',
FINISHED = 'finished',
}

12
src/types/target.ts

@ -0,0 +1,12 @@
import { Datasource } from '@corpglory/tsdb-kit';
export class Target {
constructor(
public panelUrl: string,
public panelTitle: string,
public panelId: number,
public datasource: Datasource,
public targets: Array<object>,
public datasourceName: string,
) {}
}

11
src/utils.ts

@ -0,0 +1,11 @@
export async function promisify(method: (...params: any[]) => Promise<any> | void, ...params: any[]) {
return new Promise((resolve, reject) => {
method(...params, (err, result) => {
if(err) {
reject(err);
} else {
resolve(result);
}
})
});
}
Loading…
Cancel
Save