corpglory
/
grafana-data-exporter
2
0
Fork 0
Code Issues Pull Requests Projects Releases 5 Wiki Activity
Browse Source

"Export all" to one file (#34)

pull/1/head
Dmitry Nalobin 4 years ago committed by GitHub
parent
3a670c9000
commit
f93aaf2b10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 214 additions and 133 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 20
      src/routes/tasks.ts
  2. 9
      src/services/exporter.factory.ts
  3. 163
      src/services/exporter.ts
  4. 128
      src/target.ts
  5. 4
      src/types/export-status.ts
  6. 12
      src/types/target.ts
  7. 11
      src/utils.ts

20
src/routes/tasks.ts
Unescape View File

@ -1,7 +1,8 @@
import { Target } from '../target'
import { Target } from '../types/target'
import * as express from 'express'
import { Datasource } from '@corpglory/tsdb-kit';
import { exporterFactory } from '../services/exporter.factory';
type TRequest = {
body: {
@ -9,6 +10,8 @@ type TRequest = {
to: string,
data: Array<{
panelUrl: string,
panelTitle: string,
panelId: number,
datasourceRequest: Datasource,
datasourceName: string,
target: object,
@ -32,10 +35,17 @@ async function addTask(req: TRequest, res) {
const names = data.map(item => item.datasourceName).join(', ');
res.status(200).send(`Exporting ${names} data from ${new Date(from).toLocaleString()} to ${new Date(to).toLocaleString()}`);
data.forEach(request => {
const target = new Target(request.panelUrl, user, request.datasourceRequest, [request.target], from, to, request.datasourceName);
target.export();
});
const targets = data.map(item => new Target(
item.panelUrl,
item.panelTitle,
item.panelId,
item.datasourceRequest,
[item.target],
item.datasourceName,
));
const exporter = exporterFactory.getExporter();
exporter.export(targets, user, from, to);
}
}

9
src/services/exporter.factory.ts
Unescape View File

@ -0,0 +1,9 @@
import { Exporter } from './exporter';
class ExporterFactory {
getExporter() {
return new Exporter();
}
}
export const exporterFactory = new ExporterFactory();

163
src/services/exporter.ts
Unescape View File

@ -0,0 +1,163 @@
import { Target } from '../types/target';
import { URL } from 'url';
import { apiKeys } from '../config';
import { promisify } from '../utils';
import { ExportStatus } from '../types/export-status';
import { Metric, queryByMetric } from '@corpglory/tsdb-kit';
import * as moment from 'moment';
import * as csv from 'fast-csv';
import * as fs from 'fs';
import * as path from 'path';
import * as _ from 'lodash';
const MS_IN_DAY = 24 * 60 * 60 * 1000;
const TIMESTAMP_COLUMN = 'timestamp';
export class Exporter {
private exportedRows = 0;
private createdTimestamp: number;
private user: string;
private datasource: string;
private initCsvStream() {
const csvStream = csv.createWriteStream({ headers: true });
const writableStream = fs.createWriteStream(this.getFilePath('csv'));
csvStream.pipe(writableStream);
writableStream.on('finish', async () => {
console.log(`Everything is written to ${this.getFilename('csv')}`);
await this.updateStatus(ExportStatus.FINISHED, 1);
})
return csvStream;
}
public async updateStatus(status: string, progress: number) {
try {
let time = moment().valueOf();
let data = {
time,
user: this.user,
exportedRows: this.exportedRows,
progress: progress.toLocaleString('en', { style: 'percent' }),
status,
datasourceName: this.datasource,
};
await promisify(fs.writeFile, this.getFilePath('json'), JSON.stringify(data), 'utf8')
} catch(err) {
console.error(err);
throw new Error('Can`t write file');
}
}
public async export(data: Target[], user: string, from: number, to: number) {
this.user = user;
this.validateTargets(data);
const targets = data.map(target => ({
...target,
metric: new Metric(target.datasource, target.targets)
}));
this.datasource = data.length === 1 ? data[0].datasourceName : 'all';
const stream = this.initCsvStream();
const days = Math.ceil((to - from) / MS_IN_DAY);
console.log(`Total days: ${days}`);
for(let day = 0; day < days; day++) {
to = from + MS_IN_DAY;
console.log(`${day} day: ${from}ms -> ${to}ms`);
const columns = [TIMESTAMP_COLUMN];
const values = {};
for(const [index, target] of targets.entries()) {
const host = new URL(target.panelUrl).origin;
const apiKey = apiKeys[host];
const datasourceMetrics = await queryByMetric(target.metric, target.panelUrl, from, to, apiKey);
const column = `${target.panelId}` +
`-${target.panelTitle.replace(' ', '-')}-${datasourceMetrics.columns[1]}`;
columns.push(column);
for(const row of datasourceMetrics.values) {
const [timestamp, value] = row;
if(values[timestamp] === undefined) {
values[timestamp] = new Array(targets.length);
}
values[timestamp][index] = value;
}
}
const metricsValues = [];
Object.keys(values).forEach(timestamp => {
metricsValues.push([timestamp, ...values[timestamp]]);
});
if(metricsValues.length > 0) {
this.writeCsv(stream, {
columns,
values: metricsValues,
});
}
await this.updateStatus(ExportStatus.EXPORTING, (day + 1) / days);
from += MS_IN_DAY;
}
stream.end();
}
private validateTargets(targets: Target[]) {
if(!targets || !Array.isArray(targets)) {
throw new Error('Incorrect targets format');
}
for(const target of targets) {
const host = new URL(target.panelUrl).origin;
const apiKey = apiKeys[host];
if(apiKey === undefined || apiKey === '') {
throw new Error(`Please configure API key for ${host}`);
}
}
}
private writeCsv(stream, series) {
for(let row of series.values) {
const isEmpty = _.every(
_.slice(row, 1),
val => val === null
);
if(!isEmpty) {
let csvRow = {};
for(let col in series.columns) {
csvRow[series.columns[col]] = row[col];
}
stream.write(csvRow);
this.exportedRows++;
}
}
}
private getFilename(extension) {
if(this.createdTimestamp === undefined) {
this.createdTimestamp = moment().valueOf();
}
return `${this.createdTimestamp}.${this.datasource}.${extension}`;
}
private getFilePath(extension) {
let filename = this.getFilename(extension);
return path.join(__dirname, `../exported/${filename}`);
}
}

128
src/target.ts
Unescape View File

@ -1,128 +0,0 @@
import { apiKeys } from './config';
import { queryByMetric, Datasource, Metric } from '@corpglory/tsdb-kit';
import * as csv from 'fast-csv';
import * as path from 'path';
import * as fs from 'fs';
import * as moment from 'moment';
import { URL } from 'url';
const MS_IN_DAY = 24 * 60 * 60 * 1000;
export class Target {
private exportedRows: number;
private days: number;
private day: number;
private csvStream: any;
private metric: Metric;
private createdTimestamp: number;
constructor(
private panelUrl: string,
private user: string,
datasource: Datasource,
targets: Array<Object>,
private from: number,
private to: number,
private datasourceName: string,
) {
this.metric = new Metric(datasource, targets);
}
public updateStatus(status) {
let time = moment().valueOf();
let data = {
time,
user: this.user,
exportedRows: this.exportedRows,
progress: (this.day / this.days).toLocaleString('en', { style: 'percent' }),
status,
datasourceName: this.datasourceName
};
return new Promise((resolve, reject) => {
fs.writeFile(this.getFilePath('json'), JSON.stringify(data), 'utf8', err => {
if(err) {
console.error(err);
reject('Can`t write file');
} else {
resolve();
}
});
});
}
public async export() {
this.exportedRows = 0;
this.days = Math.ceil((this.to - this.from) / MS_IN_DAY);
this.day = 0;
this.initCsvStream();
let to = this.to;
let from = this.from;
console.log(`Total days: ${this.days}`);
while(this.day < this.days) {
this.day++;
to = from + MS_IN_DAY;
console.log(`${this.day} day: ${from}ms -> ${to}ms`);
let host = new URL(this.panelUrl).origin;
let apiKey = apiKeys[host];
if(apiKey === undefined || apiKey === '') {
throw new Error(`Please configure API key for ${host}`);
}
let metrics = await queryByMetric(this.metric, this.panelUrl, from, to, apiKey);
if(metrics.values.length > 0) {
if(metrics !== undefined) {
this.writeCsv(metrics);
}
}
await this.updateStatus('exporting');
from += MS_IN_DAY;
}
this.csvStream.end();
}
// TODO: move csv-related stuff to a service
private initCsvStream() {
this.csvStream = csv.createWriteStream({ headers: true });
let writableStream = fs.createWriteStream(this.getFilePath('csv'));
this.csvStream.pipe(writableStream);
writableStream.on('finish', async () => {
console.log(`Everything is written to ${this.getFilename('csv')}`);
await this.updateStatus('finished');
})
}
private writeCsv(series) {
for(let val of series.values) {
if(val[1] !== null) {
let row = {};
for(let col in series.columns) {
row[series.columns[col]] = val[col];
}
this.csvStream.write(row);
this.exportedRows++;
}
}
}
private getFilename(extension) {
if(this.createdTimestamp === undefined) {
this.createdTimestamp = moment().valueOf();
}
return `${this.createdTimestamp}.${this.datasourceName}.${extension}`;
}
private getFilePath(extension) {
let filename = this.getFilename(extension);
return path.join(__dirname, `../exported/${filename}`);
}
}

4
src/types/export-status.ts
Unescape View File

@ -0,0 +1,4 @@
export enum ExportStatus {
EXPORTING = 'exporting',
FINISHED = 'finished',
}

12
src/types/target.ts
Unescape View File

@ -0,0 +1,12 @@
import { Datasource } from '@corpglory/tsdb-kit';
export class Target {
constructor(
public panelUrl: string,
public panelTitle: string,
public panelId: number,
public datasource: Datasource,
public targets: Array<object>,
public datasourceName: string,
) {}
}

11
src/utils.ts
Unescape View File

@ -0,0 +1,11 @@
export async function promisify(method: (...params: any[]) => Promise<any> | void, ...params: any[]) {
return new Promise((resolve, reject) => {
method(...params, (err, result) => {
if(err) {
reject(err);
} else {
resolve(result);
}
})
});
}
Write Preview
Loading…
Cancel
Save