diff --git a/package.json b/package.json
index 4b4ddf2..41827b4 100644
--- a/package.json
+++ b/package.json
@@ -11,7 +11,7 @@
"license": "ISC",
"dependencies": {},
"devDependencies": {
- "@corpglory/tsdb-kit": "^2.0.1",
+ "@corpglory/tsdb-kit": "^2.0.2",
"axios": "^1.2.1",
"express": "^4.18.2",
"fast-csv": "^4.3.6",
diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts
index b4359d7..9c5f646 100644
--- a/src/routes/tasks.ts
+++ b/src/routes/tasks.ts
@@ -2,7 +2,7 @@ import { Target } from '../types/target';
import { exporterFactory } from '../services/exporter.factory';
import { EXPORTED_PATH } from '../config';
-import { Datasource } from '@corpglory/tsdb-kit';
+import { Task, TaskTableRowConfig } from '../types';
import * as express from 'express';
@@ -13,55 +13,76 @@ type TRequest = {
body: {
from: string,
to: string,
- data: Array<{
- panelUrl: string,
- panelTitle: string,
- panelId: number,
- datasourceRequest: Datasource,
- datasourceName: string,
- target: object,
- }>,
- user: string,
- }
+ username: string,
+ tasks: Task[],
+ url: string,
+ },
};
async function getTasks(req, res) {
- res.status(200).send([{
- timestamp: 12343567,
- user: 'admin',
- datasource: 'postgres',
- rowsCount: 2345,
- progress: 100,
- status: 'Success',
- }]);
+ const resp: TaskTableRowConfig[] = [];
+ fs.readdir(EXPORTED_PATH, (err, items) => {
+ if(err) {
+ console.error(err);
+ res.status(500).send('Something went wrong');
+ } else {
+ for(let item of items) {
+ let file = path.parse(item);
+ if(file.ext !== '.json') {
+ continue;
+ }
+ // TODO: read async
+ let data = fs.readFileSync(path.join(EXPORTED_PATH, item), 'utf8');
+ let status = JSON.parse(data);
+
+ let requestedUrl = `http://${req.headers.host}`;
+ let downloadLink = '';
+ let deleteLink = '';
+ if(status.status === 'finished') {
+ downloadLink = ``;
+ }
+ resp.push({
+ timestamp: status.time,
+ user: status.user,
+ datasourceRef: status.datasourceRef,
+ rowsCount: status.exportedRows,
+ progress: status.progress,
+ status: status.status,
+ downloadLink,
+ });
+ }
+
+ res.status(200).send(resp);
+ }
+ });
}
async function addTask(req: TRequest, res) {
+
const body = req.body;
+
+ const clientUrl = body.url;
const from = parseInt(body.from);
const to = parseInt(body.to);
- const data = body.data;
- const user = body.user;
+ const username = body.username;
+ const tasks = body.tasks;
+
+ const datasourceUrl = `${new URL(clientUrl).origin}/api/ds/query`;
if(isNaN(from) || isNaN(to)) {
res.status(400).send('Range error: please fill both "from" and "to" fields');
} else if(from >= to) {
res.status(400).send('Range error: "from" should be less than "to"');
} else {
- const names = data.map(item => item.datasourceName).join(', ');
+ const names = tasks.map(item => item.datasource.name).join(', ');
res.status(200).send(`Exporting ${names} data from ${new Date(from).toLocaleString()} to ${new Date(to).toLocaleString()}`);
- const targets = data.map(item => new Target(
- item.panelUrl,
- item.panelTitle,
- item.panelId,
- item.datasourceRequest,
- [item.target],
- item.datasourceName,
+ const targets = tasks.map((task: Task) => new Target(
+ task.panel,
+ task.datasource,
));
-
const exporter = exporterFactory.getExporter();
- exporter.export(targets, user, from, to);
+ exporter.export(targets, datasourceUrl, username, from, to);
}
}
diff --git a/src/services/exporter.ts b/src/services/exporter.ts
index ab0b766..e7899b2 100644
--- a/src/services/exporter.ts
+++ b/src/services/exporter.ts
@@ -3,6 +3,7 @@ import { URL } from 'url';
import { apiKeys } from '../config';
import { promisify } from '../utils';
import { ExportStatus } from '../types/export-status';
+import { DataSourceRef } from '../types';
import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit';
// TODO: export QueryType directly from @corpglory/tsdb-kit
@@ -21,11 +22,12 @@ export class Exporter {
private exportedRows = 0;
private createdTimestamp: number;
private user: string;
- private datasource: string;
+ private datasourceRef: DataSourceRef;
private initCsvStream() {
- // @ts-ignore
- const csvStream = csv.createWriteStream({ headers: true });
+ const csvStream = csv.parse({ headers: true })
+ .on('error', error => console.error(error));
+
const writableStream = fs.createWriteStream(this.getFilePath('csv'));
csvStream.pipe(writableStream);
@@ -46,7 +48,7 @@ export class Exporter {
exportedRows: this.exportedRows,
progress: progress.toLocaleString('en', { style: 'percent' }),
status,
- datasourceName: this.datasource,
+ datasourceRef: this.datasourceRef,
};
await promisify(fs.writeFile, this.getFilePath('json'), JSON.stringify(data), 'utf8')
@@ -56,16 +58,31 @@ export class Exporter {
}
}
- public async export(data: Target[], user: string, from: number, to: number) {
+ public async export(data: Target[], datasourceUrl: string, user: string, from: number, to: number) {
this.user = user;
- this.validateTargets(data);
- const targets = data.map(target => ({
- ...target,
- metric: new QueryConfig(QueryType.GRAFANA, target.datasource, target.targets)
- }));
+ this.validateTargets(datasourceUrl, data);
+
+ // console.log('ds', data[0].datasource)
+ const targets = data.map(target => {
+ console.log({
+ ...target.datasource,
+ url: datasourceUrl
+ })
+ return {
+ ...target,
+ metric: new QueryConfig(
+ QueryType.GRAFANA,
+ {
+ ...target.datasource,
+ url: datasourceUrl
+ },
+ target.panel.targets
+ )
+ }
+ });
- this.datasource = data.length === 1 ? data[0].datasourceName : 'all';
+ this.datasourceRef = data.length === 1 ? data[0].datasource : { uid: 'all', type: 'all' };
const stream = this.initCsvStream();
const days = Math.ceil((to - from) / MS_IN_DAY);
@@ -81,13 +98,13 @@ export class Exporter {
const values = {};
for(const [index, target] of targets.entries()) {
- const host = new URL(target.panelUrl).origin;
+ const host = new URL(datasourceUrl).origin;
const apiKey = apiKeys[host];
- const datasourceMetrics = await queryByConfig(target.metric, target.panelUrl, from, to, apiKey);
+ const datasourceMetrics = await queryByConfig(target.metric, datasourceUrl, from, to, apiKey);
- const column = `${target.panelId}` +
- `-${target.panelTitle.replace(' ', '-')}-${datasourceMetrics.columns[1]}`;
+ const column = `${target.panel.id}` +
+ `-${target.panel.title.replace(' ', '-')}-${datasourceMetrics.columns[1]}`;
columns.push(column);
@@ -120,13 +137,13 @@ export class Exporter {
stream.end();
}
- private validateTargets(targets: Target[]) {
+ private validateTargets(datasourceUrl, targets: Target[]) {
if(!targets || !Array.isArray(targets)) {
throw new Error('Incorrect targets format');
}
for(const target of targets) {
- const host = new URL(target.panelUrl).origin;
+ const host = new URL(datasourceUrl).origin;
const apiKey = apiKeys[host];
if(apiKey === undefined || apiKey === '') {
@@ -156,7 +173,7 @@ export class Exporter {
if(this.createdTimestamp === undefined) {
this.createdTimestamp = moment().valueOf();
}
- return `${this.createdTimestamp}.${this.datasource}.${extension}`;
+ return `${this.createdTimestamp}.${this.datasourceRef.uid}.${extension}`;
}
private getFilePath(extension) {
diff --git a/src/types/index.ts b/src/types/index.ts
new file mode 100644
index 0000000..df63d77
--- /dev/null
+++ b/src/types/index.ts
@@ -0,0 +1,97 @@
+import { DatasourceType } from '@corpglory/tsdb-kit';
+
+export interface DataSourceRef {
+ /** The plugin type-id */
+ type?: string;
+ /** Specific datasource instance */
+ uid?: string;
+}
+
+export interface DataQuery {
+ /**
+ * A - Z
+ */
+ refId: string;
+ /**
+ * true if query is disabled (ie should not be returned to the dashboard)
+ */
+ hide?: boolean;
+ /**
+ * Unique, guid like, string used in explore mode
+ */
+ key?: string;
+ /**
+ * Specify the query flavor
+ */
+ queryType?: string;
+ /**
+ * For mixed data sources the selected datasource is on the query level.
+ * For non mixed scenarios this is undefined.
+ */
+ datasource?: DataSourceRef | null;
+}
+
+/**
+ * Data Source instance edit model. This is returned from:
+ * /api/datasources
+ */
+export interface DataSourceSettings {
+ id: number;
+ uid: string;
+ orgId: number;
+ name: string;
+ typeLogoUrl: string;
+ type: DatasourceType;
+ typeName: string;
+ access: string;
+ url: string;
+ user: string;
+ database: string;
+ basicAuth: boolean;
+ basicAuthUser: string;
+ isDefault: boolean;
+ jsonData: any;
+ secureJsonData?: any;
+ secureJsonFields: any;
+ readOnly: boolean;
+ withCredentials: boolean;
+ version?: number;
+ accessControl?: any;
+}
+
+export interface PanelModel {
+ /** ID of the panel within the current dashboard */
+ id: number;
+ /** Panel title */
+ title?: string;
+ /** Description */
+ description?: string;
+ /** Panel options */
+ options: any;
+ /** Field options configuration */
+ fieldConfig: any;
+ /** Version of the panel plugin */
+ pluginVersion?: string;
+ /** The datasource used in all targets */
+ datasource?: DataSourceRef | null;
+ /** The queries in a panel */
+ targets?: DataQuery[];
+ /** alerting v1 object */
+ alert?: any;
+}
+
+export type Task = DataQuery & {
+ selected: boolean;
+ panel: PanelModel;
+ datasource: DataSourceSettings;
+};
+
+export type TaskTableRowConfig = {
+ timestamp: number;
+ user: string;
+ datasourceRef: DataSourceRef;
+ rowsCount: number;
+ progress: number;
+ status: string;
+ downloadLink: string;
+};
diff --git a/src/types/target.ts b/src/types/target.ts
index 4ad80ae..d6639b3 100644
--- a/src/types/target.ts
+++ b/src/types/target.ts
@@ -1,12 +1,8 @@
-import { Datasource } from '@corpglory/tsdb-kit';
+import { DataSourceSettings, PanelModel } from '.';
export class Target {
constructor(
- public panelUrl: string,
- public panelTitle: string,
- public panelId: number,
- public datasource: Datasource,
- public targets: Array