From 580bbfd6a3103134e063852c59dd9fcc18be0b03 Mon Sep 17 00:00:00 2001 From: rozetko Date: Tue, 11 Sep 2018 20:50:47 +0300 Subject: [PATCH 1/3] grafana-datasource-kit usage start --- .../grafana_metric_model.ts | 0 src/grafana-datasource-kit/grafana_service.ts | 63 ++++++++++++++++++ src/grafana_api.ts | 66 ------------------- src/target.ts | 23 +++++-- 4 files changed, 79 insertions(+), 73 deletions(-) rename src/{ => grafana-datasource-kit}/grafana_metric_model.ts (100%) create mode 100644 src/grafana-datasource-kit/grafana_service.ts delete mode 100644 src/grafana_api.ts diff --git a/src/grafana_metric_model.ts b/src/grafana-datasource-kit/grafana_metric_model.ts similarity index 100% rename from src/grafana_metric_model.ts rename to src/grafana-datasource-kit/grafana_metric_model.ts diff --git a/src/grafana-datasource-kit/grafana_service.ts b/src/grafana-datasource-kit/grafana_service.ts new file mode 100644 index 0000000..a2108f9 --- /dev/null +++ b/src/grafana-datasource-kit/grafana_service.ts @@ -0,0 +1,63 @@ +import { GrafanaMetric } from './grafana_metric_model'; +import { getApiKey } from '../config'; +import { URL } from 'url'; +import axios from 'axios'; + + +const CHUNK_SIZE = 50000; + + +/** + * @param metric to query to Grafana + * @returns [time, value][] array + */ +export async function queryByMetric( + metric: GrafanaMetric, panelUrl: string, from: number, to: number +): Promise<[number, number][]> { + + let datasource = metric.datasource; + + let origin = new URL(panelUrl).origin; + let url = `${origin}/${datasource.url}`; + + let params = datasource.params + let data = []; + + let chunkParams = Object.assign({}, params); + while(true) { + chunkParams.q = metric.metricQuery.getQuery(from, to, CHUNK_SIZE, data.length); + var chunk = await queryGrafana(url, chunkParams); + data = data.concat(chunk); + if(chunk.length < CHUNK_SIZE) { + // because if we get less that we could, then there is nothing more + break; + } + } + + return data; +} + +async function queryGrafana(url: string, params: any) { + let headers = { Authorization: `Bearer ${getApiKey(url)}` }; + + try { + var res = await axios.get(url, { params, headers }); + } catch (e) { + if(e.response.status === 401) { + throw new Error('Unauthorized. Check the $HASTIC_API_KEY.'); + } + throw new Error(e.message); + } + + if (res.data.results === undefined) { + throw new Error('results field is undefined in response.'); + } + + // TODO: support more than 1 metric (each res.data.results item is a metric) + let results = res.data.results[0]; + if (results.series === undefined) { + return []; + } + + return results.series[0].values as [number, number][]; +} diff --git a/src/grafana_api.ts b/src/grafana_api.ts deleted file mode 100644 index 76a8efd..0000000 --- a/src/grafana_api.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { getApiKey } from './config'; - -import axios from 'axios'; - - -export class GrafanaAPI { - private apiKey; - constructor(private grafanaUrl) { - getApiKey(grafanaUrl) - .then(key => { - this.apiKey = key; - console.log(this.apiKey); - }); - } - - private get _headers() { - return { - 'Authorization': `Bearer ${this.apiKey}`, - 'Accept': 'application/json', - 'Content-Type': 'application/json' - }; - } - - private async _getDatasourceByName(name) { - return fetch(`${this.grafanaUrl}/api/datasources/name/${name}`, { - method: 'GET', - headers: this._headers - }) - .then(data => data.json()); - } - - public async queryDatasource(datasourceName, measurement, query) { - let datasource = await this._getDatasourceByName(datasourceName); - - return this._queryGrafana(`${this.grafanaUrl}/api/datasources/proxy/${datasource.id}/query`, { - q: encodeURIComponent(query), - db: datasource.database, - epoch: 'ms' - }); - } - - private async _queryGrafana(url: string, params: any) { - try { - var res = await axios.get(url, { params, headers: this._headers }); - } catch (e) { - if(e.response.status === 401) { - throw new Error('Unauthorized. Check the $HASTIC_API_KEY.'); - } - throw new Error(e.message); - } - - if (res.data.results === undefined) { - throw new Error('results field is undefined in response.'); - } - - // TODO: support more than 1 metric (each res.data.results item is a metric) - let results = res.data.results[0]; - if (results.series === undefined) { - return []; - } - - return results.series[0].values as [number, number][]; - } - -} - diff --git a/src/target.ts b/src/target.ts index 50b2819..bdfb6ed 100644 --- a/src/target.ts +++ b/src/target.ts @@ -1,4 +1,5 @@ -import { GrafanaAPI } from './grafana_api'; +import { queryByMetric } from './grafana-datasource-kit/grafana_service'; +import { GrafanaDatasource } from './grafana-datasource-kit/grafana_metric_model'; import * as csv from 'fast-csv'; import * as path from 'path'; @@ -13,18 +14,26 @@ export class Target { private days: number; private day: number; private csvStream: any; - private grafana: GrafanaAPI; + private _datasource: GrafanaDatasource; constructor( - private grafanaUrl: string, + panelUrl: string, private user: string, - private datasource: string, + datasource: string, private measurement: string, private query: string, private from: number, private to: number - ) { - this.grafana = new GrafanaAPI(this.grafanaUrl); + ) { + this._datasource = { + url: panelUrl, + type: type, + params: { + db: string, + q: string, + epoch: string + } + } } public updateStatus(status) { @@ -67,7 +76,7 @@ export class Target { console.log(`${this.day} day: ${from}ms -> ${to}ms`); let currentQuery = this.query.replace('$timeFilter', `time >= ${from}ms AND time <= ${to}ms`).replace('$__interval', '1s'); - let metrics = await this.grafana.queryDatasource(this.datasource, this.measurement, currentQuery); + let metrics = await queryByMetric(this.datasource, currentQuery); console.log(metrics); if(metrics.length > 0) { From 3d4fa76c234385efcf6a30b80c9c9a1f201f5a9b Mon Sep 17 00:00:00 2001 From: rozetko Date: Wed, 12 Sep 2018 19:11:50 +0300 Subject: [PATCH 2/3] it works --- .gitignore | 3 +- src/grafana-datasource-kit/grafana_service.ts | 21 +++++--- src/routes/tasks.ts | 7 ++- src/target.ts | 49 +++++++------------ 4 files changed, 37 insertions(+), 43 deletions(-) diff --git a/.gitignore b/.gitignore index d64dacc..1beaf89 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ node_modules/ dist/ -exported/*.csv -exported/*.json +exported/ api-keys.json package-lock.json diff --git a/src/grafana-datasource-kit/grafana_service.ts b/src/grafana-datasource-kit/grafana_service.ts index a2108f9..e105066 100644 --- a/src/grafana-datasource-kit/grafana_service.ts +++ b/src/grafana-datasource-kit/grafana_service.ts @@ -13,7 +13,7 @@ const CHUNK_SIZE = 50000; */ export async function queryByMetric( metric: GrafanaMetric, panelUrl: string, from: number, to: number -): Promise<[number, number][]> { +) { let datasource = metric.datasource; @@ -21,14 +21,20 @@ export async function queryByMetric( let url = `${origin}/${datasource.url}`; let params = datasource.params - let data = []; + let data = { + values: [], + columns: [] + }; let chunkParams = Object.assign({}, params); while(true) { - chunkParams.q = metric.metricQuery.getQuery(from, to, CHUNK_SIZE, data.length); + chunkParams.q = metric.metricQuery.getQuery(from, to, CHUNK_SIZE, data.values.length); var chunk = await queryGrafana(url, chunkParams); - data = data.concat(chunk); - if(chunk.length < CHUNK_SIZE) { + let values = chunk.values; + data.values = data.values.concat(values); + data.columns = chunk.columns; + + if(values.length < CHUNK_SIZE) { // because if we get less that we could, then there is nothing more break; } @@ -38,7 +44,8 @@ export async function queryByMetric( } async function queryGrafana(url: string, params: any) { - let headers = { Authorization: `Bearer ${getApiKey(url)}` }; + let origin = new URL(url).origin; + let headers = { Authorization: `Bearer ${getApiKey(origin)}` }; try { var res = await axios.get(url, { params, headers }); @@ -59,5 +66,5 @@ async function queryGrafana(url: string, params: any) { return []; } - return results.series[0].values as [number, number][]; + return results.series[0]; } diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts index d3af128..c609b90 100644 --- a/src/routes/tasks.ts +++ b/src/routes/tasks.ts @@ -7,6 +7,10 @@ async function addTask(req, res) { let body = req.body; let from = parseInt(body.from); let to = parseInt(body.to); + let panelUrl = body.panelUrl; + let targets = [body.target]; + let datasource = body.datasourceRequest; + let user = body.user; if(isNaN(from) || isNaN(to)) { res.status(500).send('Range error: please fill both "from" and "to" fields'); @@ -14,8 +18,7 @@ async function addTask(req, res) { res.status(500).send('Range error: "from" should be less than "to"'); } else { res.status(200).send('Task added'); - let grafanaUrl = req.get('origin'); - let target = new Target(grafanaUrl, body.user, body.datasource, body.measurement, body.query, from, to); + let target = new Target(panelUrl, user, datasource, targets, from, to); target.export(); } } diff --git a/src/target.ts b/src/target.ts index bdfb6ed..2c3fc53 100644 --- a/src/target.ts +++ b/src/target.ts @@ -1,5 +1,5 @@ import { queryByMetric } from './grafana-datasource-kit/grafana_service'; -import { GrafanaDatasource } from './grafana-datasource-kit/grafana_metric_model'; +import { GrafanaDatasource, GrafanaMetric } from './grafana-datasource-kit/grafana_metric_model'; import * as csv from 'fast-csv'; import * as path from 'path'; @@ -14,26 +14,17 @@ export class Target { private days: number; private day: number; private csvStream: any; - private _datasource: GrafanaDatasource; + private metric: GrafanaMetric; constructor( - panelUrl: string, + private panelUrl: string, private user: string, - datasource: string, - private measurement: string, - private query: string, + datasource: GrafanaDatasource, + targets: Array, private from: number, private to: number - ) { - this._datasource = { - url: panelUrl, - type: type, - params: { - db: string, - q: string, - epoch: string - } - } + ) { + this.metric = new GrafanaMetric(datasource, targets); } public updateStatus(status) { @@ -41,8 +32,6 @@ export class Target { let data = { time, user: this.user, - datasource: this.datasource, - measurement: this.measurement, exportedRows: this.exportedRows, progress: (this.day / this.days).toLocaleString('en', { style: 'percent' }), status @@ -75,11 +64,9 @@ export class Target { console.log(`${this.day} day: ${from}ms -> ${to}ms`); - let currentQuery = this.query.replace('$timeFilter', `time >= ${from}ms AND time <= ${to}ms`).replace('$__interval', '1s'); - let metrics = await queryByMetric(this.datasource, currentQuery); + let metrics = await queryByMetric(this.metric, this.panelUrl, from, to); - console.log(metrics); - if(metrics.length > 0) { + if(metrics.values.length > 0) { if(metrics !== undefined) { this.writeCsv(metrics); } @@ -103,22 +90,20 @@ export class Target { } private writeCsv(series) { - for(let serie of series) { - for(let val of serie.values) { - if(val[1] !== null) { - let row = {}; - for(let col in serie.columns) { - row[serie.columns[col]] = val[col]; - } - this.csvStream.write(row); - this.exportedRows++; + for(let val of series.values) { + if(val[1] !== null) { + let row = {}; + for(let col in series.columns) { + row[series.columns[col]] = val[col]; } + this.csvStream.write(row); + this.exportedRows++; } } } private getFilename(extension) { - return `${this.datasource}.${this.measurement}.${this.from}-${this.to}.${extension}`; + return `${this.from}-${this.to}.${extension}`; } private getFilePath(extension) { From 6cca9cbb3a4f12a40760c3f1de03f83b01444bf8 Mon Sep 17 00:00:00 2001 From: rozetko Date: Thu, 13 Sep 2018 13:00:11 +0300 Subject: [PATCH 3/3] fix filename --- src/target.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/target.ts b/src/target.ts index 2c3fc53..5eb8852 100644 --- a/src/target.ts +++ b/src/target.ts @@ -103,7 +103,9 @@ export class Target { } private getFilename(extension) { - return `${this.from}-${this.to}.${extension}`; + // TODO: use something unique instead of measurement in filename + // as measurement field exists only in influxDB metric + return `${this.metric.targets[0].measurement}.${this.from}-${this.to}.${extension}`; } private getFilePath(extension) {