diff --git a/.gitignore b/.gitignore index d64dacc..1beaf89 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ node_modules/ dist/ -exported/*.csv -exported/*.json +exported/ api-keys.json package-lock.json diff --git a/src/grafana_metric_model.ts b/src/grafana-datasource-kit/grafana_metric_model.ts similarity index 100% rename from src/grafana_metric_model.ts rename to src/grafana-datasource-kit/grafana_metric_model.ts diff --git a/src/grafana-datasource-kit/grafana_service.ts b/src/grafana-datasource-kit/grafana_service.ts new file mode 100644 index 0000000..e105066 --- /dev/null +++ b/src/grafana-datasource-kit/grafana_service.ts @@ -0,0 +1,70 @@ +import { GrafanaMetric } from './grafana_metric_model'; +import { getApiKey } from '../config'; +import { URL } from 'url'; +import axios from 'axios'; + + +const CHUNK_SIZE = 50000; + + +/** + * @param metric to query to Grafana + * @returns [time, value][] array + */ +export async function queryByMetric( + metric: GrafanaMetric, panelUrl: string, from: number, to: number +) { + + let datasource = metric.datasource; + + let origin = new URL(panelUrl).origin; + let url = `${origin}/${datasource.url}`; + + let params = datasource.params + let data = { + values: [], + columns: [] + }; + + let chunkParams = Object.assign({}, params); + while(true) { + chunkParams.q = metric.metricQuery.getQuery(from, to, CHUNK_SIZE, data.values.length); + var chunk = await queryGrafana(url, chunkParams); + let values = chunk.values; + data.values = data.values.concat(values); + data.columns = chunk.columns; + + if(values.length < CHUNK_SIZE) { + // because if we get less that we could, then there is nothing more + break; + } + } + + return data; +} + +async function queryGrafana(url: string, params: any) { + let origin = new URL(url).origin; + let headers = { Authorization: `Bearer ${getApiKey(origin)}` }; + + try { + var res = await axios.get(url, { params, headers }); + } catch (e) { + if(e.response.status === 401) { + throw new Error('Unauthorized. Check the $HASTIC_API_KEY.'); + } + throw new Error(e.message); + } + + if (res.data.results === undefined) { + throw new Error('results field is undefined in response.'); + } + + // TODO: support more than 1 metric (each res.data.results item is a metric) + let results = res.data.results[0]; + if (results.series === undefined) { + return []; + } + + return results.series[0]; +} diff --git a/src/grafana_api.ts b/src/grafana_api.ts deleted file mode 100644 index 76a8efd..0000000 --- a/src/grafana_api.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { getApiKey } from './config'; - -import axios from 'axios'; - - -export class GrafanaAPI { - private apiKey; - constructor(private grafanaUrl) { - getApiKey(grafanaUrl) - .then(key => { - this.apiKey = key; - console.log(this.apiKey); - }); - } - - private get _headers() { - return { - 'Authorization': `Bearer ${this.apiKey}`, - 'Accept': 'application/json', - 'Content-Type': 'application/json' - }; - } - - private async _getDatasourceByName(name) { - return fetch(`${this.grafanaUrl}/api/datasources/name/${name}`, { - method: 'GET', - headers: this._headers - }) - .then(data => data.json()); - } - - public async queryDatasource(datasourceName, measurement, query) { - let datasource = await this._getDatasourceByName(datasourceName); - - return this._queryGrafana(`${this.grafanaUrl}/api/datasources/proxy/${datasource.id}/query`, { - q: encodeURIComponent(query), - db: datasource.database, - epoch: 'ms' - }); - } - - private async _queryGrafana(url: string, params: any) { - try { - var res = await axios.get(url, { params, headers: this._headers }); - } catch (e) { - if(e.response.status === 401) { - throw new Error('Unauthorized. Check the $HASTIC_API_KEY.'); - } - throw new Error(e.message); - } - - if (res.data.results === undefined) { - throw new Error('results field is undefined in response.'); - } - - // TODO: support more than 1 metric (each res.data.results item is a metric) - let results = res.data.results[0]; - if (results.series === undefined) { - return []; - } - - return results.series[0].values as [number, number][]; - } - -} - diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts index d3af128..c609b90 100644 --- a/src/routes/tasks.ts +++ b/src/routes/tasks.ts @@ -7,6 +7,10 @@ async function addTask(req, res) { let body = req.body; let from = parseInt(body.from); let to = parseInt(body.to); + let panelUrl = body.panelUrl; + let targets = [body.target]; + let datasource = body.datasourceRequest; + let user = body.user; if(isNaN(from) || isNaN(to)) { res.status(500).send('Range error: please fill both "from" and "to" fields'); @@ -14,8 +18,7 @@ async function addTask(req, res) { res.status(500).send('Range error: "from" should be less than "to"'); } else { res.status(200).send('Task added'); - let grafanaUrl = req.get('origin'); - let target = new Target(grafanaUrl, body.user, body.datasource, body.measurement, body.query, from, to); + let target = new Target(panelUrl, user, datasource, targets, from, to); target.export(); } } diff --git a/src/target.ts b/src/target.ts index 50b2819..5eb8852 100644 --- a/src/target.ts +++ b/src/target.ts @@ -1,4 +1,5 @@ -import { GrafanaAPI } from './grafana_api'; +import { queryByMetric } from './grafana-datasource-kit/grafana_service'; +import { GrafanaDatasource, GrafanaMetric } from './grafana-datasource-kit/grafana_metric_model'; import * as csv from 'fast-csv'; import * as path from 'path'; @@ -13,18 +14,17 @@ export class Target { private days: number; private day: number; private csvStream: any; - private grafana: GrafanaAPI; + private metric: GrafanaMetric; constructor( - private grafanaUrl: string, + private panelUrl: string, private user: string, - private datasource: string, - private measurement: string, - private query: string, + datasource: GrafanaDatasource, + targets: Array, private from: number, private to: number ) { - this.grafana = new GrafanaAPI(this.grafanaUrl); + this.metric = new GrafanaMetric(datasource, targets); } public updateStatus(status) { @@ -32,8 +32,6 @@ export class Target { let data = { time, user: this.user, - datasource: this.datasource, - measurement: this.measurement, exportedRows: this.exportedRows, progress: (this.day / this.days).toLocaleString('en', { style: 'percent' }), status @@ -66,11 +64,9 @@ export class Target { console.log(`${this.day} day: ${from}ms -> ${to}ms`); - let currentQuery = this.query.replace('$timeFilter', `time >= ${from}ms AND time <= ${to}ms`).replace('$__interval', '1s'); - let metrics = await this.grafana.queryDatasource(this.datasource, this.measurement, currentQuery); + let metrics = await queryByMetric(this.metric, this.panelUrl, from, to); - console.log(metrics); - if(metrics.length > 0) { + if(metrics.values.length > 0) { if(metrics !== undefined) { this.writeCsv(metrics); } @@ -94,22 +90,22 @@ export class Target { } private writeCsv(series) { - for(let serie of series) { - for(let val of serie.values) { - if(val[1] !== null) { - let row = {}; - for(let col in serie.columns) { - row[serie.columns[col]] = val[col]; - } - this.csvStream.write(row); - this.exportedRows++; + for(let val of series.values) { + if(val[1] !== null) { + let row = {}; + for(let col in series.columns) { + row[series.columns[col]] = val[col]; } + this.csvStream.write(row); + this.exportedRows++; } } } private getFilename(extension) { - return `${this.datasource}.${this.measurement}.${this.from}-${this.to}.${extension}`; + // TODO: use something unique instead of measurement in filename + // as measurement field exists only in influxDB metric + return `${this.metric.targets[0].measurement}.${this.from}-${this.to}.${extension}`; } private getFilePath(extension) {