Merge pull request #1 from CorpGlory/use-grafana-datasource-kit

.gitignore

@@ -1,6 +1,5 @@
 node_modules/
 dist/
-exported/*.csv
+exported/
-exported/*.json
 api-keys.json
 package-lock.json

src/grafana-datasource-kit/grafana_service.ts

@@ -0,0 +1,70 @@
+import { GrafanaMetric } from './grafana_metric_model';
+import { getApiKey } from '../config';
+import { URL } from 'url';
+import axios from 'axios';
+
+
+const CHUNK_SIZE = 50000;
+
+
+/**
+ * @param metric to query to Grafana
+ * @returns [time, value][] array
+ */
+export async function queryByMetric(
+  metric: GrafanaMetric, panelUrl: string, from: number, to: number
+) {
+
+  let datasource = metric.datasource;
+
+  let origin = new URL(panelUrl).origin;
+  let url = `${origin}/${datasource.url}`;
+
+  let params = datasource.params
+  let data = {
+    values: [],
+    columns: []
+  };
+
+  let chunkParams = Object.assign({}, params);
+  while(true) {
+    chunkParams.q = metric.metricQuery.getQuery(from, to, CHUNK_SIZE, data.values.length);
+    var chunk = await queryGrafana(url, chunkParams);
+    let values = chunk.values;
+    data.values = data.values.concat(values);
+    data.columns = chunk.columns;
+
+    if(values.length < CHUNK_SIZE) {
+      // because if we get less that we could, then there is nothing more
+      break;
+    }
+  }
+
+  return data;
+}
+
+async function queryGrafana(url: string, params: any) {
+  let origin = new URL(url).origin;
+  let headers = { Authorization: `Bearer ${getApiKey(origin)}` };
+
+  try {
+    var res = await axios.get(url, { params, headers });
+  } catch (e) {
+    if(e.response.status === 401) {
+      throw new Error('Unauthorized. Check the $HASTIC_API_KEY.');
+    }
+    throw new Error(e.message);
+  }
+
+  if (res.data.results === undefined) {
+    throw new Error('results field is undefined in response.');
+  }
+
+  // TODO: support more than 1 metric (each res.data.results item is a metric)
+  let results = res.data.results[0];
+  if (results.series === undefined) {
+    return [];
+  }
+
+  return results.series[0];
+}

src/grafana_api.ts

@@ -1,66 +0,0 @@
-import { getApiKey } from './config';
-
-import axios from 'axios';
-
-
-export class GrafanaAPI {
-  private apiKey;
-  constructor(private grafanaUrl) {
-    getApiKey(grafanaUrl)
-      .then(key => {
-        this.apiKey = key;
-        console.log(this.apiKey);
-      });
-  }
-
-  private get _headers() {
-    return {
-      'Authorization': `Bearer ${this.apiKey}`,
-      'Accept': 'application/json',
-      'Content-Type': 'application/json'
-    };
-  }
-
-  private async _getDatasourceByName(name) {
-    return fetch(`${this.grafanaUrl}/api/datasources/name/${name}`, {
-      method: 'GET',
-      headers: this._headers
-    })
-      .then(data => data.json());
-  }
-
-  public async queryDatasource(datasourceName, measurement, query) {
-    let datasource = await this._getDatasourceByName(datasourceName);
-
-    return this._queryGrafana(`${this.grafanaUrl}/api/datasources/proxy/${datasource.id}/query`, {
-      q: encodeURIComponent(query),
-      db: datasource.database,
-      epoch: 'ms'
-    });
-  }
-
-  private async _queryGrafana(url: string, params: any) {
-    try {
-      var res = await axios.get(url, { params, headers: this._headers });
-    } catch (e) {
-      if(e.response.status === 401) {
-        throw new Error('Unauthorized. Check the $HASTIC_API_KEY.');
-      }
-      throw new Error(e.message);
-    }
-
-    if (res.data.results === undefined) {
-      throw new Error('results field is undefined in response.');
-    }
-
-    // TODO: support more than 1 metric (each res.data.results item is a metric)
-    let results = res.data.results[0];
-    if (results.series === undefined) {
-      return [];
-    }
-
-    return results.series[0].values as [number, number][];
-  }
-
-}
-

src/routes/tasks.ts

@@ -7,6 +7,10 @@ async function addTask(req, res) {
   let body = req.body;
   let from = parseInt(body.from);
   let to = parseInt(body.to);
+  let panelUrl = body.panelUrl;
+  let targets = [body.target];
+  let datasource = body.datasourceRequest;
+  let user = body.user;
 
   if(isNaN(from) || isNaN(to)) {
     res.status(500).send('Range error: please fill both "from" and "to" fields');
@@ -14,8 +18,7 @@ async function addTask(req, res) {
     res.status(500).send('Range error: "from" should be less than "to"');
   } else {
     res.status(200).send('Task added');
-    let grafanaUrl = req.get('origin');
+    let target = new Target(panelUrl, user, datasource, targets, from, to);
-    let target = new Target(grafanaUrl, body.user, body.datasource, body.measurement, body.query, from, to);
     target.export();
   }
 }

src/target.ts

@@ -1,4 +1,5 @@
-import { GrafanaAPI } from './grafana_api';
+import { queryByMetric } from './grafana-datasource-kit/grafana_service';
+import { GrafanaDatasource, GrafanaMetric } from './grafana-datasource-kit/grafana_metric_model';
 
 import * as csv from 'fast-csv';
 import * as path from 'path';
@@ -13,18 +14,17 @@ export class Target {
   private days: number;
   private day: number;
   private csvStream: any;
-  private grafana: GrafanaAPI;
+  private metric: GrafanaMetric;
 
   constructor(
-    private grafanaUrl: string,
+    private panelUrl: string,
     private user: string,
-    private datasource: string,
+    datasource: GrafanaDatasource,
-    private measurement: string,
+    targets: Array<Object>,
-    private query: string,
     private from: number,
     private to: number
   ) {
-    this.grafana = new GrafanaAPI(this.grafanaUrl);
+    this.metric = new GrafanaMetric(datasource, targets);
   }
 
   public updateStatus(status) {
@@ -32,8 +32,6 @@ export class Target {
     let data = {
       time,
       user: this.user,
-      datasource: this.datasource,
-      measurement: this.measurement,
       exportedRows: this.exportedRows,
       progress: (this.day / this.days).toLocaleString('en', { style: 'percent' }),
       status
@@ -66,11 +64,9 @@ export class Target {
 
       console.log(`${this.day} day: ${from}ms -> ${to}ms`);
 
-      let currentQuery = this.query.replace('$timeFilter', `time >= ${from}ms AND time <= ${to}ms`).replace('$__interval', '1s');
+      let metrics = await queryByMetric(this.metric, this.panelUrl, from, to);
-      let metrics = await this.grafana.queryDatasource(this.datasource, this.measurement, currentQuery);
 
-      console.log(metrics);
+      if(metrics.values.length > 0) {
-      if(metrics.length > 0) {
         if(metrics !== undefined) {
           this.writeCsv(metrics);
         }
@@ -94,22 +90,22 @@ export class Target {
   }
 
   private writeCsv(series) {
-    for(let serie of series) {
+    for(let val of series.values) {
-      for(let val of serie.values) {
+      if(val[1] !== null) {
-        if(val[1] !== null) {
+        let row = {};
-          let row = {};
+        for(let col in series.columns) {
-          for(let col in serie.columns) {
+          row[series.columns[col]] = val[col];
-            row[serie.columns[col]] = val[col];
-          }
-          this.csvStream.write(row);
-          this.exportedRows++;
         }
+        this.csvStream.write(row);
+        this.exportedRows++;
       }
     }
   }
 
   private getFilename(extension) {
-    return `${this.datasource}.${this.measurement}.${this.from}-${this.to}.${extension}`;
+    // TODO: use something unique instead of measurement in filename
+    // as measurement field exists only in influxDB metric
+    return `${this.metric.targets[0].measurement}.${this.from}-${this.to}.${extension}`;
   }
 
   private getFilePath(extension) {