Get all data from Grafana's datasource by chunks (#111)

server/src/services/grafana_service.ts

@@ -3,10 +3,11 @@ import { Metric } from '../models/metric_model';
 import { HASTIC_API_KEY } from '../config';
 
 import { URL } from 'url';
-import { stringify } from 'querystring';
 import axios from 'axios';
 
 
+const CHUNK_SIZE = 50000;
+
 export type Timestamp = number;
 /**
  * @param metric to query to Grafana
@@ -15,25 +16,63 @@ export type Timestamp = number;
 export async function queryByMetric(metric: Metric, panelUrl: string): Promise<[number, number][]> {
   let datasource = metric.datasource;
 
-  if(datasource.type !== 'influxdb') {
+  if (datasource.type !== 'influxdb') {
     throw new Error(`${datasource.type} queries are not supported yet`);
   }
-  var params = {};
+
   let origin = new URL(panelUrl).origin;
-  let url = `${origin}/${datasource.url}?${stringify(params)}`;
+  let url = `${origin}/${datasource.url}`;
-  console.log(url)
+
+  let params = datasource.params
+  let records = await getRecordsCount(url, params);
+
+  let limit = Math.min(records, CHUNK_SIZE);
+  let offset = 0;
+
+  let data = [];
+  while (offset <= records) {
+    let paramsClone = Object.assign({}, params);
+    paramsClone.q = paramsClone.q.replace(/(WHERE time >[^A-Z]+)/, `LIMIT ${limit} OFFSET ${offset}`);
+
+    let chunk = await queryGrafana(url, paramsClone);
+    data = data.concat(chunk);
+
+    offset += CHUNK_SIZE;
+  }
 
-  let headers = { 'Authorization': 'Bearer ' + HASTIC_API_KEY };
+  return data;
+}
+
+async function getRecordsCount(url: string, params: any) {
+  let paramsClone = Object.assign({}, params);
+  let query = paramsClone.q;
+
+  let field = query.match(/"(\w+)"\)*\sFROM/)[1];
+  let measurement = query.match(/FROM\s"(\w+)"/)[1];
+  paramsClone.q = `SELECT COUNT(${field}) FROM ${measurement}`;
+  let result = await queryGrafana(url, paramsClone);
+  return result[0][1];
+}
 
-  let res = await axios.get(url, { headers });
+async function queryGrafana(url: string, params: any) {
+  let headers = { Authorization: `Bearer ${HASTIC_API_KEY}` };
 
-  let results = res.data['results'];
+  let res;
-  console.log(results)
+  try {
-  if(results === undefined) {
+    res = await axios.get(url, { params, headers });
-    throw new Error('reuslts field is undefined in response');
+  } catch (e) {
+    console.error(`Error while getting data from Grafana: ${e}`);
   }
-  if(results.series === undefined) {
+
+  if (res.data.results === undefined) {
+    throw new Error('results field is undefined in response');
+  }
+
+  // TODO: support more than 1 metric (each res.data.results item is a metric)
+  let results = res.data.results[0];
+  if (results.series === undefined) {
     return [];
   }
-  return res['series'][0];
+
+  return results.series[0].values;
 }