From 64bbcb8065b75f6948333e63ca48609cd3de54ef Mon Sep 17 00:00:00 2001 From: Alexey Velikiy Date: Sat, 25 Aug 2018 19:36:03 +0300 Subject: [PATCH] better metrics trnslation / quering --- server/src/models/analytic_unit_model.ts | 6 +- server/src/models/grafana_metric_model.ts | 85 +++++++++++++++++++++++ server/src/models/metric_model.ts | 48 ------------- server/src/services/grafana_service.ts | 22 ++---- 4 files changed, 94 insertions(+), 67 deletions(-) create mode 100644 server/src/models/grafana_metric_model.ts delete mode 100644 server/src/models/metric_model.ts diff --git a/server/src/models/analytic_unit_model.ts b/server/src/models/analytic_unit_model.ts index 4c552b6..559d4e7 100644 --- a/server/src/models/analytic_unit_model.ts +++ b/server/src/models/analytic_unit_model.ts @@ -1,4 +1,4 @@ -import { Metric } from './metric_model'; +import { GrafanaMetric } from './grafana_metric_model'; import { Collection, makeDBQ } from '../services/data_service'; @@ -19,7 +19,7 @@ export class AnalyticUnit { public name: string, public panelUrl: string, public type: string, - public metric: Metric, + public metric: GrafanaMetric, public id?: AnalyticUnitId, public lastPredictionTime?: number, public status?: AnalyticUnitStatus, @@ -60,7 +60,7 @@ export class AnalyticUnit { obj.name, obj.panelUrl, obj.type, - Metric.fromObject(obj.metric), + GrafanaMetric.fromObject(obj.metric), obj._id, obj.lastPredictionTime, obj.status as AnalyticUnitStatus, diff --git a/server/src/models/grafana_metric_model.ts b/server/src/models/grafana_metric_model.ts new file mode 100644 index 0000000..7f8c7f7 --- /dev/null +++ b/server/src/models/grafana_metric_model.ts @@ -0,0 +1,85 @@ +export type GrafanaDatasource = { + url: string, + type: string, + params: { + db: string, + q: string, + epoch: string + } +} + +export type GrafanaMetricId = string; + +export class GrafanaMetric { + + private _metricQuery: MetricQuery = undefined; + + constructor( + public datasource: GrafanaDatasource, + public targets: any[], + public id?: GrafanaMetricId + ) { + if(datasource === undefined) { + throw new Error('datasource is undefined'); + } + if(targets === undefined) { + throw new Error('targets is undefined'); + } + if(targets.length === 0) { + throw new Error('targets is empty'); + } + } + + public get metricQuery() { + if(this._metricQuery === undefined) { + this._metricQuery = new MetricQuery(this); + } + return this._metricQuery; + } + + public toObject() { + return { + datasource: this.datasource, + targets: this.targets, + _id: this.id + }; + } + + static fromObject(obj: any): GrafanaMetric { + if(obj === undefined) { + throw new Error('obj is undefined'); + } + return new GrafanaMetric( + obj.datasource, + obj.targets, + obj._id + ); + } +} + +export class MetricQuery { + + private static INFLUX_QUERY_TIME_REGEX = /time >[^A-Z]+/; + + private _queryParts: string[]; + + constructor(metric: GrafanaMetric) { + if (metric.datasource.type !== 'influxdb') { + throw new Error(`Queries of type "${metric.datasource.type}" are not supported yet.`); + } + var queryStr = metric.datasource.params.q; + this._queryParts = queryStr.split(MetricQuery.INFLUX_QUERY_TIME_REGEX); + if(this._queryParts.length == 1) { + throw new Error( + `Query "${queryStr}" is not replaced with LIMIT/OFFSET oeprators. Missing time clause.` + ); + } + if(this._queryParts.length > 2) { + throw new Error(`Query "${queryStr}" has multiple time clauses. Can't parse.`); + } + } + + getQuery(limit: number, offset: number): string { + return `${this._queryParts[0]} TRUE ${this._queryParts[1]} LIMIT ${limit} OFFSET ${offset}`; + } +} diff --git a/server/src/models/metric_model.ts b/server/src/models/metric_model.ts deleted file mode 100644 index d1c1936..0000000 --- a/server/src/models/metric_model.ts +++ /dev/null @@ -1,48 +0,0 @@ -type Datasource = { - url: string, - type: string, - params: { - db: string, - q: string, - epoch: string - } -} - -export type MetricId = string; - -export class Metric { - constructor( - public datasource: Datasource, - public targets: any[], - public id?: MetricId - ) { - if(datasource === undefined) { - throw new Error('datasource is undefined'); - } - if(targets === undefined) { - throw new Error('targets is undefined'); - } - if(targets.length === 0) { - throw new Error('targets is empty'); - } - } - - public toObject() { - return { - datasource: this.datasource, - targets: this.targets, - _id: this.id - }; - } - - static fromObject(obj: any): Metric { - if(obj === undefined) { - throw new Error('obj is undefined'); - } - return new Metric( - obj.datasource, - obj.targets, - obj._id - ); - } -} diff --git a/server/src/services/grafana_service.ts b/server/src/services/grafana_service.ts index 2d05223..1b37785 100644 --- a/server/src/services/grafana_service.ts +++ b/server/src/services/grafana_service.ts @@ -1,23 +1,19 @@ -import { Metric } from '../models/metric_model'; +import { GrafanaMetric } from '../models/grafana_metric_model'; import { HASTIC_API_KEY } from '../config'; import { URL } from 'url'; import axios from 'axios'; const CHUNK_SIZE = 50000; -const QUERY_TIME_REPLACE_REGEX = /(WHERE time >[^A-Z]+)/; + /** * @param metric to query to Grafana * @returns [time, value][] array */ -export async function queryByMetric(metric: Metric, panelUrl: string): Promise<[number, number][]> { +export async function queryByMetric(metric: GrafanaMetric, panelUrl: string): Promise<[number, number][]> { let datasource = metric.datasource; - if (datasource.type !== 'influxdb') { - throw new Error(`${datasource.type} queries are not supported yet`); - } - let origin = new URL(panelUrl).origin; let url = `${origin}/${datasource.url}`; @@ -30,15 +26,9 @@ export async function queryByMetric(metric: Metric, panelUrl: string): Promise<[ let data = []; while (offset <= records) { let paramsClone = Object.assign({}, params); - let replacedQ = paramsClone.q.replace(QUERY_TIME_REPLACE_REGEX, `LIMIT ${limit} OFFSET ${offset}`); - if(replacedQ === paramsClone.q) { - throw new Error(`Query "${paramsClone.q}" is not replaced with LIMIT/OFFSET oeprators`) - } - paramsClone.q = replacedQ; - + paramsClone.q = metric.metricQuery.getQuery(limit, offset); let chunk = await queryGrafana(url, paramsClone); data = data.concat(chunk); - offset += CHUNK_SIZE; } @@ -64,13 +54,13 @@ async function queryGrafana(url: string, params: any) { res = await axios.get(url, { params, headers }); } catch (e) { if(e.response.status === 401) { - throw new Error('Unauthorized. Check the $HASTIC_API_KEY'); + throw new Error('Unauthorized. Check the $HASTIC_API_KEY.'); } throw new Error(e.message); } if (res.data.results === undefined) { - throw new Error('results field is undefined in response'); + throw new Error('results field is undefined in response.'); } // TODO: support more than 1 metric (each res.data.results item is a metric)