hastic
/
tsdb-kit
4
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity
Browse Source

better names

pull/5/head
rozetko 2 years ago
parent
8112975b12
commit
5e20b4de96
16 changed files with 132 additions and 138 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      spec/elasticsearch.jest.ts
  2. 7
      spec/graphite.jest.ts
  3. 6
      spec/postgres.jest.ts
  4. 15
      spec/targets.jest.ts
  5. 29
      src/index.ts
  6. 22
      src/metrics/elasticsearch_metric.ts
  7. 12
      src/metrics/graphite_metric.ts
  8. 13
      src/metrics/influxdb_metric.ts
  9. 14
      src/metrics/metric.ts
  10. 62
      src/metrics/metrics_factory.ts
  11. 14
      src/metrics/prometheus_metric.ts
  12. 12
      src/metrics/sql_metric.ts
  13. 45
      src/models/query_config.ts
  14. 4
      src/services/direct_service.ts
  15. 4
      src/services/grafana_service.ts
  16. 9
      src/tsdb-kit/index.ts

2
spec/elasticsearch.jest.ts
Unescape View File

@ -265,6 +265,6 @@ describe('simple query', function(){
]
};
expect(elasticMetric.getResults(result)).toEqual(expectedResult);
expect(elasticMetric.parseResponse(result)).toEqual(expectedResult);
});
});

7
spec/graphite.jest.ts
Unescape View File

@ -1,5 +1,4 @@
import { Datasource, Metric } from '../src/index';
import { DatasourceType } from '../src/metrics/metric';
import { Datasource, QueryConfig, DatasourceType } from '../src/index';
import 'jest';
@ -17,10 +16,10 @@ describe('correct Graphite query', function() {
};
let target = `target=template(hosts.$hostname.cpu, hostname="worker1")`;
let query = new Metric(datasource, [target]);
let queryConfig = new QueryConfig(datasource, [target]);
it("test simple query with time clause", function () {
expect(query.metricQuery.getQuery(1534809600000, 1537488000000, 500, 0).url).toBe(
expect(queryConfig.datasourceConnector.getQuery(1534809600000, 1537488000000, 500, 0).url).toBe(
`${datasource.url}?target=${target}&from=1534809600&until=1537488000&maxDataPoints=500`
)
});

6
spec/postgres.jest.ts
Unescape View File

@ -1,5 +1,5 @@
import { PostgresMetric } from '../src/metrics/postgres_metric';
import { DatasourceType, MetricQuery } from '../src/metrics/metric';
import { DatasourceType, DatasourceQuery } from '../src/metrics/metric';
import 'jest';
import * as _ from 'lodash';
@ -12,7 +12,7 @@ describe('Test query creation', function() {
let from = 1542983750857;
let to = 1542984313292;
let postgres = getMetricForSqlQuery();
let mQuery: MetricQuery = postgres.getQuery(from, to, limit, offset);
let mQuery: DatasourceQuery = postgres.getQuery(from, to, limit, offset);
it('test that payload placed to data field', function() {
expect('data' in mQuery.schema).toBeTruthy();
@ -58,7 +58,7 @@ describe('Test result parsing', function() {
}
}
let result = postgres.getResults(response);
let result = postgres.parseResponse(response);
it('check results columns order', function() {
let timestampColumnNumber = result.columns.indexOf('timestamp');

15
spec/targets.jest.ts
Unescape View File

@ -1,5 +1,4 @@
import { Datasource, Metric } from '../src/index';
import { DatasourceType } from '../src/metrics/metric';
import { Datasource, DatasourceType, QueryConfig } from '../src/index';
import 'jest';
@ -18,24 +17,24 @@ describe('Correct InfluxDB query', function() {
let target = 'mean("value")';
it("test query with two time expressions", function() {
let query = new Metric(datasource, [target]);
expect(query.metricQuery.getQuery(1534809600,1537488000,666,10).schema.params.q).toBe(
const queryConfig = new QueryConfig(datasource, [target]);
expect(queryConfig.datasourceConnector.getQuery(1534809600,1537488000,666,10).schema.params.q).toBe(
`SELECT mean("value") FROM "db" WHERE time >= 1534809600ms AND time <= 1537488000ms LIMIT 666 OFFSET 10`
)
});
it('test query with one time expression', function() {
datasource.params.q = `SELECT mean("value") FROM "cpu_value" WHERE time >= now() - 6h GROUP BY time(30s) fill(null)`;
let query = new Metric(datasource, [target]);
expect(query.metricQuery.getQuery(1534809600,1537488000,666,10).schema.params.q).toBe(
const queryConfig = new QueryConfig(datasource, [target]);
expect(queryConfig.datasourceConnector.getQuery(1534809600,1537488000,666,10).schema.params.q).toBe(
`SELECT mean("value") FROM "cpu_value" WHERE time >= 1534809600ms AND time <= 1537488000ms GROUP BY time(30s) fill(null) LIMIT 666 OFFSET 10`
)
});
it('test query with time expression', function() {
datasource.params.q = `SELECT mean("value") FROM "cpu_value" WHERE time>= now() - 6h AND time<xxx GROUP BY time(30s) fill(null)`;
let query = new Metric(datasource, [target]);
expect(query.metricQuery.getQuery(1534809600,1537488000,666,10).schema.params.q).toBe(
const queryConfig = new QueryConfig(datasource, [target]);
expect(queryConfig.datasourceConnector.getQuery(1534809600,1537488000,666,10).schema.params.q).toBe(
`SELECT mean("value") FROM "cpu_value" WHERE time >= 1534809600ms AND time <= 1537488000ms GROUP BY time(30s) fill(null) LIMIT 666 OFFSET 10`
)
});

29
src/index.ts
Unescape View File

@ -1,33 +1,32 @@
import { MetricResults, QueryType } from './metrics/metric';
import { Metric } from './metrics/metrics_factory';
import { DataTable, QueryType } from './metrics/metric';
import { QueryConfig } from './models/query_config';
import { queryDirect } from './services/direct_service';
import { queryGrafana } from './services/grafana_service';
import { BadRange } from './types';
export { Metric } from './metrics/metrics_factory';
export { Datasource } from './metrics/metric'
export { QueryConfig } from './models/query_config';
export { Datasource, DatasourceType, DataTable } from './metrics/metric'
export { DatasourceUnavailable } from './types';
export { GrafanaUnavailable } from './services/grafana_service';
const CHUNK_SIZE = 50000;
/**
* @param metric to query to Grafana
* @param queryConfig
* @returns { values: [time, value][], columns: string[] }
*/
export async function queryByMetric(
export async function queryByConfig(
// TODO: check how did we wanna use `url` field
metric: Metric, url: string, from: number, to: number, queryType: QueryType,
queryConfig: QueryConfig, url: string, from: number, to: number, queryType: QueryType,
// TODO: we need an abstract DatasourceConfig class which will differ in direct and grafana queries
apiKey?: string
): Promise<MetricResults> {
): Promise<DataTable> {
if(from > to) {
throw new BadRange(
`Data-kit got wrong range: from ${from} > to ${to}`,
metric.datasource.type,
queryConfig.datasource.type,
url
);
}
@ -36,26 +35,26 @@ export async function queryByMetric(
console.warn(`Data-kit got from === to`);
}
let data: MetricResults = {
let data: DataTable = {
values: [],
columns: []
};
while(true) {
let query = metric.metricQuery.getQuery(from, to, CHUNK_SIZE, data.values.length);
let query = queryConfig.datasourceConnector.getQuery(from, to, CHUNK_SIZE, data.values.length);
let res: any;
// TODO: use polymorphic `query` method instead
switch(queryType) {
case QueryType.GRAFANA:
res = await queryGrafana(query, apiKey as string, metric.datasource);
res = await queryGrafana(query, apiKey as string, queryConfig.datasource);
break;
case QueryType.DIRECT:
res = await queryDirect(query, metric.datasource);
res = await queryDirect(query, queryConfig.datasource);
break;
default:
throw new Error(`Unknown query type: ${queryType}`);
}
let chunk = metric.metricQuery.getResults(res);
let chunk = queryConfig.datasourceConnector.parseResponse(res);
let values = chunk.values;
data.values = data.values.concat(values);
data.columns = chunk.columns;

22
src/metrics/elasticsearch_metric.ts
Unescape View File

@ -1,4 +1,4 @@
import { AbstractMetric, Datasource, MetricId, MetricQuery, MetricResults } from './metric';
import { DatasourceConnector, Datasource, DatasourceQuery, DataTable } from './metric';
import { TsdbKitError } from '../types';
import * as _ from 'lodash';
@ -6,7 +6,7 @@ import * as _ from 'lodash';
export type RangeFilter = { range: { [key: string]: { gte: String, lte: String } } };
export type QueryStringFilter = { query_string: { analyze_wildcard: Boolean, query: String } };
export type QueryConfig = {
export type ElasticsearchQuery = {
size: number,
query: {
bool: {
@ -28,23 +28,23 @@ export type Aggregation = {
const DATE_HISTOGRAM_FIELD = 'date_histogram';
export class ElasticsearchMetric extends AbstractMetric {
constructor(datasource: Datasource, targets: any[], id?: MetricId) {
super(datasource, targets, id);
export class ElasticsearchMetric extends DatasourceConnector {
constructor(datasource: Datasource, targets: any[]) {
super(datasource, targets);
}
getQuery(from: number, to: number, limit: number, offset: number): MetricQuery {
getQuery(from: number, to: number, limit: number, offset: number): DatasourceQuery {
let data = this.datasource.data.split('\n').map(d => d === '' ? d: JSON.parse(d));
if(data.length === 0) {
throw new TsdbKitError('Datasource data is empty');
}
const queryConfig: QueryConfig = data[1];
const query: ElasticsearchQuery = data[1];
queryConfig.size = 0;
query.size = 0;
let timeField: string | null = null;
let aggs = _.filter(queryConfig.aggs, f => _.has(f, DATE_HISTOGRAM_FIELD));
let aggs = _.filter(query.aggs, f => _.has(f, DATE_HISTOGRAM_FIELD));
_.each(aggs, (agg: Aggregation) => {
agg[DATE_HISTOGRAM_FIELD].extended_bounds = {
min: from.toString(),
@ -63,7 +63,7 @@ export class ElasticsearchMetric extends AbstractMetric {
throw new Error('datasource time field not found');
}
let filters = queryConfig.query.bool.filter.filter(f => _.has(f, 'range')) as RangeFilter[];
let filters = query.query.bool.filter.filter(f => _.has(f, 'range')) as RangeFilter[];
if(filters.length === 0) {
throw new TsdbKitError('Empty filters');
}
@ -86,7 +86,7 @@ export class ElasticsearchMetric extends AbstractMetric {
}
}
getResults(res): MetricResults {
parseResponse(res): DataTable {
let columns = ['timestamp', 'target'];
let values = [];

12
src/metrics/graphite_metric.ts
Unescape View File

@ -1,14 +1,14 @@
import { AbstractMetric, Datasource, MetricId, MetricQuery, MetricResults } from './metric';
import { DatasourceConnector, Datasource, DatasourceQuery, DataTable } from './metric';
import * as _ from 'lodash';
export class GraphiteMetric extends AbstractMetric {
constructor(datasource: Datasource, targets: any[], id?: MetricId) {
super(datasource, targets, id);
export class GraphiteMetric extends DatasourceConnector {
constructor(datasource: Datasource, targets: any[]) {
super(datasource, targets);
}
getQuery(from: number, to: number, limit: number, offset: number): MetricQuery {
getQuery(from: number, to: number, limit: number, offset: number): DatasourceQuery {
let fromDate = Math.floor(from / 1000);
let toDate = Math.floor(to / 1000);
@ -42,7 +42,7 @@ export class GraphiteMetric extends AbstractMetric {
}
}
getResults(res): MetricResults {
parseResponse(res): DataTable {
if(res.data === undefined || res.data.length < 1) {
console.log('datasource return empty response, no data');

13
src/metrics/influxdb_metric.ts
Unescape View File

@ -1,16 +1,15 @@
import { AbstractMetric, Datasource, MetricId, MetricQuery, MetricResults } from "./metric";
import { DatasourceConnector, Datasource, DatasourceQuery, DataTable } from './metric';
import { processSQLLimitOffset } from './utils';
const INFLUX_QUERY_TIME_REGEX = /time ?[><=]+ ?[^A-Z]+(AND ?time ?[><=]+ ?[^A-Z]+)?/;
export class InfluxdbMetric extends AbstractMetric {
export class InfluxdbMetric extends DatasourceConnector {
private _queryParts: string[];
constructor(datasource: Datasource, targets: any[], id?: MetricId) {
super(datasource, targets, id);
constructor(datasource: Datasource, targets: any[]) {
super(datasource, targets);
var queryStr = datasource.params.q;
this._queryParts = queryStr.split(INFLUX_QUERY_TIME_REGEX);
@ -24,7 +23,7 @@ export class InfluxdbMetric extends AbstractMetric {
}
}
getQuery(from: number, to: number, limit: number, offset: number): MetricQuery {
getQuery(from: number, to: number, limit: number, offset: number): DatasourceQuery {
let timeClause = `time >= ${from}ms AND time <= ${to}ms`;
let q = `${this._queryParts[0]} ${timeClause} ${this._queryParts[2]}`;
q = processSQLLimitOffset(q, limit, offset);
@ -41,7 +40,7 @@ export class InfluxdbMetric extends AbstractMetric {
}
}
getResults(res): MetricResults {
parseResponse(res): DataTable {
let emptyResult = {
columns: ['timestamp', 'target'],
values: []

14
src/metrics/metric.ts
Unescape View File

@ -24,31 +24,29 @@ export declare type Datasource = {
datasourceId?: string;
};
export type MetricQuery = {
export type DatasourceQuery = {
url: string;
method: string;
schema: any;
headers?: any;
}
export type MetricResults = {
export type DataTable = {
values: (number | null)[][];
columns: string[];
}
export type MetricId = string;
export abstract class AbstractMetric {
export abstract class DatasourceConnector {
constructor(
public datasource: Datasource,
// TODO: Target type
public targets: any[],
public id?: MetricId
) {};
abstract getQuery(from: number, to: number, limit: number, offset: number): MetricQuery;
/*
from / to - timestamp in ms
limit - max number of items in result
offset - number of items to skip from timerange start
*/
abstract getResults(res): MetricResults;
abstract getQuery(from: number, to: number, limit: number, offset: number): DatasourceQuery;
abstract parseResponse(res): DataTable;
}

62
src/metrics/metrics_factory.ts
Unescape View File

@ -1,17 +1,17 @@
import { InfluxdbMetric } from './influxdb_metric';
import { GraphiteMetric } from './graphite_metric';
import { AbstractMetric, Datasource, DatasourceType, MetricId } from './metric';
import { DatasourceConnector, DatasourceType } from './metric';
import { PrometheusMetric } from './prometheus_metric';
import { PostgresMetric } from './postgres_metric';
import { ElasticsearchMetric } from './elasticsearch_metric';
import { MysqlMetric } from './mysql_metric';
export function metricFactory(
datasource: Datasource,
targets: any[],
id?: MetricId
): AbstractMetric {
import { QueryConfig } from '../models/query_config';
export function connectorFactory(
queryConfig: QueryConfig,
): DatasourceConnector {
const classMap = {
[DatasourceType.INFLUXDB]: InfluxdbMetric,
[DatasourceType.GRAPHITE]: GraphiteMetric,
@ -20,56 +20,12 @@ export function metricFactory(
[DatasourceType.ELASTICSEARCH]: ElasticsearchMetric,
[DatasourceType.MYSQL]: MysqlMetric,
};
const datasource = queryConfig.datasource;
const targets = queryConfig.targets;
if(classMap[datasource.type] === undefined) {
console.error(`Datasources of type ${datasource.type} are not supported currently`);
throw new Error(`Datasources of type ${datasource.type} are not supported currently`);
} else {
return new classMap[datasource.type](datasource, targets, id);
}
}
export class Metric {
datasource: Datasource;
targets: any[];
id?: MetricId;
private _metricQuery?: AbstractMetric;
constructor(datasource: Datasource, targets: any[], id?: MetricId) {
if(datasource === undefined) {
throw new Error('datasource is undefined');
}
if(targets === undefined) {
throw new Error('targets is undefined');
}
this.datasource = datasource;
this.targets = targets;
this.id = id;
}
public get metricQuery() {
if(this._metricQuery === undefined) {
this._metricQuery = metricFactory(this.datasource, this.targets, this.id);
}
return this._metricQuery;
}
public toObject() {
return {
datasource: this.datasource,
targets: this.targets,
_id: this.id
};
}
static fromObject(obj: any): Metric {
if(obj === undefined) {
throw new Error('obj is undefined');
}
return new Metric(
obj.datasource,
obj.targets,
obj._id
);
return new classMap[datasource.type](datasource, targets);
}
}

14
src/metrics/prometheus_metric.ts
Unescape View File

@ -1,15 +1,15 @@
import { AbstractMetric, Datasource, MetricId, MetricQuery, MetricResults } from './metric';
import { DatasourceConnector, Datasource, DatasourceQuery, DataTable } from './metric';
const QUERY_TIME_REGEX = /\&start=[^\&]*\&end=[^\&]*\&/;
export class PrometheusMetric extends AbstractMetric {
export class PrometheusMetric extends DatasourceConnector {
constructor(datasource: Datasource, targets: any[], id?: MetricId) {
super(datasource, targets, id);
constructor(datasource: Datasource, targets: any[]) {
super(datasource, targets);
}
getQuery(from: number, to: number, limit: number, offset: number): MetricQuery {
getQuery(from: number, to: number, limit: number, offset: number): DatasourceQuery {
let url = this.datasource.url;
from = Math.floor(from / 1000); //prometheus uses seconds for timestamp
to = Math.floor(to / 1000);
@ -25,7 +25,7 @@ export class PrometheusMetric extends AbstractMetric {
}
}
getResults(res): MetricResults {
parseResponse(res): DataTable {
if(res.data === undefined || res.data.data.result.length < 1) {
console.log('datasource return empty response, no data');
@ -36,7 +36,7 @@ export class PrometheusMetric extends AbstractMetric {
}
let result = res.data.data.result;
let result_matrix: MetricResults = {
let result_matrix: DataTable = {
columns: ['timestamp'],
values: []
};

12
src/metrics/sql_metric.ts
Unescape View File

@ -1,15 +1,15 @@
import { AbstractMetric, Datasource, MetricId, MetricQuery, MetricResults } from './metric';
import { DatasourceConnector, Datasource, DatasourceQuery, DataTable } from './metric';
import { processSQLLimitOffset } from './utils';
import * as _ from 'lodash';
// for 26.09.2020 it works for all SQL datasources
export class SqlMetric extends AbstractMetric {
export class SqlMetric extends DatasourceConnector {
private _targetName: string; //save first target name, while multi metric not implemented
private url: string = 'api/tsdb/query';
constructor(datasource: Datasource, targets: any[], id?: MetricId) {
super(datasource, targets, id);
constructor(datasource: Datasource, targets: any[]) {
super(datasource, targets);
if(targets.length === 0) {
throw Error('got empty targets list');
@ -17,7 +17,7 @@ export class SqlMetric extends AbstractMetric {
this._targetName = targets[0].refId;
}
getQuery(from: number, to: number, limit: number, offset: number): MetricQuery {
getQuery(from: number, to: number, limit: number, offset: number): DatasourceQuery {
let queries = this.targets;
_.forEach(queries, q => {
@ -40,7 +40,7 @@ export class SqlMetric extends AbstractMetric {
};
}
getResults(res): MetricResults {
parseResponse(res): DataTable {
let emptyResult = {
columns: ['timestamp', 'target'],
values: []

45
src/models/query_config.ts
Unescape View File

@ -0,0 +1,45 @@
import { Datasource, DatasourceConnector } from '../metrics/metric';
import { connectorFactory } from '../metrics/metrics_factory';
export class QueryConfig {
datasource: Datasource;
// TODO: Target type (depends on datasource type)
targets: any[];
private _datasourceConnector?: DatasourceConnector;
constructor(datasource: Datasource, targets: any[]) {
if(datasource === undefined) {
throw new Error('datasource is undefined');
}
if(targets === undefined) {
throw new Error('targets is undefined');
}
this.datasource = datasource;
this.targets = targets;
}
get datasourceConnector(): DatasourceConnector {
if(this._datasourceConnector === undefined) {
this._datasourceConnector = connectorFactory(this);
}
return this._datasourceConnector;
}
public toObject() {
return {
datasource: this.datasource,
targets: this.targets,
};
}
static fromObject(obj: any): QueryConfig {
if(obj === undefined) {
throw new Error('obj is undefined');
}
return new QueryConfig(
obj.datasource,
obj.targets,
);
}
}

4
src/services/direct_service.ts
Unescape View File

@ -1,5 +1,5 @@
import { DatasourceUnavailable } from '../types';
import { Datasource, MetricQuery } from '../metrics/metric';
import { Datasource, DatasourceQuery } from '../metrics/metric';
import axios from 'axios';
import * as _ from 'lodash';
@ -7,7 +7,7 @@ import * as _ from 'lodash';
// TODO: support direct queries auth
// TODO: move to class and inherit from QueryService abstract class
export async function queryDirect(query: MetricQuery, datasource: Datasource) {
export async function queryDirect(query: DatasourceQuery, datasource: Datasource) {
let axiosQuery = {
url: query.url,
method: query.method,

4
src/services/grafana_service.ts
Unescape View File

@ -1,4 +1,4 @@
import { Datasource, MetricQuery } from '../metrics/metric';
import { Datasource, DatasourceQuery } from '../metrics/metric';
import { TsdbKitError, DatasourceUnavailable } from '../types';
import axios from 'axios';
@ -8,7 +8,7 @@ import * as _ from 'lodash';
export class GrafanaUnavailable extends TsdbKitError { };
// TODO: move to class and inherit from QueryService abstract class
export async function queryGrafana(query: MetricQuery, apiKey: string, datasource: Datasource) {
export async function queryGrafana(query: DatasourceQuery, apiKey: string, datasource: Datasource) {
let headers = { Authorization: `Bearer ${apiKey}` };
const grafanaUrl = getGrafanaUrl(query.url);

9
src/tsdb-kit/index.ts
Unescape View File

@ -1,13 +1,12 @@
import { queryByMetric, Metric } from '..';
import { queryByConfig, QueryConfig } from '..';
import { PrometheusMetric } from '../metrics/prometheus_metric';
import { DatasourceType, QueryType } from '../metrics/metric';
import * as _ from 'lodash';
// TODO: these `const`s should be CLI arguments
const PROMETHEUS_URL = 'http://localhost:9090';
const PROMETHEUS_URL = 'http://prometheus.metrics.corpglory.com';
const QUERY = '100-(avg by (instance) (irate(node_cpu_seconds_total{job="nvicta-ai-node-exporter",mode="idle"}[5m])) * 100)';
const FROM = 1660307430000; // ms
const TO = 1660307437000; // ms
@ -17,8 +16,8 @@ const datasource = {
url: `${PROMETHEUS_URL}/api/v1/query_range?query=${QUERY}&start=1543411320&end=1543432950&step=30`
}
const targets = [];
const metric = new Metric(datasource, targets);
queryByMetric(metric as any, PROMETHEUS_URL, FROM, TO, QueryType.DIRECT)
const queryConfig = new QueryConfig(datasource, targets);
queryByConfig(queryConfig, PROMETHEUS_URL, FROM, TO, QueryType.DIRECT)
.then(res => {
console.log(res);
})

Write Preview
Loading…
Cancel
Save