From 82dfc20b2f61286b66f7734ee5c7f765a4d29777 Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 19 Aug 2022 14:54:39 +0300 Subject: [PATCH 01/11] init CLI arguments --- package.json | 1 + src/tsdb-kit/index.ts | 22 +++++++++++++++++----- webpack.config.js | 2 +- yarn.lock | 5 +++++ 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 069505a..bcba604 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "devDependencies": { "@types/jest": "^26.0.15", "@types/lodash": "^4.14.165", + "argparse": "^2.0.1", "jest": "^26.6.3", "ts-jest": "^26.4.4", "ts-loader": "^9.3.1", diff --git a/src/tsdb-kit/index.ts b/src/tsdb-kit/index.ts index 66e54ef..04f99c0 100644 --- a/src/tsdb-kit/index.ts +++ b/src/tsdb-kit/index.ts @@ -2,14 +2,26 @@ import { queryByConfig, QueryConfig } from '..'; import { DatasourceType, QueryType } from '../connectors'; +import { ArgumentParser } from 'argparse'; import * as _ from 'lodash'; +const parser = new ArgumentParser(); -// TODO: these `const`s should be CLI arguments -const PROMETHEUS_URL = 'http://localhost:9090'; -const QUERY = '100-(avg by (instance) (irate(node_cpu_seconds_total{job="nvicta-ai-node-exporter",mode="idle"}[5m])) * 100)'; -const FROM = 1660670020000; // ms -const TO = 1660670026000; // ms +// TODO: parse version from package.json +parser.add_argument('-v', '--version', { action: 'version', version: '0.1.1' }); +parser.add_argument('-u', '--url', { help: 'Datasource URL', required: true }); +parser.add_argument('-q', '--query', { help: 'Query Template', required: true }); +parser.add_argument('-f', '--from', { help: 'From timestamp (ms), e.g. 1660670020000. If not specified, `now-5m` is used' }); +parser.add_argument('-t', '--to', { help: 'To timestamp (ms), e.g. 1660670026000. If not specified, `now` is used' }); + +const args = parser.parse_args(); + +const timeNowInMs = new Date().getTime(); + +const PROMETHEUS_URL = args.url; +const QUERY = args.query; +const FROM = args.from || timeNowInMs - 5 * 60 * 1000; +const TO = args.to || timeNowInMs; const datasource = { type: DatasourceType.PROMETHEUS, diff --git a/webpack.config.js b/webpack.config.js index 96ee9fd..2cd6b25 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -18,7 +18,7 @@ module.exports = { module: { rules: [ { - test: /.ts?$/, + test: /.ts$/, loader: 'ts-loader', options: { configFile: 'bin.tsconfig.json' diff --git a/yarn.lock b/yarn.lock index 67afc9d..e539f99 100644 --- a/yarn.lock +++ b/yarn.lock @@ -905,6 +905,11 @@ argparse@^1.0.7: dependencies: sprintf-js "~1.0.2" +argparse@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" + integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== + arr-diff@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/arr-diff/-/arr-diff-4.0.0.tgz#d6461074febfec71e7e15235761a329a5dc7c520" From d68faaa8ec0f977f778244379b541c048cb4bab5 Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 19 Aug 2022 15:02:59 +0300 Subject: [PATCH 02/11] upd readme --- README.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0c70acd..1ddbfa0 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,14 @@ # tsdb-kit -[![Build Status](https://travis-ci.org/CorpGlory/tsdb-kit.svg?branch=master)](https://travis-ci.org/CorpGlory/tsdb-kit) +Node.js library and CLI-tool for querying timeseries-datasources from backend directly or using Grafana as proxy. -Node.js library and utilities for running Grafana datasources on backend. -You can send your datasource metrics from Grafana to compile it on Node.js and query your datasource via Grafana API in background. +User gets a unified interface to all datasources. Library gives single output format: fields order, time units, etc. -User gets a unified interface to all datasources. Library gives single output format: fields order, time units, etc +## Supported direct datasources -## Supported datasources +* Prometheus + +## Supported Grafana datasources * Influxdb * Graphite @@ -15,7 +16,15 @@ User gets a unified interface to all datasources. Library gives single output fo * PostgreSQL / TimescaleDB / MySQL * ElasticSearch -Please write us at ping@corpglory.com if you want your datasource to be supported: +Please write us at ping@corpglory.com if you want your datasource to be supported + +## CLI usage + +For now, CLI supports only direct Prometheus queries without auth + +Example: + +`npx tsdb-kit -u http://localhost:9090 -q '100-(avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)'` ## Projects based on library * [grafana-data-exporter](https://github.com/CorpGlory/grafana-data-exporter) From 5212d7a89a106efe53a548e57e7812258842a7ce Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 19 Aug 2022 15:03:11 +0300 Subject: [PATCH 03/11] 1.2.0-beta --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bcba604..2973872 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@corpglory/tsdb-kit", - "version": "1.1.1", + "version": "1.2.0-beta", "description": "", "scripts": { "build": "yarn build:lib && yarn build:bin", From 6448f520952760bb1e7f9515beb5e4d9bd6a322a Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 19 Aug 2022 15:04:35 +0300 Subject: [PATCH 04/11] upd npmignore --- .npmignore | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.npmignore b/.npmignore index 01b7b93..05431c7 100644 --- a/.npmignore +++ b/.npmignore @@ -2,7 +2,8 @@ src spec .travis.yml jest.config.js -tsconfig.lib.json -tsconfig.bin.json -tsconfig.jest.json - +lib.tsconfig.json +bin.tsconfig.json +webpack.config.js +yarn.lock +.vscode From 8629f863325d8d9af7a35ce2a832974113aa0583 Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 19 Aug 2022 15:14:08 +0300 Subject: [PATCH 05/11] shabang to bin --- webpack.config.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/webpack.config.js b/webpack.config.js index 2cd6b25..cee3958 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,3 +1,4 @@ +const webpack = require('webpack'); const path = require('path'); @@ -12,6 +13,9 @@ module.exports = { path: path.resolve(__dirname, './bin'), filename: 'tsdb-kit.js' }, + plugins: [ + new webpack.BannerPlugin({ banner: "#!/usr/bin/env node", raw: true }), + ], resolve: { extensions: ['.ts', '.js'], }, From 7b487f9fbbe240eb83d0c21e8ea4b817c7a0d59d Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 19 Aug 2022 15:14:20 +0300 Subject: [PATCH 06/11] 1.2.0-beta2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2973872..18e278b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@corpglory/tsdb-kit", - "version": "1.2.0-beta", + "version": "1.2.0-beta2", "description": "", "scripts": { "build": "yarn build:lib && yarn build:bin", From afcf696ae56a6524d9fe5faf195be51128c6dd5a Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 19 Aug 2022 15:26:12 +0300 Subject: [PATCH 07/11] new `dev` scripts --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 18e278b..8ab9edb 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "build": "yarn build:lib && yarn build:bin", "build:lib": "tsc --p lib.tsconfig.json", "build:bin": "webpack --config webpack.config.js", - "dev": "tsc -w", + "dev:lib": "tsc --p lib.tsconfig.json -w", + "dev:bin": "webpack --watch --config webpack.config.js", "test": "jest" }, "repository": { From 864537766d722136125e052e04662e1afb9acafa Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 19 Aug 2022 15:28:50 +0300 Subject: [PATCH 08/11] version from package.json --- src/tsdb-kit/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsdb-kit/index.ts b/src/tsdb-kit/index.ts index 04f99c0..161b314 100644 --- a/src/tsdb-kit/index.ts +++ b/src/tsdb-kit/index.ts @@ -2,13 +2,13 @@ import { queryByConfig, QueryConfig } from '..'; import { DatasourceType, QueryType } from '../connectors'; +const { version } = require('../../package.json') import { ArgumentParser } from 'argparse'; import * as _ from 'lodash'; const parser = new ArgumentParser(); -// TODO: parse version from package.json -parser.add_argument('-v', '--version', { action: 'version', version: '0.1.1' }); +parser.add_argument('-v', '--version', { action: 'version', version }); parser.add_argument('-u', '--url', { help: 'Datasource URL', required: true }); parser.add_argument('-q', '--query', { help: 'Query Template', required: true }); parser.add_argument('-f', '--from', { help: 'From timestamp (ms), e.g. 1660670020000. If not specified, `now-5m` is used' }); From eac608eab8561193de99e9956f6d9fc7dc768c6c Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 19 Aug 2022 15:36:09 +0300 Subject: [PATCH 09/11] upd readme --- README.md | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1ddbfa0..e626c7d 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,20 @@ # tsdb-kit -Node.js library and CLI-tool for querying timeseries-datasources from backend directly or using Grafana as proxy. +TSDB-kit is a node.js library and CLI-tool for querying timeseries-datasources. -User gets a unified interface to all datasources. Library gives single output format: fields order, time units, etc. +## Features -## Supported direct datasources +- can query datasources directly or using Grafana as proxy +- can be used as a lib from your node.js-code or as a CLI-tool +- user gets a unified interface to all datasources. Library gives single output format: fields order, time units, etc. -* Prometheus +## Supported datasources + +### Direct -## Supported Grafana datasources +* Prometheus (limited, auth is not supported yet) + +### Grafana * Influxdb * Graphite @@ -18,13 +24,20 @@ User gets a unified interface to all datasources. Library gives single output fo Please write us at ping@corpglory.com if you want your datasource to be supported -## CLI usage +## Usage + +### Lib (TODO) + +### CLI + +For now, CLI supports only direct Prometheus queries + +For example: -For now, CLI supports only direct Prometheus queries without auth +`npx @corpglory/tsdb-kit -u http://localhost:9090 -q '100-(avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)'` -Example: -`npx tsdb-kit -u http://localhost:9090 -q '100-(avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)'` +## Development (TODO) ## Projects based on library * [grafana-data-exporter](https://github.com/CorpGlory/grafana-data-exporter) From 4b38cbdb34bea8c21ec02861791c8a6da324888e Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 19 Aug 2022 18:40:48 +0300 Subject: [PATCH 10/11] Prometheus: support auth --- README.md | 6 +++--- spec/prometheus.jest.ts | 13 ++++++++++++- src/connectors/index.ts | 6 ++++++ src/connectors/prometheus.ts | 3 ++- src/services/query_service/direct.ts | 3 +-- src/tsdb-kit/index.ts | 15 ++++++++++++--- 6 files changed, 36 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index e626c7d..060cdce 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # tsdb-kit -TSDB-kit is a node.js library and CLI-tool for querying timeseries-datasources. +TSDB-kit is a node.js library and CLI-tool for querying timeseries-datasources. ## Features @@ -12,7 +12,7 @@ TSDB-kit is a node.js library and CLI-tool for querying timeseries-datasources. ### Direct -* Prometheus (limited, auth is not supported yet) +* Prometheus ### Grafana @@ -34,7 +34,7 @@ For now, CLI supports only direct Prometheus queries For example: -`npx @corpglory/tsdb-kit -u http://localhost:9090 -q '100-(avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)'` +`npx @corpglory/tsdb-kit -U http://localhost:9090 -q '100-(avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)' -u my_user -p my_password` ## Development (TODO) diff --git a/spec/prometheus.jest.ts b/spec/prometheus.jest.ts index 456c24c..3df0134 100644 --- a/spec/prometheus.jest.ts +++ b/spec/prometheus.jest.ts @@ -7,7 +7,11 @@ import 'jest'; describe('Test Prometheus time range processing', function() { let datasource = { type: DatasourceType.PROMETHEUS, - url: 'api/datasources/proxy/4/api/v1/query_range?query=node_disk_io_time_ms&start=1543411320&end=1543432950&step=30' + url: 'api/datasources/proxy/4/api/v1/query_range?query=node_disk_io_time_ms&start=1543411320&end=1543432950&step=30', + auth: { + username: 'my_user', + password: 'my_password', + } } let targets = []; let prometheus = new PrometheusConnector(datasource, targets); @@ -19,4 +23,11 @@ describe('Test Prometheus time range processing', function() { expect(query.url.indexOf(`start=${Math.floor(from / 1000)}`) !== -1).toBeTruthy(); expect(query.url.indexOf(`end=${Math.floor(to / 1000)}`) !== -1).toBeTruthy(); }); + + it('check that username/password present in query', function() { + let query = prometheus.getQuery(0, 0, 1000, 0); + + expect(query.auth?.username).toBe('my_user'); + expect(query.auth?.password).toBe('my_password'); + }) }); diff --git a/src/connectors/index.ts b/src/connectors/index.ts index b15fadc..0f89291 100644 --- a/src/connectors/index.ts +++ b/src/connectors/index.ts @@ -12,6 +12,7 @@ export enum DatasourceType { MYSQL = 'mysql', } +// TODO: Datasource: type -> class export declare type Datasource = { url: string; type: DatasourceType; @@ -22,6 +23,7 @@ export declare type Datasource = { }; data?: any; datasourceId?: string; + auth?: any; }; export type DatasourceQuery = { @@ -29,6 +31,10 @@ export type DatasourceQuery = { method: string; schema: any; headers?: any; + auth?: { + username: string; + password: string; + }; } export type DataTable = { diff --git a/src/connectors/prometheus.ts b/src/connectors/prometheus.ts index 37ad0f1..b48f753 100644 --- a/src/connectors/prometheus.ts +++ b/src/connectors/prometheus.ts @@ -21,7 +21,8 @@ export class PrometheusConnector extends DatasourceConnector { method: 'GET', schema: { params: this.datasource.params - } + }, + auth: this.datasource.auth, } } diff --git a/src/services/query_service/direct.ts b/src/services/query_service/direct.ts index acf3102..5cd89f7 100644 --- a/src/services/query_service/direct.ts +++ b/src/services/query_service/direct.ts @@ -14,8 +14,7 @@ export class DirectQueryService extends QueryService { async query(query: DatasourceQuery): Promise> { // TODO: support auth let axiosQuery = { - url: query.url, - method: query.method, + ...query, }; _.defaults(axiosQuery, query.schema); diff --git a/src/tsdb-kit/index.ts b/src/tsdb-kit/index.ts index 161b314..47854a7 100644 --- a/src/tsdb-kit/index.ts +++ b/src/tsdb-kit/index.ts @@ -9,10 +9,12 @@ import * as _ from 'lodash'; const parser = new ArgumentParser(); parser.add_argument('-v', '--version', { action: 'version', version }); -parser.add_argument('-u', '--url', { help: 'Datasource URL', required: true }); +parser.add_argument('-U', '--url', { help: 'Datasource URL', required: true }); parser.add_argument('-q', '--query', { help: 'Query Template', required: true }); parser.add_argument('-f', '--from', { help: 'From timestamp (ms), e.g. 1660670020000. If not specified, `now-5m` is used' }); parser.add_argument('-t', '--to', { help: 'To timestamp (ms), e.g. 1660670026000. If not specified, `now` is used' }); +parser.add_argument('-u', '--username', { help: 'Basic Auth Username' }); +parser.add_argument('-p', '--password', { help: 'Basic Auth Password' }); const args = parser.parse_args(); @@ -22,12 +24,19 @@ const PROMETHEUS_URL = args.url; const QUERY = args.query; const FROM = args.from || timeNowInMs - 5 * 60 * 1000; const TO = args.to || timeNowInMs; +const USERNAME = args.username; +const PASSWORD = args.password; +let auth; +if(USERNAME && PASSWORD) { + auth = { username: USERNAME, password: PASSWORD }; +} const datasource = { type: DatasourceType.PROMETHEUS, // TODO: remove PROMETHEUS_URL from here - url: `${PROMETHEUS_URL}/api/v1/query_range?query=${QUERY}&start=1543411320&end=1543432950&step=30` -} + url: `${PROMETHEUS_URL}/api/v1/query_range?query=${QUERY}&start=1543411320&end=1543432950&step=30`, + auth, +}; const targets = []; const queryConfig = new QueryConfig(QueryType.DIRECT, datasource, targets); queryByConfig(queryConfig, PROMETHEUS_URL, FROM, TO) From fe51c32f2a2b4fcf6bd0e852ca2e11861084bd77 Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 19 Aug 2022 18:45:52 +0300 Subject: [PATCH 11/11] 2.0.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8ab9edb..ba85bb0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@corpglory/tsdb-kit", - "version": "1.2.0-beta2", + "version": "2.0.0", "description": "", "scripts": { "build": "yarn build:lib && yarn build:bin",