From 1c2b6936d6fd38ec53e7fcbd785373b666d91f2c Mon Sep 17 00:00:00 2001 From: rozetko Date: Thu, 19 Jan 2023 22:53:20 +0300 Subject: [PATCH 1/6] return version number from package.json --- src/routes/status.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/routes/status.ts b/src/routes/status.ts index c7d9dc4..65016a5 100644 --- a/src/routes/status.ts +++ b/src/routes/status.ts @@ -1,10 +1,10 @@ import * as express from 'express' -async function deleteTask(req, res) { - res.status(200).send({ version: 123 }); +async function getStatus(req, res) { + res.status(200).send({ version: process.env.npm_package_version }); } export const router = express.Router(); -router.get('/', deleteTask); +router.get('/', getStatus); From 164b265031bf0bcb2bbd7d6c8103fd3538e552db Mon Sep 17 00:00:00 2001 From: rozetko Date: Thu, 19 Jan 2023 23:48:53 +0300 Subject: [PATCH 2/6] auto api key configuration --- .gitignore | 3 ++- Dockerfile | 2 +- config-example.json | 3 --- docker-compose.yml | 13 ++++++++++ exported/.gitkeep | 0 src/config.ts | 18 ++++++++------ src/routes/api.ts | 8 +++--- src/routes/connect.ts | 53 ++++++++++++++++++++++++++++++++++++++++ src/routes/tasks.ts | 23 +++++++++++++---- src/services/api_keys.ts | 41 +++++++++++++++++++++++++++++++ src/services/exporter.ts | 20 +++++---------- src/utils.ts | 4 +++ 12 files changed, 152 insertions(+), 36 deletions(-) create mode 100644 docker-compose.yml delete mode 100644 exported/.gitkeep create mode 100644 src/routes/connect.ts create mode 100644 src/services/api_keys.ts diff --git a/.gitignore b/.gitignore index e89fcc3..26b66cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ node_modules/ dist/ -exported/ +data/ config.json package-lock.json +api-keys.json diff --git a/Dockerfile b/Dockerfile index 5e2bb4e..af668d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ ENV BUILD_PATH=$build_path # Expose port 80 EXPOSE 8000 -VOLUME [ "/var/www/exported" ] +VOLUME [ "/var/www/data" ] # Copy custom configuration file from the current directory diff --git a/config-example.json b/config-example.json index 4707b2d..8034c59 100644 --- a/config-example.json +++ b/config-example.json @@ -1,6 +1,3 @@ { - "apiKeys" : { - "http://localhost:3000": "eyJrIjoiTjlUcmtLSFRNcTdqeXBaQjB5REk2TFkyUDBDM0Z1bWciLCJuIjoiZXhwb3J0LW1hbmFnZXIiLCJpZCI6MX0=" - }, "port": "8000" } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ccd8b43 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +version: '2' + +services: + exporter: + image: corpglory/grafana-data-exporter:latest + restart: always + ports: + - 8000: 8000 + volumes: + - data:/var/www/data + +volumes: + data diff --git a/exported/.gitkeep b/exported/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/config.ts b/src/config.ts index f27ade2..444d055 100644 --- a/src/config.ts +++ b/src/config.ts @@ -2,17 +2,20 @@ import * as path from 'path'; import * as fs from 'fs'; import * as _ from 'lodash'; + const DEFAULT_CONFIG = { - 'apiKeys': { - 'http://localhost:3000': '' - }, 'port': '8000' }; -export const EXPORTED_PATH = path.join(__dirname, '../exported'); -if(!fs.existsSync(EXPORTED_PATH)) { - console.log(`${EXPORTED_PATH} doesn't exist, creating`); - fs.mkdirSync(EXPORTED_PATH); +export const DATA_PATH = path.join(__dirname, '../data'); +if(!fs.existsSync(DATA_PATH)) { + console.log(`${DATA_PATH} doesn't exist, creating`); + fs.mkdirSync(DATA_PATH); +} +export const CSV_PATH = path.join(DATA_PATH, 'csv'); +if(!fs.existsSync(CSV_PATH)) { + console.log(`${CSV_PATH} doesn't exist, creating`); + fs.mkdirSync(CSV_PATH); } function getConfigField(field: string, defaultVal?: any) { @@ -38,4 +41,3 @@ function getConfigField(field: string, defaultVal?: any) { } export const port = getConfigField('port', '8000'); -export const apiKeys = getConfigField('apiKeys'); diff --git a/src/routes/api.ts b/src/routes/api.ts index 49a57b3..ca40d86 100644 --- a/src/routes/api.ts +++ b/src/routes/api.ts @@ -1,6 +1,7 @@ -import { EXPORTED_PATH } from '../config'; +import { CSV_PATH } from '../config'; import { router as tasksRouter } from '../routes/tasks'; import { router as statusRouter } from '../routes/status'; +import { router as connectRouter } from '../routes/connect'; import * as express from 'express'; @@ -8,7 +9,6 @@ import * as express from 'express'; export const router = express.Router(); router.use('/status', statusRouter); - +router.use('/connect', connectRouter); router.use('/task', tasksRouter); - -router.use('/static', express.static(EXPORTED_PATH)); +router.use('/static', express.static(CSV_PATH)); diff --git a/src/routes/connect.ts b/src/routes/connect.ts new file mode 100644 index 0000000..1ad79ae --- /dev/null +++ b/src/routes/connect.ts @@ -0,0 +1,53 @@ +import { upsertApiKey, validateGrafanaUrl } from '../services/api_keys'; + +import * as express from 'express' + +import * as _ from 'lodash'; + + +async function checkConnection(req, res) { + const params = req.params; + + const clientUrl = params.url; + if (_.isEmpty(clientUrl)) { + res.status(400).send('"url" field is required'); + return; + } + + try { + validateGrafanaUrl(clientUrl); + } catch (e) { + res.status(500).send(e.message); + return; + } + + res.status(200).send({ version: process.env.npm_package_version }); +} + +async function connectPlugin(req, res) { + const body = req.body; + + const clientUrl = body.url; + if (_.isEmpty(clientUrl)) { + res.status(400).send('"url" field is required'); + return; + } + const apiToken = body.apiToken; + if (_.isEmpty(apiToken)) { + res.status(400).send('"apiToken" field is required'); + return; + } + + const grafanaUrl = new URL(clientUrl).origin; + + upsertApiKey(grafanaUrl, apiToken); + + console.log(`Grafana at ${grafanaUrl} is connected`); + + res.status(200).send({ version: process.env.npm_package_version }); +} + +export const router = express.Router(); + +router.get('/', checkConnection); +router.post('/', connectPlugin); diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts index ecf19be..1193625 100644 --- a/src/routes/tasks.ts +++ b/src/routes/tasks.ts @@ -1,5 +1,6 @@ import { exporterFactory } from '../services/exporter.factory'; -import { EXPORTED_PATH } from '../config'; +import { CSV_PATH } from '../config'; +import { validateGrafanaUrl } from '../services/api_keys'; import { ExportTask } from '../types'; @@ -10,6 +11,7 @@ import * as _ from 'lodash'; import * as path from 'path'; import * as fs from 'fs'; + type TRequest = { body: { task: ExportTask, @@ -20,7 +22,7 @@ type TRequest = { async function getTasks(req, res) { const resp: ExportTask[] = []; - fs.readdir(EXPORTED_PATH, (err, items) => { + fs.readdir(CSV_PATH, (err, items) => { if(err) { console.error(err); res.status(500).send(err.message); @@ -31,7 +33,7 @@ async function getTasks(req, res) { continue; } // TODO: read async - let data = fs.readFileSync(path.join(EXPORTED_PATH, item), 'utf8'); + let data = fs.readFileSync(path.join(CSV_PATH, item), 'utf8'); try { let status = JSON.parse(data); resp.push(status); @@ -51,10 +53,19 @@ async function addTask(req: TRequest, res) { const clientUrl = body.url; if (_.isEmpty(clientUrl)) { res.status(400).send('"url" field is required'); + return; } const task = body.task; if (_.isEmpty(task)) { res.status(400).send('"task" field is required'); + return; + } + + try { + validateGrafanaUrl(clientUrl); + } catch(e) { + res.status(500).send(e.message); + return; } const datasourceUrl = `${new URL(clientUrl).origin}/api/ds/query`; @@ -63,8 +74,10 @@ async function addTask(req: TRequest, res) { const to = +task.timeRange.to; if (isNaN(from) || isNaN(to)) { res.status(400).send('Range error: please fill both "from" and "to" fields'); + return; } else if (from >= to) { res.status(400).send('Range error: "from" should be less than "to"'); + return; } const exporter = exporterFactory.getExporter(); @@ -74,8 +87,8 @@ async function addTask(req: TRequest, res) { async function deleteTask(req, res) { let taskId = req.body.taskId; - let csvFilePath = path.join(EXPORTED_PATH, `${taskId}.csv`); - let jsonFilePath = path.join(EXPORTED_PATH, `${taskId}.json`); + let csvFilePath = path.join(CSV_PATH, `${taskId}.csv`); + let jsonFilePath = path.join(CSV_PATH, `${taskId}.json`); if(fs.existsSync(csvFilePath)) { fs.unlink(csvFilePath, err => console.error(err)); diff --git a/src/services/api_keys.ts b/src/services/api_keys.ts new file mode 100644 index 0000000..9e7eccf --- /dev/null +++ b/src/services/api_keys.ts @@ -0,0 +1,41 @@ +import { DATA_PATH } from '../config'; + +import * as path from 'path'; +import * as fs from 'fs'; +import * as _ from 'lodash'; + + +const API_KEYS_FILE = path.join(DATA_PATH, 'api-keys.json'); +if(!fs.existsSync(API_KEYS_FILE)) { + console.log(`${API_KEYS_FILE} doesn't exist, creating`); + fs.writeFileSync(API_KEYS_FILE, JSON.stringify({}), 'utf8'); +} + +export function getApiKey(grafanaUrl: string): string | null { + const data = fs.readFileSync(API_KEYS_FILE, 'utf8'); + const apiKey = JSON.parse(data)[grafanaUrl]; + if(_.isNil(apiKey)) { + return null; + } + + return apiKey; +} + +export function upsertApiKey(grafanaUrl: string, apiKey: string): void { + const data = fs.readFileSync(API_KEYS_FILE, 'utf8'); + const apiKeys = JSON.parse(data); + + apiKeys[grafanaUrl] = apiKey; + + fs.writeFileSync(API_KEYS_FILE, JSON.stringify(apiKeys), 'utf8'); +} + +// TODO: query Grafana API if a key exists && remove the key if it doesn't work for specified grafanaUrl +export function validateGrafanaUrl(grafanaUrl: string) { + const host = new URL(grafanaUrl).origin; + const apiKey = getApiKey(host); + + if (_.isNil(apiKey) || apiKey === '') { + throw new Error(`Please configure API key for ${host}`); + } +} diff --git a/src/services/exporter.ts b/src/services/exporter.ts index 8d63013..2c33bf0 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -1,7 +1,8 @@ -import { URL } from 'url'; -import { apiKeys } from '../config'; +// TODO: import promisify from 'util' +import { getApiKey } from './api_keys'; import { promisify, toIsoString } from '../utils'; import { DashboardQuery, ExportProgress, ExportStatus, ExportTask } from '../types'; +import { CSV_PATH } from '../config'; import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit'; // TODO: export QueryType directly from @corpglory/tsdb-kit @@ -13,6 +14,7 @@ import * as moment from 'moment'; import * as csv from 'fast-csv'; import * as fs from 'fs'; import * as path from 'path'; +import { URL } from 'url'; import * as _ from 'lodash'; const MS_IN_DAY = 24 * 60 * 60 * 1000; @@ -33,7 +35,6 @@ export class Exporter { this._task.progress = _.cloneDeep(DEFAULT_PROGRESS); this._validateQueries(task.queries); - this._validateDatasourceUrl(datasourceUrl); await this._updateProgress(); @@ -66,7 +67,7 @@ export class Exporter { for(const queryConfig of queryConfigs) { const host = new URL(datasourceUrl).origin; - const apiKey = apiKeys[host]; + const apiKey = getApiKey(host); const datasourceMetrics = await queryByConfig(queryConfig, datasourceUrl, from, to, apiKey); @@ -156,15 +157,6 @@ export class Exporter { } } - private _validateDatasourceUrl(datasourceUrl: string) { - const host = new URL(datasourceUrl).origin; - const apiKey = apiKeys[host]; - - if(apiKey === undefined || apiKey === '') { - throw new Error(`Please configure API key for ${host}`); - } - } - private _writeCsv(stream, series) { for(let row of series.values) { let csvRow = {}; @@ -182,6 +174,6 @@ export class Exporter { private _getFilePath(extension: string): string { let filename = this._getFilename(extension); - return path.join(__dirname, `../exported/${filename}`); + return path.join(CSV_PATH, filename); } } diff --git a/src/utils.ts b/src/utils.ts index a1fcc7f..9e1e5d4 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,9 @@ +import { getApiKey } from './services/api_keys'; + import * as moment from 'moment-timezone'; +import * as _ from 'lodash'; + export async function promisify(method: (...params: any[]) => Promise | void, ...params: any[]) { return new Promise((resolve, reject) => { method(...params, (err, result) => { From 8ab0a95b732f4380b6633a856cb667075bbf7d0c Mon Sep 17 00:00:00 2001 From: rozetko Date: Thu, 19 Jan 2023 23:54:10 +0300 Subject: [PATCH 3/6] hotfix --- src/utils.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 9e1e5d4..a1fcc7f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,9 +1,5 @@ -import { getApiKey } from './services/api_keys'; - import * as moment from 'moment-timezone'; -import * as _ from 'lodash'; - export async function promisify(method: (...params: any[]) => Promise | void, ...params: any[]) { return new Promise((resolve, reject) => { method(...params, (err, result) => { From 09b0652973c04a13557862800542e8a7ca584059 Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 20 Jan 2023 00:00:04 +0300 Subject: [PATCH 4/6] hotfix --- src/routes/connect.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/routes/connect.ts b/src/routes/connect.ts index 1ad79ae..0e71fe8 100644 --- a/src/routes/connect.ts +++ b/src/routes/connect.ts @@ -6,9 +6,9 @@ import * as _ from 'lodash'; async function checkConnection(req, res) { - const params = req.params; + const query = req.query; - const clientUrl = params.url; + const clientUrl = query.url; if (_.isEmpty(clientUrl)) { res.status(400).send('"url" field is required'); return; From 8c9d237906488c6530a5b112883b3c3633147d43 Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 20 Jan 2023 15:51:56 +0300 Subject: [PATCH 5/6] rm promisify from utils --- src/services/exporter.ts | 6 +++--- src/utils.ts | 12 ------------ 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/services/exporter.ts b/src/services/exporter.ts index 2c33bf0..f353d15 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -1,6 +1,6 @@ -// TODO: import promisify from 'util' +import { promisify } from 'util' import { getApiKey } from './api_keys'; -import { promisify, toIsoString } from '../utils'; +import { toIsoString } from '../utils'; import { DashboardQuery, ExportProgress, ExportStatus, ExportTask } from '../types'; import { CSV_PATH } from '../config'; @@ -135,7 +135,7 @@ export class Exporter { progress: _.assign(this._task.progress, progress, { time }), }; - await promisify(fs.writeFile, this._getFilePath('json'), JSON.stringify(data), 'utf8'); + await promisify(fs.writeFile)(this._getFilePath('json'), JSON.stringify(data), 'utf8'); } catch(err) { console.error(err); throw new Error('Can`t write file'); diff --git a/src/utils.ts b/src/utils.ts index a1fcc7f..1377075 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,17 +1,5 @@ import * as moment from 'moment-timezone'; -export async function promisify(method: (...params: any[]) => Promise | void, ...params: any[]) { - return new Promise((resolve, reject) => { - method(...params, (err, result) => { - if(err) { - reject(err); - } else { - resolve(result); - } - }) - }); -} - export function toIsoString(msTimestamp: number, timeZone: string): string { return moment.tz(msTimestamp, timeZone).format('YYYY-MM-DD HH:mm:ssZ').replace(/:00$/, ''); } From 25b1862ad6762c92297442668d05b30fa12c5150 Mon Sep 17 00:00:00 2001 From: rozetko Date: Fri, 20 Jan 2023 16:12:53 +0300 Subject: [PATCH 6/6] codestyle fix --- src/services/exporter.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/services/exporter.ts b/src/services/exporter.ts index f353d15..ff5c4bb 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -1,4 +1,3 @@ -import { promisify } from 'util' import { getApiKey } from './api_keys'; import { toIsoString } from '../utils'; import { DashboardQuery, ExportProgress, ExportStatus, ExportTask } from '../types'; @@ -15,6 +14,8 @@ import * as csv from 'fast-csv'; import * as fs from 'fs'; import * as path from 'path'; import { URL } from 'url'; +import { promisify } from 'util'; + import * as _ from 'lodash'; const MS_IN_DAY = 24 * 60 * 60 * 1000;