diff --git a/.gitignore b/.gitignore index e89fcc3..26b66cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ node_modules/ dist/ -exported/ +data/ config.json package-lock.json +api-keys.json diff --git a/Dockerfile b/Dockerfile index 5e2bb4e..af668d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ ENV BUILD_PATH=$build_path # Expose port 80 EXPOSE 8000 -VOLUME [ "/var/www/exported" ] +VOLUME [ "/var/www/data" ] # Copy custom configuration file from the current directory diff --git a/config-example.json b/config-example.json index 4707b2d..8034c59 100644 --- a/config-example.json +++ b/config-example.json @@ -1,6 +1,3 @@ { - "apiKeys" : { - "http://localhost:3000": "eyJrIjoiTjlUcmtLSFRNcTdqeXBaQjB5REk2TFkyUDBDM0Z1bWciLCJuIjoiZXhwb3J0LW1hbmFnZXIiLCJpZCI6MX0=" - }, "port": "8000" } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ccd8b43 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +version: '2' + +services: + exporter: + image: corpglory/grafana-data-exporter:latest + restart: always + ports: + - 8000: 8000 + volumes: + - data:/var/www/data + +volumes: + data diff --git a/exported/.gitkeep b/exported/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/config.ts b/src/config.ts index f27ade2..444d055 100644 --- a/src/config.ts +++ b/src/config.ts @@ -2,17 +2,20 @@ import * as path from 'path'; import * as fs from 'fs'; import * as _ from 'lodash'; + const DEFAULT_CONFIG = { - 'apiKeys': { - 'http://localhost:3000': '' - }, 'port': '8000' }; -export const EXPORTED_PATH = path.join(__dirname, '../exported'); -if(!fs.existsSync(EXPORTED_PATH)) { - console.log(`${EXPORTED_PATH} doesn't exist, creating`); - fs.mkdirSync(EXPORTED_PATH); +export const DATA_PATH = path.join(__dirname, '../data'); +if(!fs.existsSync(DATA_PATH)) { + console.log(`${DATA_PATH} doesn't exist, creating`); + fs.mkdirSync(DATA_PATH); +} +export const CSV_PATH = path.join(DATA_PATH, 'csv'); +if(!fs.existsSync(CSV_PATH)) { + console.log(`${CSV_PATH} doesn't exist, creating`); + fs.mkdirSync(CSV_PATH); } function getConfigField(field: string, defaultVal?: any) { @@ -38,4 +41,3 @@ function getConfigField(field: string, defaultVal?: any) { } export const port = getConfigField('port', '8000'); -export const apiKeys = getConfigField('apiKeys'); diff --git a/src/routes/api.ts b/src/routes/api.ts index 49a57b3..ca40d86 100644 --- a/src/routes/api.ts +++ b/src/routes/api.ts @@ -1,6 +1,7 @@ -import { EXPORTED_PATH } from '../config'; +import { CSV_PATH } from '../config'; import { router as tasksRouter } from '../routes/tasks'; import { router as statusRouter } from '../routes/status'; +import { router as connectRouter } from '../routes/connect'; import * as express from 'express'; @@ -8,7 +9,6 @@ import * as express from 'express'; export const router = express.Router(); router.use('/status', statusRouter); - +router.use('/connect', connectRouter); router.use('/task', tasksRouter); - -router.use('/static', express.static(EXPORTED_PATH)); +router.use('/static', express.static(CSV_PATH)); diff --git a/src/routes/connect.ts b/src/routes/connect.ts new file mode 100644 index 0000000..0e71fe8 --- /dev/null +++ b/src/routes/connect.ts @@ -0,0 +1,53 @@ +import { upsertApiKey, validateGrafanaUrl } from '../services/api_keys'; + +import * as express from 'express' + +import * as _ from 'lodash'; + + +async function checkConnection(req, res) { + const query = req.query; + + const clientUrl = query.url; + if (_.isEmpty(clientUrl)) { + res.status(400).send('"url" field is required'); + return; + } + + try { + validateGrafanaUrl(clientUrl); + } catch (e) { + res.status(500).send(e.message); + return; + } + + res.status(200).send({ version: process.env.npm_package_version }); +} + +async function connectPlugin(req, res) { + const body = req.body; + + const clientUrl = body.url; + if (_.isEmpty(clientUrl)) { + res.status(400).send('"url" field is required'); + return; + } + const apiToken = body.apiToken; + if (_.isEmpty(apiToken)) { + res.status(400).send('"apiToken" field is required'); + return; + } + + const grafanaUrl = new URL(clientUrl).origin; + + upsertApiKey(grafanaUrl, apiToken); + + console.log(`Grafana at ${grafanaUrl} is connected`); + + res.status(200).send({ version: process.env.npm_package_version }); +} + +export const router = express.Router(); + +router.get('/', checkConnection); +router.post('/', connectPlugin); diff --git a/src/routes/status.ts b/src/routes/status.ts index c7d9dc4..65016a5 100644 --- a/src/routes/status.ts +++ b/src/routes/status.ts @@ -1,10 +1,10 @@ import * as express from 'express' -async function deleteTask(req, res) { - res.status(200).send({ version: 123 }); +async function getStatus(req, res) { + res.status(200).send({ version: process.env.npm_package_version }); } export const router = express.Router(); -router.get('/', deleteTask); +router.get('/', getStatus); diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts index ecf19be..1193625 100644 --- a/src/routes/tasks.ts +++ b/src/routes/tasks.ts @@ -1,5 +1,6 @@ import { exporterFactory } from '../services/exporter.factory'; -import { EXPORTED_PATH } from '../config'; +import { CSV_PATH } from '../config'; +import { validateGrafanaUrl } from '../services/api_keys'; import { ExportTask } from '../types'; @@ -10,6 +11,7 @@ import * as _ from 'lodash'; import * as path from 'path'; import * as fs from 'fs'; + type TRequest = { body: { task: ExportTask, @@ -20,7 +22,7 @@ type TRequest = { async function getTasks(req, res) { const resp: ExportTask[] = []; - fs.readdir(EXPORTED_PATH, (err, items) => { + fs.readdir(CSV_PATH, (err, items) => { if(err) { console.error(err); res.status(500).send(err.message); @@ -31,7 +33,7 @@ async function getTasks(req, res) { continue; } // TODO: read async - let data = fs.readFileSync(path.join(EXPORTED_PATH, item), 'utf8'); + let data = fs.readFileSync(path.join(CSV_PATH, item), 'utf8'); try { let status = JSON.parse(data); resp.push(status); @@ -51,10 +53,19 @@ async function addTask(req: TRequest, res) { const clientUrl = body.url; if (_.isEmpty(clientUrl)) { res.status(400).send('"url" field is required'); + return; } const task = body.task; if (_.isEmpty(task)) { res.status(400).send('"task" field is required'); + return; + } + + try { + validateGrafanaUrl(clientUrl); + } catch(e) { + res.status(500).send(e.message); + return; } const datasourceUrl = `${new URL(clientUrl).origin}/api/ds/query`; @@ -63,8 +74,10 @@ async function addTask(req: TRequest, res) { const to = +task.timeRange.to; if (isNaN(from) || isNaN(to)) { res.status(400).send('Range error: please fill both "from" and "to" fields'); + return; } else if (from >= to) { res.status(400).send('Range error: "from" should be less than "to"'); + return; } const exporter = exporterFactory.getExporter(); @@ -74,8 +87,8 @@ async function addTask(req: TRequest, res) { async function deleteTask(req, res) { let taskId = req.body.taskId; - let csvFilePath = path.join(EXPORTED_PATH, `${taskId}.csv`); - let jsonFilePath = path.join(EXPORTED_PATH, `${taskId}.json`); + let csvFilePath = path.join(CSV_PATH, `${taskId}.csv`); + let jsonFilePath = path.join(CSV_PATH, `${taskId}.json`); if(fs.existsSync(csvFilePath)) { fs.unlink(csvFilePath, err => console.error(err)); diff --git a/src/services/api_keys.ts b/src/services/api_keys.ts new file mode 100644 index 0000000..9e7eccf --- /dev/null +++ b/src/services/api_keys.ts @@ -0,0 +1,41 @@ +import { DATA_PATH } from '../config'; + +import * as path from 'path'; +import * as fs from 'fs'; +import * as _ from 'lodash'; + + +const API_KEYS_FILE = path.join(DATA_PATH, 'api-keys.json'); +if(!fs.existsSync(API_KEYS_FILE)) { + console.log(`${API_KEYS_FILE} doesn't exist, creating`); + fs.writeFileSync(API_KEYS_FILE, JSON.stringify({}), 'utf8'); +} + +export function getApiKey(grafanaUrl: string): string | null { + const data = fs.readFileSync(API_KEYS_FILE, 'utf8'); + const apiKey = JSON.parse(data)[grafanaUrl]; + if(_.isNil(apiKey)) { + return null; + } + + return apiKey; +} + +export function upsertApiKey(grafanaUrl: string, apiKey: string): void { + const data = fs.readFileSync(API_KEYS_FILE, 'utf8'); + const apiKeys = JSON.parse(data); + + apiKeys[grafanaUrl] = apiKey; + + fs.writeFileSync(API_KEYS_FILE, JSON.stringify(apiKeys), 'utf8'); +} + +// TODO: query Grafana API if a key exists && remove the key if it doesn't work for specified grafanaUrl +export function validateGrafanaUrl(grafanaUrl: string) { + const host = new URL(grafanaUrl).origin; + const apiKey = getApiKey(host); + + if (_.isNil(apiKey) || apiKey === '') { + throw new Error(`Please configure API key for ${host}`); + } +} diff --git a/src/services/exporter.ts b/src/services/exporter.ts index 8d63013..ff5c4bb 100644 --- a/src/services/exporter.ts +++ b/src/services/exporter.ts @@ -1,7 +1,7 @@ -import { URL } from 'url'; -import { apiKeys } from '../config'; -import { promisify, toIsoString } from '../utils'; +import { getApiKey } from './api_keys'; +import { toIsoString } from '../utils'; import { DashboardQuery, ExportProgress, ExportStatus, ExportTask } from '../types'; +import { CSV_PATH } from '../config'; import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit'; // TODO: export QueryType directly from @corpglory/tsdb-kit @@ -13,6 +13,9 @@ import * as moment from 'moment'; import * as csv from 'fast-csv'; import * as fs from 'fs'; import * as path from 'path'; +import { URL } from 'url'; +import { promisify } from 'util'; + import * as _ from 'lodash'; const MS_IN_DAY = 24 * 60 * 60 * 1000; @@ -33,7 +36,6 @@ export class Exporter { this._task.progress = _.cloneDeep(DEFAULT_PROGRESS); this._validateQueries(task.queries); - this._validateDatasourceUrl(datasourceUrl); await this._updateProgress(); @@ -66,7 +68,7 @@ export class Exporter { for(const queryConfig of queryConfigs) { const host = new URL(datasourceUrl).origin; - const apiKey = apiKeys[host]; + const apiKey = getApiKey(host); const datasourceMetrics = await queryByConfig(queryConfig, datasourceUrl, from, to, apiKey); @@ -134,7 +136,7 @@ export class Exporter { progress: _.assign(this._task.progress, progress, { time }), }; - await promisify(fs.writeFile, this._getFilePath('json'), JSON.stringify(data), 'utf8'); + await promisify(fs.writeFile)(this._getFilePath('json'), JSON.stringify(data), 'utf8'); } catch(err) { console.error(err); throw new Error('Can`t write file'); @@ -156,15 +158,6 @@ export class Exporter { } } - private _validateDatasourceUrl(datasourceUrl: string) { - const host = new URL(datasourceUrl).origin; - const apiKey = apiKeys[host]; - - if(apiKey === undefined || apiKey === '') { - throw new Error(`Please configure API key for ${host}`); - } - } - private _writeCsv(stream, series) { for(let row of series.values) { let csvRow = {}; @@ -182,6 +175,6 @@ export class Exporter { private _getFilePath(extension: string): string { let filename = this._getFilename(extension); - return path.join(__dirname, `../exported/${filename}`); + return path.join(CSV_PATH, filename); } } diff --git a/src/utils.ts b/src/utils.ts index a1fcc7f..1377075 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,17 +1,5 @@ import * as moment from 'moment-timezone'; -export async function promisify(method: (...params: any[]) => Promise | void, ...params: any[]) { - return new Promise((resolve, reject) => { - method(...params, (err, result) => { - if(err) { - reject(err); - } else { - resolve(result); - } - }) - }); -} - export function toIsoString(msTimestamp: number, timeZone: string): string { return moment.tz(msTimestamp, timeZone).format('YYYY-MM-DD HH:mm:ssZ').replace(/:00$/, ''); }