Merge pull request 'Auto API-key configuration' (#7) from auto-api-key-configuration into master

Reviewed-on: #7

.gitignore

@@ -1,5 +1,6 @@
 node_modules/
 dist/
-exported/
+data/
 config.json
 package-lock.json
+api-keys.json

Dockerfile

@@ -7,7 +7,7 @@ ENV BUILD_PATH=$build_path
 # Expose port 80
 EXPOSE 8000
 
-VOLUME [ "/var/www/exported" ]
+VOLUME [ "/var/www/data" ]
 
 # Copy custom configuration file from the current directory
 

config-example.json

@@ -1,6 +1,3 @@
 {
-  "apiKeys" : { 
-    "http://localhost:3000": "eyJrIjoiTjlUcmtLSFRNcTdqeXBaQjB5REk2TFkyUDBDM0Z1bWciLCJuIjoiZXhwb3J0LW1hbmFnZXIiLCJpZCI6MX0="
-  },
   "port": "8000"
 }

docker-compose.yml

@@ -0,0 +1,13 @@
+version: '2'
+
+services:
+  exporter:
+    image: corpglory/grafana-data-exporter:latest
+    restart: always
+    ports:
+      - 8000: 8000
+    volumes:
+      - data:/var/www/data
+
+volumes:
+  data

src/config.ts

@@ -2,17 +2,20 @@ import * as path from 'path';
 import * as fs from 'fs';
 import * as _ from 'lodash';
 
+
 const DEFAULT_CONFIG = {
-  'apiKeys': {
-    'http://localhost:3000': ''
-  },
   'port': '8000'
 };
 
-export const EXPORTED_PATH = path.join(__dirname, '../exported');
+export const DATA_PATH = path.join(__dirname, '../data');
-if(!fs.existsSync(EXPORTED_PATH)) {
+if(!fs.existsSync(DATA_PATH)) {
-  console.log(`${EXPORTED_PATH} doesn't exist, creating`);
+  console.log(`${DATA_PATH} doesn't exist, creating`);
-  fs.mkdirSync(EXPORTED_PATH);
+  fs.mkdirSync(DATA_PATH);
+}
+export const CSV_PATH = path.join(DATA_PATH, 'csv');
+if(!fs.existsSync(CSV_PATH)) {
+  console.log(`${CSV_PATH} doesn't exist, creating`);
+  fs.mkdirSync(CSV_PATH);
 }
 
 function getConfigField(field: string, defaultVal?: any) {
@@ -38,4 +41,3 @@ function getConfigField(field: string, defaultVal?: any) {
 }
 
 export const port = getConfigField('port', '8000');
-export const apiKeys = getConfigField('apiKeys');

src/routes/api.ts

@@ -1,6 +1,7 @@
-import { EXPORTED_PATH } from '../config';
+import { CSV_PATH } from '../config';
 import { router as tasksRouter } from '../routes/tasks';
 import { router as statusRouter } from '../routes/status';
+import { router as connectRouter } from '../routes/connect';
 
 import * as express from 'express';
 
@@ -8,7 +9,6 @@ import * as express from 'express';
 export const router = express.Router();
 
 router.use('/status', statusRouter);
-
+router.use('/connect', connectRouter);
 router.use('/task', tasksRouter);
-
+router.use('/static', express.static(CSV_PATH));
-router.use('/static', express.static(EXPORTED_PATH));

src/routes/connect.ts

@@ -0,0 +1,53 @@
+import { upsertApiKey, validateGrafanaUrl } from '../services/api_keys';
+
+import * as express from 'express'
+
+import * as _ from 'lodash';
+
+
+async function checkConnection(req, res) {
+  const query = req.query;
+
+  const clientUrl = query.url;
+  if (_.isEmpty(clientUrl)) {
+    res.status(400).send('"url" field is required');
+    return;
+  }
+
+  try {
+    validateGrafanaUrl(clientUrl);
+  } catch (e) {
+    res.status(500).send(e.message);
+    return;
+  }
+
+  res.status(200).send({ version: process.env.npm_package_version });
+}
+
+async function connectPlugin(req, res) {
+  const body = req.body;
+
+  const clientUrl = body.url;
+  if (_.isEmpty(clientUrl)) {
+    res.status(400).send('"url" field is required');
+    return;
+  }
+  const apiToken = body.apiToken;
+  if (_.isEmpty(apiToken)) {
+    res.status(400).send('"apiToken" field is required');
+    return;
+  }
+
+  const grafanaUrl = new URL(clientUrl).origin;
+
+  upsertApiKey(grafanaUrl, apiToken);
+
+  console.log(`Grafana at ${grafanaUrl} is connected`);
+
+  res.status(200).send({ version: process.env.npm_package_version });
+}
+
+export const router = express.Router();
+
+router.get('/', checkConnection);
+router.post('/', connectPlugin);

src/routes/status.ts

@@ -1,10 +1,10 @@
 import * as express from 'express'
 
 
-async function deleteTask(req, res) {
+async function getStatus(req, res) {
-  res.status(200).send({ version: 123 });
+  res.status(200).send({ version: process.env.npm_package_version });
 }
 
 export const router = express.Router();
 
-router.get('/', deleteTask);
+router.get('/', getStatus);

src/routes/tasks.ts

@@ -1,5 +1,6 @@
 import { exporterFactory } from '../services/exporter.factory';
-import { EXPORTED_PATH } from '../config';
+import { CSV_PATH } from '../config';
+import { validateGrafanaUrl } from '../services/api_keys';
 
 import { ExportTask } from '../types';
 
@@ -10,6 +11,7 @@ import * as _ from 'lodash';
 import * as path from 'path';
 import * as fs from 'fs';
 
+
 type TRequest = {
   body: {
     task: ExportTask,
@@ -20,7 +22,7 @@ type TRequest = {
 
 async function getTasks(req, res) {
   const resp: ExportTask[] = [];
-  fs.readdir(EXPORTED_PATH, (err, items) => {
+  fs.readdir(CSV_PATH, (err, items) => {
     if(err) {
       console.error(err);
       res.status(500).send(err.message);
@@ -31,7 +33,7 @@ async function getTasks(req, res) {
           continue;
         }
         // TODO: read async
-        let data = fs.readFileSync(path.join(EXPORTED_PATH, item), 'utf8');
+        let data = fs.readFileSync(path.join(CSV_PATH, item), 'utf8');
         try {
           let status = JSON.parse(data);
           resp.push(status);
@@ -51,10 +53,19 @@ async function addTask(req: TRequest, res) {
   const clientUrl = body.url;
   if (_.isEmpty(clientUrl)) {
     res.status(400).send('"url" field is required');
+    return;
   }
   const task = body.task;
   if (_.isEmpty(task)) {
     res.status(400).send('"task" field is required');
+    return;
+  }
+
+  try {
+    validateGrafanaUrl(clientUrl);
+  } catch(e) {
+    res.status(500).send(e.message);
+    return;
   }
 
   const datasourceUrl = `${new URL(clientUrl).origin}/api/ds/query`;
@@ -63,8 +74,10 @@ async function addTask(req: TRequest, res) {
   const to = +task.timeRange.to;
   if (isNaN(from) || isNaN(to)) {
     res.status(400).send('Range error: please fill both "from" and "to" fields');
+    return;
   } else if (from >= to) {
     res.status(400).send('Range error: "from" should be less than "to"');
+    return;
   }
 
   const exporter = exporterFactory.getExporter();
@@ -74,8 +87,8 @@ async function addTask(req: TRequest, res) {
 
 async function deleteTask(req, res) {
   let taskId = req.body.taskId;
-  let csvFilePath = path.join(EXPORTED_PATH, `${taskId}.csv`);
+  let csvFilePath = path.join(CSV_PATH, `${taskId}.csv`);
-  let jsonFilePath = path.join(EXPORTED_PATH, `${taskId}.json`);
+  let jsonFilePath = path.join(CSV_PATH, `${taskId}.json`);
 
   if(fs.existsSync(csvFilePath)) {
     fs.unlink(csvFilePath, err => console.error(err));

src/services/api_keys.ts

@@ -0,0 +1,41 @@
+import { DATA_PATH } from '../config';
+
+import * as path from 'path';
+import * as fs from 'fs';
+import * as _ from 'lodash';
+
+
+const API_KEYS_FILE = path.join(DATA_PATH, 'api-keys.json');
+if(!fs.existsSync(API_KEYS_FILE)) {
+  console.log(`${API_KEYS_FILE} doesn't exist, creating`);
+  fs.writeFileSync(API_KEYS_FILE, JSON.stringify({}), 'utf8');
+}
+
+export function getApiKey(grafanaUrl: string): string | null {
+  const data = fs.readFileSync(API_KEYS_FILE, 'utf8');
+  const apiKey = JSON.parse(data)[grafanaUrl];
+  if(_.isNil(apiKey)) {
+    return null;
+  }
+
+  return apiKey;
+}
+
+export function upsertApiKey(grafanaUrl: string, apiKey: string): void {
+  const data = fs.readFileSync(API_KEYS_FILE, 'utf8');
+  const apiKeys = JSON.parse(data);
+
+  apiKeys[grafanaUrl] = apiKey;
+
+  fs.writeFileSync(API_KEYS_FILE, JSON.stringify(apiKeys), 'utf8');
+}
+
+// TODO: query Grafana API if a key exists && remove the key if it doesn't work for specified grafanaUrl
+export function validateGrafanaUrl(grafanaUrl: string) {
+  const host = new URL(grafanaUrl).origin;
+  const apiKey = getApiKey(host);
+
+  if (_.isNil(apiKey) || apiKey === '') {
+    throw new Error(`Please configure API key for ${host}`);
+  }
+}

src/services/exporter.ts

@@ -1,7 +1,7 @@
-import { URL } from 'url';
+import { getApiKey } from './api_keys';
-import { apiKeys } from '../config';
+import {  toIsoString } from '../utils';
-import { promisify, toIsoString } from '../utils';
 import { DashboardQuery, ExportProgress, ExportStatus, ExportTask } from '../types';
+import { CSV_PATH } from '../config';
 
 import { QueryConfig, queryByConfig } from '@corpglory/tsdb-kit';
 // TODO: export QueryType directly from @corpglory/tsdb-kit
@@ -13,6 +13,9 @@ import * as moment from 'moment';
 import * as csv from 'fast-csv';
 import * as fs from 'fs';
 import * as path from 'path';
+import { URL } from 'url';
+import { promisify } from 'util';
+
 import * as _ from 'lodash';
 
 const MS_IN_DAY = 24 * 60 * 60 * 1000;
@@ -33,7 +36,6 @@ export class Exporter {
       this._task.progress = _.cloneDeep(DEFAULT_PROGRESS);
 
       this._validateQueries(task.queries);
-      this._validateDatasourceUrl(datasourceUrl);
 
       await this._updateProgress();
 
@@ -66,7 +68,7 @@ export class Exporter {
 
         for(const queryConfig of queryConfigs) {
           const host = new URL(datasourceUrl).origin;
-          const apiKey = apiKeys[host];
+          const apiKey = getApiKey(host);
 
           const datasourceMetrics = await queryByConfig(queryConfig, datasourceUrl, from, to, apiKey);
 
@@ -134,7 +136,7 @@ export class Exporter {
         progress: _.assign(this._task.progress, progress, { time }),
       };
 
-      await promisify(fs.writeFile, this._getFilePath('json'), JSON.stringify(data), 'utf8');
+      await promisify(fs.writeFile)(this._getFilePath('json'), JSON.stringify(data), 'utf8');
     } catch(err) {
       console.error(err);
       throw new Error('Can`t write file');
@@ -156,15 +158,6 @@ export class Exporter {
     }
   }
 
-  private _validateDatasourceUrl(datasourceUrl: string) {
-    const host = new URL(datasourceUrl).origin;
-    const apiKey = apiKeys[host];
-
-    if(apiKey === undefined || apiKey === '') {
-      throw new Error(`Please configure API key for ${host}`);
-    }
-  }
-
   private _writeCsv(stream, series) {
     for(let row of series.values) {
       let csvRow = {};
@@ -182,6 +175,6 @@ export class Exporter {
 
   private _getFilePath(extension: string): string {
     let filename = this._getFilename(extension);
-    return path.join(__dirname, `../exported/${filename}`);
+    return path.join(CSV_PATH, filename);
   }
 }

src/utils.ts

@@ -1,17 +1,5 @@
 import * as moment from 'moment-timezone';
 
-export async function promisify(method: (...params: any[]) => Promise<any> | void, ...params: any[]) {
-  return new Promise((resolve, reject) => {
-    method(...params, (err, result) => {
-      if(err) {
-        reject(err);
-      } else {
-        resolve(result);
-      }
-    })
-  });
-}
-
 export function toIsoString(msTimestamp: number, timeZone: string): string {
   return moment.tz(msTimestamp, timeZone).format('YYYY-MM-DD HH:mm:ssZ').replace(/:00$/, '');
 }