Initial commit

.gitignore

@@ -0,0 +1,6 @@
+node_modules/
+dist/
+exported/*.csv
+exported/*.json
+api-keys.json
+package-lock.json

Dockerfile

@@ -0,0 +1,21 @@
+# Pull node base image
+FROM node:latest
+
+ARG build_path=/var/www
+ENV BUILD_PATH=$build_path
+
+# Expose port 80
+EXPOSE 80
+
+VOLUME [ "/var/www/exported" ]
+
+# Copy custom configuration file from the current directory
+
+WORKDIR ${BUILD_PATH}
+COPY . ${BUILD_PATH}
+
+RUN npm install
+RUN npm run build
+
+# Start up node server
+CMD ["npm", "start"]

README.md

@@ -0,0 +1,50 @@
+# Export-manager backend
+
+Server for fetching data from Grafana datasources
+
+Running on 80 port in Docker container.
+
+Dockerfile has 2 volumes:
+
+- `/var/www/api-keys.json` (required, contains API keys for your Grafana hosts, see [example](api-keys-example.json))
+- `/var/www/exported` (optional, directory which contains exported csv files and info about them)
+
+## Build
+
+```
+npm install
+npm run build
+```
+
+## Run
+
+```
+npm start
+```
+
+## Development
+
+You should have `nodemon` module installed to run development server.
+
+```
+npm i -g nodemon
+npm run dev
+```
+
+## Changelog
+
+### [0.3.1] - 2018-05-14
+#### Added
+- Show confirmation modal on task delete.
+
+### [0.3.0] - 2018-05-10
+#### Added
+- Save user that initialized export.
+- Support different grafana URLs.
+- Delete tasks.
+
+### [0.2.0] - 2018-05-09
+#### Added
+- Fetch data from Grafana API and save it to CSV.
+- Endpoint for showing task status in Simple-JSON datasource format.
+- CSV download URL on task finish.

api-keys-example.json

@@ -0,0 +1,4 @@
+{
+  "http://grafana.example.com": "eyJrIjoiNFRDNms3Z3RZRmtoU2hKd002dFJUS2FCOHEySUlSZzEiLCJuIjoiZXhwb3J0IiwiaWQiOjF9",
+  "http://localhost:3500": "eyJrIjoiTjlUcmtLSFRNcTdqeXBaQjB5REk2TFkyUDBDM0Z1bWciLCJuIjoiZXhwb3J0LW1hbmFnZXIiLCJpZCI6MX0="
+}

build/dev-server.js

@@ -0,0 +1,10 @@
+const { spawn } = require('child_process');
+
+const webpack = spawn('webpack', ['--config', 'build/webpack.dev.conf.js'], {
+  stdio: 'inherit',
+  shell: true
+});
+//webpack.stdout.pipe(process.stdout);
+
+const nodemon = spawn('nodemon', ['../dist/server', '--watch', 'server.js']);
+nodemon.stdout.pipe(process.stdout);

build/webpack.base.conf.js

@@ -0,0 +1,52 @@
+const path = require('path');
+const fs = require('fs');
+
+const webpack = require('webpack');
+
+
+function resolve(p) {
+  return path.join(__dirname, '/../', p);
+}
+
+module.exports = {
+  target: 'node',
+  node: {
+    __dirname: false,
+    __filename: false,
+  },
+  context: resolve('./src'),
+  entry: './index',
+  devtool: 'inline-source-map',
+  output: {
+    filename: "server.js",
+    path: resolve('dist')
+  },
+  externals: [
+    function(context, request, callback) {
+      if(request[0] == '.') {
+        callback();
+      } else {
+        callback(null, "require('" + request + "')");
+      }
+    }
+  ],
+  plugins: [
+    new webpack.optimize.OccurrenceOrderPlugin(),
+    new webpack.HotModuleReplacementPlugin(),
+    new webpack.DefinePlugin({
+      'process.env.NODE_ENV': JSON.stringify('development')
+    })
+  ],
+  resolve: {
+    extensions: [".ts", ".js"]
+  },
+  module: {
+    rules: [
+      {
+        test: /\.ts$/,
+        loader: "ts-loader",
+        exclude: /node_modules/
+      }
+    ]
+  }
+}

build/webpack.dev.conf.js

@@ -0,0 +1,4 @@
+var base = require('./webpack.base.conf');
+
+base.watch = true;
+module.exports = base;

build/webpack.prod.conf.js

@@ -0,0 +1,3 @@
+var base = require('./webpack.base.conf');
+
+module.exports = base;

package.json

@@ -0,0 +1,25 @@
+{
+  "name": "export-manager-server",
+  "version": "0.3.1",
+  "description": "Server for fetching data from Grafana datasources",
+  "scripts": {
+    "start": "node dist/server.js",
+    "dev": "node build/dev-server.js",
+    "build": "webpack --config build/webpack.prod.conf.js"
+  },
+  "author": "CorpGlory",
+  "license": "ISC",
+  "dependencies": {},
+  "devDependencies": {
+    "@types/express": "^4.11.1",
+    "axios": "^0.18.0",
+    "express": "^4.16.3",
+    "fast-csv": "^2.4.1",
+    "moment": "^2.22.1",
+    "node-fetch": "^2.1.2",
+    "nodemon": "^1.17.3",
+    "ts-loader": "^3.5.0",
+    "typescript": "^2.8.3",
+    "webpack": "^3.5.6"
+  }
+}

src/config.ts

@@ -0,0 +1,16 @@
+import * as path from 'path';
+import * as fs from 'fs';
+
+export const EXPORTED_PATH = path.join(__dirname, '../exported');
+
+
+export function getApiKey(host) {
+  return new Promise((resolve, reject) => {
+    fs.readFile(path.join(__dirname, '../api-keys.json'), 'utf8', (err, data) => {
+      if(err) {
+        reject(err);
+      }
+      resolve(JSON.parse(data)[host]);
+    });
+  });
+}

src/grafana_api.ts

@@ -0,0 +1,66 @@
+import { getApiKey } from './config';
+
+import axios from 'axios';
+
+
+export class GrafanaAPI {
+  private apiKey;
+  constructor(private grafanaUrl) {
+    getApiKey(grafanaUrl)
+      .then(key => {
+        this.apiKey = key;
+        console.log(this.apiKey);
+      });
+  }
+
+  private get _headers() {
+    return {
+      'Authorization': `Bearer ${this.apiKey}`,
+      'Accept': 'application/json',
+      'Content-Type': 'application/json'
+    };
+  }
+
+  private async _getDatasourceByName(name) {
+    return fetch(`${this.grafanaUrl}/api/datasources/name/${name}`, {
+      method: 'GET',
+      headers: this._headers
+    })
+      .then(data => data.json());
+  }
+
+  public async queryDatasource(datasourceName, measurement, query) {
+    let datasource = await this._getDatasourceByName(datasourceName);
+
+    return this._queryGrafana(`${this.grafanaUrl}/api/datasources/proxy/${datasource.id}/query`, {
+      q: encodeURIComponent(query),
+      db: datasource.database,
+      epoch: 'ms'
+    });
+  }
+
+  private async _queryGrafana(url: string, params: any) {
+    try {
+      var res = await axios.get(url, { params, headers: this._headers });
+    } catch (e) {
+      if(e.response.status === 401) {
+        throw new Error('Unauthorized. Check the $HASTIC_API_KEY.');
+      }
+      throw new Error(e.message);
+    }
+
+    if (res.data.results === undefined) {
+      throw new Error('results field is undefined in response.');
+    }
+
+    // TODO: support more than 1 metric (each res.data.results item is a metric)
+    let results = res.data.results[0];
+    if (results.series === undefined) {
+      return [];
+    }
+
+    return results.series[0].values as [number, number][];
+  }
+
+}
+

src/grafana_metric_model.ts

@@ -0,0 +1,88 @@
+export type GrafanaDatasource = {
+  url: string,
+  type: string,
+  params: {
+    db: string,
+    q: string,
+    epoch: string
+  }
+}
+
+export type GrafanaMetricId = string;
+
+export class GrafanaMetric {
+
+  private _metricQuery: MetricQuery = undefined;
+
+  constructor(
+    public datasource: GrafanaDatasource,
+    public targets: any[],
+    public id?: GrafanaMetricId
+  ) {
+    if(datasource === undefined) {
+      throw new Error('datasource is undefined');
+    }
+    if(targets === undefined) {
+      throw new Error('targets is undefined');
+    }
+    if(targets.length === 0) {
+      throw new Error('targets is empty');
+    }
+  }
+
+  public get metricQuery() {
+    if(this._metricQuery === undefined) {
+      this._metricQuery = new MetricQuery(this);
+    }
+    return this._metricQuery;
+  }
+
+  public toObject() {
+    return {
+      datasource: this.datasource,
+      targets: this.targets,
+      _id: this.id
+    };
+  }
+
+  static fromObject(obj: any): GrafanaMetric {
+    if(obj === undefined) {
+      throw new Error('obj is undefined');
+    }
+    return new GrafanaMetric(
+      obj.datasource,
+      obj.targets,
+      obj._id
+    );
+  }
+}
+
+export class MetricQuery {
+
+  private static INFLUX_QUERY_TIME_REGEX = /time >[^A-Z]+/;
+
+  private _queryParts: string[];
+  private _type: string;
+
+  constructor(metric: GrafanaMetric) {
+    this._type = metric.datasource.type;
+    if (this._type !== 'influxdb') {
+      throw new Error(`Queries of type "${metric.datasource.type}" are not supported yet.`);
+    }
+    var queryStr = metric.datasource.params.q;
+    this._queryParts = queryStr.split(MetricQuery.INFLUX_QUERY_TIME_REGEX);
+    if(this._queryParts.length == 1) {
+      throw new Error(
+        `Query "${queryStr}" is not replaced with LIMIT/OFFSET oeprators. Missing time clause.`
+      );
+    }
+    if(this._queryParts.length > 2) {
+      throw new Error(`Query "${queryStr}" has multiple time clauses. Can't parse.`);
+    }
+  }
+
+  getQuery(from: number, to: number, limit: number, offset: number): string {
+    let timeClause = `time >= ${from}ms AND time <= ${to}ms`;
+    return `${this._queryParts[0]} ${timeClause} ${this._queryParts[1]} LIMIT ${limit} OFFSET ${offset}`;
+  }
+}

src/index.ts

@@ -0,0 +1,31 @@
+import { EXPORTED_PATH } from './config';
+import { router as tasksRouter } from './routes/tasks';
+import { router as datasourceRouter } from './routes/datasource';
+import { router as deleteRouter } from './routes/delete';
+
+import * as express from 'express';
+import * as bodyParser from 'body-parser';
+
+const app = express();
+const PORT = 80;
+
+app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({ extended: true }));
+
+app.use(function(req, res, next) {
+  res.header('Access-Control-Allow-Origin', '*');
+  res.header('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, PATCH, OPTIONS');
+  res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept');
+  next();
+})
+
+app.use('/tasks', tasksRouter);
+app.use('/datasource', datasourceRouter);
+app.use('/delete', deleteRouter);
+
+app.use('/static', express.static(EXPORTED_PATH));
+app.use('/', (req, res) => { res.send('Export-manager backend server works') });
+
+app.listen(PORT, () => {
+  console.log(`Server is running on :${PORT}`);
+})

src/routes/datasource.ts

@@ -0,0 +1,123 @@
+import { EXPORTED_PATH } from '../config';
+
+import * as express from 'express';
+import * as path from 'path';
+import * as fs from 'fs';
+
+
+function sendOk(req, res) {
+  res.status(200).send('Datasource works');
+}
+
+function search(req, res) {
+  fs.readdir(EXPORTED_PATH, (err, items) => {
+    if(err) {
+      console.error(err);
+      res.status(500).send('Something went wrong');
+    } else {
+      res.status(200).send(['All tasks']);
+    }
+  })
+}
+
+function query(req, res) {
+  let body = req.body;
+  let targets = body.targets;
+
+  let resp = {
+    target: targets[0].target,
+    type: 'table',
+    columns: [
+      {
+        text: 'Time',
+        type: 'time'
+      },
+      {
+        text: 'User',
+        type: 'string'
+      },
+      {
+        text: 'Datasource',
+        type: 'string'
+      },
+      {
+        text: 'Measurement',
+        type: 'string'
+      },
+      {
+        text: 'Exported Rows',
+        type: 'number'
+      },
+      {
+        text: 'Progress',
+        type: 'string'
+      },
+      {
+        text: 'Download CSV',
+        type: 'string'
+      },
+      {
+        text: 'Status',
+        type: 'string'
+      },
+      {
+        text: 'Delete task',
+        type: 'string'
+      }
+    ],
+    rows: []
+  };
+
+  for(let target of targets) {
+    fs.readdir(EXPORTED_PATH, (err, items) => {
+      if(err) {
+        console.error(err);
+        res.status(500).send('Something went wrong');
+      } else {
+        for(let item of items) {
+          let file = path.parse(item);
+          if(file.ext !== '.json') {
+            continue;
+          }
+          // TODO: read async
+          let data = fs.readFileSync(path.join(EXPORTED_PATH, item), 'utf8')
+          let status = JSON.parse(data)
+
+          let requestedUrl = `http://${req.headers.host}`;
+          let downloadLink = '';
+          let deleteLink = '';
+          if(status.status === 'finished') {
+            downloadLink = `<a class="download-csv" href="${requestedUrl}/static/${file.name}.csv" target="_blank"><i class="fa fa-download"></i></a>`;
+            deleteLink = `<a class="delete-task" href="${requestedUrl}/delete?filename=${file.name}"><i class="fa fa-times"></i></a>`;
+          }
+          resp.rows.push([
+            status.time,
+            status.user,
+            status.datasource,
+            status.measurement,
+            status.exportedRows,
+            status.progress,
+            downloadLink,
+            status.status,
+            deleteLink
+          ]);
+        }
+
+        res.status(200).send([resp]);
+      }
+    })
+  }
+
+}
+
+function sendAnnotations(req, res) {
+  res.status(200).send([]);
+}
+
+export const router = express.Router();
+
+router.get('/', sendOk);
+router.post('/search', search);
+router.post('/query', query);
+router.post('/annotations', sendAnnotations);
+

src/routes/delete.ts

@@ -0,0 +1,25 @@
+import { EXPORTED_PATH } from '../config'
+
+import * as express from 'express'
+import * as fs from 'fs'
+import * as path from 'path'
+
+
+async function deleteTask(req, res) {
+  let filename = req.query.filename;
+  let csvFilePath = path.join(EXPORTED_PATH, `${filename}.csv`);
+  let jsonFilePath = path.join(EXPORTED_PATH, `${filename}.json`);
+  
+  if(fs.existsSync(csvFilePath)) {
+    fs.unlink(csvFilePath, err => console.error(err));
+  }
+  if(fs.existsSync(jsonFilePath)) {
+    fs.unlink(jsonFilePath, err => console.error(err));
+  }
+
+  res.status(200).send({ status: 'OK' });
+}
+
+export const router = express.Router();
+
+router.get('/', deleteTask);

src/routes/tasks.ts

@@ -0,0 +1,25 @@
+import { Target } from '../target'
+
+import * as express from 'express'
+
+
+async function addTask(req, res) {
+  let body = req.body;
+  let from = parseInt(body.from);
+  let to = parseInt(body.to);
+
+  if(isNaN(from) || isNaN(to)) {
+    res.status(500).send('Range error: please fill both "from" and "to" fields');
+  } else if(from >= to) {
+    res.status(500).send('Range error: "from" should be less than "to"');
+  } else {
+    res.status(200).send('Task added');
+    let grafanaUrl = req.get('origin');
+    let target = new Target(grafanaUrl, body.user, body.datasource, body.measurement, body.query, from, to);
+    target.export();
+  }
+}
+
+export const router = express.Router();
+
+router.post('/', addTask);

src/target.ts

@@ -0,0 +1,120 @@
+import { GrafanaAPI } from './grafana_api';
+
+import * as csv from 'fast-csv';
+import * as path from 'path';
+import * as fs from 'fs';
+import * as moment from 'moment';
+
+
+const MS_IN_DAY = 24 * 60 * 60 * 1000;
+
+export class Target {
+  private exportedRows: number;
+  private days: number;
+  private day: number;
+  private csvStream: any;
+  private grafana: GrafanaAPI;
+
+  constructor(
+    private grafanaUrl: string,
+    private user: string,
+    private datasource: string,
+    private measurement: string,
+    private query: string,
+    private from: number,
+    private to: number
+  ) {
+    this.grafana = new GrafanaAPI(this.grafanaUrl);
+  }
+
+  public updateStatus(status) {
+    let time = moment().valueOf();
+    let data = {
+      time,
+      user: this.user,
+      datasource: this.datasource,
+      measurement: this.measurement,
+      exportedRows: this.exportedRows,
+      progress: (this.day / this.days).toLocaleString('en', { style: 'percent' }),
+      status
+    };
+    return new Promise((resolve, reject) => {
+      fs.writeFile(this.getFilePath('json'), JSON.stringify(data), 'utf8', err => {
+        if(err) {
+          console.error(err);
+          reject('Can`t write file');
+        } else {
+          resolve();
+        }
+      });
+    });
+  }
+
+  public async export() {
+    this.exportedRows = 0;
+    this.days = (this.to - this.from) / MS_IN_DAY;
+    this.day = 0;
+    this.initCsvStream();
+
+    let to = this.to;
+    let from = this.from;
+
+    console.log(`Total days: ${this.days}`);
+    while(this.day < this.days) {
+      this.day++;
+      to = from + MS_IN_DAY;
+
+      console.log(`${this.day} day: ${from}ms -> ${to}ms`);
+
+      let currentQuery = this.query.replace('$timeFilter', `time >= ${from}ms AND time <= ${to}ms`).replace('$__interval', '1s');
+      let metrics = await this.grafana.queryDatasource(this.datasource, this.measurement, currentQuery);
+
+      console.log(metrics);
+      if(metrics.length > 0) {
+        if(metrics !== undefined) {
+          this.writeCsv(metrics);
+        }
+      }
+      await this.updateStatus('exporting');
+
+      from += MS_IN_DAY;
+    }
+    this.csvStream.end();
+  }
+
+  private initCsvStream() {
+    this.csvStream = csv.createWriteStream({ headers: true });
+    let writableStream = fs.createWriteStream(this.getFilePath('csv'));
+
+    this.csvStream.pipe(writableStream);
+    writableStream.on('finish', async () => {
+      console.log('Everything is written');
+      await this.updateStatus('finished');
+    })
+  }
+
+  private writeCsv(series) {
+    for(let serie of series) {
+      for(let val of serie.values) {
+        if(val[1] !== null) {
+          let row = {};
+          for(let col in serie.columns) {
+            row[serie.columns[col]] = val[col];
+          }
+          this.csvStream.write(row);
+          this.exportedRows++;
+        }
+      }
+    }
+  }
+
+  private getFilename(extension) {
+    return `${this.datasource}.${this.measurement}.${this.from}-${this.to}.${extension}`;
+  }
+
+  private getFilePath(extension) {
+    let filename = this.getFilename(extension);
+    return path.join(__dirname, `../exported/${filename}`);
+  }
+
+}

tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "outDir": "./dist/",
+    "sourceMap": true,
+    "noImplicitAny": false,
+    "module": "commonjs",
+    "target": "es2015",
+    "allowJs": true
+  }
+}