From 02e4168497c3de3ee077e8ca263f18018c9a596a Mon Sep 17 00:00:00 2001 From: rozetko Date: Mon, 10 Sep 2018 20:53:30 +0300 Subject: [PATCH] Initial commit --- .gitignore | 6 ++ Dockerfile | 21 ++++++ README.md | 50 +++++++++++++++ api-keys-example.json | 4 ++ build/dev-server.js | 10 +++ build/webpack.base.conf.js | 52 +++++++++++++++ build/webpack.dev.conf.js | 4 ++ build/webpack.prod.conf.js | 3 + exported/.gitkeep | 0 package.json | 25 ++++++++ src/config.ts | 16 +++++ src/grafana_api.ts | 66 +++++++++++++++++++ src/grafana_metric_model.ts | 88 ++++++++++++++++++++++++++ src/index.ts | 31 +++++++++ src/routes/datasource.ts | 123 ++++++++++++++++++++++++++++++++++++ src/routes/delete.ts | 25 ++++++++ src/routes/tasks.ts | 25 ++++++++ src/target.ts | 120 +++++++++++++++++++++++++++++++++++ tsconfig.json | 10 +++ 19 files changed, 679 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 api-keys-example.json create mode 100644 build/dev-server.js create mode 100644 build/webpack.base.conf.js create mode 100644 build/webpack.dev.conf.js create mode 100644 build/webpack.prod.conf.js create mode 100644 exported/.gitkeep create mode 100644 package.json create mode 100644 src/config.ts create mode 100644 src/grafana_api.ts create mode 100644 src/grafana_metric_model.ts create mode 100644 src/index.ts create mode 100644 src/routes/datasource.ts create mode 100644 src/routes/delete.ts create mode 100644 src/routes/tasks.ts create mode 100644 src/target.ts create mode 100644 tsconfig.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d64dacc --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +node_modules/ +dist/ +exported/*.csv +exported/*.json +api-keys.json +package-lock.json diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a92df0a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +# Pull node base image +FROM node:latest + +ARG build_path=/var/www +ENV BUILD_PATH=$build_path + +# Expose port 80 +EXPOSE 80 + +VOLUME [ "/var/www/exported" ] + +# Copy custom configuration file from the current directory + +WORKDIR ${BUILD_PATH} +COPY . ${BUILD_PATH} + +RUN npm install +RUN npm run build + +# Start up node server +CMD ["npm", "start"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..56cb453 --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ +# Export-manager backend + +Server for fetching data from Grafana datasources + +Running on 80 port in Docker container. + +Dockerfile has 2 volumes: + +- `/var/www/api-keys.json` (required, contains API keys for your Grafana hosts, see [example](api-keys-example.json)) +- `/var/www/exported` (optional, directory which contains exported csv files and info about them) + +## Build + +``` +npm install +npm run build +``` + +## Run + +``` +npm start +``` + +## Development + +You should have `nodemon` module installed to run development server. + +``` +npm i -g nodemon +npm run dev +``` + +## Changelog + +### [0.3.1] - 2018-05-14 +#### Added +- Show confirmation modal on task delete. + +### [0.3.0] - 2018-05-10 +#### Added +- Save user that initialized export. +- Support different grafana URLs. +- Delete tasks. + +### [0.2.0] - 2018-05-09 +#### Added +- Fetch data from Grafana API and save it to CSV. +- Endpoint for showing task status in Simple-JSON datasource format. +- CSV download URL on task finish. diff --git a/api-keys-example.json b/api-keys-example.json new file mode 100644 index 0000000..d88fee9 --- /dev/null +++ b/api-keys-example.json @@ -0,0 +1,4 @@ +{ + "http://grafana.example.com": "eyJrIjoiNFRDNms3Z3RZRmtoU2hKd002dFJUS2FCOHEySUlSZzEiLCJuIjoiZXhwb3J0IiwiaWQiOjF9", + "http://localhost:3500": "eyJrIjoiTjlUcmtLSFRNcTdqeXBaQjB5REk2TFkyUDBDM0Z1bWciLCJuIjoiZXhwb3J0LW1hbmFnZXIiLCJpZCI6MX0=" +} diff --git a/build/dev-server.js b/build/dev-server.js new file mode 100644 index 0000000..76ed9d3 --- /dev/null +++ b/build/dev-server.js @@ -0,0 +1,10 @@ +const { spawn } = require('child_process'); + +const webpack = spawn('webpack', ['--config', 'build/webpack.dev.conf.js'], { + stdio: 'inherit', + shell: true +}); +//webpack.stdout.pipe(process.stdout); + +const nodemon = spawn('nodemon', ['../dist/server', '--watch', 'server.js']); +nodemon.stdout.pipe(process.stdout); diff --git a/build/webpack.base.conf.js b/build/webpack.base.conf.js new file mode 100644 index 0000000..b910b69 --- /dev/null +++ b/build/webpack.base.conf.js @@ -0,0 +1,52 @@ +const path = require('path'); +const fs = require('fs'); + +const webpack = require('webpack'); + + +function resolve(p) { + return path.join(__dirname, '/../', p); +} + +module.exports = { + target: 'node', + node: { + __dirname: false, + __filename: false, + }, + context: resolve('./src'), + entry: './index', + devtool: 'inline-source-map', + output: { + filename: "server.js", + path: resolve('dist') + }, + externals: [ + function(context, request, callback) { + if(request[0] == '.') { + callback(); + } else { + callback(null, "require('" + request + "')"); + } + } + ], + plugins: [ + new webpack.optimize.OccurrenceOrderPlugin(), + new webpack.HotModuleReplacementPlugin(), + new webpack.DefinePlugin({ + 'process.env.NODE_ENV': JSON.stringify('development') + }) + ], + resolve: { + extensions: [".ts", ".js"] + }, + module: { + rules: [ + { + test: /\.ts$/, + loader: "ts-loader", + exclude: /node_modules/ + } + ] + } +} diff --git a/build/webpack.dev.conf.js b/build/webpack.dev.conf.js new file mode 100644 index 0000000..7be3655 --- /dev/null +++ b/build/webpack.dev.conf.js @@ -0,0 +1,4 @@ +var base = require('./webpack.base.conf'); + +base.watch = true; +module.exports = base; \ No newline at end of file diff --git a/build/webpack.prod.conf.js b/build/webpack.prod.conf.js new file mode 100644 index 0000000..b5b325e --- /dev/null +++ b/build/webpack.prod.conf.js @@ -0,0 +1,3 @@ +var base = require('./webpack.base.conf'); + +module.exports = base; \ No newline at end of file diff --git a/exported/.gitkeep b/exported/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/package.json b/package.json new file mode 100644 index 0000000..7b75b87 --- /dev/null +++ b/package.json @@ -0,0 +1,25 @@ +{ + "name": "export-manager-server", + "version": "0.3.1", + "description": "Server for fetching data from Grafana datasources", + "scripts": { + "start": "node dist/server.js", + "dev": "node build/dev-server.js", + "build": "webpack --config build/webpack.prod.conf.js" + }, + "author": "CorpGlory", + "license": "ISC", + "dependencies": {}, + "devDependencies": { + "@types/express": "^4.11.1", + "axios": "^0.18.0", + "express": "^4.16.3", + "fast-csv": "^2.4.1", + "moment": "^2.22.1", + "node-fetch": "^2.1.2", + "nodemon": "^1.17.3", + "ts-loader": "^3.5.0", + "typescript": "^2.8.3", + "webpack": "^3.5.6" + } +} diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..b8dea22 --- /dev/null +++ b/src/config.ts @@ -0,0 +1,16 @@ +import * as path from 'path'; +import * as fs from 'fs'; + +export const EXPORTED_PATH = path.join(__dirname, '../exported'); + + +export function getApiKey(host) { + return new Promise((resolve, reject) => { + fs.readFile(path.join(__dirname, '../api-keys.json'), 'utf8', (err, data) => { + if(err) { + reject(err); + } + resolve(JSON.parse(data)[host]); + }); + }); +} diff --git a/src/grafana_api.ts b/src/grafana_api.ts new file mode 100644 index 0000000..76a8efd --- /dev/null +++ b/src/grafana_api.ts @@ -0,0 +1,66 @@ +import { getApiKey } from './config'; + +import axios from 'axios'; + + +export class GrafanaAPI { + private apiKey; + constructor(private grafanaUrl) { + getApiKey(grafanaUrl) + .then(key => { + this.apiKey = key; + console.log(this.apiKey); + }); + } + + private get _headers() { + return { + 'Authorization': `Bearer ${this.apiKey}`, + 'Accept': 'application/json', + 'Content-Type': 'application/json' + }; + } + + private async _getDatasourceByName(name) { + return fetch(`${this.grafanaUrl}/api/datasources/name/${name}`, { + method: 'GET', + headers: this._headers + }) + .then(data => data.json()); + } + + public async queryDatasource(datasourceName, measurement, query) { + let datasource = await this._getDatasourceByName(datasourceName); + + return this._queryGrafana(`${this.grafanaUrl}/api/datasources/proxy/${datasource.id}/query`, { + q: encodeURIComponent(query), + db: datasource.database, + epoch: 'ms' + }); + } + + private async _queryGrafana(url: string, params: any) { + try { + var res = await axios.get(url, { params, headers: this._headers }); + } catch (e) { + if(e.response.status === 401) { + throw new Error('Unauthorized. Check the $HASTIC_API_KEY.'); + } + throw new Error(e.message); + } + + if (res.data.results === undefined) { + throw new Error('results field is undefined in response.'); + } + + // TODO: support more than 1 metric (each res.data.results item is a metric) + let results = res.data.results[0]; + if (results.series === undefined) { + return []; + } + + return results.series[0].values as [number, number][]; + } + +} + diff --git a/src/grafana_metric_model.ts b/src/grafana_metric_model.ts new file mode 100644 index 0000000..9991233 --- /dev/null +++ b/src/grafana_metric_model.ts @@ -0,0 +1,88 @@ +export type GrafanaDatasource = { + url: string, + type: string, + params: { + db: string, + q: string, + epoch: string + } +} + +export type GrafanaMetricId = string; + +export class GrafanaMetric { + + private _metricQuery: MetricQuery = undefined; + + constructor( + public datasource: GrafanaDatasource, + public targets: any[], + public id?: GrafanaMetricId + ) { + if(datasource === undefined) { + throw new Error('datasource is undefined'); + } + if(targets === undefined) { + throw new Error('targets is undefined'); + } + if(targets.length === 0) { + throw new Error('targets is empty'); + } + } + + public get metricQuery() { + if(this._metricQuery === undefined) { + this._metricQuery = new MetricQuery(this); + } + return this._metricQuery; + } + + public toObject() { + return { + datasource: this.datasource, + targets: this.targets, + _id: this.id + }; + } + + static fromObject(obj: any): GrafanaMetric { + if(obj === undefined) { + throw new Error('obj is undefined'); + } + return new GrafanaMetric( + obj.datasource, + obj.targets, + obj._id + ); + } +} + +export class MetricQuery { + + private static INFLUX_QUERY_TIME_REGEX = /time >[^A-Z]+/; + + private _queryParts: string[]; + private _type: string; + + constructor(metric: GrafanaMetric) { + this._type = metric.datasource.type; + if (this._type !== 'influxdb') { + throw new Error(`Queries of type "${metric.datasource.type}" are not supported yet.`); + } + var queryStr = metric.datasource.params.q; + this._queryParts = queryStr.split(MetricQuery.INFLUX_QUERY_TIME_REGEX); + if(this._queryParts.length == 1) { + throw new Error( + `Query "${queryStr}" is not replaced with LIMIT/OFFSET oeprators. Missing time clause.` + ); + } + if(this._queryParts.length > 2) { + throw new Error(`Query "${queryStr}" has multiple time clauses. Can't parse.`); + } + } + + getQuery(from: number, to: number, limit: number, offset: number): string { + let timeClause = `time >= ${from}ms AND time <= ${to}ms`; + return `${this._queryParts[0]} ${timeClause} ${this._queryParts[1]} LIMIT ${limit} OFFSET ${offset}`; + } +} diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..f827a03 --- /dev/null +++ b/src/index.ts @@ -0,0 +1,31 @@ +import { EXPORTED_PATH } from './config'; +import { router as tasksRouter } from './routes/tasks'; +import { router as datasourceRouter } from './routes/datasource'; +import { router as deleteRouter } from './routes/delete'; + +import * as express from 'express'; +import * as bodyParser from 'body-parser'; + +const app = express(); +const PORT = 80; + +app.use(bodyParser.json()); +app.use(bodyParser.urlencoded({ extended: true })); + +app.use(function(req, res, next) { + res.header('Access-Control-Allow-Origin', '*'); + res.header('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, PATCH, OPTIONS'); + res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept'); + next(); +}) + +app.use('/tasks', tasksRouter); +app.use('/datasource', datasourceRouter); +app.use('/delete', deleteRouter); + +app.use('/static', express.static(EXPORTED_PATH)); +app.use('/', (req, res) => { res.send('Export-manager backend server works') }); + +app.listen(PORT, () => { + console.log(`Server is running on :${PORT}`); +}) diff --git a/src/routes/datasource.ts b/src/routes/datasource.ts new file mode 100644 index 0000000..7cb94f1 --- /dev/null +++ b/src/routes/datasource.ts @@ -0,0 +1,123 @@ +import { EXPORTED_PATH } from '../config'; + +import * as express from 'express'; +import * as path from 'path'; +import * as fs from 'fs'; + + +function sendOk(req, res) { + res.status(200).send('Datasource works'); +} + +function search(req, res) { + fs.readdir(EXPORTED_PATH, (err, items) => { + if(err) { + console.error(err); + res.status(500).send('Something went wrong'); + } else { + res.status(200).send(['All tasks']); + } + }) +} + +function query(req, res) { + let body = req.body; + let targets = body.targets; + + let resp = { + target: targets[0].target, + type: 'table', + columns: [ + { + text: 'Time', + type: 'time' + }, + { + text: 'User', + type: 'string' + }, + { + text: 'Datasource', + type: 'string' + }, + { + text: 'Measurement', + type: 'string' + }, + { + text: 'Exported Rows', + type: 'number' + }, + { + text: 'Progress', + type: 'string' + }, + { + text: 'Download CSV', + type: 'string' + }, + { + text: 'Status', + type: 'string' + }, + { + text: 'Delete task', + type: 'string' + } + ], + rows: [] + }; + + for(let target of targets) { + fs.readdir(EXPORTED_PATH, (err, items) => { + if(err) { + console.error(err); + res.status(500).send('Something went wrong'); + } else { + for(let item of items) { + let file = path.parse(item); + if(file.ext !== '.json') { + continue; + } + // TODO: read async + let data = fs.readFileSync(path.join(EXPORTED_PATH, item), 'utf8') + let status = JSON.parse(data) + + let requestedUrl = `http://${req.headers.host}`; + let downloadLink = ''; + let deleteLink = ''; + if(status.status === 'finished') { + downloadLink = ``; + deleteLink = ``; + } + resp.rows.push([ + status.time, + status.user, + status.datasource, + status.measurement, + status.exportedRows, + status.progress, + downloadLink, + status.status, + deleteLink + ]); + } + + res.status(200).send([resp]); + } + }) + } + +} + +function sendAnnotations(req, res) { + res.status(200).send([]); +} + +export const router = express.Router(); + +router.get('/', sendOk); +router.post('/search', search); +router.post('/query', query); +router.post('/annotations', sendAnnotations); + diff --git a/src/routes/delete.ts b/src/routes/delete.ts new file mode 100644 index 0000000..3079222 --- /dev/null +++ b/src/routes/delete.ts @@ -0,0 +1,25 @@ +import { EXPORTED_PATH } from '../config' + +import * as express from 'express' +import * as fs from 'fs' +import * as path from 'path' + + +async function deleteTask(req, res) { + let filename = req.query.filename; + let csvFilePath = path.join(EXPORTED_PATH, `${filename}.csv`); + let jsonFilePath = path.join(EXPORTED_PATH, `${filename}.json`); + + if(fs.existsSync(csvFilePath)) { + fs.unlink(csvFilePath, err => console.error(err)); + } + if(fs.existsSync(jsonFilePath)) { + fs.unlink(jsonFilePath, err => console.error(err)); + } + + res.status(200).send({ status: 'OK' }); +} + +export const router = express.Router(); + +router.get('/', deleteTask); diff --git a/src/routes/tasks.ts b/src/routes/tasks.ts new file mode 100644 index 0000000..d3af128 --- /dev/null +++ b/src/routes/tasks.ts @@ -0,0 +1,25 @@ +import { Target } from '../target' + +import * as express from 'express' + + +async function addTask(req, res) { + let body = req.body; + let from = parseInt(body.from); + let to = parseInt(body.to); + + if(isNaN(from) || isNaN(to)) { + res.status(500).send('Range error: please fill both "from" and "to" fields'); + } else if(from >= to) { + res.status(500).send('Range error: "from" should be less than "to"'); + } else { + res.status(200).send('Task added'); + let grafanaUrl = req.get('origin'); + let target = new Target(grafanaUrl, body.user, body.datasource, body.measurement, body.query, from, to); + target.export(); + } +} + +export const router = express.Router(); + +router.post('/', addTask); diff --git a/src/target.ts b/src/target.ts new file mode 100644 index 0000000..50b2819 --- /dev/null +++ b/src/target.ts @@ -0,0 +1,120 @@ +import { GrafanaAPI } from './grafana_api'; + +import * as csv from 'fast-csv'; +import * as path from 'path'; +import * as fs from 'fs'; +import * as moment from 'moment'; + + +const MS_IN_DAY = 24 * 60 * 60 * 1000; + +export class Target { + private exportedRows: number; + private days: number; + private day: number; + private csvStream: any; + private grafana: GrafanaAPI; + + constructor( + private grafanaUrl: string, + private user: string, + private datasource: string, + private measurement: string, + private query: string, + private from: number, + private to: number + ) { + this.grafana = new GrafanaAPI(this.grafanaUrl); + } + + public updateStatus(status) { + let time = moment().valueOf(); + let data = { + time, + user: this.user, + datasource: this.datasource, + measurement: this.measurement, + exportedRows: this.exportedRows, + progress: (this.day / this.days).toLocaleString('en', { style: 'percent' }), + status + }; + return new Promise((resolve, reject) => { + fs.writeFile(this.getFilePath('json'), JSON.stringify(data), 'utf8', err => { + if(err) { + console.error(err); + reject('Can`t write file'); + } else { + resolve(); + } + }); + }); + } + + public async export() { + this.exportedRows = 0; + this.days = (this.to - this.from) / MS_IN_DAY; + this.day = 0; + this.initCsvStream(); + + let to = this.to; + let from = this.from; + + console.log(`Total days: ${this.days}`); + while(this.day < this.days) { + this.day++; + to = from + MS_IN_DAY; + + console.log(`${this.day} day: ${from}ms -> ${to}ms`); + + let currentQuery = this.query.replace('$timeFilter', `time >= ${from}ms AND time <= ${to}ms`).replace('$__interval', '1s'); + let metrics = await this.grafana.queryDatasource(this.datasource, this.measurement, currentQuery); + + console.log(metrics); + if(metrics.length > 0) { + if(metrics !== undefined) { + this.writeCsv(metrics); + } + } + await this.updateStatus('exporting'); + + from += MS_IN_DAY; + } + this.csvStream.end(); + } + + private initCsvStream() { + this.csvStream = csv.createWriteStream({ headers: true }); + let writableStream = fs.createWriteStream(this.getFilePath('csv')); + + this.csvStream.pipe(writableStream); + writableStream.on('finish', async () => { + console.log('Everything is written'); + await this.updateStatus('finished'); + }) + } + + private writeCsv(series) { + for(let serie of series) { + for(let val of serie.values) { + if(val[1] !== null) { + let row = {}; + for(let col in serie.columns) { + row[serie.columns[col]] = val[col]; + } + this.csvStream.write(row); + this.exportedRows++; + } + } + } + } + + private getFilename(extension) { + return `${this.datasource}.${this.measurement}.${this.from}-${this.to}.${extension}`; + } + + private getFilePath(extension) { + let filename = this.getFilename(extension); + return path.join(__dirname, `../exported/${filename}`); + } + +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..987f1c7 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "outDir": "./dist/", + "sourceMap": true, + "noImplicitAny": false, + "module": "commonjs", + "target": "es2015", + "allowJs": true + } +}