From 231967948c5ac7f3941d7321f1b37933030658f6 Mon Sep 17 00:00:00 2001 From: Evgeny Smyshlyaev Date: Fri, 22 Feb 2019 20:25:22 +0300 Subject: [PATCH] Webhook about missing connection to grafana #412 (#431) --- .../src/controllers/analytics_controller.ts | 7 +++- server/src/services/alert_service.ts | 16 +++++++- server/src/services/analytics_service.ts | 2 +- server/src/services/data_puller.ts | 39 +++++++------------ server/src/utils/reporter.ts | 23 +++++++++++ 5 files changed, 59 insertions(+), 28 deletions(-) create mode 100644 server/src/utils/reporter.ts diff --git a/server/src/controllers/analytics_controller.ts b/server/src/controllers/analytics_controller.ts index 7bb1254..adcfef5 100644 --- a/server/src/controllers/analytics_controller.ts +++ b/server/src/controllers/analytics_controller.ts @@ -24,6 +24,7 @@ const taskResolvers = new Map(); let analyticsService: AnalyticsService = undefined; let alertService: AlertService = undefined; +let grafanaAvailableWebhok: Function = undefined; let dataPuller: DataPuller; @@ -83,6 +84,7 @@ export function init() { analyticsService = new AnalyticsService(onMessage); alertService = new AlertService(); + grafanaAvailableWebhok = alertService.getGrafanaAvailableReporter(); alertService.startAlerting(); dataPuller = new DataPuller(analyticsService); @@ -137,9 +139,12 @@ async function query(analyticUnit: AnalyticUnit.AnalyticUnit, detector: Analytic HASTIC_API_KEY ); data = queryResult.values; + grafanaAvailableWebhok(true); } catch(e) { if(e instanceof ConnectionRefused) { - throw new Error(`Can't connect Grafana: ${e.message}, check GRAFANA_URL`); + const msg = `Can't connect Grafana: ${e.message}, check GRAFANA_URL`; + grafanaAvailableWebhok(false); + throw new Error(msg); } throw e; } diff --git a/server/src/services/alert_service.ts b/server/src/services/alert_service.ts index 5a5cb69..f2c05ba 100644 --- a/server/src/services/alert_service.ts +++ b/server/src/services/alert_service.ts @@ -3,6 +3,7 @@ import { sendAnalyticWebhook, sendInfoWebhook } from './notification_service'; import * as _ from 'lodash'; import * as AnalyticUnit from '../models/analytic_unit_model'; import { Segment } from '../models/segment_model'; +import { availableReporter } from '../utils/reporter'; export class Alert { @@ -59,6 +60,7 @@ export class AlertService { private _alerts: { [id: string]: Alert; }; private _alertingEnable: boolean; + private _grafanaAvailableReporter: Function; constructor() { this._alerts = {} @@ -78,13 +80,25 @@ export class AlertService { this._alerts[id].receive(segment); }; - public onStateChange(message: string, optionalInfo = {}) { + public sendMsg(message: string, optionalInfo = {}) { let message_payload = { message }; sendInfoWebhook(Object.assign(message_payload, optionalInfo)); } + public getGrafanaAvailableReporter() { + if(!this._grafanaAvailableReporter) { + this._grafanaAvailableReporter = availableReporter( + 'Grafana available', + 'Grafana unavailable for pulling data', + this.sendMsg, + this.sendMsg + ); + } + return this._grafanaAvailableReporter; + } + public addAnalyticUnit(analyticUnit: AnalyticUnit.AnalyticUnit) { let detector = AnalyticUnit.getDetectorByType(analyticUnit.type); let alertsType = {}; diff --git a/server/src/services/analytics_service.ts b/server/src/services/analytics_service.ts index 6a5d580..0b51db8 100644 --- a/server/src/services/analytics_service.ts +++ b/server/src/services/analytics_service.ts @@ -171,7 +171,7 @@ export class AnalyticsService { private async _onAnalyticsDown() { let msg = 'Analytics is down'; console.log(msg); - this._alertService.onStateChange(msg); + this._alertService.sendMsg(msg); if(this._productionMode && !this._inDocker) { await AnalyticsService._runAnalyticsProcess(this._zmqConnectionString); } diff --git a/server/src/services/data_puller.ts b/server/src/services/data_puller.ts index 89c210f..203762b 100644 --- a/server/src/services/data_puller.ts +++ b/server/src/services/data_puller.ts @@ -3,6 +3,8 @@ import * as AnalyticUnit from '../models/analytic_unit_model'; import * as AnalyticUnitCache from '../models/analytic_unit_cache_model'; import { AnalyticsService } from './analytics_service'; import { HASTIC_API_KEY, GRAFANA_URL } from '../config'; +import { availableReporter } from '../utils/reporter'; +import { AlertService } from './alert_service'; import { queryByMetric, ConnectionRefused } from 'grafana-datasource-kit'; @@ -15,38 +17,23 @@ const PULL_PERIOD_MS = 5000; export class DataPuller { - private _availableReporter = (positiveMsg: string|null, negativeMsg: string|null) => { - let reported = false; - return available => { - if(available && reported) { - reported = false; - if(positiveMsg) { - console.log(positiveMsg); - } - } - - if(!available && !reported) { - reported = true; - if(negativeMsg) { - console.error(negativeMsg); - } - } - } - }; - - private _analyticReadyReporter = this._availableReporter( + private _analyticReadyConsoleReporter = availableReporter( 'data puller: analytic ready, start pushing', 'data puller: analytic service not ready, return empty result' ); - private _grafanaConnectionRefusedReporter = this._availableReporter( + private _grafanaAvailableConsoleReporter = availableReporter( 'data puller: connected to Grafana', `data puller: can't connect to Grafana. Check GRAFANA_URL` ); private _unitTimes: { [analyticUnitId: string]: number } = {}; + private _grafanaAvailableWebhook: Function; - constructor(private analyticsService: AnalyticsService) {}; + constructor(private analyticsService: AnalyticsService) { + const _alertService = new AlertService(); + this._grafanaAvailableWebhook = _alertService.getGrafanaAvailableReporter(); + }; public addUnit(analyticUnit: AnalyticUnit.AnalyticUnit) { console.log(`start pulling analytic unit ${analyticUnit.id}`); @@ -155,7 +142,7 @@ export class DataPuller { AsyncIterableIterator { const getData = async () => { - this._analyticReadyReporter(this.analyticsService.ready); + this._analyticReadyConsoleReporter(this.analyticsService.ready); if(!this.analyticsService.ready) { return { columns: [], @@ -167,12 +154,14 @@ export class DataPuller { const time = this._unitTimes[analyticUnit.id] const now = Date.now(); const res = await this.pullData(analyticUnit, time, now); - this._grafanaConnectionRefusedReporter(true); + this._grafanaAvailableConsoleReporter(true); + this._grafanaAvailableWebhook(true); return res; } catch(err) { if(err instanceof ConnectionRefused) { - this._grafanaConnectionRefusedReporter(false); + this._grafanaAvailableConsoleReporter(false); + this._grafanaAvailableWebhook(false); } else { console.error(`error while pulling data: ${err.message}`); } diff --git a/server/src/utils/reporter.ts b/server/src/utils/reporter.ts new file mode 100644 index 0000000..3a9195a --- /dev/null +++ b/server/src/utils/reporter.ts @@ -0,0 +1,23 @@ +export function availableReporter( + positiveMsg: string|null, + negativeMsg: string|null, + positiveAction = console.log, + negativeAction = console.error, +) { + let reported = false; + return available => { + if(available && reported) { + reported = false; + if(positiveMsg) { + positiveAction(positiveMsg); + } + } + + if(!available && !reported) { + reported = true; + if(negativeMsg) { + negativeAction(negativeMsg); + } + } + } +};