Webhook about missing connection to grafana #412 (#431)

server/src/controllers/analytics_controller.ts

@@ -24,6 +24,7 @@ const taskResolvers = new Map<AnalyticsTaskId, TaskResolver>();
 
 let analyticsService: AnalyticsService = undefined;
 let alertService: AlertService = undefined;
+let grafanaAvailableWebhok: Function = undefined;
 let dataPuller: DataPuller;
 
 
@@ -83,6 +84,7 @@ export function init() {
   analyticsService = new AnalyticsService(onMessage);
 
   alertService = new AlertService();
+  grafanaAvailableWebhok = alertService.getGrafanaAvailableReporter();
   alertService.startAlerting();
 
   dataPuller = new DataPuller(analyticsService);
@@ -137,9 +139,12 @@ async function query(analyticUnit: AnalyticUnit.AnalyticUnit, detector: Analytic
       HASTIC_API_KEY
     );
     data = queryResult.values;
+    grafanaAvailableWebhok(true);
   } catch(e) {
     if(e instanceof ConnectionRefused) {
-      throw new Error(`Can't connect Grafana: ${e.message}, check GRAFANA_URL`);
+      const msg = `Can't connect Grafana: ${e.message}, check GRAFANA_URL`;
+      grafanaAvailableWebhok(false);
+      throw new Error(msg);
     }
     throw e;
   }

server/src/services/alert_service.ts

@@ -3,6 +3,7 @@ import { sendAnalyticWebhook, sendInfoWebhook } from './notification_service';
 import * as _ from 'lodash';
 import * as AnalyticUnit from '../models/analytic_unit_model';
 import { Segment } from '../models/segment_model';
+import { availableReporter } from '../utils/reporter';
 
 
 export class Alert {
@@ -59,6 +60,7 @@ export class AlertService {
 
   private _alerts: { [id: string]: Alert; };
   private _alertingEnable: boolean;
+  private _grafanaAvailableReporter: Function;
 
   constructor() {
     this._alerts = {}
@@ -78,13 +80,25 @@ export class AlertService {
     this._alerts[id].receive(segment);
   };
 
-  public onStateChange(message: string, optionalInfo = {}) {
+  public sendMsg(message: string, optionalInfo = {}) {
     let message_payload = {
       message
     };
     sendInfoWebhook(Object.assign(message_payload, optionalInfo));
   }
 
+  public getGrafanaAvailableReporter() {
+    if(!this._grafanaAvailableReporter) {
+      this._grafanaAvailableReporter = availableReporter(
+        'Grafana available',
+        'Grafana unavailable for pulling data',
+        this.sendMsg,
+        this.sendMsg
+      );
+    }
+    return this._grafanaAvailableReporter;
+  }
+
   public addAnalyticUnit(analyticUnit: AnalyticUnit.AnalyticUnit) {
     let detector = AnalyticUnit.getDetectorByType(analyticUnit.type);
     let alertsType = {};

server/src/services/analytics_service.ts

@@ -171,7 +171,7 @@ export class AnalyticsService {
   private async _onAnalyticsDown() {
     let msg = 'Analytics is down';
     console.log(msg);
-    this._alertService.onStateChange(msg);
+    this._alertService.sendMsg(msg);
     if(this._productionMode && !this._inDocker) {
       await AnalyticsService._runAnalyticsProcess(this._zmqConnectionString);
     }

server/src/services/data_puller.ts

@@ -3,6 +3,8 @@ import * as AnalyticUnit from '../models/analytic_unit_model';
 import * as AnalyticUnitCache from '../models/analytic_unit_cache_model';
 import { AnalyticsService } from './analytics_service';
 import { HASTIC_API_KEY, GRAFANA_URL } from '../config';
+import { availableReporter } from '../utils/reporter';
+import { AlertService } from './alert_service';
 
 import { queryByMetric, ConnectionRefused } from 'grafana-datasource-kit';
 
@@ -15,38 +17,23 @@ const PULL_PERIOD_MS = 5000;
 
 export class DataPuller {
 
-  private _availableReporter = (positiveMsg: string|null, negativeMsg: string|null) => {
+  private _analyticReadyConsoleReporter = availableReporter(
-    let reported = false;
-    return available => {
-      if(available && reported) {
-        reported = false;
-        if(positiveMsg) {
-          console.log(positiveMsg);
-        }
-      }
-
-      if(!available && !reported) {
-        reported = true;
-        if(negativeMsg) {
-          console.error(negativeMsg);
-        }
-      }
-    }
-  };
-
-  private _analyticReadyReporter = this._availableReporter(
     'data puller: analytic ready, start pushing',
     'data puller: analytic service not ready, return empty result'
   );
 
-  private _grafanaConnectionRefusedReporter = this._availableReporter(
+  private _grafanaAvailableConsoleReporter = availableReporter(
     'data puller: connected to Grafana',
     `data puller: can't connect to Grafana. Check GRAFANA_URL`
   );
 
   private _unitTimes: { [analyticUnitId: string]: number } = {};
+  private _grafanaAvailableWebhook: Function;
 
-  constructor(private analyticsService: AnalyticsService) {};
+  constructor(private analyticsService: AnalyticsService) {
+    const _alertService = new AlertService();
+    this._grafanaAvailableWebhook = _alertService.getGrafanaAvailableReporter();
+  };
 
   public addUnit(analyticUnit: AnalyticUnit.AnalyticUnit) {
     console.log(`start pulling analytic unit ${analyticUnit.id}`);
@@ -155,7 +142,7 @@ export class DataPuller {
     AsyncIterableIterator<MetricDataChunk> {
 
     const getData = async () => {
-      this._analyticReadyReporter(this.analyticsService.ready);
+      this._analyticReadyConsoleReporter(this.analyticsService.ready);
       if(!this.analyticsService.ready) {
         return {
           columns: [],
@@ -167,12 +154,14 @@ export class DataPuller {
         const time = this._unitTimes[analyticUnit.id]
         const now = Date.now();
         const res = await this.pullData(analyticUnit, time, now);
-        this._grafanaConnectionRefusedReporter(true);
+        this._grafanaAvailableConsoleReporter(true);
+        this._grafanaAvailableWebhook(true);
         return res;
       } catch(err) {
 
         if(err instanceof ConnectionRefused) {
-          this._grafanaConnectionRefusedReporter(false);
+          this._grafanaAvailableConsoleReporter(false);
+          this._grafanaAvailableWebhook(false);
         } else {
           console.error(`error while pulling data: ${err.message}`);
         }

server/src/utils/reporter.ts

@@ -0,0 +1,23 @@
+export function availableReporter(
+  positiveMsg: string|null,
+  negativeMsg: string|null,
+  positiveAction = console.log,
+  negativeAction = console.error,
+) {
+  let reported = false;
+  return available => {
+    if(available && reported) {
+      reported = false;
+      if(positiveMsg) {
+        positiveAction(positiveMsg);
+      }
+    }
+
+    if(!available && !reported) {
+      reported = true;
+      if(negativeMsg) {
+        negativeAction(negativeMsg);
+      }
+    }
+  }
+};