Compatibility with Prometheus Alertmanager #482 (#802)

server/spec/setup_tests.ts

@@ -9,7 +9,9 @@ jest.mock('../src/config.ts', () => ({
   HASTIC_API_KEY: 'fake-key',
   ZMQ_IPC_PATH: 'fake-zmq-path',
   HASTIC_DB_CONNECTION_TYPE: 'nedb',
-  HASTIC_IN_MEMORY_PERSISTANCE: true
+  HASTIC_IN_MEMORY_PERSISTANCE: true,
+  HASTIC_ALERT_TYPE: 'webhook',
+  AlertTypes: jest.requireActual('../src/config').AlertTypes,
 }));
 
 jest.mock('deasync', () => ({ loopWhile: jest.fn() }));

server/src/config.ts

@@ -2,10 +2,12 @@ import { getJsonDataSync } from './services/json_service';
 import { normalizeUrl } from './utils/url';
 import { parseTimeZone } from './utils/time';
 
+import * as _ from 'lodash';
+import * as moment from 'moment';
 import * as path from 'path';
 import * as fs from 'fs';
 import * as os from 'os';
-import * as moment from 'moment';
+
 
 let configFile = path.join(__dirname, '../../config.json');
 let configExists = fs.existsSync(configFile);
@@ -21,7 +23,8 @@ export type DBConfig = {
 export const ANALYTICS_PATH = path.join(__dirname, '../../analytics');
 
 export const HASTIC_DB_IN_MEMORY = getConfigField('HASTIC_IN_MEMORY_PERSISTANCE', false);
-export const HASTIC_DB_CONNECTION_TYPE = getConfigField('HASTIC_DB_CONNECTION_TYPE', 'nedb'); //nedb or mongodb
+// TODO: enum for DB types
+export const HASTIC_DB_CONNECTION_TYPE = getConfigField('HASTIC_DB_CONNECTION_TYPE', 'nedb', ['nedb', 'mongodb']);
 
 //connection string syntax: <db_user>:<db_password>@<db_url>/<db_name>
 export const HASTIC_DB_CONNECTION_STRING = getConfigField(
@@ -45,14 +48,24 @@ export const ZMQ_DEV_PORT = getConfigField('ZMQ_DEV_PORT', '8002');
 export const ZMQ_HOST = getConfigField('ZMQ_HOST', '127.0.0.1');
 export const HASTIC_API_KEY = getConfigField('HASTIC_API_KEY');
 export const GRAFANA_URL = normalizeUrl(getConfigField('GRAFANA_URL', null));
+
 // TODO: save orgId in analytic_units.db
 export const ORG_ID = getConfigField('ORG_ID', 1);
+
+export enum AlertTypes {
+  WEBHOOK = 'webhook',
+  ALERTMANAGER = 'alertmanager'
+};
+export const HASTIC_ALERT_TYPE = getConfigField('HASTIC_ALERT_TYPE', AlertTypes.WEBHOOK, _.values(AlertTypes));
+export const HASTIC_ALERT_IMAGE = getConfigField('HASTIC_ALERT_IMAGE', false);
+
 export const HASTIC_WEBHOOK_URL = getConfigField('HASTIC_WEBHOOK_URL', null);
 export const HASTIC_WEBHOOK_TYPE = getConfigField('HASTIC_WEBHOOK_TYPE', 'application/json');
 export const HASTIC_WEBHOOK_SECRET = getConfigField('HASTIC_WEBHOOK_SECRET', null);
-export const HASTIC_WEBHOOK_IMAGE_ENABLED = getConfigField('HASTIC_WEBHOOK_IMAGE', false);
 export const TIMEZONE_UTC_OFFSET = getTimeZoneOffset();
 
+export const HASTIC_ALERTMANAGER_URL = getConfigField('HASTIC_ALERTMANAGER_URL', null);
+
 export const ANLYTICS_PING_INTERVAL = 500; // ms
 export const PACKAGE_VERSION = getPackageVersion();
 export const GIT_INFO = getGitInfo();
@@ -63,7 +76,7 @@ export const ZMQ_CONNECTION_STRING = createZMQConnectionString();
 export const HASTIC_INSTANCE_NAME = getConfigField('HASTIC_INSTANCE_NAME', os.hostname());
 
 
-function getConfigField(field: string, defaultVal?: any) {
+function getConfigField(field: string, defaultVal?: any, allowedVals?: any[]) {
   let val;
 
   if(process.env[field] !== undefined) {
@@ -82,6 +95,11 @@ function getConfigField(field: string, defaultVal?: any) {
     }
     val = defaultVal;
   }
+
+  if(allowedVals !== undefined && !_.includes(allowedVals, val)) {
+    throw new Error(`${field} value must be in ${allowedVals}, got ${val}`);
+  }
+
   console.log(`${field}: ${val}`);
   return val;
 }

server/src/services/alert_service.ts

@@ -1,14 +1,14 @@
-import { sendNotification, MetaInfo, AnalyticMeta, WebhookType, Notification } from './notification_service';
+import { getNotifier, AnalyticMeta, WebhookType, Notification, MetaInfo } from './notification_service';
 import * as AnalyticUnit from '../models/analytic_units';
 import { Segment } from '../models/segment_model';
 import { availableReporter } from '../utils/reporter';
 import { toTimeZone } from '../utils/time';
-import { ORG_ID, HASTIC_API_KEY, HASTIC_WEBHOOK_IMAGE_ENABLED } from '../config';
+import { ORG_ID, HASTIC_API_KEY, HASTIC_ALERT_IMAGE } from '../config';
 
 import axios from 'axios';
 import * as _ from 'lodash';
 
-
+const Notifier = getNotifier();
 export class Alert {
   public enabled = true;
   constructor(protected analyticUnit: AnalyticUnit.AnalyticUnit) {};
@@ -21,7 +21,7 @@ export class Alert {
   protected async send(segment) {
     const notification = await this.makeNotification(segment);
     try {
-      await sendNotification(notification);
+      await Notifier.sendNotification(notification);
     } catch(error) {
       console.error(`can't send notification ${error}`);
     };
@@ -31,7 +31,7 @@ export class Alert {
     const meta = this.makeMeta(segment);
     const text = this.makeMessage(meta);
     let result: Notification = { meta, text };
-    if(HASTIC_WEBHOOK_IMAGE_ENABLED) {
+    if(HASTIC_ALERT_IMAGE) {
       try {
         const image = await this.loadImage();
         result.image = image;
@@ -212,7 +212,10 @@ export class AlertService {
       from: now,
       to: now
     }
-    sendNotification({ text, meta: infoAlert });
+
+    Notifier.sendNotification({ text, meta: infoAlert }).catch((err) => {
+      console.error(`can't send message ${err.message}`);
+    });
   }
 
   public sendGrafanaAvailableWebhook() {

server/src/services/notification_service.ts

@@ -1,5 +1,5 @@
 import * as AnalyticUnit from '../models/analytic_units';
-import { HASTIC_WEBHOOK_URL, HASTIC_WEBHOOK_TYPE, HASTIC_INSTANCE_NAME } from '../config';
+import * as config from '../config';
 
 import axios from 'axios';
 import * as querystring from 'querystring';
@@ -17,7 +17,14 @@ export enum WebhookType {
   MESSAGE = 'MESSAGE'
 }
 
-export declare type AnalyticMeta = {
+export type MetaInfo = {
+  type: WebhookType,
+  from: number,
+  to: number,
+  params?: any
+}
+
+export type AnalyticMeta = {
   type: WebhookType,
   analyticUnitType: string,
   analyticUnitName: string,
@@ -28,47 +35,124 @@ export declare type AnalyticMeta = {
   message?: any
 }
 
-export declare type MetaInfo = {
-  type: WebhookType,
-  from: number,
-  to: number,
-  params?: any
-}
-
 export declare type Notification = {
   text: string,
   meta: MetaInfo | AnalyticMeta,
   image?: any
 }
 
-export async function sendNotification(notification: Notification) {
-  if(HASTIC_WEBHOOK_URL === null) {
+// TODO: split notifiers into 3 files
+export interface Notifier {
+  sendNotification(notification: Notification): Promise<void>;
+}
+
+// TODO: singleton
+export function getNotifier(): Notifier {
+  if(config.HASTIC_ALERT_TYPE === config.AlertTypes.WEBHOOK) {
+    return new WebhookNotifier();
+  }
+
+  if(config.HASTIC_ALERT_TYPE === config.AlertTypes.ALERTMANAGER) {
+    return new AlertManagerNotifier();
+  }
+
+  throw new Error(`${config.HASTIC_ALERT_TYPE} alert type not supported`);
+}
+
+class WebhookNotifier implements Notifier {
+  async sendNotification(notification: Notification) {
+    if(config.HASTIC_WEBHOOK_URL === null) {
       console.log(`HASTIC_WEBHOOK_URL is not set, skip sending notification: ${notification.text}`);
       return;
     }
   
-  notification.text += `\nInstance: ${HASTIC_INSTANCE_NAME}`;
+    notification.text += `\nInstance: ${config.HASTIC_INSTANCE_NAME}`;
   
     let data;
-  if(HASTIC_WEBHOOK_TYPE === ContentType.JSON) {
+    if(config.HASTIC_WEBHOOK_TYPE === ContentType.JSON) {
       data = JSON.stringify(notification);
-  } else if(HASTIC_WEBHOOK_TYPE === ContentType.URLENCODED) {
+    } else if(config.HASTIC_WEBHOOK_TYPE === ContentType.URLENCODED) {
       data = querystring.stringify(notification);
     } else {
-    throw new Error(`Unknown webhook type: ${HASTIC_WEBHOOK_TYPE}`);
+      throw new Error(`Unknown webhook type: ${config.HASTIC_WEBHOOK_TYPE}`);
     }
   
     // TODO: use HASTIC_WEBHOOK_SECRET
     const options = {
       method: 'POST',
-    url: HASTIC_WEBHOOK_URL,
+      url: config.HASTIC_WEBHOOK_URL,
       data,
-    headers: { 'Content-Type': HASTIC_WEBHOOK_TYPE }
+      headers: { 'Content-Type': config.HASTIC_WEBHOOK_TYPE }
+    };
+
+    await axios(options);
+  }
+}
+
+type PostableAlertLabels = {
+  alertname: string;
+  [key: string]: string
+};
+
+type PostableAlertAnnotations = {
+  message?: string;
+  summary?: string;
+};
+
+type PostableAlert = {
+  labels: PostableAlertLabels,
+  annotations: PostableAlertAnnotations
+  generatorURL?: string,
+  endsAt?: string
+};
+
+class AlertManagerNotifier implements Notifier {
+
+  /**
+   * @throws {Error} from axios if query fails
+   */
+  async sendNotification(notification: Notification) {
+    if(config.HASTIC_ALERTMANAGER_URL === null) {
+      console.log(`HASTIC_ALERTMANAGER_URL is not set, skip sending notification: ${notification.text}`);
+      return;
+    }
+
+    let generatorURL: string;
+    let labels: PostableAlertLabels = {
+      alertname:  notification.meta.type,
+      instance: config.HASTIC_INSTANCE_NAME
+    };
+    let annotations: PostableAlertAnnotations = {
+      message: notification.text
+    };
+
+    if(_.has(notification.meta, 'grafanaUrl')) {
+      generatorURL = (notification.meta as AnalyticMeta).grafanaUrl;
+      labels.alertname = (notification.meta as AnalyticMeta).analyticUnitName;
+      labels.analyticUnitId = (notification.meta as AnalyticMeta).analyticUnitId;
+      labels.analyticUnitType = (notification.meta as AnalyticMeta).analyticUnitType;
+      annotations.message = `${(notification.meta as AnalyticMeta).message}\n${generatorURL}`;
+    }
+    
+    let alertData: PostableAlert = {
+      labels,
+      annotations,
+      generatorURL
     };
 
-  try {
+    let options = {
+      method: 'POST',
+      url: `${config.HASTIC_ALERTMANAGER_URL}/api/v2/alerts`,
+      data: JSON.stringify([alertData]),
+      headers: { 'Content-Type': ContentType.JSON }
+    };
+  
+    //first part: send start request
+    await axios(options);
+    //TODO: resolve FAILURE alert only after RECOVERY event
+    //second part: send end request
+    alertData.endsAt = (new Date()).toISOString();
+    options.data = JSON.stringify([alertData]);
     await axios(options);
-  } catch(err) {
-    console.error(`Can't send notification to ${HASTIC_WEBHOOK_URL}. Error: ${err.message}`);
   }
 }