|
|
|
|
@ -1,9 +1,10 @@
|
|
|
|
|
import { sendAnalyticWebhook, sendInfoWebhook, InfoAlert, AnalyticAlert, WebhookType } from './notification_service'; |
|
|
|
|
import { sendNotification, InfoMeta, AnalyticMeta, WebhookType, Notification } from './notification_service'; |
|
|
|
|
|
|
|
|
|
import * as _ from 'lodash'; |
|
|
|
|
import * as AnalyticUnit from '../models/analytic_units'; |
|
|
|
|
import { Segment } from '../models/segment_model'; |
|
|
|
|
import { availableReporter } from '../utils/reporter'; |
|
|
|
|
import { ORG_ID } from '../config'; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
export class Alert { |
|
|
|
|
@ -11,24 +12,43 @@ export class Alert {
|
|
|
|
|
constructor(protected analyticUnit: AnalyticUnit.AnalyticUnit) {}; |
|
|
|
|
public receive(segment: Segment) { |
|
|
|
|
if(this.enabled) { |
|
|
|
|
const alert = this.makeAlert(segment); |
|
|
|
|
sendAnalyticWebhook(alert); |
|
|
|
|
sendNotification(this.makeNotification(segment)); |
|
|
|
|
} |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
protected makeAlert(segment): AnalyticAlert { |
|
|
|
|
const alert: AnalyticAlert = { |
|
|
|
|
protected makeNotification(segment: Segment): Notification { |
|
|
|
|
const meta = this.makeMeta(segment); |
|
|
|
|
const message = this.makeMessage(meta); |
|
|
|
|
return { meta, message }; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
protected makeMeta(segment: Segment): AnalyticMeta { |
|
|
|
|
const datshdoardId = this.analyticUnit.panelId.split('/')[0]; |
|
|
|
|
const panelId = this.analyticUnit.panelId.split('/')[1]; |
|
|
|
|
const grafanaUrl = `${this.analyticUnit.grafanaUrl}/d/${datshdoardId}?panelId=${panelId}&edit=true&fullscreen=true?orgId=${ORG_ID}`; |
|
|
|
|
const alert: AnalyticMeta = { |
|
|
|
|
type: WebhookType.DETECT, |
|
|
|
|
analyticUnitType: this.analyticUnit.type, |
|
|
|
|
analyticUnitName: this.analyticUnit.name, |
|
|
|
|
analyticUnitId: this.analyticUnit.id, |
|
|
|
|
grafanaUrl: this.analyticUnit.grafanaUrl, |
|
|
|
|
grafanaUrl, |
|
|
|
|
from: segment.from, |
|
|
|
|
to: segment.to
|
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
return alert; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
protected makeMessage(meta: AnalyticMeta): string { |
|
|
|
|
return [ |
|
|
|
|
`[${meta.analyticUnitType.toUpperCase()} ALERTING] ${meta.analyticUnitName}`, |
|
|
|
|
`URL: ${meta.grafanaUrl}`, |
|
|
|
|
``, |
|
|
|
|
`From: ${new Date(meta.from)}`, |
|
|
|
|
`To: ${new Date(meta.to)}`, |
|
|
|
|
`ID: ${meta.analyticUnitId}` |
|
|
|
|
].join('\n'); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
class PatternAlert extends Alert { |
|
|
|
|
@ -39,14 +59,27 @@ class PatternAlert extends Alert {
|
|
|
|
|
if(this.lastSentSegment === undefined || !segment.equals(this.lastSentSegment) ) { |
|
|
|
|
this.lastSentSegment = segment; |
|
|
|
|
if(this.enabled) { |
|
|
|
|
sendAnalyticWebhook(this.makeAlert(segment)); |
|
|
|
|
sendNotification(this.makeNotification(segment)); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
protected makeMessage(meta: AnalyticMeta): string { |
|
|
|
|
return [ |
|
|
|
|
`[PATTERN DETECTED] ${meta.analyticUnitName}`, |
|
|
|
|
`URL: ${meta.grafanaUrl}`, |
|
|
|
|
``, |
|
|
|
|
`From: ${new Date(meta.from)}`, |
|
|
|
|
`To: ${new Date(meta.to)}`, |
|
|
|
|
`ID: ${meta.analyticUnitId}` |
|
|
|
|
].join('\n'); |
|
|
|
|
} |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class ThresholdAlert extends Alert { |
|
|
|
|
// TODO: configure threshold timing in panel like Grafana's alerts (`evaluate` time, `for` time)
|
|
|
|
|
// TODO: make events for start and end of threshold
|
|
|
|
|
EXPIRE_PERIOD_MS = 60000; |
|
|
|
|
lastOccurence = 0; |
|
|
|
|
|
|
|
|
|
@ -54,31 +87,60 @@ class ThresholdAlert extends Alert {
|
|
|
|
|
if(this.lastOccurence === 0) { |
|
|
|
|
this.lastOccurence = segment.from; |
|
|
|
|
if(this.enabled) { |
|
|
|
|
sendAnalyticWebhook(this.makeAlert(segment)); |
|
|
|
|
sendNotification(this.makeNotification(segment)); |
|
|
|
|
} |
|
|
|
|
} else { |
|
|
|
|
|
|
|
|
|
if(segment.from - this.lastOccurence > this.EXPIRE_PERIOD_MS) { |
|
|
|
|
if(this.enabled) { |
|
|
|
|
console.log(`time between threshold occurences ${segment.from - this.lastOccurence}ms, send alert`); |
|
|
|
|
sendAnalyticWebhook(this.makeAlert(segment)); |
|
|
|
|
sendNotification(this.makeNotification(segment)); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
this.lastOccurence = segment.from; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
protected makeMessage(meta: AnalyticMeta): string { |
|
|
|
|
let message = [ |
|
|
|
|
`[THRESHOLD ALERTING] ${meta.analyticUnitName}`, |
|
|
|
|
`URL: ${meta.grafanaUrl}`, |
|
|
|
|
``, |
|
|
|
|
`Starts at: ${new Date(meta.from)}`, |
|
|
|
|
`ID: ${meta.analyticUnitId}` |
|
|
|
|
].join('\n'); |
|
|
|
|
|
|
|
|
|
if(meta.params !== undefined) { |
|
|
|
|
const metrics = ` |
|
|
|
|
Metrics: |
|
|
|
|
${this.analyticUnit.metric.targets[0].expr}: ${meta.params.value} |
|
|
|
|
`;
|
|
|
|
|
message += metrics; |
|
|
|
|
} |
|
|
|
|
return message; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
export class AlertService { |
|
|
|
|
|
|
|
|
|
private _alerts: { [id: string]: Alert; }; |
|
|
|
|
// TODO: object -> Map
|
|
|
|
|
private _alerts: { [id: string]: Alert }; |
|
|
|
|
private _alertingEnable: boolean; |
|
|
|
|
private _grafanaAvailableReporter: Function; |
|
|
|
|
private _datasourceAvailableReporters: Map<string, Function>; |
|
|
|
|
|
|
|
|
|
constructor() { |
|
|
|
|
this._alerts = {} |
|
|
|
|
this._alerts = {}; |
|
|
|
|
this._datasourceAvailableReporters = new Map(); |
|
|
|
|
|
|
|
|
|
this._grafanaAvailableReporter = availableReporter( |
|
|
|
|
['[OK] Grafana available', WebhookType.RECOVERY], |
|
|
|
|
['[FAILURE] Grafana unavailable for pulling data', WebhookType.FAILURE], |
|
|
|
|
this.sendMsg, |
|
|
|
|
this.sendMsg |
|
|
|
|
); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public receiveAlert(analyticUnit: AnalyticUnit.AnalyticUnit, segment: Segment) { |
|
|
|
|
@ -97,46 +159,41 @@ export class AlertService {
|
|
|
|
|
|
|
|
|
|
public sendMsg(message: string, type: WebhookType, optionalInfo = {}) { |
|
|
|
|
const now = Date.now(); |
|
|
|
|
const infoAlert: InfoAlert = { |
|
|
|
|
message, |
|
|
|
|
const infoAlert: InfoMeta = { |
|
|
|
|
params: optionalInfo, |
|
|
|
|
type, |
|
|
|
|
from: now, |
|
|
|
|
to: now |
|
|
|
|
} |
|
|
|
|
sendInfoWebhook(infoAlert); |
|
|
|
|
sendNotification({ message, meta: infoAlert }); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public getGrafanaAvailableReporter() { |
|
|
|
|
if(!this._grafanaAvailableReporter) { |
|
|
|
|
this._grafanaAvailableReporter = availableReporter( |
|
|
|
|
['Grafana available', WebhookType.RECOVERY], |
|
|
|
|
['Grafana unavailable for pulling data', WebhookType.FAILURE], |
|
|
|
|
this.sendMsg, |
|
|
|
|
this.sendMsg |
|
|
|
|
); |
|
|
|
|
} |
|
|
|
|
return this._grafanaAvailableReporter; |
|
|
|
|
public sendGrafanaAvailableWebhook() { |
|
|
|
|
this._grafanaAvailableReporter(true); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public getAvailableWebhook(recoveryMsg: string, failureMsg: string) { |
|
|
|
|
return availableReporter( |
|
|
|
|
[recoveryMsg, WebhookType.RECOVERY], |
|
|
|
|
[failureMsg, WebhookType.FAILURE], |
|
|
|
|
this.sendMsg, |
|
|
|
|
this.sendMsg |
|
|
|
|
); |
|
|
|
|
public sendGrafanaUnavailableWebhook() { |
|
|
|
|
this._grafanaAvailableReporter(false); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public sendDatasourceAvailableWebhook(url: string) { |
|
|
|
|
const reporter = this._getDatasourceAvailableReporter(url); |
|
|
|
|
reporter(true); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public sendDatasourceUnavailableWebhook(url: string) { |
|
|
|
|
const reporter = this._getDatasourceAvailableReporter(url); |
|
|
|
|
reporter(false); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public addAnalyticUnit(analyticUnit: AnalyticUnit.AnalyticUnit) { |
|
|
|
|
const detector = analyticUnit.detectorType; |
|
|
|
|
let alertsType = {}; |
|
|
|
|
|
|
|
|
|
alertsType[AnalyticUnit.DetectorType.THRESHOLD] = ThresholdAlert; |
|
|
|
|
alertsType[AnalyticUnit.DetectorType.PATTERN] = PatternAlert; |
|
|
|
|
alertsType[AnalyticUnit.DetectorType.ANOMALY] = Alert; |
|
|
|
|
|
|
|
|
|
this._alerts[analyticUnit.id] = new alertsType[detector](analyticUnit); |
|
|
|
|
this._alerts[analyticUnit.id] = new alertsType[analyticUnit.detectorType](analyticUnit); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public removeAnalyticUnit(analyticUnitId: AnalyticUnit.AnalyticUnitId) { |
|
|
|
|
@ -145,10 +202,21 @@ export class AlertService {
|
|
|
|
|
|
|
|
|
|
public stopAlerting() { |
|
|
|
|
this._alertingEnable = false; |
|
|
|
|
this._alerts = {}; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
public startAlerting() { |
|
|
|
|
this._alertingEnable = true; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private _getDatasourceAvailableReporter(url: string) { |
|
|
|
|
if(!_.has(this._datasourceAvailableReporters, url)) { |
|
|
|
|
this._datasourceAvailableReporters[url] = availableReporter( |
|
|
|
|
[`[OK] Datasource ${url} available`, WebhookType.RECOVERY], |
|
|
|
|
[`[FAILURE] Datasource ${url} unavailable`, WebhookType.FAILURE], |
|
|
|
|
this.sendMsg, |
|
|
|
|
this.sendMsg |
|
|
|
|
); |
|
|
|
|
} |
|
|
|
|
return this._datasourceAvailableReporters[url]; |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
Evgeny Smyshlyaev
6 years ago
committed by