hastic
/
hastic-server
4
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity
Browse Source

Anomaly detector detection re running in inspect mode #757 (#759) 

* fix

* fix

* fix

* Update server/src/controllers/analytics_controller.ts

Co-Authored-By: Alexey Velikiy <av@corpglory.com>

* add test

* fix

* Update server/spec/analytic_controller.jest.ts

Co-Authored-By: Alexey Velikiy <av@corpglory.com>

* fix

* fix

* fix

* Update server/spec/analytic_controller.jest.ts

Co-Authored-By: rozetko <rozetko@hastic.io>

* fix

* Update server/spec/analytic_controller.jest.ts

Co-Authored-By: Alexey Velikiy <av@corpglory.com>

* Update server/spec/analytic_controller.jest.ts

Co-Authored-By: Alexey Velikiy <av@corpglory.com>

* fix

* fix

* fix
pull/1/head
Evgeny Smyshlyaev 5 years ago committed by Alexey Velikiy
parent
b6d80015b6
commit
d469c1e98c
3 changed files with 101 additions and 88 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 148
      server/spec/analytic_controller.jest.ts
  2. 35
      server/src/controllers/analytics_controller.ts
  3. 6
      server/src/models/analytic_unit_cache_model.ts

148
server/spec/analytic_controller.jest.ts
Unescape View File

@ -3,92 +3,94 @@ import { queryByMetric } from 'grafana-datasource-kit';
jest.mock('grafana-datasource-kit', () => ( jest.mock('grafana-datasource-kit', () => (
{ {
...(jest.requireActual('grafana-datasource-kit')), ...(jest.requireActual('grafana-datasource-kit')),
queryByMetric: jest.fn((metric, url, from, to, apiKey) => {}) queryByMetric: jest.fn((metric, url, from, to, apiKey) => {
return { values:[], columns:[] }
})
} }
)); ));
import { saveAnalyticUnitFromObject, runDetect, onDetect } from '../src/controllers/analytics_controller'; import { saveAnalyticUnitFromObject, runDetect, onDetect, getHSR } from '../src/controllers/analytics_controller';
import * as AnalyticUnit from '../src/models/analytic_units'; import * as AnalyticUnit from '../src/models/analytic_units';
import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model'; import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model';
import * as Segment from '../src/models/segment_model'; import * as Segment from '../src/models/segment_model';
import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units'; import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units';
import { buildSegments, clearSegmentsDB, convertSegmentsToTimeRanges } from './utils_for_tests/segments'; import { buildSegments, clearSegmentsDB, convertSegmentsToTimeRanges } from './utils_for_tests/segments';
import { HASTIC_API_KEY } from '../src/config'; import { HASTIC_API_KEY } from '../src/config';
import * as _ from 'lodash';
describe('Check detection range', function() {
const analyticUnitObj = {
_id: 'test', const DEFAULT_ANALYTIC_UNIT_OBJECT = {
name: "test", name: "test",
grafanaUrl: "http://127.0.0.1:3000", grafanaUrl: "http://127.0.0.1:3000",
panelId: "ZLc0KfNZk/2", panelId: "ZLc0KfNZk/2",
type: "GENERAL", type: "GENERAL",
metric: { metric: {
datasource: { datasource: {
url: "api/datasources/proxy/5/query", url: "api/datasources/proxy/5/query",
method: "GET", method: "GET",
data: null, data: null,
params: { params: {
db:"dbname", db:"dbname",
q: "SELECT mean(\"value\") FROM \"autogen\".\"tcpconns_value\" WHERE time >= now() - 6h GROUP BY time(20s) fill(null)", q: "SELECT mean(\"value\") FROM \"autogen\".\"tcpconns_value\" WHERE time >= now() - 6h GROUP BY time(20s) fill(null)",
epoch: "ms" epoch: "ms"
},
type: "influxdb"
}, },
targets: [ type: "influxdb"
{
groupBy: [
{
params: ["$__interval"],
type: "time"
},
{
params: ["null"],
type: "fill"
}
],
measurement: "tcpconns_value",
orderByTime: "ASC",
policy: "autogen",
refId: "A",
resultFormat: "time_series",
select: [[{"params":["value"],"type":"field"},{"params":[],"type":"mean"}]],"tags":[]
}
]
}, },
alert: false, targets: [
labeledColor: "#FF99FF", {
deletedColor: "#00f0ff", groupBy: [
detectorType: "pattern", {
visible: true, params: ["$__interval"],
collapsed: false, type: "time"
createdAt: {"$$date":1564476040880}, },
updatedAt: {"$$date":1564476040880} {
} params: ["null"],
type: "fill"
const WINDOW_SIZE = 10; }
const TIME_STEP = 1000; ],
measurement: "tcpconns_value",
async function addTestUnitToDB(): Promise<string> { orderByTime: "ASC",
const analyticUnitId = await saveAnalyticUnitFromObject(analyticUnitObj); policy: "autogen",
await AnalyticUnit.update(analyticUnitId, {lastDetectionTime: 1000}); refId: "A",
await AnalyticUnitCache.create(analyticUnitId); resultFormat: "time_series",
await AnalyticUnitCache.setData(analyticUnitId, { select: [[{"params":["value"],"type":"field"},{"params":[],"type":"mean"}]],"tags":[]
windowSize: WINDOW_SIZE, }
timeStep: TIME_STEP ]
}); },
return analyticUnitId; alert: false,
}; labeledColor: "#FF99FF",
deletedColor: "#00f0ff",
detectorType: "pattern",
visible: true,
collapsed: false,
createdAt: {"$$date":1564476040880},
updatedAt: {"$$date":1564476040880}
}
const WINDOW_SIZE = 10;
const TIME_STEP = 1000;
async function addTestUnitToDB(analyticUnitObj: any): Promise<string> {
const analyticUnitId = await saveAnalyticUnitFromObject(analyticUnitObj);
await AnalyticUnit.update(analyticUnitId, { lastDetectionTime: 1000 });
await AnalyticUnitCache.create(analyticUnitId);
await AnalyticUnitCache.setData(analyticUnitId, {
windowSize: WINDOW_SIZE,
timeStep: TIME_STEP
});
return analyticUnitId;
};
describe('Check detection range', function() {
it('check range >= 2 * window size * timeStep', async () => { it('check range >= 2 * window size * timeStep', async () => {
const from = 1500000000000; const from = 1500000000000;
const to = 1500000000001; const to = 1500000000001;
const expectedFrom = to - WINDOW_SIZE * TIME_STEP * 2; const expectedFrom = to - WINDOW_SIZE * TIME_STEP * 2;
const id = await addTestUnitToDB(); const id = await addTestUnitToDB(DEFAULT_ANALYTIC_UNIT_OBJECT);
await runDetect(id, from, to); await runDetect(id, from, to);
expect(queryByMetric).toBeCalledWith(analyticUnitObj.metric, undefined, expectedFrom, to, HASTIC_API_KEY); expect(queryByMetric).toBeCalledWith(DEFAULT_ANALYTIC_UNIT_OBJECT.metric, undefined, expectedFrom, to, HASTIC_API_KEY);
}); });
}); });
@ -136,3 +138,15 @@ describe('onDetect', () => {
expect(detectedRanges).toEqual([[7, 8]]); expect(detectedRanges).toEqual([[7, 8]]);
}); });
}); });
describe('getHSR', function() {
it('should return nothing if unit state is LEARNING', async () => {
let unitObj = _.clone(DEFAULT_ANALYTIC_UNIT_OBJECT);
unitObj.detectorType = 'anomaly';
const analyticUnitId = await addTestUnitToDB(unitObj);
await AnalyticUnitCache.remove(analyticUnitId);
const unit = await AnalyticUnit.findById(analyticUnitId);
const result = await getHSR(unit, 9000, 100000);
expect(result).toEqual({"hsr": {"columns": [], "values": []}});
});
});

35
server/src/controllers/analytics_controller.ts
Unescape View File

@ -640,34 +640,39 @@ export async function getHSR(
analyticUnit: AnalyticUnit.AnalyticUnit, analyticUnit: AnalyticUnit.AnalyticUnit,
from: number, from: number,
to: number to: number
): Promise<{ ): Promise< HSRResult | undefined> {
hsr: TableTimeSeries,
lowerBound?: TableTimeSeries,
upperBound?: TableTimeSeries
}> {
try { try {
const grafanaUrl = getGrafanaUrl(analyticUnit.grafanaUrl); const grafanaUrl = getGrafanaUrl(analyticUnit.grafanaUrl);
const data = await queryByMetric(analyticUnit.metric, grafanaUrl, from, to, HASTIC_API_KEY);
let resultSeries: HSRResult = {
hsr: data
}
if(analyticUnit.detectorType === AnalyticUnit.DetectorType.PATTERN) { if(analyticUnit.detectorType === AnalyticUnit.DetectorType.PATTERN) {
const resultSeries: HSRResult = {
hsr: await queryByMetric(analyticUnit.metric, grafanaUrl, from, to, HASTIC_API_KEY)
};
return resultSeries; return resultSeries;
} }
let cache = await AnalyticUnitCache.findById(analyticUnit.id); let cache = await AnalyticUnitCache.findById(analyticUnit.id);
if(cache === null || cache.isCacheOutdated(analyticUnit)) {
await runLearning(analyticUnit.id, from, to); const inLearningState = analyticUnit.status === AnalyticUnit.AnalyticUnitStatus.LEARNING;
cache = await AnalyticUnitCache.findById(analyticUnit.id);
//cache.data === null when learning started but not completed yet or first learning was failed
if(inLearningState || cache === null || cache.data === null) {
const resultSeries: HSRResult = {
hsr: await queryByMetric(analyticUnit.metric, grafanaUrl, from, to, HASTIC_API_KEY)
};
return resultSeries;
//TODO: send warning: can't show HSR before learning
} }
cache = cache.data; cache = cache.data;
let resultSeries: HSRResult = {
hsr: await queryByMetric(analyticUnit.metric, grafanaUrl, from, to, HASTIC_API_KEY)
};
const analyticUnitType = analyticUnit.type; const analyticUnitType = analyticUnit.type;
const detector = analyticUnit.detectorType; const detector = analyticUnit.detectorType;
const payload = { const payload = {
data: data.values, data: resultSeries.hsr.values,
analyticUnitType, analyticUnitType,
detector, detector,
cache cache
@ -680,11 +685,11 @@ export async function getHSR(
} }
if(result.payload.lowerBound !== undefined) { if(result.payload.lowerBound !== undefined) {
resultSeries.lowerBound = { values: result.payload.lowerBound, columns: data.columns }; resultSeries.lowerBound = { values: result.payload.lowerBound, columns: resultSeries.hsr.columns };
} }
if(result.payload.upperBound !== undefined) { if(result.payload.upperBound !== undefined) {
resultSeries.upperBound = { values: result.payload.upperBound, columns: data.columns }; resultSeries.upperBound = { values: result.payload.upperBound, columns: resultSeries.hsr.columns };
} }
return resultSeries; return resultSeries;

6
server/src/models/analytic_unit_cache_model.ts
Unescape View File

@ -52,12 +52,6 @@ export class AnalyticUnitCache {
public getTimeStep(): number { public getTimeStep(): number {
return this.data.timeStep; return this.data.timeStep;
} }
public isCacheOutdated(analyticUnit: AnalyticUnit) {
return !_.every(
_.keys(analyticUnit.analyticProps).map(k => _.isEqual(analyticUnit.analyticProps[k], this.data[k]))
);
}
} }
export async function findById(id: AnalyticUnitId): Promise<AnalyticUnitCache> { export async function findById(id: AnalyticUnitId): Promise<AnalyticUnitCache> {

Write Preview
Loading…
Cancel
Save