hastic
/
hastic-server
4
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity
Browse Source

Anomaly detector detection re running in inspect mode #757 (#759) 

* fix

* fix

* fix

* Update server/src/controllers/analytics_controller.ts

Co-Authored-By: Alexey Velikiy <av@corpglory.com>

* add test

* fix

* Update server/spec/analytic_controller.jest.ts

Co-Authored-By: Alexey Velikiy <av@corpglory.com>

* fix

* fix

* fix

* Update server/spec/analytic_controller.jest.ts

Co-Authored-By: rozetko <rozetko@hastic.io>

* fix

* Update server/spec/analytic_controller.jest.ts

Co-Authored-By: Alexey Velikiy <av@corpglory.com>

* Update server/spec/analytic_controller.jest.ts

Co-Authored-By: Alexey Velikiy <av@corpglory.com>

* fix

* fix

* fix
pull/1/head
Evgeny Smyshlyaev 5 years ago committed by Alexey Velikiy
parent
b6d80015b6
commit
d469c1e98c
3 changed files with 101 additions and 88 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 148
      server/spec/analytic_controller.jest.ts
  2. 35
      server/src/controllers/analytics_controller.ts
  3. 6
      server/src/models/analytic_unit_cache_model.ts

148
server/spec/analytic_controller.jest.ts
Unescape View File

@ -3,92 +3,94 @@ import { queryByMetric } from 'grafana-datasource-kit';
jest.mock('grafana-datasource-kit', () => (
{
...(jest.requireActual('grafana-datasource-kit')),
queryByMetric: jest.fn((metric, url, from, to, apiKey) => {})
queryByMetric: jest.fn((metric, url, from, to, apiKey) => {
return { values:[], columns:[] }
})
}
));
import { saveAnalyticUnitFromObject, runDetect, onDetect } from '../src/controllers/analytics_controller';
import { saveAnalyticUnitFromObject, runDetect, onDetect, getHSR } from '../src/controllers/analytics_controller';
import * as AnalyticUnit from '../src/models/analytic_units';
import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model';
import * as Segment from '../src/models/segment_model';
import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units';
import { buildSegments, clearSegmentsDB, convertSegmentsToTimeRanges } from './utils_for_tests/segments';
import { HASTIC_API_KEY } from '../src/config';
describe('Check detection range', function() {
const analyticUnitObj = {
_id: 'test',
name: "test",
grafanaUrl: "http://127.0.0.1:3000",
panelId: "ZLc0KfNZk/2",
type: "GENERAL",
metric: {
datasource: {
url: "api/datasources/proxy/5/query",
method: "GET",
data: null,
params: {
db:"dbname",
q: "SELECT mean(\"value\") FROM \"autogen\".\"tcpconns_value\" WHERE time >= now() - 6h GROUP BY time(20s) fill(null)",
epoch: "ms"
},
type: "influxdb"
import * as _ from 'lodash';
const DEFAULT_ANALYTIC_UNIT_OBJECT = {
name: "test",
grafanaUrl: "http://127.0.0.1:3000",
panelId: "ZLc0KfNZk/2",
type: "GENERAL",
metric: {
datasource: {
url: "api/datasources/proxy/5/query",
method: "GET",
data: null,
params: {
db:"dbname",
q: "SELECT mean(\"value\") FROM \"autogen\".\"tcpconns_value\" WHERE time >= now() - 6h GROUP BY time(20s) fill(null)",
epoch: "ms"
},
targets: [
{
groupBy: [
{
params: ["$__interval"],
type: "time"
},
{
params: ["null"],
type: "fill"
}
],
measurement: "tcpconns_value",
orderByTime: "ASC",
policy: "autogen",
refId: "A",
resultFormat: "time_series",
select: [[{"params":["value"],"type":"field"},{"params":[],"type":"mean"}]],"tags":[]
}
]
type: "influxdb"
},
alert: false,
labeledColor: "#FF99FF",
deletedColor: "#00f0ff",
detectorType: "pattern",
visible: true,
collapsed: false,
createdAt: {"$$date":1564476040880},
updatedAt: {"$$date":1564476040880}
}
const WINDOW_SIZE = 10;
const TIME_STEP = 1000;
async function addTestUnitToDB(): Promise<string> {
const analyticUnitId = await saveAnalyticUnitFromObject(analyticUnitObj);
await AnalyticUnit.update(analyticUnitId, {lastDetectionTime: 1000});
await AnalyticUnitCache.create(analyticUnitId);
await AnalyticUnitCache.setData(analyticUnitId, {
windowSize: WINDOW_SIZE,
timeStep: TIME_STEP
});
return analyticUnitId;
};
targets: [
{
groupBy: [
{
params: ["$__interval"],
type: "time"
},
{
params: ["null"],
type: "fill"
}
],
measurement: "tcpconns_value",
orderByTime: "ASC",
policy: "autogen",
refId: "A",
resultFormat: "time_series",
select: [[{"params":["value"],"type":"field"},{"params":[],"type":"mean"}]],"tags":[]
}
]
},
alert: false,
labeledColor: "#FF99FF",
deletedColor: "#00f0ff",
detectorType: "pattern",
visible: true,
collapsed: false,
createdAt: {"$$date":1564476040880},
updatedAt: {"$$date":1564476040880}
}
const WINDOW_SIZE = 10;
const TIME_STEP = 1000;
async function addTestUnitToDB(analyticUnitObj: any): Promise<string> {
const analyticUnitId = await saveAnalyticUnitFromObject(analyticUnitObj);
await AnalyticUnit.update(analyticUnitId, { lastDetectionTime: 1000 });
await AnalyticUnitCache.create(analyticUnitId);
await AnalyticUnitCache.setData(analyticUnitId, {
windowSize: WINDOW_SIZE,
timeStep: TIME_STEP
});
return analyticUnitId;
};
describe('Check detection range', function() {
it('check range >= 2 * window size * timeStep', async () => {
const from = 1500000000000;
const to = 1500000000001;
const expectedFrom = to - WINDOW_SIZE * TIME_STEP * 2;
const id = await addTestUnitToDB();
const id = await addTestUnitToDB(DEFAULT_ANALYTIC_UNIT_OBJECT);
await runDetect(id, from, to);
expect(queryByMetric).toBeCalledWith(analyticUnitObj.metric, undefined, expectedFrom, to, HASTIC_API_KEY);
expect(queryByMetric).toBeCalledWith(DEFAULT_ANALYTIC_UNIT_OBJECT.metric, undefined, expectedFrom, to, HASTIC_API_KEY);
});
});
@ -136,3 +138,15 @@ describe('onDetect', () => {
expect(detectedRanges).toEqual([[7, 8]]);
});
});
describe('getHSR', function() {
it('should return nothing if unit state is LEARNING', async () => {
let unitObj = _.clone(DEFAULT_ANALYTIC_UNIT_OBJECT);
unitObj.detectorType = 'anomaly';
const analyticUnitId = await addTestUnitToDB(unitObj);
await AnalyticUnitCache.remove(analyticUnitId);
const unit = await AnalyticUnit.findById(analyticUnitId);
const result = await getHSR(unit, 9000, 100000);
expect(result).toEqual({"hsr": {"columns": [], "values": []}});
});
});

35
server/src/controllers/analytics_controller.ts
Unescape View File

@ -640,34 +640,39 @@ export async function getHSR(
analyticUnit: AnalyticUnit.AnalyticUnit,
from: number,
to: number
): Promise<{
hsr: TableTimeSeries,
lowerBound?: TableTimeSeries,
upperBound?: TableTimeSeries
}> {
): Promise< HSRResult | undefined> {
try {
const grafanaUrl = getGrafanaUrl(analyticUnit.grafanaUrl);
const data = await queryByMetric(analyticUnit.metric, grafanaUrl, from, to, HASTIC_API_KEY);
let resultSeries: HSRResult = {
hsr: data
}
if(analyticUnit.detectorType === AnalyticUnit.DetectorType.PATTERN) {
const resultSeries: HSRResult = {
hsr: await queryByMetric(analyticUnit.metric, grafanaUrl, from, to, HASTIC_API_KEY)
};
return resultSeries;
}
let cache = await AnalyticUnitCache.findById(analyticUnit.id);
if(cache === null || cache.isCacheOutdated(analyticUnit)) {
await runLearning(analyticUnit.id, from, to);
cache = await AnalyticUnitCache.findById(analyticUnit.id);
const inLearningState = analyticUnit.status === AnalyticUnit.AnalyticUnitStatus.LEARNING;
//cache.data === null when learning started but not completed yet or first learning was failed
if(inLearningState || cache === null || cache.data === null) {
const resultSeries: HSRResult = {
hsr: await queryByMetric(analyticUnit.metric, grafanaUrl, from, to, HASTIC_API_KEY)
};
return resultSeries;
//TODO: send warning: can't show HSR before learning
}
cache = cache.data;
let resultSeries: HSRResult = {
hsr: await queryByMetric(analyticUnit.metric, grafanaUrl, from, to, HASTIC_API_KEY)
};
const analyticUnitType = analyticUnit.type;
const detector = analyticUnit.detectorType;
const payload = {
data: data.values,
data: resultSeries.hsr.values,
analyticUnitType,
detector,
cache
@ -680,11 +685,11 @@ export async function getHSR(
}
if(result.payload.lowerBound !== undefined) {
resultSeries.lowerBound = { values: result.payload.lowerBound, columns: data.columns };
resultSeries.lowerBound = { values: result.payload.lowerBound, columns: resultSeries.hsr.columns };
}
if(result.payload.upperBound !== undefined) {
resultSeries.upperBound = { values: result.payload.upperBound, columns: data.columns };
resultSeries.upperBound = { values: result.payload.upperBound, columns: resultSeries.hsr.columns };
}
return resultSeries;

6
server/src/models/analytic_unit_cache_model.ts
Unescape View File

@ -52,12 +52,6 @@ export class AnalyticUnitCache {
public getTimeStep(): number {
return this.data.timeStep;
}
public isCacheOutdated(analyticUnit: AnalyticUnit) {
return !_.every(
_.keys(analyticUnit.analyticProps).map(k => _.isEqual(analyticUnit.analyticProps[k], this.data[k]))
);
}
}
export async function findById(id: AnalyticUnitId): Promise<AnalyticUnitCache> {

Write Preview
Loading…
Cancel
Save