From a42a5b85a70d8f5f38dfa7272f8b7a2f01681881 Mon Sep 17 00:00:00 2001 From: Evgeny Smyshlyaev Date: Mon, 26 Aug 2019 22:13:23 +0300 Subject: [PATCH] Class for analytic unit object structure #760 (#762) * just save * fix * fix * fix * fix * fix * fix * fix * Update server/spec/setup_tests.ts * fix * fix * fix * fix * fix * fix * fix * fix * Update server/spec/analytic_controller.jest.ts Co-Authored-By: Alexey Velikiy --- server/jest.config.js | 2 + server/spec/analytic_controller.jest.ts | 100 +++++------------- server/spec/setup_tests.ts | 21 +--- server/spec/utils_for_tests/analytic_units.ts | 82 ++++++++++++++ server/src/config.ts | 2 + server/src/services/data_service.ts | 14 +-- 6 files changed, 125 insertions(+), 96 deletions(-) diff --git a/server/jest.config.js b/server/jest.config.js index 4f0f1e2..1bf199f 100644 --- a/server/jest.config.js +++ b/server/jest.config.js @@ -19,3 +19,5 @@ module.exports = { "/spec/setup_tests.ts" ] }; + +process.env.HASTIC_DB_IN_MEMORY = 'true'; diff --git a/server/spec/analytic_controller.jest.ts b/server/spec/analytic_controller.jest.ts index 70aa8b3..2f4ae9f 100644 --- a/server/spec/analytic_controller.jest.ts +++ b/server/spec/analytic_controller.jest.ts @@ -9,88 +9,34 @@ jest.mock('grafana-datasource-kit', () => ( } )); -import { saveAnalyticUnitFromObject, runDetect, onDetect, getHSR } from '../src/controllers/analytics_controller'; -import * as AnalyticUnit from '../src/models/analytic_units'; +import { runDetect, onDetect, getHSR } from '../src/controllers/analytics_controller'; import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model'; import * as Segment from '../src/models/segment_model'; -import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units'; +import { TEST_ANALYTIC_UNIT_ID, createTestDB, clearTestDB, DEFAULT_METRIC } from './utils_for_tests/analytic_units'; import { buildSegments, clearSegmentsDB, convertSegmentsToTimeRanges } from './utils_for_tests/segments'; -import { HASTIC_API_KEY } from '../src/config'; +import { HASTIC_API_KEY, GRAFANA_URL } from '../src/config'; import * as _ from 'lodash'; - - -const DEFAULT_ANALYTIC_UNIT_OBJECT = { - name: "test", - grafanaUrl: "http://127.0.0.1:3000", - panelId: "ZLc0KfNZk/2", - type: "GENERAL", - metric: { - datasource: { - url: "api/datasources/proxy/5/query", - method: "GET", - data: null, - params: { - db:"dbname", - q: "SELECT mean(\"value\") FROM \"autogen\".\"tcpconns_value\" WHERE time >= now() - 6h GROUP BY time(20s) fill(null)", - epoch: "ms" - }, - type: "influxdb" - }, - targets: [ - { - groupBy: [ - { - params: ["$__interval"], - type: "time" - }, - { - params: ["null"], - type: "fill" - } - ], - measurement: "tcpconns_value", - orderByTime: "ASC", - policy: "autogen", - refId: "A", - resultFormat: "time_series", - select: [[{"params":["value"],"type":"field"},{"params":[],"type":"mean"}]],"tags":[] - } - ] - }, - alert: false, - labeledColor: "#FF99FF", - deletedColor: "#00f0ff", - detectorType: "pattern", - visible: true, - collapsed: false, - createdAt: {"$$date":1564476040880}, - updatedAt: {"$$date":1564476040880} -} +import * as AnalyticUnit from '../src/models/analytic_units'; const WINDOW_SIZE = 10; const TIME_STEP = 1000; -async function addTestUnitToDB(analyticUnitObj: any): Promise { - const analyticUnitId = await saveAnalyticUnitFromObject(analyticUnitObj); - await AnalyticUnit.update(analyticUnitId, { lastDetectionTime: 1000 }); - await AnalyticUnitCache.create(analyticUnitId); - await AnalyticUnitCache.setData(analyticUnitId, { - windowSize: WINDOW_SIZE, - timeStep: TIME_STEP - }); - return analyticUnitId; -}; +beforeEach(async () => { + await clearTestDB(); + await createTestDB(); +}); describe('Check detection range', function() { - it('check range >= 2 * window size * timeStep', async () => { + it('range should be >= 2 * windowSize * timeStep', async () => { const from = 1500000000000; const to = 1500000000001; const expectedFrom = to - WINDOW_SIZE * TIME_STEP * 2; - const id = await addTestUnitToDB(DEFAULT_ANALYTIC_UNIT_OBJECT); - await runDetect(id, from, to); - expect(queryByMetric).toBeCalledWith(DEFAULT_ANALYTIC_UNIT_OBJECT.metric, undefined, expectedFrom, to, HASTIC_API_KEY); + await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: TIME_STEP, windowSize: WINDOW_SIZE }); + console.log(await AnalyticUnitCache.findById(TEST_ANALYTIC_UNIT_ID)); + await runDetect(TEST_ANALYTIC_UNIT_ID, from, to); + expect(queryByMetric).toBeCalledWith(DEFAULT_METRIC, GRAFANA_URL, expectedFrom, to, HASTIC_API_KEY); }); }); @@ -130,6 +76,7 @@ describe('onDetect', () => { segments: buildSegments([[7, 8]]), lastDetectionTime: 0 }); + const detectedSegments = await Promise.all( detectedSegmentIds.map(id => Segment.findOne(id)) ); @@ -140,12 +87,21 @@ describe('onDetect', () => { }); describe('getHSR', function() { + let cacheToSave: AnalyticUnitCache.AnalyticUnitCache; + + beforeAll(async () => { + await clearTestDB(); + await createTestDB(false); + }); + + afterAll(async () => { + await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID); + await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, cacheToSave.data); + }); + it('should return nothing if unit state is LEARNING', async () => { - let unitObj = _.clone(DEFAULT_ANALYTIC_UNIT_OBJECT); - unitObj.detectorType = 'anomaly'; - const analyticUnitId = await addTestUnitToDB(unitObj); - await AnalyticUnitCache.remove(analyticUnitId); - const unit = await AnalyticUnit.findById(analyticUnitId); + const unit = await AnalyticUnit.findById(TEST_ANALYTIC_UNIT_ID); + unit.status = AnalyticUnit.AnalyticUnitStatus.LEARNING; const result = await getHSR(unit, 9000, 100000); expect(result).toEqual({"hsr": {"columns": [], "values": []}}); }); diff --git a/server/spec/setup_tests.ts b/server/spec/setup_tests.ts index cac4cef..e1712b1 100644 --- a/server/spec/setup_tests.ts +++ b/server/spec/setup_tests.ts @@ -1,31 +1,16 @@ import * as AnalyticUnit from '../src/models/analytic_units'; import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model'; -import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units'; +import { TEST_ANALYTIC_UNIT_ID, createTestDB } from './utils_for_tests/analytic_units'; import { clearSegmentsDB } from './utils_for_tests/segments'; console.log = jest.fn(); console.error = jest.fn(); jest.mock('../src/config.ts', () => ({ - HASTIC_API_KEY: 'fake-key', DATA_PATH: 'fake-data-path', + HASTIC_API_KEY: 'fake-key', ZMQ_IPC_PATH: 'fake-zmq-path' })); +clearSegmentsDB(); createTestDB(); - -async function createTestDB() { - await clearSegmentsDB(); - await AnalyticUnit.create( - AnalyticUnit.createAnalyticUnitFromObject({ - _id: TEST_ANALYTIC_UNIT_ID, - name: 'name', - grafanaUrl: 'grafanaUrl', - panelId: 'panelId', - type: 'type', - detectorType: AnalyticUnit.DetectorType.ANOMALY - }) - ); - await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID); - await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: 1 }); -} diff --git a/server/spec/utils_for_tests/analytic_units.ts b/server/spec/utils_for_tests/analytic_units.ts index bd91e78..45e4367 100644 --- a/server/spec/utils_for_tests/analytic_units.ts +++ b/server/spec/utils_for_tests/analytic_units.ts @@ -1,3 +1,85 @@ import * as AnalyticUnit from '../../src/models/analytic_units'; +import { Metric } from 'grafana-datasource-kit'; + +import * as _ from 'lodash'; +import * as AnalyticUnitCache from '../../src/models/analytic_unit_cache_model'; + + export const TEST_ANALYTIC_UNIT_ID: AnalyticUnit.AnalyticUnitId = 'testid'; + +const DEFAULT_DATASOURCE_STRUCTURE = { + url: "api/datasources/proxy/5/query", + data: null, + params: { + db:"dbname", + q: "SELECT mean(\"value\") FROM \"autogen\".\"tcpconns_value\" WHERE time >= now() - 6h GROUP BY time(20s) fill(null)", + epoch: "ms" + }, + type: "influxdb" +}; + +const DEFAULT_TARGETS_STRUCTURE = [ + { + groupBy: [ + { + params: ["$__interval"], + type: "time" + }, + { + params: ["null"], + type: "fill" + } + ], + measurement: "tcpconns_value", + orderByTime: "ASC", + policy: "autogen", + refId: "A", + resultFormat: "time_series", + select: [[{"params":["value"],"type":"field"},{"params":[],"type":"mean"}]],"tags":[] + } +]; + +export const DEFAULT_METRIC = new Metric( + DEFAULT_DATASOURCE_STRUCTURE, + DEFAULT_TARGETS_STRUCTURE +); + +export async function createTestDB(createCache = true) { + const analyticUnitObject = AnalyticUnitObject.getAnalyticUnitObject(); + const unit = AnalyticUnit.createAnalyticUnitFromObject(analyticUnitObject); + await AnalyticUnit.create(unit); + + if(createCache) { + await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID); + await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: 1 }); + } + return unit; +} + +export async function clearTestDB() { + await AnalyticUnit.remove(TEST_ANALYTIC_UNIT_ID); + await AnalyticUnitCache.remove(TEST_ANALYTIC_UNIT_ID); +} + +export class AnalyticUnitObject { + + constructor( + public _id: AnalyticUnit.AnalyticUnitId = TEST_ANALYTIC_UNIT_ID, + public name: string = 'name', + public grafanaUrl: string = 'grafanaUrl', + public panelId: string = 'panelId', + public type: string = 'type', + public metric: Metric = DEFAULT_METRIC, + public alert: boolean = false, + public labeledColor: string = '#FF99FF', + public deletedColor: string = '#00f0ff', + public detectorType: AnalyticUnit.DetectorType = AnalyticUnit.DetectorType.ANOMALY, + public visible: boolean = true, + public collapsed: boolean = false + ){}; + + static getAnalyticUnitObject(): AnalyticUnitObject { + return new AnalyticUnitObject(); + } +} diff --git a/server/src/config.ts b/server/src/config.ts index 37c67b7..9743846 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -11,6 +11,8 @@ let configExists = fs.existsSync(configFile); export const ANALYTICS_PATH = path.join(__dirname, '../../analytics'); +export const HASTIC_DB_IN_MEMORY = getConfigField('HASTIC_IN_MEMORY_PERSISTANCE', false); + export const DATA_PATH = path.join(__dirname, '../../data'); export const ANALYTIC_UNITS_DATABASE_PATH = path.join(DATA_PATH, 'analytic_units.db'); diff --git a/server/src/services/data_service.ts b/server/src/services/data_service.ts index da96880..3f72970 100644 --- a/server/src/services/data_service.ts +++ b/server/src/services/data_service.ts @@ -215,10 +215,12 @@ function checkDataFolders(): void { } checkDataFolders(); +const inMemoryOnly = config.HASTIC_DB_IN_MEMORY; + // TODO: it's better if models request db which we create if it`s needed -db.set(Collection.ANALYTIC_UNITS, new nedb({ filename: config.ANALYTIC_UNITS_DATABASE_PATH, autoload: true, timestampData: true })); -db.set(Collection.ANALYTIC_UNIT_CACHES, new nedb({ filename: config.ANALYTIC_UNIT_CACHES_DATABASE_PATH, autoload: true })); -db.set(Collection.SEGMENTS, new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true })); -db.set(Collection.THRESHOLD, new nedb({ filename: config.THRESHOLD_DATABASE_PATH, autoload: true })); -db.set(Collection.DETECTION_SPANS, new nedb({ filename: config.DETECTION_SPANS_DATABASE_PATH, autoload: true })); -db.set(Collection.DB_META, new nedb({ filename: config.DB_META_PATH, autoload: true })); +db.set(Collection.ANALYTIC_UNITS, new nedb({ filename: config.ANALYTIC_UNITS_DATABASE_PATH, autoload: true, timestampData: true, inMemoryOnly})); +db.set(Collection.ANALYTIC_UNIT_CACHES, new nedb({ filename: config.ANALYTIC_UNIT_CACHES_DATABASE_PATH, autoload: true, inMemoryOnly})); +db.set(Collection.SEGMENTS, new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true, inMemoryOnly})); +db.set(Collection.THRESHOLD, new nedb({ filename: config.THRESHOLD_DATABASE_PATH, autoload: true, inMemoryOnly})); +db.set(Collection.DETECTION_SPANS, new nedb({ filename: config.DETECTION_SPANS_DATABASE_PATH, autoload: true, inMemoryOnly})); +db.set(Collection.DB_META, new nedb({ filename: config.DB_META_PATH, autoload: true, inMemoryOnly}));