Class for analytic unit object structure #760 (#762)

* just save

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* Update server/spec/setup_tests.ts

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* Update server/spec/analytic_controller.jest.ts

Co-Authored-By: Alexey Velikiy <av@corpglory.com>

server/jest.config.js

@@ -19,3 +19,5 @@ module.exports = {
     "<rootDir>/spec/setup_tests.ts"
   ]
 };
+
+process.env.HASTIC_DB_IN_MEMORY = 'true';

server/spec/analytic_controller.jest.ts

@@ -9,88 +9,34 @@ jest.mock('grafana-datasource-kit', () => (
   }
 ));
 
-import { saveAnalyticUnitFromObject, runDetect, onDetect, getHSR } from '../src/controllers/analytics_controller';
+import { runDetect, onDetect, getHSR } from '../src/controllers/analytics_controller';
-import * as AnalyticUnit from '../src/models/analytic_units';
 import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model';
 import * as Segment from '../src/models/segment_model';
-import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units';
+import { TEST_ANALYTIC_UNIT_ID, createTestDB, clearTestDB, DEFAULT_METRIC } from './utils_for_tests/analytic_units';
 import { buildSegments, clearSegmentsDB, convertSegmentsToTimeRanges } from './utils_for_tests/segments';
-import { HASTIC_API_KEY } from '../src/config';
+import { HASTIC_API_KEY, GRAFANA_URL } from '../src/config';
 
 import * as _ from 'lodash';
-
+import * as AnalyticUnit from '../src/models/analytic_units';
-
-const DEFAULT_ANALYTIC_UNIT_OBJECT = {
-  name: "test",
-  grafanaUrl: "http://127.0.0.1:3000",
-  panelId: "ZLc0KfNZk/2",
-  type: "GENERAL",
-  metric: {
-    datasource: {
-      url: "api/datasources/proxy/5/query",
-      method: "GET",
-      data: null,
-      params: {
-        db:"dbname",
-        q: "SELECT mean(\"value\") FROM \"autogen\".\"tcpconns_value\" WHERE time >= now() - 6h GROUP BY time(20s) fill(null)",
-        epoch: "ms"
-      },
-      type: "influxdb"
-    },
-    targets: [
-      {
-        groupBy: [
-          {
-            params: ["$__interval"],
-            type: "time"
-          },
-          {
-            params: ["null"],
-            type: "fill"
-          }
-        ],
-        measurement: "tcpconns_value",
-        orderByTime: "ASC",
-        policy: "autogen",
-        refId: "A",
-        resultFormat: "time_series",
-        select: [[{"params":["value"],"type":"field"},{"params":[],"type":"mean"}]],"tags":[]
-      }
-    ]
-  },
-  alert: false,
-  labeledColor: "#FF99FF",
-  deletedColor: "#00f0ff",
-  detectorType: "pattern",
-  visible: true,
-  collapsed: false,
-  createdAt: {"$$date":1564476040880},
-  updatedAt: {"$$date":1564476040880}
-}
 
 const WINDOW_SIZE = 10;
 const TIME_STEP = 1000;
 
-async function addTestUnitToDB(analyticUnitObj: any): Promise<string> {
+beforeEach(async () => {
-  const analyticUnitId = await saveAnalyticUnitFromObject(analyticUnitObj);
+  await clearTestDB();
-  await AnalyticUnit.update(analyticUnitId, { lastDetectionTime: 1000 });
+  await createTestDB();
-  await AnalyticUnitCache.create(analyticUnitId);
-  await AnalyticUnitCache.setData(analyticUnitId, {
-    windowSize: WINDOW_SIZE,
-    timeStep: TIME_STEP
 });
-  return analyticUnitId;
-};
 
 describe('Check detection range', function() {
-  it('check range >= 2 * window size * timeStep', async () => {
+  it('range should be >= 2 * windowSize * timeStep', async () => {
     const from = 1500000000000;
     const to = 1500000000001;
     const expectedFrom = to - WINDOW_SIZE * TIME_STEP * 2;
 
-    const id = await addTestUnitToDB(DEFAULT_ANALYTIC_UNIT_OBJECT);
+    await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: TIME_STEP, windowSize: WINDOW_SIZE });
-    await runDetect(id, from, to);
+    console.log(await AnalyticUnitCache.findById(TEST_ANALYTIC_UNIT_ID));
-    expect(queryByMetric).toBeCalledWith(DEFAULT_ANALYTIC_UNIT_OBJECT.metric, undefined, expectedFrom, to, HASTIC_API_KEY);
+    await runDetect(TEST_ANALYTIC_UNIT_ID, from, to);
+    expect(queryByMetric).toBeCalledWith(DEFAULT_METRIC, GRAFANA_URL, expectedFrom, to, HASTIC_API_KEY);
   });
 });
 
@@ -130,6 +76,7 @@ describe('onDetect', () => {
       segments: buildSegments([[7, 8]]),
       lastDetectionTime: 0
     });
+
     const detectedSegments = await Promise.all(
       detectedSegmentIds.map(id => Segment.findOne(id))
     );
@@ -140,12 +87,21 @@ describe('onDetect', () => {
 });
 
 describe('getHSR', function() {
+  let cacheToSave: AnalyticUnitCache.AnalyticUnitCache;
+
+  beforeAll(async () => {
+    await clearTestDB();
+    await createTestDB(false);
+  });
+
+  afterAll(async () => {
+    await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID);
+    await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, cacheToSave.data);
+  });
+
   it('should return nothing if unit state is LEARNING', async () => {
-    let unitObj = _.clone(DEFAULT_ANALYTIC_UNIT_OBJECT);
+    const unit = await AnalyticUnit.findById(TEST_ANALYTIC_UNIT_ID);
-    unitObj.detectorType = 'anomaly';
+    unit.status = AnalyticUnit.AnalyticUnitStatus.LEARNING;
-    const analyticUnitId = await addTestUnitToDB(unitObj);
-    await AnalyticUnitCache.remove(analyticUnitId);
-    const unit = await AnalyticUnit.findById(analyticUnitId);
     const result = await getHSR(unit, 9000, 100000);
     expect(result).toEqual({"hsr": {"columns": [], "values": []}});
   });

server/spec/setup_tests.ts

@@ -1,31 +1,16 @@
 import * as AnalyticUnit from '../src/models/analytic_units';
 import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model';
-import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units';
+import { TEST_ANALYTIC_UNIT_ID, createTestDB } from './utils_for_tests/analytic_units';
 import { clearSegmentsDB } from './utils_for_tests/segments';
 
 console.log = jest.fn();
 console.error = jest.fn();
 
 jest.mock('../src/config.ts', () => ({
-  HASTIC_API_KEY: 'fake-key',
   DATA_PATH: 'fake-data-path',
+  HASTIC_API_KEY: 'fake-key',
   ZMQ_IPC_PATH: 'fake-zmq-path'
 }));
 
+clearSegmentsDB();
 createTestDB();
-
-async function createTestDB() {
-  await clearSegmentsDB();
-  await AnalyticUnit.create(
-    AnalyticUnit.createAnalyticUnitFromObject({
-      _id: TEST_ANALYTIC_UNIT_ID,
-      name: 'name',
-      grafanaUrl: 'grafanaUrl',
-      panelId: 'panelId',
-      type: 'type',
-      detectorType: AnalyticUnit.DetectorType.ANOMALY
-    })
-  );
-  await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID);
-  await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: 1 });
-}

server/spec/utils_for_tests/analytic_units.ts

@@ -1,3 +1,85 @@
 import * as AnalyticUnit from '../../src/models/analytic_units';
 
+import { Metric } from 'grafana-datasource-kit';
+
+import * as _ from 'lodash';
+import * as AnalyticUnitCache from '../../src/models/analytic_unit_cache_model';
+
+
 export const TEST_ANALYTIC_UNIT_ID: AnalyticUnit.AnalyticUnitId = 'testid';
+
+const DEFAULT_DATASOURCE_STRUCTURE = {
+  url: "api/datasources/proxy/5/query",
+  data: null,
+  params: {
+    db:"dbname",
+    q: "SELECT mean(\"value\") FROM \"autogen\".\"tcpconns_value\" WHERE time >= now() - 6h GROUP BY time(20s) fill(null)",
+    epoch: "ms"
+  },
+  type: "influxdb"
+};
+
+const DEFAULT_TARGETS_STRUCTURE = [
+  {
+    groupBy: [
+      {
+        params: ["$__interval"],
+        type: "time"
+      },
+      {
+        params: ["null"],
+        type: "fill"
+      }
+    ],
+    measurement: "tcpconns_value",
+    orderByTime: "ASC",
+    policy: "autogen",
+    refId: "A",
+    resultFormat: "time_series",
+    select: [[{"params":["value"],"type":"field"},{"params":[],"type":"mean"}]],"tags":[]
+  }
+];
+
+export const DEFAULT_METRIC = new Metric(
+  DEFAULT_DATASOURCE_STRUCTURE,
+  DEFAULT_TARGETS_STRUCTURE
+);
+
+export async function createTestDB(createCache = true) {
+  const analyticUnitObject = AnalyticUnitObject.getAnalyticUnitObject();
+  const unit = AnalyticUnit.createAnalyticUnitFromObject(analyticUnitObject);
+  await AnalyticUnit.create(unit);
+
+  if(createCache) {
+    await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID);
+    await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: 1 });
+  }
+  return unit;
+}
+
+export async function clearTestDB() {
+  await AnalyticUnit.remove(TEST_ANALYTIC_UNIT_ID);
+  await AnalyticUnitCache.remove(TEST_ANALYTIC_UNIT_ID);
+}
+
+export class AnalyticUnitObject {
+
+  constructor(
+    public _id: AnalyticUnit.AnalyticUnitId = TEST_ANALYTIC_UNIT_ID,
+    public name: string = 'name',
+    public grafanaUrl: string = 'grafanaUrl',
+    public panelId: string = 'panelId',
+    public type: string = 'type',
+    public metric: Metric = DEFAULT_METRIC,
+    public alert: boolean = false,
+    public labeledColor: string = '#FF99FF',
+    public deletedColor: string = '#00f0ff',
+    public detectorType: AnalyticUnit.DetectorType = AnalyticUnit.DetectorType.ANOMALY,
+    public visible: boolean = true,
+    public collapsed: boolean = false
+  ){};
+
+  static getAnalyticUnitObject(): AnalyticUnitObject {
+    return new AnalyticUnitObject();
+  }
+}

server/src/config.ts

@@ -11,6 +11,8 @@ let configExists = fs.existsSync(configFile);
 
 export const ANALYTICS_PATH = path.join(__dirname, '../../analytics');
 
+export const HASTIC_DB_IN_MEMORY = getConfigField('HASTIC_IN_MEMORY_PERSISTANCE', false);
+
 export const DATA_PATH = path.join(__dirname, '../../data');
 
 export const ANALYTIC_UNITS_DATABASE_PATH = path.join(DATA_PATH, 'analytic_units.db');

server/src/services/data_service.ts

@@ -215,10 +215,12 @@ function checkDataFolders(): void {
 }
 checkDataFolders();
 
+const inMemoryOnly = config.HASTIC_DB_IN_MEMORY;
+
 // TODO: it's better if models request db which we create if it`s needed
-db.set(Collection.ANALYTIC_UNITS, new nedb({ filename: config.ANALYTIC_UNITS_DATABASE_PATH, autoload: true, timestampData: true }));
+db.set(Collection.ANALYTIC_UNITS, new nedb({ filename: config.ANALYTIC_UNITS_DATABASE_PATH, autoload: true, timestampData: true, inMemoryOnly}));
-db.set(Collection.ANALYTIC_UNIT_CACHES, new nedb({ filename: config.ANALYTIC_UNIT_CACHES_DATABASE_PATH, autoload: true }));
+db.set(Collection.ANALYTIC_UNIT_CACHES, new nedb({ filename: config.ANALYTIC_UNIT_CACHES_DATABASE_PATH, autoload: true, inMemoryOnly}));
-db.set(Collection.SEGMENTS, new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true }));
+db.set(Collection.SEGMENTS, new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true, inMemoryOnly}));
-db.set(Collection.THRESHOLD, new nedb({ filename: config.THRESHOLD_DATABASE_PATH, autoload: true }));
+db.set(Collection.THRESHOLD, new nedb({ filename: config.THRESHOLD_DATABASE_PATH, autoload: true, inMemoryOnly}));
-db.set(Collection.DETECTION_SPANS, new nedb({ filename: config.DETECTION_SPANS_DATABASE_PATH, autoload: true }));
+db.set(Collection.DETECTION_SPANS, new nedb({ filename: config.DETECTION_SPANS_DATABASE_PATH, autoload: true, inMemoryOnly}));
-db.set(Collection.DB_META, new nedb({ filename: config.DB_META_PATH, autoload: true }));
+db.set(Collection.DB_META, new nedb({ filename: config.DB_META_PATH, autoload: true, inMemoryOnly}));