hastic
/
hastic-server
4
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity
Browse Source

Class for analytic unit object structure #760 (#762) 

* just save

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* Update server/spec/setup_tests.ts

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* Update server/spec/analytic_controller.jest.ts

Co-Authored-By: Alexey Velikiy <av@corpglory.com>
pull/1/head
Evgeny Smyshlyaev 5 years ago committed by Alexey Velikiy
parent
d469c1e98c
commit
a42a5b85a7
6 changed files with 125 additions and 96 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      server/jest.config.js
  2. 98
      server/spec/analytic_controller.jest.ts
  3. 21
      server/spec/setup_tests.ts
  4. 82
      server/spec/utils_for_tests/analytic_units.ts
  5. 2
      server/src/config.ts
  6. 14
      server/src/services/data_service.ts

2
server/jest.config.js
Unescape View File

@ -19,3 +19,5 @@ module.exports = {
"<rootDir>/spec/setup_tests.ts"
]
};
process.env.HASTIC_DB_IN_MEMORY = 'true';

98
server/spec/analytic_controller.jest.ts
Unescape View File

@ -9,88 +9,34 @@ jest.mock('grafana-datasource-kit', () => (
}
));
import { saveAnalyticUnitFromObject, runDetect, onDetect, getHSR } from '../src/controllers/analytics_controller';
import * as AnalyticUnit from '../src/models/analytic_units';
import { runDetect, onDetect, getHSR } from '../src/controllers/analytics_controller';
import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model';
import * as Segment from '../src/models/segment_model';
import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units';
import { TEST_ANALYTIC_UNIT_ID, createTestDB, clearTestDB, DEFAULT_METRIC } from './utils_for_tests/analytic_units';
import { buildSegments, clearSegmentsDB, convertSegmentsToTimeRanges } from './utils_for_tests/segments';
import { HASTIC_API_KEY } from '../src/config';
import { HASTIC_API_KEY, GRAFANA_URL } from '../src/config';
import * as _ from 'lodash';
const DEFAULT_ANALYTIC_UNIT_OBJECT = {
name: "test",
grafanaUrl: "http://127.0.0.1:3000",
panelId: "ZLc0KfNZk/2",
type: "GENERAL",
metric: {
datasource: {
url: "api/datasources/proxy/5/query",
method: "GET",
data: null,
params: {
db:"dbname",
q: "SELECT mean(\"value\") FROM \"autogen\".\"tcpconns_value\" WHERE time >= now() - 6h GROUP BY time(20s) fill(null)",
epoch: "ms"
},
type: "influxdb"
},
targets: [
{
groupBy: [
{
params: ["$__interval"],
type: "time"
},
{
params: ["null"],
type: "fill"
}
],
measurement: "tcpconns_value",
orderByTime: "ASC",
policy: "autogen",
refId: "A",
resultFormat: "time_series",
select: [[{"params":["value"],"type":"field"},{"params":[],"type":"mean"}]],"tags":[]
}
]
},
alert: false,
labeledColor: "#FF99FF",
deletedColor: "#00f0ff",
detectorType: "pattern",
visible: true,
collapsed: false,
createdAt: {"$$date":1564476040880},
updatedAt: {"$$date":1564476040880}
}
import * as AnalyticUnit from '../src/models/analytic_units';
const WINDOW_SIZE = 10;
const TIME_STEP = 1000;
async function addTestUnitToDB(analyticUnitObj: any): Promise<string> {
const analyticUnitId = await saveAnalyticUnitFromObject(analyticUnitObj);
await AnalyticUnit.update(analyticUnitId, { lastDetectionTime: 1000 });
await AnalyticUnitCache.create(analyticUnitId);
await AnalyticUnitCache.setData(analyticUnitId, {
windowSize: WINDOW_SIZE,
timeStep: TIME_STEP
beforeEach(async () => {
await clearTestDB();
await createTestDB();
});
return analyticUnitId;
};
describe('Check detection range', function() {
it('check range >= 2 * window size * timeStep', async () => {
it('range should be >= 2 * windowSize * timeStep', async () => {
const from = 1500000000000;
const to = 1500000000001;
const expectedFrom = to - WINDOW_SIZE * TIME_STEP * 2;
const id = await addTestUnitToDB(DEFAULT_ANALYTIC_UNIT_OBJECT);
await runDetect(id, from, to);
expect(queryByMetric).toBeCalledWith(DEFAULT_ANALYTIC_UNIT_OBJECT.metric, undefined, expectedFrom, to, HASTIC_API_KEY);
await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: TIME_STEP, windowSize: WINDOW_SIZE });
console.log(await AnalyticUnitCache.findById(TEST_ANALYTIC_UNIT_ID));
await runDetect(TEST_ANALYTIC_UNIT_ID, from, to);
expect(queryByMetric).toBeCalledWith(DEFAULT_METRIC, GRAFANA_URL, expectedFrom, to, HASTIC_API_KEY);
});
});
@ -130,6 +76,7 @@ describe('onDetect', () => {
segments: buildSegments([[7, 8]]),
lastDetectionTime: 0
});
const detectedSegments = await Promise.all(
detectedSegmentIds.map(id => Segment.findOne(id))
);
@ -140,12 +87,21 @@ describe('onDetect', () => {
});
describe('getHSR', function() {
let cacheToSave: AnalyticUnitCache.AnalyticUnitCache;
beforeAll(async () => {
await clearTestDB();
await createTestDB(false);
});
afterAll(async () => {
await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID);
await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, cacheToSave.data);
});
it('should return nothing if unit state is LEARNING', async () => {
let unitObj = _.clone(DEFAULT_ANALYTIC_UNIT_OBJECT);
unitObj.detectorType = 'anomaly';
const analyticUnitId = await addTestUnitToDB(unitObj);
await AnalyticUnitCache.remove(analyticUnitId);
const unit = await AnalyticUnit.findById(analyticUnitId);
const unit = await AnalyticUnit.findById(TEST_ANALYTIC_UNIT_ID);
unit.status = AnalyticUnit.AnalyticUnitStatus.LEARNING;
const result = await getHSR(unit, 9000, 100000);
expect(result).toEqual({"hsr": {"columns": [], "values": []}});
});

21
server/spec/setup_tests.ts
Unescape View File

@ -1,31 +1,16 @@
import * as AnalyticUnit from '../src/models/analytic_units';
import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model';
import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units';
import { TEST_ANALYTIC_UNIT_ID, createTestDB } from './utils_for_tests/analytic_units';
import { clearSegmentsDB } from './utils_for_tests/segments';
console.log = jest.fn();
console.error = jest.fn();
jest.mock('../src/config.ts', () => ({
HASTIC_API_KEY: 'fake-key',
DATA_PATH: 'fake-data-path',
HASTIC_API_KEY: 'fake-key',
ZMQ_IPC_PATH: 'fake-zmq-path'
}));
clearSegmentsDB();
createTestDB();
async function createTestDB() {
await clearSegmentsDB();
await AnalyticUnit.create(
AnalyticUnit.createAnalyticUnitFromObject({
_id: TEST_ANALYTIC_UNIT_ID,
name: 'name',
grafanaUrl: 'grafanaUrl',
panelId: 'panelId',
type: 'type',
detectorType: AnalyticUnit.DetectorType.ANOMALY
})
);
await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID);
await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: 1 });
}

82
server/spec/utils_for_tests/analytic_units.ts
Unescape View File

@ -1,3 +1,85 @@
import * as AnalyticUnit from '../../src/models/analytic_units';
import { Metric } from 'grafana-datasource-kit';
import * as _ from 'lodash';
import * as AnalyticUnitCache from '../../src/models/analytic_unit_cache_model';
export const TEST_ANALYTIC_UNIT_ID: AnalyticUnit.AnalyticUnitId = 'testid';
const DEFAULT_DATASOURCE_STRUCTURE = {
url: "api/datasources/proxy/5/query",
data: null,
params: {
db:"dbname",
q: "SELECT mean(\"value\") FROM \"autogen\".\"tcpconns_value\" WHERE time >= now() - 6h GROUP BY time(20s) fill(null)",
epoch: "ms"
},
type: "influxdb"
};
const DEFAULT_TARGETS_STRUCTURE = [
{
groupBy: [
{
params: ["$__interval"],
type: "time"
},
{
params: ["null"],
type: "fill"
}
],
measurement: "tcpconns_value",
orderByTime: "ASC",
policy: "autogen",
refId: "A",
resultFormat: "time_series",
select: [[{"params":["value"],"type":"field"},{"params":[],"type":"mean"}]],"tags":[]
}
];
export const DEFAULT_METRIC = new Metric(
DEFAULT_DATASOURCE_STRUCTURE,
DEFAULT_TARGETS_STRUCTURE
);
export async function createTestDB(createCache = true) {
const analyticUnitObject = AnalyticUnitObject.getAnalyticUnitObject();
const unit = AnalyticUnit.createAnalyticUnitFromObject(analyticUnitObject);
await AnalyticUnit.create(unit);
if(createCache) {
await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID);
await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: 1 });
}
return unit;
}
export async function clearTestDB() {
await AnalyticUnit.remove(TEST_ANALYTIC_UNIT_ID);
await AnalyticUnitCache.remove(TEST_ANALYTIC_UNIT_ID);
}
export class AnalyticUnitObject {
constructor(
public _id: AnalyticUnit.AnalyticUnitId = TEST_ANALYTIC_UNIT_ID,
public name: string = 'name',
public grafanaUrl: string = 'grafanaUrl',
public panelId: string = 'panelId',
public type: string = 'type',
public metric: Metric = DEFAULT_METRIC,
public alert: boolean = false,
public labeledColor: string = '#FF99FF',
public deletedColor: string = '#00f0ff',
public detectorType: AnalyticUnit.DetectorType = AnalyticUnit.DetectorType.ANOMALY,
public visible: boolean = true,
public collapsed: boolean = false
){};
static getAnalyticUnitObject(): AnalyticUnitObject {
return new AnalyticUnitObject();
}
}

2
server/src/config.ts
Unescape View File

@ -11,6 +11,8 @@ let configExists = fs.existsSync(configFile);
export const ANALYTICS_PATH = path.join(__dirname, '../../analytics');
export const HASTIC_DB_IN_MEMORY = getConfigField('HASTIC_IN_MEMORY_PERSISTANCE', false);
export const DATA_PATH = path.join(__dirname, '../../data');
export const ANALYTIC_UNITS_DATABASE_PATH = path.join(DATA_PATH, 'analytic_units.db');

14
server/src/services/data_service.ts
Unescape View File

@ -215,10 +215,12 @@ function checkDataFolders(): void {
}
checkDataFolders();
const inMemoryOnly = config.HASTIC_DB_IN_MEMORY;
// TODO: it's better if models request db which we create if it`s needed
db.set(Collection.ANALYTIC_UNITS, new nedb({ filename: config.ANALYTIC_UNITS_DATABASE_PATH, autoload: true, timestampData: true }));
db.set(Collection.ANALYTIC_UNIT_CACHES, new nedb({ filename: config.ANALYTIC_UNIT_CACHES_DATABASE_PATH, autoload: true }));
db.set(Collection.SEGMENTS, new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true }));
db.set(Collection.THRESHOLD, new nedb({ filename: config.THRESHOLD_DATABASE_PATH, autoload: true }));
db.set(Collection.DETECTION_SPANS, new nedb({ filename: config.DETECTION_SPANS_DATABASE_PATH, autoload: true }));
db.set(Collection.DB_META, new nedb({ filename: config.DB_META_PATH, autoload: true }));
db.set(Collection.ANALYTIC_UNITS, new nedb({ filename: config.ANALYTIC_UNITS_DATABASE_PATH, autoload: true, timestampData: true, inMemoryOnly}));
db.set(Collection.ANALYTIC_UNIT_CACHES, new nedb({ filename: config.ANALYTIC_UNIT_CACHES_DATABASE_PATH, autoload: true, inMemoryOnly}));
db.set(Collection.SEGMENTS, new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true, inMemoryOnly}));
db.set(Collection.THRESHOLD, new nedb({ filename: config.THRESHOLD_DATABASE_PATH, autoload: true, inMemoryOnly}));
db.set(Collection.DETECTION_SPANS, new nedb({ filename: config.DETECTION_SPANS_DATABASE_PATH, autoload: true, inMemoryOnly}));
db.set(Collection.DB_META, new nedb({ filename: config.DB_META_PATH, autoload: true, inMemoryOnly}));

Write Preview
Loading…
Cancel
Save