From ac626d726ed5c4cf2cc44b5905719a8534df1c88 Mon Sep 17 00:00:00 2001 From: rozetko Date: Wed, 14 Aug 2019 18:10:34 +0300 Subject: [PATCH] Large segments are not shown #744 (#745) --- server/spec/analytic_controller.jest.ts | 35 ++++-------- server/spec/models/segment_model.jest.ts | 54 +++++++++++++++++++ server/spec/segments.jest.ts | 23 ++------ server/spec/setup_tests.ts | 23 ++++++++ server/spec/utils_for_tests/analytic_units.ts | 3 ++ server/spec/utils_for_tests/segments.ts | 11 ++-- server/src/models/segment_model.ts | 22 ++++++++ server/src/routes/segments_router.ts | 17 +++--- 8 files changed, 131 insertions(+), 57 deletions(-) create mode 100644 server/spec/models/segment_model.jest.ts create mode 100644 server/spec/utils_for_tests/analytic_units.ts diff --git a/server/spec/analytic_controller.jest.ts b/server/spec/analytic_controller.jest.ts index 9148e2b..d4d6a1a 100644 --- a/server/spec/analytic_controller.jest.ts +++ b/server/spec/analytic_controller.jest.ts @@ -11,7 +11,8 @@ import { saveAnalyticUnitFromObject, runDetect, onDetect } from '../src/controll import * as AnalyticUnit from '../src/models/analytic_units'; import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model'; import * as Segment from '../src/models/segment_model'; -import { buildSegments, clearDB, TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/segments'; +import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units'; +import { buildSegments, clearSegmentsDB, convertSegmentsToTimeRanges } from './utils_for_tests/segments'; import { HASTIC_API_KEY } from '../src/config'; @@ -94,28 +95,12 @@ describe('Check detection range', function() { describe('onDetect', () => { const INITIAL_SEGMENTS = buildSegments([[0, 1], [2, 3], [4, 5]]); - beforeAll(async () => { - clearDB(); - await AnalyticUnit.create( - AnalyticUnit.createAnalyticUnitFromObject({ - _id: TEST_ANALYTIC_UNIT_ID, - name: 'name', - grafanaUrl: 'grafanaUrl', - panelId: 'panelId', - type: 'type', - detectorType: AnalyticUnit.DetectorType.ANOMALY - }) - ); - await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID); - await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: 1 }); - }); - beforeEach(async () => { await Segment.mergeAndInsertSegments(INITIAL_SEGMENTS); }); afterEach(async () => { - clearDB(); + await clearSegmentsDB(); }); it('should not send a webhook after merging', async () => { @@ -129,13 +114,12 @@ describe('onDetect', () => { } } }); - let detectedSegments = await Promise.all( + const detectedSegments = await Promise.all( detectedSegmentIds.map(id => Segment.findOne(id)) ); - expect( - detectedSegments.map(segment => [segment.from, segment.to]) - ).toEqual([]); + const detectedRanges = convertSegmentsToTimeRanges(detectedSegments); + expect(detectedRanges).toEqual([]); }); it('should send a webhook when there was no merging', async () => { @@ -144,12 +128,11 @@ describe('onDetect', () => { segments: buildSegments([[7, 8]]), lastDetectionTime: 0 }); - let detectedSegments = await Promise.all( + const detectedSegments = await Promise.all( detectedSegmentIds.map(id => Segment.findOne(id)) ); - expect( - detectedSegments.map(segment => [segment.from, segment.to]) - ).toEqual([[7, 8]]); + const detectedRanges = convertSegmentsToTimeRanges(detectedSegments); + expect(detectedRanges).toEqual([[7, 8]]); }); }); diff --git a/server/spec/models/segment_model.jest.ts b/server/spec/models/segment_model.jest.ts new file mode 100644 index 0000000..71da58b --- /dev/null +++ b/server/spec/models/segment_model.jest.ts @@ -0,0 +1,54 @@ +import { TEST_ANALYTIC_UNIT_ID } from '../utils_for_tests/analytic_units'; +import { buildSegments, clearSegmentsDB, convertSegmentsToTimeRanges } from '../utils_for_tests/segments'; + +import * as Segment from '../../src/models/segment_model'; + +afterEach(async () => { + await clearSegmentsDB(); +}); + +describe('mergeAndInsertSegments', function() { + const initialSegments = buildSegments([[0, 1], [2, 3], [4, 5]]); + + beforeEach(async () => { + await Segment.mergeAndInsertSegments(initialSegments); + }); + + it('Segments should be merged before insertion', async function() { + const segmentsToInsert = buildSegments([[1, 2]]); + await Segment.mergeAndInsertSegments(segmentsToInsert); + + const actualSegments = await Segment.findMany(TEST_ANALYTIC_UNIT_ID, {}); + const actualRanges = convertSegmentsToTimeRanges(actualSegments); + + expect(actualRanges).toEqual([[0, 3], [4, 5]]); + }); +}); + +describe('findIntersectedSegments', () => { + const initialSegments = buildSegments([[0, 3], [5, 6], [10, 13]]); + + beforeEach(async () => { + await Segment.mergeAndInsertSegments(initialSegments); + }); + + it('should find intersected segments', async () => { + const testCases = [ + { from: 1, to: 4, expected: [[0, 3]] }, + { from: 11, to: 12, expected: [[10, 13]] }, + { from: 6, to: 10, expected: [[5, 6], [10, 13]] }, + { from: 16, to: 17, expected: [] }, + { from: 5, expected: [[5, 6], [10, 13]] }, + { to: 5, expected: [[0, 3], [5, 6]] }, + { expected: [[0, 3], [5, 6], [10, 13]] } + ]; + + for(let testCase of testCases) { + const foundSegments = await Segment.findIntersectedSegments( + TEST_ANALYTIC_UNIT_ID, testCase.from, testCase.to + ); + const foundRanges = convertSegmentsToTimeRanges(foundSegments); + expect(foundRanges).toEqual(testCase.expected); + } + }); +}); diff --git a/server/spec/segments.jest.ts b/server/spec/segments.jest.ts index d2825ed..480be96 100644 --- a/server/spec/segments.jest.ts +++ b/server/spec/segments.jest.ts @@ -1,35 +1,18 @@ -import { buildSegments, clearDB, TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/segments'; +import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units'; +import { buildSegments, clearSegmentsDB, convertSegmentsToTimeRanges } from './utils_for_tests/segments'; -import * as AnalyticUnit from '../src/models/analytic_units'; import * as Segment from '../src/models/segment_model'; -import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model'; import * as _ from 'lodash'; const INITIAL_SEGMENTS = buildSegments([[0, 1], [2, 3], [4, 5]]); -beforeAll(async () => { - clearDB(); - await AnalyticUnit.create( - AnalyticUnit.createAnalyticUnitFromObject({ - _id: TEST_ANALYTIC_UNIT_ID, - name: 'name', - grafanaUrl: 'grafanaUrl', - panelId: 'panelId', - type: 'type', - detectorType: AnalyticUnit.DetectorType.ANOMALY - }) - ); - await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID); - await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: 1 }); -}); - beforeEach(async () => { await Segment.mergeAndInsertSegments(INITIAL_SEGMENTS); }); afterEach(async () => { - clearDB(); + await clearSegmentsDB(); }); describe('mergeAndInsertSegments', function() { diff --git a/server/spec/setup_tests.ts b/server/spec/setup_tests.ts index f190669..cac4cef 100644 --- a/server/spec/setup_tests.ts +++ b/server/spec/setup_tests.ts @@ -1,3 +1,8 @@ +import * as AnalyticUnit from '../src/models/analytic_units'; +import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model'; +import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units'; +import { clearSegmentsDB } from './utils_for_tests/segments'; + console.log = jest.fn(); console.error = jest.fn(); @@ -6,3 +11,21 @@ jest.mock('../src/config.ts', () => ({ DATA_PATH: 'fake-data-path', ZMQ_IPC_PATH: 'fake-zmq-path' })); + +createTestDB(); + +async function createTestDB() { + await clearSegmentsDB(); + await AnalyticUnit.create( + AnalyticUnit.createAnalyticUnitFromObject({ + _id: TEST_ANALYTIC_UNIT_ID, + name: 'name', + grafanaUrl: 'grafanaUrl', + panelId: 'panelId', + type: 'type', + detectorType: AnalyticUnit.DetectorType.ANOMALY + }) + ); + await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID); + await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: 1 }); +} diff --git a/server/spec/utils_for_tests/analytic_units.ts b/server/spec/utils_for_tests/analytic_units.ts new file mode 100644 index 0000000..bd91e78 --- /dev/null +++ b/server/spec/utils_for_tests/analytic_units.ts @@ -0,0 +1,3 @@ +import * as AnalyticUnit from '../../src/models/analytic_units'; + +export const TEST_ANALYTIC_UNIT_ID: AnalyticUnit.AnalyticUnitId = 'testid'; diff --git a/server/spec/utils_for_tests/segments.ts b/server/spec/utils_for_tests/segments.ts index bc117a5..87bca00 100644 --- a/server/spec/utils_for_tests/segments.ts +++ b/server/spec/utils_for_tests/segments.ts @@ -1,7 +1,7 @@ -import * as AnalyticUnit from '../../src/models/analytic_units'; +import { TEST_ANALYTIC_UNIT_ID } from './analytic_units'; import * as Segment from '../../src/models/segment_model'; -export const TEST_ANALYTIC_UNIT_ID: AnalyticUnit.AnalyticUnitId = 'testid'; +import * as _ from 'lodash'; export function buildSegments(times: number[][]): Segment.Segment[] { return times.map(t => { @@ -9,7 +9,12 @@ export function buildSegments(times: number[][]): Segment.Segment[] { }); } -export async function clearDB(): Promise { +export function convertSegmentsToTimeRanges(segments: Segment.Segment[]): number[][] { + const ranges = segments.map(segment => [segment.from, segment.to]); + return _.sortBy(ranges, range => range[0]); +} + +export async function clearSegmentsDB(): Promise { const segments = await Segment.findMany(TEST_ANALYTIC_UNIT_ID, { labeled: false, deleted: false }); await Segment.removeSegments(segments.map(s => s.id)); } diff --git a/server/src/models/segment_model.ts b/server/src/models/segment_model.ts index 4f3aba6..3edc9a3 100644 --- a/server/src/models/segment_model.ts +++ b/server/src/models/segment_model.ts @@ -106,6 +106,28 @@ export async function findMany(id: AnalyticUnitId, query: FindManyQuery): Promis return segs.map(Segment.fromObject); } + +/** + * If `from` and `to` are defined: @returns segments intersected with `[from; to]` + * If `to` is `undefined`: @returns segments intersected with `[-inf; from]` + * If `from` is `undefined`: @returns segments intersected with `[to: +inf]` + * If `from` and `to` are undefined: @returns all segments + */ +export async function findIntersectedSegments( + analyticUnitId: AnalyticUnit.AnalyticUnitId, + from?: number, + to?: number +): Promise { + let query: FindManyQuery = {}; + if(from !== undefined) { + query.to = { $gte: from }; + } + if(to !== undefined) { + query.from = { $lte: to }; + } + return findMany(analyticUnitId, query); +} + /** * Merges an array of segments with ones existing in the DB * Inserts resulting segments into DB diff --git a/server/src/routes/segments_router.ts b/server/src/routes/segments_router.ts index fb74280..d3e7110 100644 --- a/server/src/routes/segments_router.ts +++ b/server/src/routes/segments_router.ts @@ -6,20 +6,21 @@ import * as Segment from '../models/segment_model'; import * as Router from 'koa-router'; -async function getSegments(ctx: Router.IRouterContext) { +export async function getSegments(ctx: Router.IRouterContext) { let id: AnalyticUnitId = ctx.request.query.id; if(id === undefined || id === '') { throw new Error('analyticUnitId (id) is missing'); } - let query: Segment.FindManyQuery = {}; - - if(!isNaN(+ctx.request.query.from)) { - query.from = { $gte: +ctx.request.query.from }; + let from = +ctx.request.query.from; + if(isNaN(from)) { + from = undefined; } - if(!isNaN(+ctx.request.query.to)) { - query.to = { $lte: +ctx.request.query.to }; + let to = +ctx.request.query.to; + if(isNaN(to)) { + to = undefined; } - let segments = await Segment.findMany(id, query); + + const segments = await Segment.findIntersectedSegments(id, from, to); ctx.response.body = { segments }; }