Large segments are not shown #744 (#745)

server/spec/analytic_controller.jest.ts

@@ -11,7 +11,8 @@ import { saveAnalyticUnitFromObject, runDetect, onDetect } from '../src/controll
 import * as AnalyticUnit from '../src/models/analytic_units';
 import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model';
 import * as Segment from '../src/models/segment_model';
-import { buildSegments, clearDB, TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/segments';
+import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units';
+import { buildSegments, clearSegmentsDB, convertSegmentsToTimeRanges } from './utils_for_tests/segments';
 
 import { HASTIC_API_KEY } from '../src/config';
 
@@ -94,28 +95,12 @@ describe('Check detection range', function() {
 describe('onDetect', () => {
   const INITIAL_SEGMENTS = buildSegments([[0, 1], [2, 3], [4, 5]]);
 
-  beforeAll(async () => {
-    clearDB();
-    await AnalyticUnit.create(
-      AnalyticUnit.createAnalyticUnitFromObject({
-        _id: TEST_ANALYTIC_UNIT_ID,
-        name: 'name',
-        grafanaUrl: 'grafanaUrl',
-        panelId: 'panelId',
-        type: 'type',
-        detectorType: AnalyticUnit.DetectorType.ANOMALY
-      })
-    );
-    await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID);
-    await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: 1 });
-  });
-
   beforeEach(async () => {
     await Segment.mergeAndInsertSegments(INITIAL_SEGMENTS);
   });
 
   afterEach(async () => {
-    clearDB();
+    await clearSegmentsDB();
   });
 
   it('should not send a webhook after merging', async () => {
@@ -129,13 +114,12 @@ describe('onDetect', () => {
         }
       }
     });
-    let detectedSegments = await Promise.all(
+    const detectedSegments = await Promise.all(
       detectedSegmentIds.map(id => Segment.findOne(id))
     );
 
-    expect(
-      detectedSegments.map(segment => [segment.from, segment.to])
-    ).toEqual([]);
+    const detectedRanges = convertSegmentsToTimeRanges(detectedSegments);
+    expect(detectedRanges).toEqual([]);
   });
 
   it('should send a webhook when there was no merging', async () => {
@@ -144,12 +128,11 @@ describe('onDetect', () => {
       segments: buildSegments([[7, 8]]),
       lastDetectionTime: 0
     });
-    let detectedSegments = await Promise.all(
+    const detectedSegments = await Promise.all(
       detectedSegmentIds.map(id => Segment.findOne(id))
     );
 
-    expect(
-      detectedSegments.map(segment => [segment.from, segment.to])
-    ).toEqual([[7, 8]]);
+    const detectedRanges = convertSegmentsToTimeRanges(detectedSegments);
+    expect(detectedRanges).toEqual([[7, 8]]);
   });
 });

server/spec/models/segment_model.jest.ts

@@ -0,0 +1,54 @@
+import { TEST_ANALYTIC_UNIT_ID } from '../utils_for_tests/analytic_units';
+import { buildSegments, clearSegmentsDB, convertSegmentsToTimeRanges } from '../utils_for_tests/segments';
+
+import * as Segment from '../../src/models/segment_model';
+
+afterEach(async () => {
+  await clearSegmentsDB();
+});
+
+describe('mergeAndInsertSegments', function() {
+  const initialSegments = buildSegments([[0, 1], [2, 3], [4, 5]]);
+
+  beforeEach(async () => {
+    await Segment.mergeAndInsertSegments(initialSegments);
+  });
+
+  it('Segments should be merged before insertion', async function() {
+    const segmentsToInsert = buildSegments([[1, 2]]);
+    await Segment.mergeAndInsertSegments(segmentsToInsert);
+
+    const actualSegments = await Segment.findMany(TEST_ANALYTIC_UNIT_ID, {});
+    const actualRanges = convertSegmentsToTimeRanges(actualSegments);
+
+    expect(actualRanges).toEqual([[0, 3], [4, 5]]);
+  });
+});
+
+describe('findIntersectedSegments', () => {
+  const initialSegments = buildSegments([[0, 3], [5, 6], [10, 13]]);
+
+  beforeEach(async () => {
+    await Segment.mergeAndInsertSegments(initialSegments);
+  });
+
+  it('should find intersected segments', async () => {
+    const testCases = [
+      { from: 1, to: 4, expected: [[0, 3]] },
+      { from: 11, to: 12, expected: [[10, 13]] },
+      { from: 6, to: 10, expected: [[5, 6], [10, 13]] },
+      { from: 16, to: 17, expected: [] },
+      { from: 5, expected: [[5, 6], [10, 13]] },
+      { to: 5, expected: [[0, 3], [5, 6]] },
+      { expected: [[0, 3], [5, 6], [10, 13]] }
+    ];
+
+    for(let testCase of testCases) {
+      const foundSegments = await Segment.findIntersectedSegments(
+        TEST_ANALYTIC_UNIT_ID, testCase.from, testCase.to
+      );
+      const foundRanges = convertSegmentsToTimeRanges(foundSegments);
+      expect(foundRanges).toEqual(testCase.expected);
+    }
+  });
+});

server/spec/segments.jest.ts

@@ -1,35 +1,18 @@
-import { buildSegments, clearDB, TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/segments';
+import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units';
+import { buildSegments, clearSegmentsDB, convertSegmentsToTimeRanges } from './utils_for_tests/segments';
 
-import * as AnalyticUnit from '../src/models/analytic_units';
 import * as Segment from '../src/models/segment_model';
-import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model';
 
 import * as _ from 'lodash';
 
 const INITIAL_SEGMENTS = buildSegments([[0, 1], [2, 3], [4, 5]]);
 
-beforeAll(async () => {
-  clearDB();
-  await AnalyticUnit.create(
-    AnalyticUnit.createAnalyticUnitFromObject({
-      _id: TEST_ANALYTIC_UNIT_ID,
-      name: 'name',
-      grafanaUrl: 'grafanaUrl',
-      panelId: 'panelId',
-      type: 'type',
-      detectorType: AnalyticUnit.DetectorType.ANOMALY
-    })
-  );
-  await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID);
-  await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: 1 });
-});
-
 beforeEach(async () => {
   await Segment.mergeAndInsertSegments(INITIAL_SEGMENTS);
 });
 
 afterEach(async () => {
-  clearDB();
+  await clearSegmentsDB();
 });
 
 describe('mergeAndInsertSegments', function() {

server/spec/setup_tests.ts

@@ -1,3 +1,8 @@
+import * as AnalyticUnit from '../src/models/analytic_units';
+import * as AnalyticUnitCache from '../src/models/analytic_unit_cache_model';
+import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units';
+import { clearSegmentsDB } from './utils_for_tests/segments';
+
 console.log = jest.fn();
 console.error = jest.fn();
 
@@ -6,3 +11,21 @@ jest.mock('../src/config.ts', () => ({
   DATA_PATH: 'fake-data-path',
   ZMQ_IPC_PATH: 'fake-zmq-path'
 }));
+
+createTestDB();
+
+async function createTestDB() {
+  await clearSegmentsDB();
+  await AnalyticUnit.create(
+    AnalyticUnit.createAnalyticUnitFromObject({
+      _id: TEST_ANALYTIC_UNIT_ID,
+      name: 'name',
+      grafanaUrl: 'grafanaUrl',
+      panelId: 'panelId',
+      type: 'type',
+      detectorType: AnalyticUnit.DetectorType.ANOMALY
+    })
+  );
+  await AnalyticUnitCache.create(TEST_ANALYTIC_UNIT_ID);
+  await AnalyticUnitCache.setData(TEST_ANALYTIC_UNIT_ID, { timeStep: 1 });
+}

server/spec/utils_for_tests/analytic_units.ts

@@ -0,0 +1,3 @@
+import * as AnalyticUnit from '../../src/models/analytic_units';
+
+export const TEST_ANALYTIC_UNIT_ID: AnalyticUnit.AnalyticUnitId = 'testid';

server/spec/utils_for_tests/segments.ts

@@ -1,7 +1,7 @@
-import * as AnalyticUnit from '../../src/models/analytic_units';
+import { TEST_ANALYTIC_UNIT_ID } from './analytic_units';
 import * as Segment from '../../src/models/segment_model';
 
-export const TEST_ANALYTIC_UNIT_ID: AnalyticUnit.AnalyticUnitId = 'testid';
+import * as _ from 'lodash';
 
 export function buildSegments(times: number[][]): Segment.Segment[] {
   return times.map(t => {
@@ -9,7 +9,12 @@ export function buildSegments(times: number[][]): Segment.Segment[] {
   });
 }
 
-export async function clearDB(): Promise<void> {
+export function convertSegmentsToTimeRanges(segments: Segment.Segment[]): number[][] {
+  const ranges = segments.map(segment => [segment.from, segment.to]);
+  return _.sortBy(ranges, range => range[0]);
+}
+
+export async function clearSegmentsDB(): Promise<void> {
   const segments = await Segment.findMany(TEST_ANALYTIC_UNIT_ID, { labeled: false, deleted: false });
   await Segment.removeSegments(segments.map(s => s.id));
 }

server/src/models/segment_model.ts

@@ -106,6 +106,28 @@ export async function findMany(id: AnalyticUnitId, query: FindManyQuery): Promis
   return segs.map(Segment.fromObject);
 }
 
+
+/**
+ * If `from` and `to` are defined: @returns segments intersected with `[from; to]`
+ * If `to` is `undefined`: @returns segments intersected with `[-inf; from]`
+ * If `from` is `undefined`: @returns segments intersected with `[to: +inf]`
+ * If `from` and `to` are undefined: @returns all segments
+ */
+export async function findIntersectedSegments(
+  analyticUnitId: AnalyticUnit.AnalyticUnitId, 
+  from?: number, 
+  to?: number
+): Promise<Segment[]> {
+  let query: FindManyQuery = {};
+  if(from !== undefined) {
+    query.to = { $gte: from };
+  }
+  if(to !== undefined) {
+    query.from = { $lte: to };
+  }
+  return findMany(analyticUnitId, query);
+}
+
 /**
  * Merges an array of segments with ones existing in the DB
  * Inserts resulting segments into DB

server/src/routes/segments_router.ts

@@ -6,20 +6,21 @@ import * as Segment from '../models/segment_model';
 import * as Router from 'koa-router';
 
 
-async function getSegments(ctx: Router.IRouterContext) {
+export async function getSegments(ctx: Router.IRouterContext) {
   let id: AnalyticUnitId = ctx.request.query.id;
   if(id === undefined || id === '') {
     throw new Error('analyticUnitId (id) is missing');
   }
-  let query: Segment.FindManyQuery = {};
-
-  if(!isNaN(+ctx.request.query.from)) {
-    query.from = { $gte: +ctx.request.query.from };
+  let from = +ctx.request.query.from;
+  if(isNaN(from)) {
+    from = undefined;
   }
-  if(!isNaN(+ctx.request.query.to)) {
-    query.to = { $lte: +ctx.request.query.to };
+  let to = +ctx.request.query.to;
+  if(isNaN(to)) {
+    to = undefined;
   }
-  let segments = await Segment.findMany(id, query);
+
+  const segments = await Segment.findIntersectedSegments(id, from, to);
   ctx.response.body = { segments };
 }