Utils SpansCut to SegmentsCut, IntegerSegment and IntegerSegmentSet (#756)

5 years ago · b6d80015b6
5 changed files with 392 additions and 117 deletions
--- a/server/spec/utils/segments.jest.ts
+++ b/server/spec/utils/segments.jest.ts
@ -0,0 +1,205 @@
+import { cutSegmentWithSegments, IntegerSegment, IntegerSegmentsSet } from '../../src/utils/segments';
+
+import 'jest';
+
+
+function IS(from: number, to: number) {
+  return new IntegerSegment(from, to);
+}
+
+function ISS(xs: [number, number][]) {
+  return new IntegerSegmentsSet(xs.map(x => IS(x[0], x[1])));
+}
+
+function cutSpan(from: number, to: number, cuts: [number, number][]): [number, number][] {
+  return cutSegmentWithSegments(
+    new IntegerSegment(from, to),
+    cuts.map(([from, to]) => new IntegerSegment(from, to))
+  ).map(({ from, to }) => [from, to] as [number, number]);
+}
+
+describe('IntegerSegment', function() {
+  it('should throw an error on float from or to', function() {
+    expect(() => IS(0.1, 0)).toThrow();
+    expect(() => IS(1, 5.04)).toThrow();
+    expect(() => IS(1, 5)).not.toThrow();
+  });  
+});
+
+describe('IntegerSegment.intersect', function() {
+  it('return undefined if segments don`t intersect', function() {
+    expect(IS(4, 5).insersect(IS(6, 10))).toEqual(undefined);
+    expect(IS(7, 10).insersect(IS(1, 3))).toEqual(undefined);
+  });
+
+  it('return a point when borders intersect', function() {
+    expect(IS(4, 5).insersect(IS(5, 6))).toEqual(IS(5, 5));
+    expect(IS(4, 5).insersect(IS(4, 4))).toEqual(IS(4, 4));
+  });
+});
+
+describe('IntegerSegmentSet constructor', function() {
+  it('can construct from empty segments list', function() {
+    expect(() => ISS([])).not.toThrow();
+  });
+
+  it('should sort segments', function() {
+    expect(ISS([[10, 15], [5, 8]]).segments).toEqual([IS(5, 8), IS(10, 15)]);
+    expect(ISS([[10, 15], [-Infinity, 8]]).segments).toEqual([IS(-Infinity, 8), IS(10, 15)]);
+    expect(ISS([[10, Infinity], [-Infinity, 8]]).segments).toEqual([IS(-Infinity, 8), IS(10, Infinity)]);
+  });
+
+  it('should merge segments', function() {
+    expect(ISS([[5, 10], [7, 20]]).segments).toEqual([IS(5, 20)]); // it's because 7 <= 10
+    expect(ISS([[5, 10], [10, 20]]).segments).toEqual([IS(5, 20)]);
+    expect(ISS([[5, 10], [11, 20]]).segments).toEqual([IS(5, 20)]); // it's because [..., 10], [11, ...], 
+                                                                    // there is nothing between 10 and 11
+    expect(ISS([[3, 11], [4, 10]]).segments).toEqual([IS(3, 11)]);
+  });
+});
+
+describe('IntegerSegmentSet.inversed', function() {
+  it('should return Infinite segment whes set is empty', function() {
+    let setA = ISS([]);
+    expect(setA.inversed()).toEqual(ISS([[-Infinity, Infinity]]));
+  });
+
+  it('should return empty segment whes set is infinite', function() {
+    let setA = ISS([[-Infinity, Infinity]]);
+    expect(setA.inversed()).toEqual(ISS([]));
+  });
+
+  it('should inverse a point', function() {
+    expect(ISS([[4, 4]]).inversed()).toEqual(ISS([[-Infinity, 3], [5, Infinity]]));
+  });
+
+  it('should inverse basic cases', function() {
+    expect(ISS([[3, 10]]).inversed()).toEqual(ISS([[-Infinity, 2], [11, Infinity]]));
+    expect(ISS([[3, 10], [15, 20]]).inversed()).toEqual(ISS([[-Infinity, 2], [11, 14] , [21, Infinity]]));
+  });
+
+  it('should inverse infinites', function() {
+    expect(ISS([[3, Infinity]]).inversed()).toEqual(ISS([[-Infinity, 2]]));
+    expect(ISS([[-Infinity, 3]]).inversed()).toEqual(ISS([[4, Infinity]]));
+  });
+
+});
+
+describe('IntegerSegmentSet.intersected', function() {
+  it('should return empty set if one of intersection is empty', function() {
+    let setA = ISS([]);
+    let setB = ISS([[1, 5]]);
+    expect(setA.intersect(setB).segments).toEqual([]);
+    expect(setB.intersect(setA).segments).toEqual([]);
+  });
+
+  it('should intersect two segments', function() {
+    let setA = ISS([[2, 5]]);
+    let setB = ISS([[1, 4]]);
+    expect(setA.intersect(setB)).toEqual(ISS([[2, 4]]));
+  });
+
+  it('should intersect basic cases', function() {
+    let setA = ISS([[2, 5], [6, 10]]);
+    let setB = ISS([[1, 9]]);
+    let setC = ISS([[2, 5], [6, 10]]);
+    let setD = ISS([[4, 4], [10, 10]]);
+    let setE = ISS([[4, 4], [10, 10], [12, 15]]);
+    expect(setA.intersect(setB)).toEqual(ISS([[2, 5], [6, 9]]));
+    expect(setA.intersect(setC)).toEqual(ISS([[2, 5], [6, 10]]));
+    expect(setA.intersect(setD)).toEqual(ISS([[4, 4], [10, 10]]));
+    expect(setA.intersect(setE)).toEqual(ISS([[4, 4], [10, 10]]));
+    expect(setE.intersect(setA)).toEqual(ISS([[4, 4], [10, 10]]));
+  });
+  
+});
+
+describe('cutSpanWithSpans', function() {
+
+  it('should handle empty input spans list case', function() {
+    expect(cutSpan(4, 10, [])).toEqual([[4, 10]]);
+  });
+
+  it('should handle works fine one point results', function() {
+    expect(cutSpan(1, 10, [[2, 10]])).toEqual([[1, 1]]);
+    expect(cutSpan(1, 10, [[2, 11]])).toEqual([[1, 1]]);
+    expect(cutSpan(1, 10, [[1, 9]])).toEqual([[10, 10]]);
+    expect(cutSpan(1, 10, [[0, 9]])).toEqual([[10, 10]]);
+    expect(cutSpan(1, 10, [[1, 4], [6, 10]])).toEqual([[5, 5]]);
+    expect(cutSpan(1, 10, [[2, 9]])).toEqual([[1, 1], [10, 10]]);
+  });
+
+  it('should throw error is cut contains float border', function() {
+    expect(() => cutSpan(0, 10, [[0.1, 5]])).toThrow()
+    expect(() => cutSpan(1, 10, [[0.9, 0.0]])).toThrow();
+    expect(() => cutSpan(0.5, 10, [[1, 5]])).toThrow();
+  });
+
+  it('should handle one-point cuts', function() {
+    expect(cutSpan(1, 10, [[5, 5]])).toEqual([[1, 4], [6, 10]]);
+    expect(cutSpan(1, 10, [[1, 1]])).toEqual([[2, 10]]);
+    expect(cutSpan(1, 10, [[10, 10]])).toEqual([[1, 9]]);
+    expect(cutSpan(1, 10, [[11, 11]])).toEqual([[1, 10]]);
+    expect(cutSpan(1, 15, [[11, 11], [12, 12]])).toEqual([[1, 10], [13, 15]]);
+  });
+
+  it('should handle basic cases', function() {
+    let cutSpans = [[3, 4], [6, 8], [11, 20]] as [number, number][];
+
+    expect(cutSpan(0, 11, cutSpans)).toEqual([[0, 2], [5, 5], [9, 10]]);
+    expect(cutSpan(5, 11, cutSpans)).toEqual([[5, 5], [9, 10]]);
+    expect(cutSpan(4, 10, cutSpans)).toEqual([[5, 5], [9, 10]]);
+    expect(cutSpan(5, 10, cutSpans)).toEqual([[5, 5], [9, 10]]);
+    expect(cutSpan(4, 20, cutSpans)).toEqual([[5, 5], [9, 10]]);
+    expect(cutSpan(4, 21, cutSpans)).toEqual([[5, 5], [9, 10], [21, 21]]);
+    expect(cutSpan(2, 20, cutSpans)).toEqual([[2, 2], [5, 5], [9, 10]]);
+    expect(cutSpan(2, 21, cutSpans)).toEqual([[2, 2], [5, 5], [9, 10], [21, 21]]);
+    expect(cutSpan(3, 11, cutSpans)).toEqual([[5, 5], [9, 10]]);
+    expect(cutSpan(3, 20, cutSpans)).toEqual([[5, 5], [9, 10]]);
+    expect(cutSpan(4, 7, [[3, 5], [6, 8]])).toEqual([]);
+  });
+
+  it('should handle infitie span and infinite cuts', function() {
+    expect(cutSpan(0, Infinity, [[5, 10]])).toEqual([[0, 4], [11, Infinity]]);
+    expect(cutSpan(0, 6, [[0, Infinity]])).toEqual([]);
+    expect(cutSpan(0, 6, [[2, Infinity]])).toEqual([[0, 1]]);
+    expect(cutSpan(-Infinity, Infinity, [[-Infinity, Infinity]])).toEqual([]);
+  });
+
+  it('should handle case when from and to are inside of one big span', function() {
+    expect(cutSpan(4, 10, [[1, 20]])).toEqual([]);
+    expect(cutSpan(4, 10, [[1, 10]])).toEqual([]);
+    expect(cutSpan(4, 10, [[4, 20]])).toEqual([]);
+    expect(cutSpan(4, 10, [[4, 10]])).toEqual([]);
+  });
+
+  it('should be ready to get not-sorted cuts', function() {
+    expect(cutSpan(0, 20, [[3, 5], [1, 2]])).toEqual([[0, 0], [6, 20]]);
+    expect(cutSpan(0, 20, [[3, 5], [1, 2], [0, 0]])).toEqual([[6, 20]]);
+  });
+
+  it('should be ready to get overlayed cuts', function() {
+    expect(cutSpan(0, 20, [[3, 5], [4, 10]])).toEqual([[0, 2], [11, 20]]);
+    expect(cutSpan(0, 20, [[3, 9], [4, 9]])).toEqual([[0, 2], [10, 20]]);
+    expect(cutSpan(0, 20, [[3, 11], [4, 10]])).toEqual([[0, 2], [12, 20]]);
+    expect(cutSpan(0, 20, [[3, 11], [3, 12]])).toEqual([[0, 2], [13, 20]]);
+    expect(cutSpan(0, 20, [[3, 11], [3, 12], [3, 10], [3, 15], [3, 14]])).toEqual([[0, 2], [16, 20]]);
+    expect(cutSpan(0, 20, [[2, 11], [3, 12]])).toEqual([[0, 1], [13, 20]]);
+    expect(cutSpan(0, 20, [[2, 15], [3, 12]])).toEqual([[0, 1], [16, 20]]);
+    expect(cutSpan(0, 20, [[2, 15], [3, 12], [1, 18]])).toEqual([[0, 0], [19, 20]]);
+    expect(cutSpan(0, 20, [[2, 15], [3, Infinity], [1, 18]])).toEqual([[0, 0]]);
+    expect(cutSpan(0, 20, [[3, 3], [3, Infinity]])).toEqual([[0, 2]]);
+    expect(cutSpan(0, 20, [[3, 3], [3, Infinity], [3, 3]])).toEqual([[0, 2]]);
+    expect(cutSpan(0, 20, [[3, 3], [3, Infinity], [3, 3], [4, 4]])).toEqual([[0, 2]]);
+    expect(cutSpan(0, 20, [[3, 3], [3, Infinity], [3, 3], [4, 4], [3, 5]])).toEqual([[0, 2]]);
+    expect(cutSpan(-Infinity, Infinity, [[3, 3], [3, Infinity], [3, 3], [4, 4], [3, 5]])).toEqual([[-Infinity, 2]]);
+  });
+
+  it('should handle cuts from point span', function() {
+    expect(cutSpan(1, 1, [[1, 1]])).toEqual([]);
+    expect(cutSpan(1, 1, [[0, 2]])).toEqual([]);
+    expect(cutSpan(1, 1, [[0, 1]])).toEqual([]);
+    expect(cutSpan(1, 1, [[1, 2]])).toEqual([]);
+  });
+
+});
--- a/server/spec/utils/spans.jest.ts
+++ b/server/spec/utils/spans.jest.ts
@ -1,51 +0,0 @@
-import { cutSpanWithSpans } from '../../src/utils/spans';
-
-import 'jest';
-
-
-function cutSpan(from: number, to: number, cuts: [number, number][]): [number, number][] {
-  return cutSpanWithSpans(
-    { from: from, to: to },
-    cuts.map(([from, to]) => ({ from, to }))
-  ).map(({ from, to }) => [from, to] as [number, number]);
-}
-
-describe('cutSpanWithSpans', function() {
-
-  it('should find spans in simple non-intersected borders', function() {
-    let cutSpans = [[3, 5], [6, 8], [10, 20]] as [number, number][];
-
-    expect(cutSpan(4, 11, cutSpans)).toEqual([[5, 6], [8, 10]]);
-    expect(cutSpan(5, 11, cutSpans)).toEqual([[5, 6], [8, 10]]);
-    expect(cutSpan(4, 10, cutSpans)).toEqual([[5, 6], [8, 10]]);
-    expect(cutSpan(5, 10, cutSpans)).toEqual([[5, 6], [8, 10]]);
-    expect(cutSpan(4, 20, cutSpans)).toEqual([[5, 6], [8, 10]]);
-    expect(cutSpan(4, 21, cutSpans)).toEqual([[5, 6], [8, 10], [20, 21]]);
-    expect(cutSpan(2, 20, cutSpans)).toEqual([[2, 3], [5, 6], [8, 10]]);
-    expect(cutSpan(2, 21, cutSpans)).toEqual([[2, 3], [5, 6], [8, 10], [20, 21]]);
-    expect(cutSpan(3, 11, cutSpans)).toEqual([[5, 6], [8, 10]]);
-    expect(cutSpan(3, 20, cutSpans)).toEqual([[5, 6], [8, 10]]);
-    expect(cutSpan(4, 7, [[3, 5], [6, 8]])).toEqual([[5, 6]]);
-  });
-
-  it('should handle empty input spans list case', function() {
-    expect(cutSpan(4, 10, [])).toEqual([[4, 10]]);
-  });
-
-  it('should handle case when from and to are inside of one big span', function() {
-    expect(cutSpan(4, 10, [[1, 20]])).toEqual([]);
-    expect(cutSpan(4, 10, [[1, 10]])).toEqual([]);
-    expect(cutSpan(4, 10, [[4, 20]])).toEqual([]);
-    expect(cutSpan(4, 10, [[4, 10]])).toEqual([]);
-  });
-
-  it('should be ready to get not-sorted cuts', function() {
-    expect(cutSpan(0, 20, [[3, 5], [1, 2]])).toEqual([[0, 1], [2, 3], [5, 20]]);
-    expect(cutSpan(0, 20, [[3, 5], [1, 2], [0.1, 0.5]])).toEqual([[0, 0.1], [0.5, 1], [2, 3], [5, 20]]);
-  });
-
-  it('should be ready to get overlayed cuts', function() {
-    expect(cutSpan(0, 20, [[3, 5], [4, 10]])).toEqual([[0,3], [10, 20]]);
-  });
-
-});
--- a/server/src/controllers/analytics_controller.ts
+++ b/server/src/controllers/analytics_controller.ts
@ -10,7 +10,7 @@ import { AlertService } from '../services/alert_service';
 import { HASTIC_API_KEY } from '../config';
 import { DataPuller } from '../services/data_puller';
 import { getGrafanaUrl } from '../utils/grafana';
-import { cutSpanWithSpans } from '../utils/spans';
+import { cutSegmentWithSegments } from '../utils/segments';

 import { queryByMetric, GrafanaUnavailable, DatasourceUnavailable } from 'grafana-datasource-kit';

@ -564,7 +564,7 @@ export async function getDetectionSpans(
    }
  }

-  let newDetectionSpans = cutSpanWithSpans({ from, to }, readySpans);
+  let newDetectionSpans = cutSegmentWithSegments({ from, to }, readySpans);
  if(newDetectionSpans.length === 0) {
    return [ new Detection.DetectionSpan(analyticUnitId, from, to, Detection.DetectionStatus.READY) ];
  }
--- a/server/src/utils/segments.ts
+++ b/server/src/utils/segments.ts
@ -0,0 +1,185 @@
+//TODO: move this code to span model
+
+import * as _ from 'lodash';
+
+
+export declare type Segment = {
+  readonly from: number,
+  readonly to: number
+}
+
+export class IntegerSegment {
+  readonly from: number;
+  readonly to: number;
+
+  constructor(from: number, to: number) {
+    if(!(Number.isInteger(from) || !Number.isFinite(from))) {
+      throw new Error(`From should be an Integer or Infinity, but got ${from}`);
+    }
+    if(!(Number.isInteger(to) || !Number.isFinite(to))) {
+      throw new Error(`To should be an Integer or Infinity, but got ${from}`);
+    }
+
+    let l = IntegerSegment.lengthBetweenPoints(from, to);
+    if(l < 1) {
+      throw new Error(
+        `Length of segment is less than 1: [${from}, ${to}]. 
+        It's not possible for IntegerSegment`
+      );
+    }
+    this.from = from;
+    this.to = to;
+  }
+
+  get length(): number {
+    return IntegerSegment.lengthBetweenPoints(this.from, this.to);
+  }
+
+  insersect(segment: IntegerSegment): IntegerSegment | undefined {
+    let from = Math.max(this.from, segment.from);
+    let to = Math.min(this.to, segment.to);
+    if(IntegerSegment.lengthBetweenPoints(from, to) >= 1) {
+      return new IntegerSegment(from, to);
+    }
+    return undefined;
+  }
+
+  toString(): string {
+    return `[${this.from}, ${this.to}]`;
+  }
+
+  static lengthBetweenPoints(from: number, to: number): number {
+    let l = to - from + 1; // because [x, x] has length 1
+    if(isNaN(l)) { // when [Infinity, Infinity] or [-Infinity, -Infinity]
+      return 0;
+    } else {
+      return Math.max(l, 0); // becase [x, x - 1] we consider as zero length
+    }
+  }
+}
+
+export class IntegerSegmentsSet {
+
+  private _segments: IntegerSegment[];
+
+  constructor(segments: IntegerSegment[], noramlized: boolean = false) {
+    this._segments = segments;
+    if(noramlized !== true) {
+      this._normalize();
+    }
+  }
+
+  private _normalize() {
+    if(this._segments.length === 0) {
+      return;
+    }
+    let sortedSegments = _.sortBy(this._segments, s => s.from);
+    let lastFrom = sortedSegments[0].from;
+    let lastTo = sortedSegments[0].to;
+    let mergedSegments: IntegerSegment[] = [];
+    for(let i = 1; i < sortedSegments.length; i++) {
+      let currentSegment = sortedSegments[i];
+      if(lastTo + 1 >= currentSegment.from) { // because [a, x], [x + 1, b] is [a, b]
+        lastTo = Math.max(currentSegment.to, lastTo); // we can be inside previous
+        continue;
+      }
+      mergedSegments.push(new IntegerSegment(lastFrom, lastTo));
+      lastFrom = currentSegment.from;
+      lastTo = currentSegment.to;
+    }
+    mergedSegments.push(new IntegerSegment(lastFrom, lastTo));
+    this._segments = mergedSegments;
+  }
+
+  get segments(): IntegerSegment[] {
+    return this._segments;
+  }
+
+  inversed(): IntegerSegmentsSet {
+    var invertedSegments: IntegerSegment[] = [];
+    if(this._segments.length === 0) {
+      invertedSegments = [new IntegerSegment(-Infinity, Infinity)];
+    } else {
+      let push = (f: number, t: number) => {
+        if(IntegerSegment.lengthBetweenPoints(f, t) > 0) {
+          invertedSegments.push(new IntegerSegment(f, t));
+        }
+      }
+      _.reduce(this._segments, (prev: IntegerSegment | null, s: IntegerSegment) => {
+        if(prev === null) {
+          push(-Infinity, s.from - 1);
+        } else {
+          push(prev.to + 1, s.from - 1);
+        }
+        return s;
+      }, null);
+      push(_.last(this._segments).to + 1, Infinity);
+    }
+    return new IntegerSegmentsSet(invertedSegments, true);
+  }
+
+  intersect(other: IntegerSegmentsSet): IntegerSegmentsSet {
+    let result: IntegerSegment[] = [];
+
+    if(this._segments.length === 0 || other.segments.length === 0) {
+      return new IntegerSegmentsSet([], true);
+    }
+
+    let currentSegmentIndex = 0;
+    let withSegmentIndex = 0;
+
+    do {
+      let currentSegemet = this.segments[currentSegmentIndex];
+      let withSegment = other.segments[withSegmentIndex];
+      if(currentSegemet.to < withSegment.from) {
+        currentSegmentIndex++;
+        continue;
+      }
+      if(withSegment.to < currentSegemet.from) {
+        withSegmentIndex++;
+        continue;
+      }
+      let segmentsIntersection = currentSegemet.insersect(withSegment);
+      if(segmentsIntersection === undefined) {
+        throw new Error(
+          `Impossible condition, segments ${currentSegemet} and ${withSegment} don't interset`
+        )
+      }
+      result.push(segmentsIntersection);
+
+      if(currentSegemet.to < withSegment.to) {
+        currentSegmentIndex++;
+      } else {
+        withSegmentIndex++;
+      }
+    } while (
+      currentSegmentIndex < this._segments.length &&
+      withSegmentIndex < other.segments.length
+    )
+
+    return new IntegerSegmentsSet(result, true);
+  }
+
+  sub(other: IntegerSegmentsSet): IntegerSegmentsSet {
+    let inversed = other.inversed();
+    return this.intersect(inversed);
+  }
+
+}
+
+// TODO: move from utils and use generator
+/**
+ *
+ * @param inputSegment a big segment which we will cut
+ * @param cutSegments segments to cut the inputSegment. Segments can overlay.
+ *
+ * @returns array of segments remain after cut
+ */
+export function cutSegmentWithSegments(inputSegment: Segment, cutSegments: Segment[]): Segment[] {
+  let setA = new IntegerSegmentsSet([new IntegerSegment(inputSegment.from, inputSegment.to)]);
+  let setB = new IntegerSegmentsSet(cutSegments.map(
+    s => new IntegerSegment(s.from, s.to)
+  ));
+  let setResult = setA.sub(setB);
+  return setResult.segments.map(s => ({ from: s.from, to: s.to }));
+}
--- a/server/src/utils/spans.ts
+++ b/server/src/utils/spans.ts
@ -1,64 +0,0 @@
-//TODO: move this code to span model
-
-import * as _ from 'lodash';
-
-
-export declare type Span = {
-  from: number,
-  to: number
-}
-
-// TODO: move from utils and use generator
-/**
- *
- * @param inputSpan a big span which we will cut
- * @param cutSpans spans which to cut the inputSpan. Spans can overlay.
- *
- * @returns array of spans which are holes
- */
-export function cutSpanWithSpans(inputSpan: Span, cutSpans: Span[]): Span[] {
-  if(cutSpans.length === 0) {
-    return [inputSpan];
-  }
-
-  // we sort and merge out cuts to normalize it
-  cutSpans = _.sortBy(cutSpans, s => s.from);
-  const mergedSortedCuts =_.reduce(cutSpans, 
-    ((acc: Span[], s: Span) => {
-      if(acc.length === 0) return [s];
-      let last = acc[acc.length - 1];
-      if(s.to <= last.to) return acc;
-      if(s.from <= last.to) {
-        last.to = s.to;
-        return acc;
-      }
-      acc.push(s);
-      return acc;
-    }), []
-  );
-
-  // this is what we get if we cut `mergedSortedCuts` from (-Infinity, Infinity)
-  const holes = mergedSortedCuts.map((cut, i) => {
-    let from = -Infinity;
-    let to = cutSpans[0].from;
-    if(i > 0) {
-      from = mergedSortedCuts[i - 1].to;
-      to = cut.from;
-    }
-    return { from, to };
-  }).concat({
-    from: mergedSortedCuts[mergedSortedCuts.length - 1].to,
-    to: Infinity
-  });
-
-  const holesInsideInputSpan = _(holes).map(c => {
-    if(c.to <= inputSpan.from) return undefined;
-    if(inputSpan.to <= c.from) return undefined;
-    return {
-      from: Math.max(c.from, inputSpan.from),
-      to: Math.min(c.to, inputSpan.to),
-    }
-  }).compact().value();
-
-  return Array.from(holesInsideInputSpan);
-}