From b6d80015b6142d250acf0114938e976697f65e34 Mon Sep 17 00:00:00 2001 From: Alexey Velikiy Date: Mon, 19 Aug 2019 20:19:27 +0300 Subject: [PATCH] Utils SpansCut to SegmentsCut, IntegerSegment and IntegerSegmentSet (#756) --- server/spec/utils/segments.jest.ts | 205 ++++++++++++++++++ server/spec/utils/spans.jest.ts | 51 ----- .../src/controllers/analytics_controller.ts | 4 +- server/src/utils/segments.ts | 185 ++++++++++++++++ server/src/utils/spans.ts | 64 ------ 5 files changed, 392 insertions(+), 117 deletions(-) create mode 100644 server/spec/utils/segments.jest.ts delete mode 100644 server/spec/utils/spans.jest.ts create mode 100644 server/src/utils/segments.ts delete mode 100644 server/src/utils/spans.ts diff --git a/server/spec/utils/segments.jest.ts b/server/spec/utils/segments.jest.ts new file mode 100644 index 0000000..208d8f8 --- /dev/null +++ b/server/spec/utils/segments.jest.ts @@ -0,0 +1,205 @@ +import { cutSegmentWithSegments, IntegerSegment, IntegerSegmentsSet } from '../../src/utils/segments'; + +import 'jest'; + + +function IS(from: number, to: number) { + return new IntegerSegment(from, to); +} + +function ISS(xs: [number, number][]) { + return new IntegerSegmentsSet(xs.map(x => IS(x[0], x[1]))); +} + +function cutSpan(from: number, to: number, cuts: [number, number][]): [number, number][] { + return cutSegmentWithSegments( + new IntegerSegment(from, to), + cuts.map(([from, to]) => new IntegerSegment(from, to)) + ).map(({ from, to }) => [from, to] as [number, number]); +} + +describe('IntegerSegment', function() { + it('should throw an error on float from or to', function() { + expect(() => IS(0.1, 0)).toThrow(); + expect(() => IS(1, 5.04)).toThrow(); + expect(() => IS(1, 5)).not.toThrow(); + }); +}); + +describe('IntegerSegment.intersect', function() { + it('return undefined if segments don`t intersect', function() { + expect(IS(4, 5).insersect(IS(6, 10))).toEqual(undefined); + expect(IS(7, 10).insersect(IS(1, 3))).toEqual(undefined); + }); + + it('return a point when borders intersect', function() { + expect(IS(4, 5).insersect(IS(5, 6))).toEqual(IS(5, 5)); + expect(IS(4, 5).insersect(IS(4, 4))).toEqual(IS(4, 4)); + }); +}); + +describe('IntegerSegmentSet constructor', function() { + it('can construct from empty segments list', function() { + expect(() => ISS([])).not.toThrow(); + }); + + it('should sort segments', function() { + expect(ISS([[10, 15], [5, 8]]).segments).toEqual([IS(5, 8), IS(10, 15)]); + expect(ISS([[10, 15], [-Infinity, 8]]).segments).toEqual([IS(-Infinity, 8), IS(10, 15)]); + expect(ISS([[10, Infinity], [-Infinity, 8]]).segments).toEqual([IS(-Infinity, 8), IS(10, Infinity)]); + }); + + it('should merge segments', function() { + expect(ISS([[5, 10], [7, 20]]).segments).toEqual([IS(5, 20)]); // it's because 7 <= 10 + expect(ISS([[5, 10], [10, 20]]).segments).toEqual([IS(5, 20)]); + expect(ISS([[5, 10], [11, 20]]).segments).toEqual([IS(5, 20)]); // it's because [..., 10], [11, ...], + // there is nothing between 10 and 11 + expect(ISS([[3, 11], [4, 10]]).segments).toEqual([IS(3, 11)]); + }); +}); + +describe('IntegerSegmentSet.inversed', function() { + it('should return Infinite segment whes set is empty', function() { + let setA = ISS([]); + expect(setA.inversed()).toEqual(ISS([[-Infinity, Infinity]])); + }); + + it('should return empty segment whes set is infinite', function() { + let setA = ISS([[-Infinity, Infinity]]); + expect(setA.inversed()).toEqual(ISS([])); + }); + + it('should inverse a point', function() { + expect(ISS([[4, 4]]).inversed()).toEqual(ISS([[-Infinity, 3], [5, Infinity]])); + }); + + it('should inverse basic cases', function() { + expect(ISS([[3, 10]]).inversed()).toEqual(ISS([[-Infinity, 2], [11, Infinity]])); + expect(ISS([[3, 10], [15, 20]]).inversed()).toEqual(ISS([[-Infinity, 2], [11, 14] , [21, Infinity]])); + }); + + it('should inverse infinites', function() { + expect(ISS([[3, Infinity]]).inversed()).toEqual(ISS([[-Infinity, 2]])); + expect(ISS([[-Infinity, 3]]).inversed()).toEqual(ISS([[4, Infinity]])); + }); + +}); + +describe('IntegerSegmentSet.intersected', function() { + it('should return empty set if one of intersection is empty', function() { + let setA = ISS([]); + let setB = ISS([[1, 5]]); + expect(setA.intersect(setB).segments).toEqual([]); + expect(setB.intersect(setA).segments).toEqual([]); + }); + + it('should intersect two segments', function() { + let setA = ISS([[2, 5]]); + let setB = ISS([[1, 4]]); + expect(setA.intersect(setB)).toEqual(ISS([[2, 4]])); + }); + + it('should intersect basic cases', function() { + let setA = ISS([[2, 5], [6, 10]]); + let setB = ISS([[1, 9]]); + let setC = ISS([[2, 5], [6, 10]]); + let setD = ISS([[4, 4], [10, 10]]); + let setE = ISS([[4, 4], [10, 10], [12, 15]]); + expect(setA.intersect(setB)).toEqual(ISS([[2, 5], [6, 9]])); + expect(setA.intersect(setC)).toEqual(ISS([[2, 5], [6, 10]])); + expect(setA.intersect(setD)).toEqual(ISS([[4, 4], [10, 10]])); + expect(setA.intersect(setE)).toEqual(ISS([[4, 4], [10, 10]])); + expect(setE.intersect(setA)).toEqual(ISS([[4, 4], [10, 10]])); + }); + +}); + +describe('cutSpanWithSpans', function() { + + it('should handle empty input spans list case', function() { + expect(cutSpan(4, 10, [])).toEqual([[4, 10]]); + }); + + it('should handle works fine one point results', function() { + expect(cutSpan(1, 10, [[2, 10]])).toEqual([[1, 1]]); + expect(cutSpan(1, 10, [[2, 11]])).toEqual([[1, 1]]); + expect(cutSpan(1, 10, [[1, 9]])).toEqual([[10, 10]]); + expect(cutSpan(1, 10, [[0, 9]])).toEqual([[10, 10]]); + expect(cutSpan(1, 10, [[1, 4], [6, 10]])).toEqual([[5, 5]]); + expect(cutSpan(1, 10, [[2, 9]])).toEqual([[1, 1], [10, 10]]); + }); + + it('should throw error is cut contains float border', function() { + expect(() => cutSpan(0, 10, [[0.1, 5]])).toThrow() + expect(() => cutSpan(1, 10, [[0.9, 0.0]])).toThrow(); + expect(() => cutSpan(0.5, 10, [[1, 5]])).toThrow(); + }); + + it('should handle one-point cuts', function() { + expect(cutSpan(1, 10, [[5, 5]])).toEqual([[1, 4], [6, 10]]); + expect(cutSpan(1, 10, [[1, 1]])).toEqual([[2, 10]]); + expect(cutSpan(1, 10, [[10, 10]])).toEqual([[1, 9]]); + expect(cutSpan(1, 10, [[11, 11]])).toEqual([[1, 10]]); + expect(cutSpan(1, 15, [[11, 11], [12, 12]])).toEqual([[1, 10], [13, 15]]); + }); + + it('should handle basic cases', function() { + let cutSpans = [[3, 4], [6, 8], [11, 20]] as [number, number][]; + + expect(cutSpan(0, 11, cutSpans)).toEqual([[0, 2], [5, 5], [9, 10]]); + expect(cutSpan(5, 11, cutSpans)).toEqual([[5, 5], [9, 10]]); + expect(cutSpan(4, 10, cutSpans)).toEqual([[5, 5], [9, 10]]); + expect(cutSpan(5, 10, cutSpans)).toEqual([[5, 5], [9, 10]]); + expect(cutSpan(4, 20, cutSpans)).toEqual([[5, 5], [9, 10]]); + expect(cutSpan(4, 21, cutSpans)).toEqual([[5, 5], [9, 10], [21, 21]]); + expect(cutSpan(2, 20, cutSpans)).toEqual([[2, 2], [5, 5], [9, 10]]); + expect(cutSpan(2, 21, cutSpans)).toEqual([[2, 2], [5, 5], [9, 10], [21, 21]]); + expect(cutSpan(3, 11, cutSpans)).toEqual([[5, 5], [9, 10]]); + expect(cutSpan(3, 20, cutSpans)).toEqual([[5, 5], [9, 10]]); + expect(cutSpan(4, 7, [[3, 5], [6, 8]])).toEqual([]); + }); + + it('should handle infitie span and infinite cuts', function() { + expect(cutSpan(0, Infinity, [[5, 10]])).toEqual([[0, 4], [11, Infinity]]); + expect(cutSpan(0, 6, [[0, Infinity]])).toEqual([]); + expect(cutSpan(0, 6, [[2, Infinity]])).toEqual([[0, 1]]); + expect(cutSpan(-Infinity, Infinity, [[-Infinity, Infinity]])).toEqual([]); + }); + + it('should handle case when from and to are inside of one big span', function() { + expect(cutSpan(4, 10, [[1, 20]])).toEqual([]); + expect(cutSpan(4, 10, [[1, 10]])).toEqual([]); + expect(cutSpan(4, 10, [[4, 20]])).toEqual([]); + expect(cutSpan(4, 10, [[4, 10]])).toEqual([]); + }); + + it('should be ready to get not-sorted cuts', function() { + expect(cutSpan(0, 20, [[3, 5], [1, 2]])).toEqual([[0, 0], [6, 20]]); + expect(cutSpan(0, 20, [[3, 5], [1, 2], [0, 0]])).toEqual([[6, 20]]); + }); + + it('should be ready to get overlayed cuts', function() { + expect(cutSpan(0, 20, [[3, 5], [4, 10]])).toEqual([[0, 2], [11, 20]]); + expect(cutSpan(0, 20, [[3, 9], [4, 9]])).toEqual([[0, 2], [10, 20]]); + expect(cutSpan(0, 20, [[3, 11], [4, 10]])).toEqual([[0, 2], [12, 20]]); + expect(cutSpan(0, 20, [[3, 11], [3, 12]])).toEqual([[0, 2], [13, 20]]); + expect(cutSpan(0, 20, [[3, 11], [3, 12], [3, 10], [3, 15], [3, 14]])).toEqual([[0, 2], [16, 20]]); + expect(cutSpan(0, 20, [[2, 11], [3, 12]])).toEqual([[0, 1], [13, 20]]); + expect(cutSpan(0, 20, [[2, 15], [3, 12]])).toEqual([[0, 1], [16, 20]]); + expect(cutSpan(0, 20, [[2, 15], [3, 12], [1, 18]])).toEqual([[0, 0], [19, 20]]); + expect(cutSpan(0, 20, [[2, 15], [3, Infinity], [1, 18]])).toEqual([[0, 0]]); + expect(cutSpan(0, 20, [[3, 3], [3, Infinity]])).toEqual([[0, 2]]); + expect(cutSpan(0, 20, [[3, 3], [3, Infinity], [3, 3]])).toEqual([[0, 2]]); + expect(cutSpan(0, 20, [[3, 3], [3, Infinity], [3, 3], [4, 4]])).toEqual([[0, 2]]); + expect(cutSpan(0, 20, [[3, 3], [3, Infinity], [3, 3], [4, 4], [3, 5]])).toEqual([[0, 2]]); + expect(cutSpan(-Infinity, Infinity, [[3, 3], [3, Infinity], [3, 3], [4, 4], [3, 5]])).toEqual([[-Infinity, 2]]); + }); + + it('should handle cuts from point span', function() { + expect(cutSpan(1, 1, [[1, 1]])).toEqual([]); + expect(cutSpan(1, 1, [[0, 2]])).toEqual([]); + expect(cutSpan(1, 1, [[0, 1]])).toEqual([]); + expect(cutSpan(1, 1, [[1, 2]])).toEqual([]); + }); + +}); diff --git a/server/spec/utils/spans.jest.ts b/server/spec/utils/spans.jest.ts deleted file mode 100644 index 8bf3104..0000000 --- a/server/spec/utils/spans.jest.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { cutSpanWithSpans } from '../../src/utils/spans'; - -import 'jest'; - - -function cutSpan(from: number, to: number, cuts: [number, number][]): [number, number][] { - return cutSpanWithSpans( - { from: from, to: to }, - cuts.map(([from, to]) => ({ from, to })) - ).map(({ from, to }) => [from, to] as [number, number]); -} - -describe('cutSpanWithSpans', function() { - - it('should find spans in simple non-intersected borders', function() { - let cutSpans = [[3, 5], [6, 8], [10, 20]] as [number, number][]; - - expect(cutSpan(4, 11, cutSpans)).toEqual([[5, 6], [8, 10]]); - expect(cutSpan(5, 11, cutSpans)).toEqual([[5, 6], [8, 10]]); - expect(cutSpan(4, 10, cutSpans)).toEqual([[5, 6], [8, 10]]); - expect(cutSpan(5, 10, cutSpans)).toEqual([[5, 6], [8, 10]]); - expect(cutSpan(4, 20, cutSpans)).toEqual([[5, 6], [8, 10]]); - expect(cutSpan(4, 21, cutSpans)).toEqual([[5, 6], [8, 10], [20, 21]]); - expect(cutSpan(2, 20, cutSpans)).toEqual([[2, 3], [5, 6], [8, 10]]); - expect(cutSpan(2, 21, cutSpans)).toEqual([[2, 3], [5, 6], [8, 10], [20, 21]]); - expect(cutSpan(3, 11, cutSpans)).toEqual([[5, 6], [8, 10]]); - expect(cutSpan(3, 20, cutSpans)).toEqual([[5, 6], [8, 10]]); - expect(cutSpan(4, 7, [[3, 5], [6, 8]])).toEqual([[5, 6]]); - }); - - it('should handle empty input spans list case', function() { - expect(cutSpan(4, 10, [])).toEqual([[4, 10]]); - }); - - it('should handle case when from and to are inside of one big span', function() { - expect(cutSpan(4, 10, [[1, 20]])).toEqual([]); - expect(cutSpan(4, 10, [[1, 10]])).toEqual([]); - expect(cutSpan(4, 10, [[4, 20]])).toEqual([]); - expect(cutSpan(4, 10, [[4, 10]])).toEqual([]); - }); - - it('should be ready to get not-sorted cuts', function() { - expect(cutSpan(0, 20, [[3, 5], [1, 2]])).toEqual([[0, 1], [2, 3], [5, 20]]); - expect(cutSpan(0, 20, [[3, 5], [1, 2], [0.1, 0.5]])).toEqual([[0, 0.1], [0.5, 1], [2, 3], [5, 20]]); - }); - - it('should be ready to get overlayed cuts', function() { - expect(cutSpan(0, 20, [[3, 5], [4, 10]])).toEqual([[0,3], [10, 20]]); - }); - -}); diff --git a/server/src/controllers/analytics_controller.ts b/server/src/controllers/analytics_controller.ts index 0e12c28..5f4b3b6 100644 --- a/server/src/controllers/analytics_controller.ts +++ b/server/src/controllers/analytics_controller.ts @@ -10,7 +10,7 @@ import { AlertService } from '../services/alert_service'; import { HASTIC_API_KEY } from '../config'; import { DataPuller } from '../services/data_puller'; import { getGrafanaUrl } from '../utils/grafana'; -import { cutSpanWithSpans } from '../utils/spans'; +import { cutSegmentWithSegments } from '../utils/segments'; import { queryByMetric, GrafanaUnavailable, DatasourceUnavailable } from 'grafana-datasource-kit'; @@ -564,7 +564,7 @@ export async function getDetectionSpans( } } - let newDetectionSpans = cutSpanWithSpans({ from, to }, readySpans); + let newDetectionSpans = cutSegmentWithSegments({ from, to }, readySpans); if(newDetectionSpans.length === 0) { return [ new Detection.DetectionSpan(analyticUnitId, from, to, Detection.DetectionStatus.READY) ]; } diff --git a/server/src/utils/segments.ts b/server/src/utils/segments.ts new file mode 100644 index 0000000..01b9c00 --- /dev/null +++ b/server/src/utils/segments.ts @@ -0,0 +1,185 @@ +//TODO: move this code to span model + +import * as _ from 'lodash'; + + +export declare type Segment = { + readonly from: number, + readonly to: number +} + +export class IntegerSegment { + readonly from: number; + readonly to: number; + + constructor(from: number, to: number) { + if(!(Number.isInteger(from) || !Number.isFinite(from))) { + throw new Error(`From should be an Integer or Infinity, but got ${from}`); + } + if(!(Number.isInteger(to) || !Number.isFinite(to))) { + throw new Error(`To should be an Integer or Infinity, but got ${from}`); + } + + let l = IntegerSegment.lengthBetweenPoints(from, to); + if(l < 1) { + throw new Error( + `Length of segment is less than 1: [${from}, ${to}]. + It's not possible for IntegerSegment` + ); + } + this.from = from; + this.to = to; + } + + get length(): number { + return IntegerSegment.lengthBetweenPoints(this.from, this.to); + } + + insersect(segment: IntegerSegment): IntegerSegment | undefined { + let from = Math.max(this.from, segment.from); + let to = Math.min(this.to, segment.to); + if(IntegerSegment.lengthBetweenPoints(from, to) >= 1) { + return new IntegerSegment(from, to); + } + return undefined; + } + + toString(): string { + return `[${this.from}, ${this.to}]`; + } + + static lengthBetweenPoints(from: number, to: number): number { + let l = to - from + 1; // because [x, x] has length 1 + if(isNaN(l)) { // when [Infinity, Infinity] or [-Infinity, -Infinity] + return 0; + } else { + return Math.max(l, 0); // becase [x, x - 1] we consider as zero length + } + } +} + +export class IntegerSegmentsSet { + + private _segments: IntegerSegment[]; + + constructor(segments: IntegerSegment[], noramlized: boolean = false) { + this._segments = segments; + if(noramlized !== true) { + this._normalize(); + } + } + + private _normalize() { + if(this._segments.length === 0) { + return; + } + let sortedSegments = _.sortBy(this._segments, s => s.from); + let lastFrom = sortedSegments[0].from; + let lastTo = sortedSegments[0].to; + let mergedSegments: IntegerSegment[] = []; + for(let i = 1; i < sortedSegments.length; i++) { + let currentSegment = sortedSegments[i]; + if(lastTo + 1 >= currentSegment.from) { // because [a, x], [x + 1, b] is [a, b] + lastTo = Math.max(currentSegment.to, lastTo); // we can be inside previous + continue; + } + mergedSegments.push(new IntegerSegment(lastFrom, lastTo)); + lastFrom = currentSegment.from; + lastTo = currentSegment.to; + } + mergedSegments.push(new IntegerSegment(lastFrom, lastTo)); + this._segments = mergedSegments; + } + + get segments(): IntegerSegment[] { + return this._segments; + } + + inversed(): IntegerSegmentsSet { + var invertedSegments: IntegerSegment[] = []; + if(this._segments.length === 0) { + invertedSegments = [new IntegerSegment(-Infinity, Infinity)]; + } else { + let push = (f: number, t: number) => { + if(IntegerSegment.lengthBetweenPoints(f, t) > 0) { + invertedSegments.push(new IntegerSegment(f, t)); + } + } + _.reduce(this._segments, (prev: IntegerSegment | null, s: IntegerSegment) => { + if(prev === null) { + push(-Infinity, s.from - 1); + } else { + push(prev.to + 1, s.from - 1); + } + return s; + }, null); + push(_.last(this._segments).to + 1, Infinity); + } + return new IntegerSegmentsSet(invertedSegments, true); + } + + intersect(other: IntegerSegmentsSet): IntegerSegmentsSet { + let result: IntegerSegment[] = []; + + if(this._segments.length === 0 || other.segments.length === 0) { + return new IntegerSegmentsSet([], true); + } + + let currentSegmentIndex = 0; + let withSegmentIndex = 0; + + do { + let currentSegemet = this.segments[currentSegmentIndex]; + let withSegment = other.segments[withSegmentIndex]; + if(currentSegemet.to < withSegment.from) { + currentSegmentIndex++; + continue; + } + if(withSegment.to < currentSegemet.from) { + withSegmentIndex++; + continue; + } + let segmentsIntersection = currentSegemet.insersect(withSegment); + if(segmentsIntersection === undefined) { + throw new Error( + `Impossible condition, segments ${currentSegemet} and ${withSegment} don't interset` + ) + } + result.push(segmentsIntersection); + + if(currentSegemet.to < withSegment.to) { + currentSegmentIndex++; + } else { + withSegmentIndex++; + } + } while ( + currentSegmentIndex < this._segments.length && + withSegmentIndex < other.segments.length + ) + + return new IntegerSegmentsSet(result, true); + } + + sub(other: IntegerSegmentsSet): IntegerSegmentsSet { + let inversed = other.inversed(); + return this.intersect(inversed); + } + +} + +// TODO: move from utils and use generator +/** + * + * @param inputSegment a big segment which we will cut + * @param cutSegments segments to cut the inputSegment. Segments can overlay. + * + * @returns array of segments remain after cut + */ +export function cutSegmentWithSegments(inputSegment: Segment, cutSegments: Segment[]): Segment[] { + let setA = new IntegerSegmentsSet([new IntegerSegment(inputSegment.from, inputSegment.to)]); + let setB = new IntegerSegmentsSet(cutSegments.map( + s => new IntegerSegment(s.from, s.to) + )); + let setResult = setA.sub(setB); + return setResult.segments.map(s => ({ from: s.from, to: s.to })); +} diff --git a/server/src/utils/spans.ts b/server/src/utils/spans.ts deleted file mode 100644 index 7c58450..0000000 --- a/server/src/utils/spans.ts +++ /dev/null @@ -1,64 +0,0 @@ -//TODO: move this code to span model - -import * as _ from 'lodash'; - - -export declare type Span = { - from: number, - to: number -} - -// TODO: move from utils and use generator -/** - * - * @param inputSpan a big span which we will cut - * @param cutSpans spans which to cut the inputSpan. Spans can overlay. - * - * @returns array of spans which are holes - */ -export function cutSpanWithSpans(inputSpan: Span, cutSpans: Span[]): Span[] { - if(cutSpans.length === 0) { - return [inputSpan]; - } - - // we sort and merge out cuts to normalize it - cutSpans = _.sortBy(cutSpans, s => s.from); - const mergedSortedCuts =_.reduce(cutSpans, - ((acc: Span[], s: Span) => { - if(acc.length === 0) return [s]; - let last = acc[acc.length - 1]; - if(s.to <= last.to) return acc; - if(s.from <= last.to) { - last.to = s.to; - return acc; - } - acc.push(s); - return acc; - }), [] - ); - - // this is what we get if we cut `mergedSortedCuts` from (-Infinity, Infinity) - const holes = mergedSortedCuts.map((cut, i) => { - let from = -Infinity; - let to = cutSpans[0].from; - if(i > 0) { - from = mergedSortedCuts[i - 1].to; - to = cut.from; - } - return { from, to }; - }).concat({ - from: mergedSortedCuts[mergedSortedCuts.length - 1].to, - to: Infinity - }); - - const holesInsideInputSpan = _(holes).map(c => { - if(c.to <= inputSpan.from) return undefined; - if(inputSpan.to <= c.from) return undefined; - return { - from: Math.max(c.from, inputSpan.from), - to: Math.min(c.to, inputSpan.to), - } - }).compact().value(); - - return Array.from(holesInsideInputSpan); -}