diff --git a/.travis.yml b/.travis.yml index c6c68fa..2d360c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,9 +19,6 @@ matrix: before_script: - cd analytics - pip install -r requirements.txt - - sudo apt-get update - - sudo apt-get install -y - apt-utils gnupg curl make g++ git script: - python -m unittest discover diff --git a/server/package.json b/server/package.json index b1ac1bf..51e37a5 100644 --- a/server/package.json +++ b/server/package.json @@ -21,7 +21,7 @@ "homepage": "https://github.com/hastic/hastic-server#readme", "dependencies": {}, "devDependencies": { - "@types/jest": "^23.1.1", + "@types/jest": "^23.3.14", "@types/koa": "^2.0.46", "@types/koa-bodyparser": "^4.2.0", "@types/koa-router": "^7.0.31", diff --git a/server/spec/segments.jest.ts b/server/spec/segments.jest.ts index 480be96..667208c 100644 --- a/server/spec/segments.jest.ts +++ b/server/spec/segments.jest.ts @@ -1,5 +1,5 @@ -import { TEST_ANALYTIC_UNIT_ID } from './utils_for_tests/analytic_units'; -import { buildSegments, clearSegmentsDB, convertSegmentsToTimeRanges } from './utils_for_tests/segments'; +import { TEST_ANALYTIC_UNIT_ID, createTestDB, clearTestDB } from './utils_for_tests/analytic_units'; +import { buildSegments, clearSegmentsDB } from './utils_for_tests/segments'; import * as Segment from '../src/models/segment_model'; @@ -7,6 +7,11 @@ import * as _ from 'lodash'; const INITIAL_SEGMENTS = buildSegments([[0, 1], [2, 3], [4, 5]]); +beforeAll(async () => { + await clearTestDB(); + await createTestDB(); +}); + beforeEach(async () => { await Segment.mergeAndInsertSegments(INITIAL_SEGMENTS); }); @@ -16,7 +21,7 @@ afterEach(async () => { }); describe('mergeAndInsertSegments', function() { - it('Should be merged before insertion', async function() { + it('should be merged before insertion', async function() { const segmentsToInsert = buildSegments([[1, 2]]); await Segment.mergeAndInsertSegments(segmentsToInsert); diff --git a/server/spec/utils_for_tests/analytic_units.ts b/server/spec/utils_for_tests/analytic_units.ts index 45e4367..9c20c97 100644 --- a/server/spec/utils_for_tests/analytic_units.ts +++ b/server/spec/utils_for_tests/analytic_units.ts @@ -1,9 +1,10 @@ import * as AnalyticUnit from '../../src/models/analytic_units'; +import * as AnalyticUnitCache from '../../src/models/analytic_unit_cache_model'; + import { Metric } from 'grafana-datasource-kit'; import * as _ from 'lodash'; -import * as AnalyticUnitCache from '../../src/models/analytic_unit_cache_model'; export const TEST_ANALYTIC_UNIT_ID: AnalyticUnit.AnalyticUnitId = 'testid'; diff --git a/server/spec/utils_for_tests/segments.ts b/server/spec/utils_for_tests/segments.ts index 87bca00..9a5cdfb 100644 --- a/server/spec/utils_for_tests/segments.ts +++ b/server/spec/utils_for_tests/segments.ts @@ -16,5 +16,5 @@ export function convertSegmentsToTimeRanges(segments: Segment.Segment[]): number export async function clearSegmentsDB(): Promise { const segments = await Segment.findMany(TEST_ANALYTIC_UNIT_ID, { labeled: false, deleted: false }); - await Segment.removeSegments(segments.map(s => s.id)); + await Segment.removeSegments(_.compact(segments.map(s => s.id))); } diff --git a/server/src/models/analytic_unit_cache_model.ts b/server/src/models/analytic_unit_cache_model.ts index 2f22c46..0be76cb 100644 --- a/server/src/models/analytic_unit_cache_model.ts +++ b/server/src/models/analytic_unit_cache_model.ts @@ -54,7 +54,7 @@ export class AnalyticUnitCache { } } -export async function findById(id: AnalyticUnitId): Promise { +export async function findById(id: AnalyticUnitId): Promise { let obj = await db.findOne(id); if(obj === null) { return null; diff --git a/server/src/models/analytic_units/anomaly_analytic_unit_model.ts b/server/src/models/analytic_units/anomaly_analytic_unit_model.ts index 4f4d3be..0802f5c 100644 --- a/server/src/models/analytic_units/anomaly_analytic_unit_model.ts +++ b/server/src/models/analytic_units/anomaly_analytic_unit_model.ts @@ -75,7 +75,7 @@ export class AnomalyAnalyticUnit extends AnalyticUnit { static fromObject(obj: any) { // TODO: remove duplication - let metric: Metric; + let metric: Metric | undefined = undefined; if (obj.metric !== undefined) { metric = Metric.fromObject(obj.metric); } diff --git a/server/src/models/analytic_units/db.ts b/server/src/models/analytic_units/db.ts index a0369a8..e5a8f6f 100644 --- a/server/src/models/analytic_units/db.ts +++ b/server/src/models/analytic_units/db.ts @@ -10,7 +10,7 @@ import * as _ from 'lodash'; const db = makeDBQ(Collection.ANALYTIC_UNITS); -export async function findById(id: AnalyticUnitId): Promise { +export async function findById(id: AnalyticUnitId): Promise { let obj = await db.findOne(id); if (obj === null) { return null; diff --git a/server/src/models/analytic_units/pattern_analytic_unit_model.ts b/server/src/models/analytic_units/pattern_analytic_unit_model.ts index ea4ccf6..56f74ed 100644 --- a/server/src/models/analytic_units/pattern_analytic_unit_model.ts +++ b/server/src/models/analytic_units/pattern_analytic_unit_model.ts @@ -56,7 +56,7 @@ export class PatternAnalyticUnit extends AnalyticUnit { static fromObject(obj: any) { // TODO: remove duplication - let metric: Metric; + let metric: Metric | undefined = undefined; if(obj.metric !== undefined) { metric = Metric.fromObject(obj.metric); } diff --git a/server/src/models/analytic_units/threshold_analytic_unit_model.ts b/server/src/models/analytic_units/threshold_analytic_unit_model.ts index e241668..d5a830a 100644 --- a/server/src/models/analytic_units/threshold_analytic_unit_model.ts +++ b/server/src/models/analytic_units/threshold_analytic_unit_model.ts @@ -72,7 +72,7 @@ export class ThresholdAnalyticUnit extends AnalyticUnit { static fromObject(obj: any) { // TODO: remove duplication - let metric: Metric; + let metric: Metric | undefined = undefined; if (obj.metric !== undefined) { metric = Metric.fromObject(obj.metric); } diff --git a/server/src/models/segment_model.ts b/server/src/models/segment_model.ts index 3edc9a3..ee19920 100644 --- a/server/src/models/segment_model.ts +++ b/server/src/models/segment_model.ts @@ -114,8 +114,8 @@ export async function findMany(id: AnalyticUnitId, query: FindManyQuery): Promis * If `from` and `to` are undefined: @returns all segments */ export async function findIntersectedSegments( - analyticUnitId: AnalyticUnit.AnalyticUnitId, - from?: number, + analyticUnitId: AnalyticUnit.AnalyticUnitId, + from?: number, to?: number ): Promise { let query: FindManyQuery = {}; @@ -128,6 +128,9 @@ export async function findIntersectedSegments( return findMany(analyticUnitId, query); } + +// TODO: rewrite all this horrible function +// TODO: use utils.segments.IntegerSegmentsSet /** * Merges an array of segments with ones existing in the DB * Inserts resulting segments into DB @@ -142,6 +145,18 @@ export async function mergeAndInsertSegments(segments: Segment[]): Promise<{ return { addedIds: [], removedIds: [] }; } const analyticUnitId: AnalyticUnitId = segments[0].analyticUnitId; + const unit = await AnalyticUnit.findById(analyticUnitId); + if(unit === null) { + throw new Error('Can`t find analytic unit ' + analyticUnitId); + } + const cache = await AnalyticUnitCache.findById(analyticUnitId); + + if(cache === null) { + throw new Error('Can`t find cache for analytic unit ' + analyticUnitId); + } + + const detector = unit.detectorType; + let segmentIdsToRemove: SegmentId[] = []; let segmentsToInsert: Segment[] = []; @@ -156,11 +171,7 @@ export async function mergeAndInsertSegments(segments: Segment[]): Promise<{ } } - let cache = await AnalyticUnitCache.findById(analyticUnitId); - let unit = await AnalyticUnit.findById(analyticUnitId); - const detector = unit.detectorType; - - let intersectedSegments: Segment[]; + let intersectedSegments: Segment[] = []; if(detector === AnalyticUnit.DetectorType.PATTERN) { intersectedSegments = await findMany(analyticUnitId, { to: { $gte: segment.from }, @@ -179,12 +190,24 @@ export async function mergeAndInsertSegments(segments: Segment[]): Promise<{ } if(intersectedSegments.length > 0) { - let from = _.minBy(intersectedSegments.concat(segment), s => s.from).from; - let to = _.maxBy(intersectedSegments.concat(segment), s => s.to).to; + let intersectedIds = intersectedSegments.map(s => s.id); + let minFromSegment = _.minBy(intersectedSegments.concat(segment), s => s.from); + let maxToSegment = _.maxBy(intersectedSegments.concat(segment), s => s.to); + + if(minFromSegment === undefined) { + throw new Error('minFromSegment is undefined'); + } + + if(maxToSegment === undefined) { + throw new Error('maxToSegment is undefined'); + } + + let from = minFromSegment.from; + let to = maxToSegment.to; let newSegment = Segment.fromObject(segment.toObject()); newSegment.from = from; newSegment.to = to; - segmentIdsToRemove = segmentIdsToRemove.concat(intersectedSegments.map(s => s.id)); + segmentIdsToRemove = segmentIdsToRemove.concat(_.compact(intersectedIds)); segmentsToInsert.push(newSegment); } else { segmentsToInsert.push(segment); diff --git a/server/src/services/data_service.ts b/server/src/services/data_service.ts index 3f72970..20a4c0f 100644 --- a/server/src/services/data_service.ts +++ b/server/src/services/data_service.ts @@ -31,16 +31,24 @@ export type DBQ = { removeMany: (query: string[] | object) => Promise } +function nedbCollectionFromCollection(collection: Collection): nedb { + let nedbCollection = db.get(collection); + if(nedbCollection === undefined) { + throw new Error('Can`t find collection ' + collection); + } + return nedbCollection; +} + export function makeDBQ(collection: Collection): DBQ { return { - findOne: dbFindOne.bind(null, collection), - findMany: dbFindMany.bind(null, collection), - insertOne: dbInsertOne.bind(null, collection), - insertMany: dbInsertMany.bind(null, collection), - updateOne: dbUpdateOne.bind(null, collection), - updateMany: dbUpdateMany.bind(null, collection), - removeOne: dbRemoveOne.bind(null, collection), - removeMany: dbRemoveMany.bind(null, collection) + findOne: dbFindOne.bind(null, nedbCollectionFromCollection(collection)), + findMany: dbFindMany.bind(null, nedbCollectionFromCollection(collection)), + insertOne: dbInsertOne.bind(null, nedbCollectionFromCollection(collection)), + insertMany: dbInsertMany.bind(null, nedbCollectionFromCollection(collection)), + updateOne: dbUpdateOne.bind(null, nedbCollectionFromCollection(collection)), + updateMany: dbUpdateMany.bind(null, nedbCollectionFromCollection(collection)), + removeOne: dbRemoveOne.bind(null, nedbCollectionFromCollection(collection)), + removeMany: dbRemoveMany.bind(null, nedbCollectionFromCollection(collection)) } } @@ -58,6 +66,7 @@ function wrapIdsToQuery(query: string[] | object): any { return query; } +// TODO: move to utils function isEmptyArray(obj: any): boolean { if(!Array.isArray(obj)) { return false; @@ -67,24 +76,25 @@ function isEmptyArray(obj: any): boolean { const db = new Map(); -let dbInsertOne = (collection: Collection, doc: object) => { + +async function dbInsertOne(nd: nedb, doc: object): Promise { return new Promise((resolve, reject) => { - db.get(collection).insert(doc, (err, newDoc: any) => { + nd.insert(doc, (err, newDoc: any) => { if(err) { reject(err); } else { resolve(newDoc._id); } - }); + }) }); } -let dbInsertMany = (collection: Collection, docs: object[]) => { +async function dbInsertMany(nd: nedb, docs: object[]): Promise { if(docs.length === 0) { return Promise.resolve([]); } return new Promise((resolve, reject) => { - db.get(collection).insert(docs, (err, newDocs: any[]) => { + nd.insert(docs, (err, newDocs: any[]) => { if(err) { reject(err); } else { @@ -94,12 +104,12 @@ let dbInsertMany = (collection: Collection, docs: object[]) => { }); } -let dbUpdateOne = (collection: Collection, query: string | object, updateQuery: object) => { +async function dbUpdateOne(nd: nedb, query: string | object, updateQuery: object): Promise { // https://github.com/louischatriot/nedb#updating-documents let nedbUpdateQuery = { $set: updateQuery } query = wrapIdToQuery(query); return new Promise((resolve, reject) => { - db.get(collection).update( + nd.update( query, nedbUpdateQuery, { returnUpdatedDocs: true }, @@ -114,7 +124,7 @@ let dbUpdateOne = (collection: Collection, query: string | object, updateQuery: }); } -let dbUpdateMany = (collection: Collection, query: string[] | object, updateQuery: object) => { +async function dbUpdateMany(nd: nedb, query: string[] | object, updateQuery: object): Promise { // https://github.com/louischatriot/nedb#updating-documents if(isEmptyArray(query)) { return Promise.resolve([]); @@ -122,7 +132,7 @@ let dbUpdateMany = (collection: Collection, query: string[] | object, updateQuer let nedbUpdateQuery = { $set: updateQuery }; query = wrapIdsToQuery(query); return new Promise((resolve, reject) => { - db.get(collection).update( + nd.update( query, nedbUpdateQuery, { returnUpdatedDocs: true, multi: true }, @@ -137,10 +147,10 @@ let dbUpdateMany = (collection: Collection, query: string[] | object, updateQuer }); } -let dbFindOne = (collection: Collection, query: string | object) => { +async function dbFindOne(nd: nedb, query: string | object): Promise { query = wrapIdToQuery(query); return new Promise((resolve, reject) => { - db.get(collection).findOne(query, (err, doc) => { + nd.findOne(query, (err, doc) => { if(err) { reject(err); } else { @@ -150,13 +160,13 @@ let dbFindOne = (collection: Collection, query: string | object) => { }); } -let dbFindMany = (collection: Collection, query: string[] | object, sortQuery: object = {}) => { +async function dbFindMany(nd: nedb, query: string[] | object, sortQuery: object = {}): Promise { if(isEmptyArray(query)) { return Promise.resolve([]); } query = wrapIdsToQuery(query); return new Promise((resolve, reject) => { - db.get(collection).find(query).sort(sortQuery).exec((err, docs: any[]) => { + nd.find(query).sort(sortQuery).exec((err, docs: any[]) => { if(err) { reject(err); } else { @@ -166,10 +176,10 @@ let dbFindMany = (collection: Collection, query: string[] | object, sortQuery: o }); } -let dbRemoveOne = (collection: Collection, query: string | object) => { +async function dbRemoveOne(nd: nedb, query: string | object): Promise { query = wrapIdToQuery(query); return new Promise((resolve, reject) => { - db.get(collection).remove(query, { /* options */ }, (err, numRemoved) => { + nd.remove(query, { /* options */ }, (err, numRemoved) => { if(err) { reject(err); } else { @@ -183,13 +193,13 @@ let dbRemoveOne = (collection: Collection, query: string | object) => { }); } -let dbRemoveMany = (collection: Collection, query: string[] | object) => { +async function dbRemoveMany(nd: nedb, query: string[] | object): Promise { if(isEmptyArray(query)) { - return Promise.resolve([]); + return Promise.resolve(0); } query = wrapIdsToQuery(query); return new Promise((resolve, reject) => { - db.get(collection).remove(query, { multi: true }, (err, numRemoved) => { + nd.remove(query, { multi: true }, (err, numRemoved) => { if(err) { reject(err); } else { diff --git a/server/src/utils/segments.ts b/server/src/utils/segments.ts index 01b9c00..b94aec3 100644 --- a/server/src/utils/segments.ts +++ b/server/src/utils/segments.ts @@ -113,7 +113,7 @@ export class IntegerSegmentsSet { } return s; }, null); - push(_.last(this._segments).to + 1, Infinity); + push(this._segments[this._segments.length - 1].to + 1, Infinity); } return new IntegerSegmentsSet(invertedSegments, true); } diff --git a/server/tsconfig.json b/server/tsconfig.json index d5cef27..291b73f 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -2,6 +2,7 @@ "compilerOptions": { "sourceMap": true, "module": "commonjs", - "target": "es6" + "target": "es6", + // "strict": true } }