Browse Source

Merge pull request #106 from hastic/NeDB-instead-of-files-for-analytics-#58

Ne db instead of files for analytics #58
pull/1/head
Alexey Velikiy 6 years ago committed by GitHub
parent
commit
42bfc9bb24
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 3
      server/.vscode/launch.json
  2. 15
      server/src/config.ts
  3. 2
      server/src/controllers/alerts_controller.ts
  4. 156
      server/src/controllers/analytics_controller.ts
  5. 30
      server/src/controllers/metrics_controler.ts
  6. 77
      server/src/controllers/segments_controller.ts
  7. 12
      server/src/index.ts
  8. 84
      server/src/models/analytic_unit.ts
  9. 98
      server/src/models/analytic_unit_model.ts
  10. 32
      server/src/models/metric_model.ts
  11. 87
      server/src/models/segment_model.ts
  12. 32
      server/src/models/task_model.ts
  13. 94
      server/src/routes/analytic_units_router.ts
  14. 64
      server/src/routes/segments_router.ts
  15. 8
      server/src/services/analytics_service.ts
  16. 179
      server/src/services/data_service.ts
  17. 6
      server/src/services/notification_service.ts
  18. 12
      server/src/services/process_service.ts
  19. 4
      server/tsconfig.json

3
server/.vscode/launch.json vendored

@ -17,7 +17,8 @@
//"outFiles": [ "/home/alex/Projects/hastic-server/server.js" ], //"outFiles": [ "/home/alex/Projects/hastic-server/server.js" ],
"port": 9229, "port": 9229,
"restart": true, "restart": true,
"trace": true "trace": true,
"timeout": 100000
} }
] ]
} }

15
server/src/config.ts

@ -1,5 +1,6 @@
import * as path from 'path'; import * as path from 'path';
import * as fs from 'fs'; import * as fs from 'fs';
import * as os from 'os';
import { getJsonDataSync } from './services/json_service'; import { getJsonDataSync } from './services/json_service';
@ -10,25 +11,17 @@ export const ANALYTICS_PATH = path.join(__dirname, '../../analytics');
export const DATA_PATH = path.join(__dirname, '../../data'); export const DATA_PATH = path.join(__dirname, '../../data');
export const ANALYTIC_UNITS_DATABASE_PATH = path.join(DATA_PATH, 'analyticUnits.db'); export const ANALYTIC_UNITS_DATABASE_PATH = path.join(DATA_PATH, 'analytic_units.db');
export const METRICS_DATABASE_PATH = path.join(DATA_PATH, 'metrics.db');
export const SEGMENTS_DATABASE_PATH = path.join(DATA_PATH, 'segments.db'); export const SEGMENTS_DATABASE_PATH = path.join(DATA_PATH, 'segments.db');
export const FILES_DATABASE_PATH = path.join(DATA_PATH, 'files.db');
export const DATASETS_PATH = path.join(DATA_PATH, 'datasets');
//export const ANALYTIC_UNITS_PATH = path.join(DATA_PATH, 'analytic_units');
export const MODELS_PATH = path.join(DATA_PATH, 'models');
export const METRICS_PATH = path.join(DATA_PATH, 'metrics');
export const SEGMENTS_PATH = path.join(DATA_PATH, 'segments');
export const HASTIC_PORT = getConfigField('HASTIC_PORT', '8000'); export const HASTIC_PORT = getConfigField('HASTIC_PORT', '8000');
export const ZMQ_CONNECTION_STRING = getConfigField('ZMQ_CONNECTION_STRING', null); export const ZMQ_CONNECTION_STRING = getConfigField('ZMQ_CONNECTION_STRING', null);
export const ZMQ_IPC_PATH = getConfigField('ZMQ_IPC_PATH', path.join('/tmp', 'hastic')); export const ZMQ_IPC_PATH = getConfigField('ZMQ_IPC_PATH', path.join(os.tmpdir(), 'hastic'));
export const ZMQ_DEV_PORT = getConfigField('ZMQ_DEV_PORT', '8002'); export const ZMQ_DEV_PORT = getConfigField('ZMQ_DEV_PORT', '8002');
export const ANLYTICS_PING_INTERVAL = 500; // ms export const ANLYTICS_PING_INTERVAL = 500; // ms
function getConfigField(field, defaultVal?) { function getConfigField(field: string, defaultVal?: any) {
let val = defaultVal; let val = defaultVal;
if(process.env[field] !== undefined) { if(process.env[field] !== undefined) {

2
server/src/controllers/alerts_controller.ts

@ -2,7 +2,7 @@
* Alarting is not supported yet * Alarting is not supported yet
*/ */
throw new console.error("Not supported"); throw new Error('not supported');
// import { runPredict } from './analytics_controller'; // import { runPredict } from './analytics_controller';

156
server/src/controllers/analytics_controller.ts

@ -1,11 +1,10 @@
import * as DataService from '../services/data_service'; import { Task } from '../models/task_model';
import { getTarget } from './metrics_controler'; import * as SegmentsController from '../models/segment_model';
import { getLabeledSegments, insertSegments, removeSegments } from './segments_controller'; import * as AnalyticUnit from '../models/analytic_unit_model';
import * as AnalyticUnit from '../models/analytic_unit'
import { AnalyticsService, AnalyticsMessage } from '../services/analytics_service'; import { AnalyticsService, AnalyticsMessage } from '../services/analytics_service';
const taskMap = {}; const taskMap = new Map<string, any>();
let nextTaskId = 0; let nextTaskId = 0;
let analyticsService: AnalyticsService = undefined; let analyticsService: AnalyticsService = undefined;
@ -16,21 +15,13 @@ function onTaskResult(taskResult: any) {
let status = taskResult.status; let status = taskResult.status;
if(status === 'SUCCESS' || status === 'FAILED') { if(status === 'SUCCESS' || status === 'FAILED') {
if(taskId in taskMap) { if(taskId in taskMap) {
let resolver = taskMap[taskId]; let resolver: any = taskMap.get(taskId);
resolver(taskResult); resolver(taskResult);
delete taskMap[taskId]; taskMap.delete(taskId);
} }
} }
} }
async function onFileSave(payload: any): Promise<any> {
return DataService.saveFile(payload.filename, payload.content);
}
async function onFileLoad(payload: any): Promise<any> {
return DataService.loadFile(payload.filename);
}
async function onMessage(message: AnalyticsMessage) { async function onMessage(message: AnalyticsMessage) {
let responsePayload = null; let responsePayload = null;
let resolvedMethod = false; let resolvedMethod = false;
@ -40,15 +31,6 @@ async function onMessage(message: AnalyticsMessage) {
resolvedMethod = true; resolvedMethod = true;
} }
if(message.method === 'FILE_SAVE') {
responsePayload = await onFileSave(message.payload);
resolvedMethod = true;
}
if(message.method === 'FILE_LOAD') {
responsePayload = await onFileLoad(message.payload);
resolvedMethod = true;
}
if(!resolvedMethod) { if(!resolvedMethod) {
throw new TypeError('Unknown method ' + message.method); throw new TypeError('Unknown method ' + message.method);
} }
@ -68,76 +50,82 @@ export function terminate() {
analyticsService.close(); analyticsService.close();
} }
async function runTask(task): Promise<any> { async function runTask(task: Task): Promise<any> {
let anomaly: AnalyticUnit.AnalyticUnit = AnalyticUnit.findById(task.analyticUnitId); // let anomaly: AnalyticUnit.AnalyticUnit = await AnalyticUnit.findById(task.analyticUnitId);
task.metric = { // task.metric = {
datasource: anomaly.metric.datasource, // datasource: anomaly.metric.datasource,
targets: anomaly.metric.targets.map(getTarget) // targets: anomaly.metric.targets.map(getTarget)
}; // };
task._taskId = nextTaskId++; // task._taskId = nextTaskId++;
await analyticsService.sendTask(task); // await analyticsService.sendTask(task);
return new Promise<void>(resolve => { // return new Promise<void>(resolve => {
taskMap[task._taskId] = resolve; // taskMap[task._taskId] = resolve;
}) // })
} }
export async function runLearning(id: AnalyticUnit.AnalyticUnitId) { export async function runLearning(id: AnalyticUnit.AnalyticUnitId) {
let segments = getLabeledSegments(id); // let segments = getLabeledSegments(id);
AnalyticUnit.setStatus(id, 'LEARNING'); // AnalyticUnit.setStatus(id, 'LEARNING');
let unit = AnalyticUnit.findById(id); // let unit = await AnalyticUnit.findById(id);
let pattern = unit.type; // let pattern = unit.type;
let task = { // let task = {
analyticUnitId: id, // analyticUnitId: id,
type: 'LEARN', // type: 'LEARN',
pattern, // pattern,
segments: segments // segments: segments
}; // };
let result = await runTask(task); // let result = await runTask(task);
if (result.status === 'SUCCESS') { // if (result.status === 'SUCCESS') {
AnalyticUnit.setStatus(id, 'READY'); // AnalyticUnit.setStatus(id, 'READY');
insertSegments(id, result.segments, false); // insertSegments(id, result.segments, false);
AnalyticUnit.setPredictionTime(id, result.lastPredictionTime); // AnalyticUnit.setPredictionTime(id, result.lastPredictionTime);
} else { // } else {
AnalyticUnit.setStatus(id, 'FAILED', result.error); // AnalyticUnit.setStatus(id, 'FAILED', result.error);
// }
} }
}
export async function runPredict(id: AnalyticUnit.AnalyticUnitId) { export async function runPredict(id: AnalyticUnit.AnalyticUnitId) {
let unit = AnalyticUnit.findById(id); // let unit = await AnalyticUnit.findById(id);
let pattern = unit.type; // let pattern = unit.type;
let task = { // let task = {
type: 'predict', // type: 'PREDICT',
analyticUnitId: id, // analyticUnitId: id,
pattern, // pattern,
lastPredictionTime: unit.lastPredictionTime // lastPredictionTime: unit.lastPredictionTime
}; // };
let result = await runTask(task); // let result = await runTask(task);
if(result.status === 'FAILED') { // if(result.status === 'FAILED') {
return []; // return [];
} // }
// Merging segments // // Merging segments
let segments = getLabeledSegments(id); // let segments = getLabeledSegments(id);
if(segments.length > 0 && result.segments.length > 0) { // if(segments.length > 0 && result.segments.length > 0) {
let lastOldSegment = segments[segments.length - 1]; // let lastOldSegment = segments[segments.length - 1];
let firstNewSegment = result.segments[0]; // let firstNewSegment = result.segments[0];
if(firstNewSegment.start <= lastOldSegment.finish) { // if(firstNewSegment.start <= lastOldSegment.finish) {
result.segments[0].start = lastOldSegment.start; // result.segments[0].start = lastOldSegment.start;
removeSegments(id, [lastOldSegment.id]); // removeSegments(id, [lastOldSegment.id]);
} // }
} // }
insertSegments(id, result.segments, false); // insertSegments(id, result.segments, false);
AnalyticUnit.setPredictionTime(id, result.lastPredictionTime); // AnalyticUnit.setPredictionTime(id, result.lastPredictionTime);
return result.segments; // return result.segments;
} }
export function isAnalyticReady(): boolean { export function isAnalyticReady(): boolean {
return analyticsService.ready; return analyticsService.ready;
} }
export async function createAnalyticUnitFromObject(obj: any): Promise<AnalyticUnit.AnalyticUnitId> {
let unit: AnalyticUnit.AnalyticUnit = AnalyticUnit.AnalyticUnit.fromObject(obj);
let id = await AnalyticUnit.create(unit);
// runLearning(unit);
return id;
}

30
server/src/controllers/metrics_controler.ts

@ -1,30 +0,0 @@
import { getJsonDataSync, writeJsonDataSync } from '../services/json_service';
import { METRICS_PATH } from '../config';
import * as crypto from 'crypto';
import * as path from 'path';
function saveTargets(targets) {
let metrics = [];
for (let target of targets) {
metrics.push(saveTarget(target));
}
return metrics;
}
function saveTarget(target) {
//const md5 = crypto.createHash('md5')
const targetId = crypto.createHash('md5').update(JSON.stringify(target)).digest('hex');
let filename = path.join(METRICS_PATH, `${targetId}.json`);
writeJsonDataSync(filename, target);
return targetId;
}
function getTarget(targetId) {
let filename = path.join(METRICS_PATH, `${targetId}.json`);
return getJsonDataSync(filename);
}
export { saveTargets, getTarget }

77
server/src/controllers/segments_controller.ts

@ -1,77 +0,0 @@
import { getJsonDataSync, writeJsonDataSync } from '../services/json_service';
import { AnalyticUnitId, findById, save } from '../models/analytic_unit';
import { SEGMENTS_PATH } from '../config';
import * as _ from 'lodash';
import * as path from 'path';
import * as fs from 'fs';
export function getLabeledSegments(id: AnalyticUnitId) {
let filename = path.join(SEGMENTS_PATH, `${id}_labeled.json`);
if(!fs.existsSync(filename)) {
return [];
} else {
let segments = getJsonDataSync(filename);
for(let segment of segments) {
if(segment.labeled === undefined) {
segment.labeled = false;
}
}
return segments;
}
}
export function getPredictedSegments(id: AnalyticUnitId) {
let filename = path.join(SEGMENTS_PATH, `${id}_segments.json`);
let jsonData;
try {
jsonData = getJsonDataSync(filename);
} catch(e) {
console.error(e.message);
jsonData = [];
}
return jsonData;
}
export function saveSegments(id: AnalyticUnitId, segments) {
let filename = path.join(SEGMENTS_PATH, `${id}_labeled.json`);
try {
return writeJsonDataSync(filename, _.uniqBy(segments, 'start'));
} catch(e) {
console.error(e.message);
throw new Error('Can`t write to db');
}
}
export function insertSegments(id: AnalyticUnitId, addedSegments, labeled: boolean) {
// Set status
let info = findById(id);
let segments = getLabeledSegments(id);
let nextId = info.nextId;
let addedIds = []
for (let segment of addedSegments) {
segment.id = nextId;
segment.labeled = labeled;
addedIds.push(nextId);
nextId++;
segments.push(segment);
}
info.nextId = nextId;
saveSegments(id, segments);
save(id, info);
return addedIds;
}
export function removeSegments(id: AnalyticUnitId, removedSegments) {
let segments = getLabeledSegments(id);
for (let segmentId of removedSegments) {
segments = segments.filter(el => el.id !== segmentId);
}
saveSegments(id, segments);
}

12
server/src/index.ts

@ -1,10 +1,9 @@
import { router as anomaliesRouter } from './routes/analytic_units_router'; import { router as anomaliesRouter } from './routes/analytic_units_router';
import { router as segmentsRouter } from './routes/segments_router'; import { router as segmentsRouter } from './routes/segments_router';
//import { router as alertsRouter } from './routes/alerts_router';
import * as AnalyticsController from './controllers/analytics_controller'; import * as AnalyticsController from './controllers/analytics_controller';
import * as Data from './services/data_service';
import * as ProcessService from './services/process_service'; import * as ProcessService from './services/process_service';
import { HASTIC_PORT } from './config'; import { HASTIC_PORT } from './config';
@ -14,19 +13,24 @@ import * as Router from 'koa-router';
import * as bodyParser from 'koa-bodyparser'; import * as bodyParser from 'koa-bodyparser';
Data.checkDataFolders();
AnalyticsController.init(); AnalyticsController.init();
ProcessService.registerExitHandler(AnalyticsController.terminate); ProcessService.registerExitHandler(AnalyticsController.terminate);
var app = new Koa(); var app = new Koa();
app.on('error', (err, ctx) => {
console.log('got server error:');
console.log(err);
});
app.use(bodyParser()) app.use(bodyParser())
app.use(async function(ctx, next) { app.use(async function(ctx, next) {
ctx.set('Access-Control-Allow-Origin', '*'); ctx.set('Access-Control-Allow-Origin', '*');
ctx.set('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, PATCH, OPTIONS'); ctx.set('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, PATCH, OPTIONS');
ctx.set('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept'); ctx.set('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept');
next(); await next();
}); });

84
server/src/models/analytic_unit.ts

@ -1,84 +0,0 @@
import { getJsonDataSync, writeJsonDataSync } from '../services/json_service'
import { ANALYTIC_UNITS_PATH } from '../config'
import * as crypto from 'crypto';
import * as path from 'path'
import * as fs from 'fs'
export type Datasource = {
method: string,
data: Object,
params: Object,
type: string,
url: string
}
export type Metric = {
datasource: string,
targets: string[]
}
export type AnalyticUnitId = string;
export type AnalyticUnit = {
id?: AnalyticUnitId,
name: string,
panelUrl: string,
type: string,
metric: Metric,
datasource: Datasource
status: string,
error?: string,
lastPredictionTime: number,
nextId: number
}
export function createItem(item: AnalyticUnit): AnalyticUnitId {
const hashString = item.name + (new Date()).toString();
const newId: AnalyticUnitId = crypto.createHash('md5').update(hashString).digest('hex');
let filename = path.join(ANALYTIC_UNITS_PATH, `${newId}.json`);
if(fs.existsSync(filename)) {
throw new Error(`Can't create item with id ${newId}`);
}
save(newId, item);
item.id = newId;
return newId;
}
export function remove(id: AnalyticUnitId) {
let filename = path.join(ANALYTIC_UNITS_PATH, `${id}.json`);
fs.unlinkSync(filename);
}
export function save(id: AnalyticUnitId, unit: AnalyticUnit) {
let filename = path.join(ANALYTIC_UNITS_PATH, `${id}.json`);
return writeJsonDataSync(filename, unit);
}
// TODO: make async
export function findById(id: AnalyticUnitId): AnalyticUnit {
let filename = path.join(ANALYTIC_UNITS_PATH, `${id}.json`);
if(!fs.existsSync(filename)) {
throw new Error(`Can't find Analytic Unit with id ${id}`);
}
return getJsonDataSync(filename);
}
export function setStatus(predictorId: AnalyticUnitId, status: string, error?: string) {
let info = findById(predictorId);
info.status = status;
if(error !== undefined) {
info.error = error;
} else {
info.error = '';
}
save(predictorId, info);
}
export function setPredictionTime(id: AnalyticUnitId, time: number) {
let info = findById(id);
info.lastPredictionTime = time;
save(id, info);
}

98
server/src/models/analytic_unit_model.ts

@ -0,0 +1,98 @@
import { Metric } from './metric_model';
import { Collection, makeDBQ } from '../services/data_service';
let db = makeDBQ(Collection.ANALYTIC_UNITS);
export type AnalyticUnitId = string;
export class AnalyticUnit {
constructor(
public name: string,
public panelUrl: string,
public type: string,
public metric: Metric,
public id?: AnalyticUnitId,
public lastPredictionTime?: number,
public status?: string,
public error?: string,
) {
if(name === undefined) {
throw new Error(`Missing field "name"`);
}
if(panelUrl === undefined) {
throw new Error(`Missing field "panelUrl"`);
}
if(type === undefined) {
throw new Error(`Missing field "type"`);
}
if(metric === undefined) {
throw new Error(`Missing field "metric"`);
}
}
public toObject() {
return {
_id: this.id,
name: this.name,
panelUrl: this.panelUrl,
type: this.type,
metric: this.metric.toObject(),
lastPredictionTime: this.lastPredictionTime,
status: this.status,
error: this.error
};
}
static fromObject(obj: any): AnalyticUnit {
if(obj === undefined) {
throw new Error('obj is undefined');
}
return new AnalyticUnit(
obj.name,
obj.panelUrl,
obj.type,
Metric.fromObject(obj.metric),
obj.status,
obj.lastPredictionTime,
obj._id,
obj.error,
);
}
}
export async function findById(id: AnalyticUnitId): Promise<AnalyticUnit> {
return AnalyticUnit.fromObject(await db.findOne(id));
}
/**
* Creates and updates new unit.id
*
* @param unit to create
* @returns unit.id
*/
export async function create(unit: AnalyticUnit): Promise<AnalyticUnitId> {
var obj = unit.toObject();
var r = await db.insertOne(obj);
return r;
}
export async function remove(id: AnalyticUnitId): Promise<void> {
await db.removeOne(id);
return;
}
export async function update(id: AnalyticUnitId, unit: AnalyticUnit) {
return db.updateOne(id, unit);
}
export async function setStatus(id: AnalyticUnitId, status: string, error?: string) {
return db.updateOne(id, { status, error });
}
export async function setPredictionTime(id: AnalyticUnitId, lastPredictionTime: number) {
return db.updateOne(id, { lastPredictionTime });
}

32
server/src/models/metric_model.ts

@ -0,0 +1,32 @@
export class Metric {
constructor(public datasource: string, public targets: any[]) {
if(datasource === undefined) {
throw new Error('datasource is undefined');
}
if(targets === undefined) {
throw new Error('targets is undefined');
}
if(targets.length === 0) {
throw new Error('targets is empty');
}
}
public toObject() {
return {
datasource: this.datasource,
targets: this.targets
};
}
static fromObject(obj: any): Metric {
if(obj === undefined) {
throw new Error('obj is undefined');
}
return new Metric(
obj.datasource,
obj.targets
);
}
}

87
server/src/models/segment_model.ts

@ -0,0 +1,87 @@
import { AnalyticUnitId } from './analytic_unit_model';
import { Collection, makeDBQ } from '../services/data_service';
let db = makeDBQ(Collection.SEGMENTS);
export type SegmentId = string;
export class Segment {
constructor(
public analyticUnitId: AnalyticUnitId,
public from: number,
public to: number,
public labeled: boolean,
public id?: SegmentId
) {
if(analyticUnitId === undefined) {
throw new Error('AnalyticUnitId is undefined');
}
if(from === undefined) {
throw new Error('from is undefined');
}
if(isNaN(from)) {
throw new Error('from is NaN');
}
if(to === undefined) {
throw new Error('to is undefined');
}
if(isNaN(to)) {
throw new Error('to is NaN');
}
if(labeled === undefined) {
throw new Error('labeled is undefined');
}
}
public toObject() {
return {
_id: this.id,
analyticUnitId: this.analyticUnitId,
from: this.from,
to: this.to,
labeled: this.labeled
};
}
static fromObject(obj: any): Segment {
if(obj === undefined) {
throw new Error('obj is undefined');
}
return new Segment(
obj.analyticUnitId,
+obj.from, +obj.to,
obj.labeled, obj._id
);
}
}
export type FindManyQuery = {
timeFromGTE?: number,
timeToLTE?: number,
intexGT?: number
}
export async function findMany(id: AnalyticUnitId, query: FindManyQuery): Promise<Segment[]> {
var dbQuery: any = { analyticUnitId: id };
if(query.timeFromGTE !== undefined) {
dbQuery.from = { $gte: query.timeFromGTE };
}
if(query.timeToLTE !== undefined) {
dbQuery.to = { $lte: query.timeToLTE };
}
let segs = await db.findMany(dbQuery);
if(segs === null) {
return [];
}
return segs.map(Segment.fromObject);
}
export async function insertSegments(segments: Segment[]) {
return db.insertMany(segments.map(s => s.toObject()));
}
export function removeSegments(idsToRemove: SegmentId[]) {
return db.removeMany(idsToRemove);
}

32
server/src/models/task_model.ts

@ -0,0 +1,32 @@
import { AnalyticUnitId } from "./analytic_unit_model";
export type TaskId = string;
export class Task {
constructor(
public analyticUnitId: AnalyticUnitId,
public id?: TaskId
) {
if(analyticUnitId === undefined) {
throw new Error('analyticUnitId is undefined');
}
}
public toObject() {
return {
_id: this.id,
analyticUnitId: this.analyticUnitId
};
}
static fromObject(obj: any): Task {
if(obj === undefined) {
throw new Error('obj is undefined');
}
return new Task(
obj.analyticUnitId,
obj._id,
);
}
}

94
server/src/routes/analytic_units_router.ts

@ -1,32 +1,22 @@
import * as Router from 'koa-router'; import * as Router from 'koa-router';
import * as AnalyticUnit from '../models/analytic_unit'; import * as AnalyticUnit from '../models/analytic_unit_model';
import { runLearning } from '../controllers/analytics_controller' import { createAnalyticUnitFromObject } from '../controllers/analytics_controller'
import { saveTargets } from '../controllers/metrics_controler';
async function sendStatus(ctx: Router.IRouterContext) {
try {
let id = ctx.request.query.id;
if(id === undefined) {
throw new Error('Id is undefined');
}
let unit = AnalyticUnit.findById(id);
if(unit.status === undefined) { async function getStatus(ctx: Router.IRouterContext) {
throw new Error('status is undefined'); try {
} ctx.response.body = { status: 'READY', errorMessage: undefined };
ctx.response.body = { status: unit.status, errorMessage: unit.error };
} catch(e) { } catch(e) {
console.error(e); console.error(e);
// TODO: better send 404 when we know than isn`t found // TODO: better send 404 when we know than isn`t found
ctx.response.status = 500; ctx.response.status = 500;
ctx.response.body = { error: 'Can`t return anything' }; ctx.response.body = { error: 'Can`t return anything' };
} }
} }
async function findItem(ctx: Router.IRouterContext) { async function getUnit(ctx: Router.IRouterContext) {
try { try {
let id = ctx.request.query.id; let id = ctx.request.query.id;
@ -34,7 +24,7 @@ async function findItem(ctx: Router.IRouterContext) {
throw new Error('No id param in query'); throw new Error('No id param in query');
} }
let unit: AnalyticUnit.AnalyticUnit = AnalyticUnit.findById(id); let unit: AnalyticUnit.AnalyticUnit = await AnalyticUnit.findById(id);
ctx.response.body = { ctx.response.body = {
name: unit.name, name: unit.name,
@ -50,58 +40,10 @@ async function findItem(ctx: Router.IRouterContext) {
} }
} }
async function createItem(ctx: Router.IRouterContext) { async function createUnit(ctx: Router.IRouterContext) {
try { try {
let newId = await createAnalyticUnitFromObject(ctx.request.body);
let body = ctx.request.body;
if(body.type === undefined) {
throw new Error(`Missing field "type"`);
}
if(body.name === undefined) {
throw new Error(`Missing field "name"`);
}
if(body.panelUrl === undefined) {
throw new Error(`Missing field "panelUrl"`);
}
if(body.metric === undefined) {
throw new Error(`Missing field "datasource"`);
}
if(body.metric.datasource === undefined) {
throw new Error(`Missing field "metric.datasource"`);
}
if(body.metric.targets === undefined) {
throw new Error(`Missing field "metric.targets"`);
}
const metric: AnalyticUnit.Metric = {
datasource: body.metric.datasource,
targets: saveTargets(body.metric.targets)
};
const unit: AnalyticUnit.AnalyticUnit = {
name: body.name,
panelUrl: body.panelUrl,
type: body.type,
datasource: body.datasource,
metric: metric,
status: 'learning',
lastPredictionTime: 0,
nextId: 0
};
let newId = AnalyticUnit.createItem(unit);
if(newId === null) {
ctx.response.status = 403;
ctx.response.body = {
code: 403,
message: 'Item exists'
};
}
ctx.response.body = { id: newId }; ctx.response.body = { id: newId };
runLearning(newId);
} catch(e) { } catch(e) {
ctx.response.status = 500; ctx.response.status = 500;
ctx.response.body = { ctx.response.body = {
@ -109,16 +51,12 @@ async function createItem(ctx: Router.IRouterContext) {
message: `Creation error: ${e.message}` message: `Creation error: ${e.message}`
}; };
} }
}
function deleteItem(ctx: Router.IRouterContext) {
try {
let id = ctx.request.query.id;
if(id !== undefined) {
AnalyticUnit.remove(id);
} }
async function deleteUnit(ctx: Router.IRouterContext) {
try {
await AnalyticUnit.remove(ctx.request.query.id);
ctx.response.body = { ctx.response.body = {
code: 200, code: 200,
message: 'Success' message: 'Success'
@ -135,7 +73,7 @@ function deleteItem(ctx: Router.IRouterContext) {
export var router = new Router(); export var router = new Router();
router.get('/status', sendStatus); router.get('/', getUnit);
router.get('/', findItem); router.get('/status', getStatus);
router.post('/', createItem); router.post('/', createUnit);
router.delete('/', deleteItem); router.delete('/', deleteUnit);

64
server/src/routes/segments_router.ts

@ -1,49 +1,53 @@
import * as Router from 'koa-router'; import * as Router from 'koa-router';
import { AnalyticUnitId } from '../models/analytic_unit'; import { AnalyticUnitId } from '../models/analytic_unit_model';
import { import * as SegmentModel from '../models/segment_model';
getLabeledSegments,
insertSegments,
removeSegments,
} from '../controllers/segments_controller';
import { runLearning } from '../controllers/analytics_controller'; import { runLearning } from '../controllers/analytics_controller';
async function sendSegments(ctx: Router.IRouterContext) { async function getSegments(ctx: Router.IRouterContext) {
let id: AnalyticUnitId = ctx.request.query.id; let id: AnalyticUnitId = ctx.request.query.id;
if(id === undefined || id === '') {
let lastSegmentId = ctx.request.query.lastSegmentId; throw new Error('analyticUnitId (id) is missing');
let timeFrom = ctx.request.query.from;
let timeTo = ctx.request.query.to;
let segments = getLabeledSegments(id);
// Id filtering
if(lastSegmentId !== undefined) {
segments = segments.filter(el => el.id > lastSegmentId);
} }
let query: SegmentModel.FindManyQuery = {};
// Time filtering if(!isNaN(+ctx.request.query.lastSegmentId)) {
if(timeFrom !== undefined) { query.intexGT = +ctx.request.query.lastSegmentId;
segments = segments.filter(el => el.finish > timeFrom);
} }
if(!isNaN(+ctx.request.query.from)) {
if(timeTo !== undefined) { query.timeFromGTE = +ctx.request.query.from;
segments = segments.filter(el => el.start < timeTo); }
if(!isNaN(+ctx.request.query.to)) {
query.timeToLTE = +ctx.request.query.to;
} }
ctx.response.body = { segments } let segments = await SegmentModel.findMany(id, query);
ctx.response.body = { segments };
} }
async function updateSegments(ctx: Router.IRouterContext) { async function updateSegments(ctx: Router.IRouterContext) {
try { try {
let segmentsUpdate = ctx.request.body;
let id = segmentsUpdate.id; let {
let addedIds = insertSegments(id, segmentsUpdate.addedSegments, true); addedSegments, id, removedSegments: removedIds
removeSegments(id, segmentsUpdate.removedSegments); } = ctx.request.body as {
ctx.response.body = { addedIds }; addedSegments: any[], id: AnalyticUnitId, removedSegments: SegmentModel.SegmentId[]
};
let segmentsToInsert: SegmentModel.Segment[] = addedSegments.map(
s => SegmentModel.Segment.fromObject({ analyticUnitId: id, labeled: true, ...s })
);
let [addedIds, removed] = await Promise.all([
SegmentModel.insertSegments(segmentsToInsert),
SegmentModel.removeSegments(removedIds)
]);
ctx.response.body = { addedIds, removed };
runLearning(id); runLearning(id);
} catch(e) { } catch(e) {
ctx.response.status = 500; ctx.response.status = 500;
@ -56,5 +60,5 @@ async function updateSegments(ctx: Router.IRouterContext) {
export const router = new Router(); export const router = new Router();
router.get('/', sendSegments); router.get('/', getSegments);
router.patch('/', updateSegments); router.patch('/', updateSegments);

8
server/src/services/analytics_service.ts

@ -29,8 +29,8 @@ export class AnalyticsService {
private _requester: any; private _requester: any;
private _ready: boolean = false; private _ready: boolean = false;
private _pingResponded = false; private _pingResponded = false;
private _zmqConnectionString = null; private _zmqConnectionString: string = null;
private _ipcPath = null; private _ipcPath: string = null;
private _analyticsPinger: NodeJS.Timer = null; private _analyticsPinger: NodeJS.Timer = null;
private _isClosed = false; private _isClosed = false;
@ -55,7 +55,7 @@ export class AnalyticsService {
strMessage = 'PING'; strMessage = 'PING';
} }
return new Promise<void>((resolve, reject) => { return new Promise<void>((resolve, reject) => {
this._requester.send(strMessage, undefined, (err) => { this._requester.send(strMessage, undefined, (err: any) => {
if(err) { if(err) {
reject(err); reject(err);
} else { } else {
@ -184,7 +184,7 @@ export class AnalyticsService {
} }
} }
private _onAnalyticsMessage(data: any, error) { private _onAnalyticsMessage(data: any) {
if(data.toString() === 'PONG') { if(data.toString() === 'PONG') {
this._pingResponded = true; this._pingResponded = true;
if(!this._ready) { if(!this._ready) {

179
server/src/services/data_service.ts

@ -3,75 +3,178 @@ import * as config from '../config';
import * as nedb from 'nedb'; import * as nedb from 'nedb';
import * as fs from 'fs'; import * as fs from 'fs';
export const db = {
analyticUnits: new nedb({ filename: config.ANALYTIC_UNITS_DATABASE_PATH, autoload: true }),
metrics: new nedb({ filename: config.METRICS_DATABASE_PATH, autoload: true }),
segments: new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true }),
files: new nedb({ filename: config.FILES_DATABASE_PATH, autoload: true })
};
export enum Collection { ANALYTIC_UNITS, SEGMENTS };
let dbUpsertFile = (query: any, updateQuery: any) => {
/**
* Class which helps to make queries to your collection
*
* @param { string | object } query: a key as a string or mongodb-style query
*/
export type DBQ = {
findOne: (query: string | object) => Promise<any>,
findMany: (query: string[] | object) => Promise<any[]>,
insertOne: (document: object) => Promise<string>,
insertMany: (documents: object[]) => Promise<string[]>,
updateOne: (query: string | object, updateQuery: any) => Promise<void>,
removeOne: (query: string) => Promise<boolean>
removeMany: (query: string[] | object) => Promise<number>
}
export function makeDBQ(collection: Collection): DBQ {
return {
findOne: dbFindOne.bind(null, collection),
findMany: dbFindMany.bind(null, collection),
insertOne: dbInsertOne.bind(null, collection),
insertMany: dbInsertMany.bind(null, collection),
updateOne: dbUpdateOne.bind(null, collection),
removeOne: dbRemoveOne.bind(null, collection),
removeMany: dbRemoveMany.bind(null, collection)
}
}
function wrapIdToQuery(query: string | object): any {
if(typeof query === 'string') {
return { _id: query };
}
return query;
}
function wrapIdsToQuery(query: string[] | object): any {
if(Array.isArray(query)) {
return { _id: { $in: query } };
}
return query;
}
function isEmptyArray(obj: any): boolean {
if(!Array.isArray(obj)) {
return false;
}
return obj.length == 0;
}
const db = new Map<Collection, nedb>();
let dbInsertOne = (collection: Collection, doc: object) => {
return new Promise<string>((resolve, reject) => {
db.get(collection).insert(doc, (err, newDoc: any) => {
if(err) {
reject(err);
} else {
resolve(newDoc._id);
}
});
});
}
let dbInsertMany = (collection: Collection, docs: object[]) => {
if(docs.length === 0) {
return Promise.resolve([]);
}
return new Promise<string[]>((resolve, reject) => {
db.get(collection).insert(docs, (err, newDocs: any[]) => {
if(err) {
reject(err);
} else {
resolve(newDocs.map(d => d._id));
}
});
});
}
let dbUpdateOne = (collection: Collection, query: string | object, updateQuery: object) => {
query = wrapIdToQuery(query);
return new Promise<void>((resolve, reject) => { return new Promise<void>((resolve, reject) => {
db.files.update(query, updateQuery, { upsert: true }, (err: Error) => { db.get(collection).update(query, updateQuery, { /* options */ }, (err: Error) => {
if(err) { if(err) {
reject(err); reject(err);
} else { } else {
console.log('saved shit with query ');
console.log(query);
console.log('saved shit with updateQuery ');
console.log(updateQuery);
resolve(); resolve();
} }
}); });
}); });
} }
let dbLoadFile = (query: any) => { let dbFindOne = (collection: Collection, query: string | object) => {
query = wrapIdToQuery(query);
return new Promise<any>((resolve, reject) => { return new Promise<any>((resolve, reject) => {
db.files.findOne(query, (err, doc) => { db.get(collection).findOne(query, (err, doc) => {
if(err) { if(err) {
reject(err); reject(err);
} else { } else {
console.log('got shit with query');
console.log(query);
console.log('doc');
console.log(doc);
resolve(doc); resolve(doc);
} }
}); });
}); });
} }
function maybeCreate(path: string): void { let dbFindMany = (collection: Collection, query: string[] | object) => {
if(fs.existsSync(path)) { if(isEmptyArray(query)) {
return; return Promise.resolve([]);
} }
console.log('mkdir: ' + path); query = wrapIdsToQuery(query);
fs.mkdirSync(path); return new Promise<any[]>((resolve, reject) => {
console.log('exists: ' + fs.existsSync(path)); db.get(collection).find(query, (err, docs: any[]) => {
if(err) {
reject(err);
} else {
resolve(docs);
}
});
});
}
let dbRemoveOne = (collection: Collection, query: string | object) => {
query = wrapIdToQuery(query);
return new Promise<boolean>((resolve, reject) => {
db.get(collection).remove(query, { /* options */ }, (err, numRemoved) => {
if(err) {
reject(err);
} else {
if(numRemoved > 1) {
throw new Error(`Removed ${numRemoved} elements with query: ${JSON.stringify(query)}. Only one is Ok.`);
} else {
resolve(numRemoved == 1);
}
}
});
});
} }
export async function saveFile(filename: string, content: string): Promise<void> { let dbRemoveMany = (collection: Collection, query: string[] | object) => {
return dbUpsertFile({ filename } , { filename, content }); if(isEmptyArray(query)) {
return Promise.resolve([]);
}
query = wrapIdsToQuery(query);
return new Promise<number>((resolve, reject) => {
db.get(collection).remove(query, { multi: true }, (err, numRemoved) => {
if(err) {
reject(err);
} else {
resolve(numRemoved);
}
});
});
} }
export async function loadFile(filename: string): Promise<string> { function maybeCreateDir(path: string): void {
let doc = await dbLoadFile({ filename }); if(fs.existsSync(path)) {
if(doc === null) { return;
return null;
} }
return doc.content; console.log('mkdir: ' + path);
fs.mkdirSync(path);
} }
export function checkDataFolders(): void { function checkDataFolders(): void {
[ [
config.DATA_PATH, config.DATA_PATH,
config.DATASETS_PATH,
config.ANALYTIC_UNITS_PATH,
config.MODELS_PATH,
config.METRICS_PATH,
config.SEGMENTS_PATH,
config.ZMQ_IPC_PATH config.ZMQ_IPC_PATH
].forEach(maybeCreate); ].forEach(maybeCreateDir);
} }
checkDataFolders();
// TODO: it's better if models request db which we create if it`s needed
db.set(Collection.ANALYTIC_UNITS, new nedb({ filename: config.ANALYTIC_UNITS_DATABASE_PATH, autoload: true }));
db.set(Collection.SEGMENTS, new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true }));

6
server/src/services/notification_service.ts

@ -1,11 +1,11 @@
import { findById } from '../models/analytic_unit'; import { findById, AnalyticUnitId } from '../models/analytic_unit_model';
import axios from 'axios'; import axios from 'axios';
// TODO: send notification with payload without dep to AnalyticUnit // TODO: send notification with payload without dep to AnalyticUnit
export async function sendNotification(predictorId, active) { export async function sendNotification(id: AnalyticUnitId, active: boolean) {
let anomalyName = findById(predictorId).name; let anomalyName = (await findById(id)).name
console.log('Notification ' + anomalyName); console.log('Notification ' + anomalyName);
let notification = { let notification = {

12
server/src/services/process_service.ts

@ -1,6 +1,6 @@
var exitHandlers = [] var exitHandlers: (() => void)[] = [];
var exitHandled = false; var exitHandled = false;
/** /**
@ -12,7 +12,7 @@ export function registerExitHandler(callback: () => void) {
exitHandlers.push(callback); exitHandlers.push(callback);
} }
function exitHandler(options, err) { function exitHandler(options: any, err?: any) {
if(exitHandled) { if(exitHandled) {
return; return;
} }
@ -24,6 +24,12 @@ function exitHandler(options, err) {
process.exit(); process.exit();
} }
function catchException(options: any, err: any) {
console.log('Server exception:');
console.log(err);
exitHandler({ exit: true });
}
//do something when app is closing //do something when app is closing
process.on('exit', exitHandler.bind(null, { cleanup:true })); process.on('exit', exitHandler.bind(null, { cleanup:true }));
@ -35,4 +41,4 @@ process.on('SIGUSR1', exitHandler.bind(null, {exit:true}));
process.on('SIGUSR2', exitHandler.bind(null, { exit:true })); process.on('SIGUSR2', exitHandler.bind(null, { exit:true }));
//catches uncaught exceptions //catches uncaught exceptions
process.on('uncaughtException', exitHandler.bind(null, {exit:true})); process.on('uncaughtException', catchException.bind(null, { exit:true }));

4
server/tsconfig.json

@ -1,9 +1,7 @@
{ {
"compilerOptions": { "compilerOptions": {
"sourceMap": true, "sourceMap": true,
"noImplicitAny": false,
"module": "commonjs", "module": "commonjs",
"target": "es6", "target": "es6"
"allowJs": true
} }
} }

Loading…
Cancel
Save