Alexey Velikiy
6 years ago
committed by
GitHub
19 changed files with 846 additions and 753 deletions
@ -1,30 +0,0 @@ |
|||||||
import { getJsonDataSync, writeJsonDataSync } from '../services/json_service'; |
|
||||||
import { METRICS_PATH } from '../config'; |
|
||||||
|
|
||||||
import * as crypto from 'crypto'; |
|
||||||
|
|
||||||
import * as path from 'path'; |
|
||||||
|
|
||||||
|
|
||||||
function saveTargets(targets) { |
|
||||||
let metrics = []; |
|
||||||
for (let target of targets) { |
|
||||||
metrics.push(saveTarget(target)); |
|
||||||
} |
|
||||||
return metrics; |
|
||||||
} |
|
||||||
|
|
||||||
function saveTarget(target) { |
|
||||||
//const md5 = crypto.createHash('md5')
|
|
||||||
const targetId = crypto.createHash('md5').update(JSON.stringify(target)).digest('hex'); |
|
||||||
let filename = path.join(METRICS_PATH, `${targetId}.json`); |
|
||||||
writeJsonDataSync(filename, target); |
|
||||||
return targetId; |
|
||||||
} |
|
||||||
|
|
||||||
function getTarget(targetId) { |
|
||||||
let filename = path.join(METRICS_PATH, `${targetId}.json`); |
|
||||||
return getJsonDataSync(filename); |
|
||||||
} |
|
||||||
|
|
||||||
export { saveTargets, getTarget } |
|
||||||
@ -1,77 +0,0 @@ |
|||||||
import { getJsonDataSync, writeJsonDataSync } from '../services/json_service'; |
|
||||||
import { AnalyticUnitId, findById, save } from '../models/analytic_unit'; |
|
||||||
import { SEGMENTS_PATH } from '../config'; |
|
||||||
|
|
||||||
import * as _ from 'lodash'; |
|
||||||
|
|
||||||
import * as path from 'path'; |
|
||||||
import * as fs from 'fs'; |
|
||||||
|
|
||||||
|
|
||||||
export function getLabeledSegments(id: AnalyticUnitId) { |
|
||||||
let filename = path.join(SEGMENTS_PATH, `${id}_labeled.json`); |
|
||||||
|
|
||||||
if(!fs.existsSync(filename)) { |
|
||||||
return []; |
|
||||||
} else { |
|
||||||
let segments = getJsonDataSync(filename); |
|
||||||
for(let segment of segments) { |
|
||||||
if(segment.labeled === undefined) { |
|
||||||
segment.labeled = false; |
|
||||||
} |
|
||||||
} |
|
||||||
return segments; |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
export function getPredictedSegments(id: AnalyticUnitId) { |
|
||||||
let filename = path.join(SEGMENTS_PATH, `${id}_segments.json`); |
|
||||||
|
|
||||||
let jsonData; |
|
||||||
try { |
|
||||||
jsonData = getJsonDataSync(filename); |
|
||||||
} catch(e) { |
|
||||||
console.error(e.message); |
|
||||||
jsonData = []; |
|
||||||
} |
|
||||||
return jsonData; |
|
||||||
} |
|
||||||
|
|
||||||
export function saveSegments(id: AnalyticUnitId, segments) { |
|
||||||
let filename = path.join(SEGMENTS_PATH, `${id}_labeled.json`); |
|
||||||
|
|
||||||
try { |
|
||||||
return writeJsonDataSync(filename, _.uniqBy(segments, 'start')); |
|
||||||
} catch(e) { |
|
||||||
console.error(e.message); |
|
||||||
throw new Error('Can`t write to db'); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
export function insertSegments(id: AnalyticUnitId, addedSegments, labeled: boolean) { |
|
||||||
// Set status
|
|
||||||
let info = findById(id); |
|
||||||
let segments = getLabeledSegments(id); |
|
||||||
|
|
||||||
let nextId = info.nextId; |
|
||||||
let addedIds = [] |
|
||||||
for (let segment of addedSegments) { |
|
||||||
segment.id = nextId; |
|
||||||
segment.labeled = labeled; |
|
||||||
addedIds.push(nextId); |
|
||||||
nextId++; |
|
||||||
segments.push(segment); |
|
||||||
} |
|
||||||
info.nextId = nextId; |
|
||||||
saveSegments(id, segments); |
|
||||||
save(id, info); |
|
||||||
return addedIds; |
|
||||||
} |
|
||||||
|
|
||||||
export function removeSegments(id: AnalyticUnitId, removedSegments) { |
|
||||||
let segments = getLabeledSegments(id); |
|
||||||
for (let segmentId of removedSegments) { |
|
||||||
segments = segments.filter(el => el.id !== segmentId); |
|
||||||
} |
|
||||||
saveSegments(id, segments); |
|
||||||
} |
|
||||||
@ -1,84 +0,0 @@ |
|||||||
import { getJsonDataSync, writeJsonDataSync } from '../services/json_service' |
|
||||||
import { ANALYTIC_UNITS_PATH } from '../config' |
|
||||||
|
|
||||||
import * as crypto from 'crypto'; |
|
||||||
|
|
||||||
import * as path from 'path' |
|
||||||
import * as fs from 'fs' |
|
||||||
|
|
||||||
|
|
||||||
export type Datasource = { |
|
||||||
method: string, |
|
||||||
data: Object, |
|
||||||
params: Object, |
|
||||||
type: string, |
|
||||||
url: string |
|
||||||
} |
|
||||||
|
|
||||||
export type Metric = { |
|
||||||
datasource: string, |
|
||||||
targets: string[] |
|
||||||
} |
|
||||||
|
|
||||||
export type AnalyticUnitId = string; |
|
||||||
|
|
||||||
export type AnalyticUnit = { |
|
||||||
id?: AnalyticUnitId, |
|
||||||
name: string, |
|
||||||
panelUrl: string, |
|
||||||
type: string, |
|
||||||
metric: Metric, |
|
||||||
datasource: Datasource |
|
||||||
status: string, |
|
||||||
error?: string, |
|
||||||
lastPredictionTime: number, |
|
||||||
nextId: number |
|
||||||
} |
|
||||||
|
|
||||||
export function createItem(item: AnalyticUnit): AnalyticUnitId { |
|
||||||
const hashString = item.name + (new Date()).toString(); |
|
||||||
const newId: AnalyticUnitId = crypto.createHash('md5').update(hashString).digest('hex'); |
|
||||||
let filename = path.join(ANALYTIC_UNITS_PATH, `${newId}.json`); |
|
||||||
if(fs.existsSync(filename)) { |
|
||||||
throw new Error(`Can't create item with id ${newId}`); |
|
||||||
} |
|
||||||
save(newId, item); |
|
||||||
item.id = newId; |
|
||||||
return newId; |
|
||||||
} |
|
||||||
|
|
||||||
export function remove(id: AnalyticUnitId) { |
|
||||||
let filename = path.join(ANALYTIC_UNITS_PATH, `${id}.json`); |
|
||||||
fs.unlinkSync(filename); |
|
||||||
} |
|
||||||
|
|
||||||
export function save(id: AnalyticUnitId, unit: AnalyticUnit) { |
|
||||||
let filename = path.join(ANALYTIC_UNITS_PATH, `${id}.json`); |
|
||||||
return writeJsonDataSync(filename, unit); |
|
||||||
} |
|
||||||
|
|
||||||
// TODO: make async
|
|
||||||
export function findById(id: AnalyticUnitId): AnalyticUnit { |
|
||||||
let filename = path.join(ANALYTIC_UNITS_PATH, `${id}.json`); |
|
||||||
if(!fs.existsSync(filename)) { |
|
||||||
throw new Error(`Can't find Analytic Unit with id ${id}`); |
|
||||||
} |
|
||||||
return getJsonDataSync(filename); |
|
||||||
} |
|
||||||
|
|
||||||
export function setStatus(predictorId: AnalyticUnitId, status: string, error?: string) { |
|
||||||
let info = findById(predictorId); |
|
||||||
info.status = status; |
|
||||||
if(error !== undefined) { |
|
||||||
info.error = error; |
|
||||||
} else { |
|
||||||
info.error = ''; |
|
||||||
} |
|
||||||
save(predictorId, info); |
|
||||||
} |
|
||||||
|
|
||||||
export function setPredictionTime(id: AnalyticUnitId, time: number) { |
|
||||||
let info = findById(id); |
|
||||||
info.lastPredictionTime = time; |
|
||||||
save(id, info); |
|
||||||
} |
|
||||||
@ -0,0 +1,98 @@ |
|||||||
|
import { Metric } from './metric_model'; |
||||||
|
import { Collection, makeDBQ } from '../services/data_service'; |
||||||
|
|
||||||
|
|
||||||
|
let db = makeDBQ(Collection.ANALYTIC_UNITS); |
||||||
|
|
||||||
|
|
||||||
|
export type AnalyticUnitId = string; |
||||||
|
|
||||||
|
export class AnalyticUnit { |
||||||
|
constructor( |
||||||
|
public name: string, |
||||||
|
public panelUrl: string, |
||||||
|
public type: string, |
||||||
|
public metric: Metric, |
||||||
|
public id?: AnalyticUnitId, |
||||||
|
public lastPredictionTime?: number, |
||||||
|
public status?: string, |
||||||
|
public error?: string, |
||||||
|
) { |
||||||
|
if(name === undefined) { |
||||||
|
throw new Error(`Missing field "name"`); |
||||||
|
} |
||||||
|
if(panelUrl === undefined) { |
||||||
|
throw new Error(`Missing field "panelUrl"`); |
||||||
|
} |
||||||
|
if(type === undefined) { |
||||||
|
throw new Error(`Missing field "type"`); |
||||||
|
} |
||||||
|
if(metric === undefined) { |
||||||
|
throw new Error(`Missing field "metric"`); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public toObject() { |
||||||
|
return { |
||||||
|
_id: this.id, |
||||||
|
name: this.name, |
||||||
|
panelUrl: this.panelUrl, |
||||||
|
type: this.type, |
||||||
|
metric: this.metric.toObject(), |
||||||
|
lastPredictionTime: this.lastPredictionTime, |
||||||
|
status: this.status, |
||||||
|
error: this.error |
||||||
|
}; |
||||||
|
} |
||||||
|
|
||||||
|
static fromObject(obj: any): AnalyticUnit { |
||||||
|
if(obj === undefined) { |
||||||
|
throw new Error('obj is undefined'); |
||||||
|
} |
||||||
|
return new AnalyticUnit( |
||||||
|
obj.name, |
||||||
|
obj.panelUrl, |
||||||
|
obj.type, |
||||||
|
Metric.fromObject(obj.metric), |
||||||
|
obj.status, |
||||||
|
obj.lastPredictionTime, |
||||||
|
obj._id, |
||||||
|
obj.error, |
||||||
|
); |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
export async function findById(id: AnalyticUnitId): Promise<AnalyticUnit> { |
||||||
|
return AnalyticUnit.fromObject(await db.findOne(id)); |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* Creates and updates new unit.id |
||||||
|
* |
||||||
|
* @param unit to create |
||||||
|
* @returns unit.id |
||||||
|
*/ |
||||||
|
export async function create(unit: AnalyticUnit): Promise<AnalyticUnitId> { |
||||||
|
var obj = unit.toObject(); |
||||||
|
var r = await db.insertOne(obj); |
||||||
|
return r; |
||||||
|
} |
||||||
|
|
||||||
|
export async function remove(id: AnalyticUnitId): Promise<void> { |
||||||
|
await db.removeOne(id); |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
export async function update(id: AnalyticUnitId, unit: AnalyticUnit) { |
||||||
|
return db.updateOne(id, unit); |
||||||
|
} |
||||||
|
|
||||||
|
export async function setStatus(id: AnalyticUnitId, status: string, error?: string) { |
||||||
|
return db.updateOne(id, { status, error }); |
||||||
|
} |
||||||
|
|
||||||
|
export async function setPredictionTime(id: AnalyticUnitId, lastPredictionTime: number) { |
||||||
|
return db.updateOne(id, { lastPredictionTime }); |
||||||
|
} |
||||||
@ -0,0 +1,32 @@ |
|||||||
|
export class Metric { |
||||||
|
constructor(public datasource: string, public targets: any[]) { |
||||||
|
if(datasource === undefined) { |
||||||
|
throw new Error('datasource is undefined'); |
||||||
|
} |
||||||
|
if(targets === undefined) { |
||||||
|
throw new Error('targets is undefined'); |
||||||
|
} |
||||||
|
if(targets.length === 0) { |
||||||
|
throw new Error('targets is empty'); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public toObject() { |
||||||
|
return { |
||||||
|
datasource: this.datasource, |
||||||
|
targets: this.targets |
||||||
|
}; |
||||||
|
} |
||||||
|
|
||||||
|
static fromObject(obj: any): Metric { |
||||||
|
if(obj === undefined) { |
||||||
|
throw new Error('obj is undefined'); |
||||||
|
} |
||||||
|
return new Metric( |
||||||
|
obj.datasource, |
||||||
|
obj.targets |
||||||
|
); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
@ -0,0 +1,87 @@ |
|||||||
|
import { AnalyticUnitId } from './analytic_unit_model'; |
||||||
|
|
||||||
|
import { Collection, makeDBQ } from '../services/data_service'; |
||||||
|
|
||||||
|
let db = makeDBQ(Collection.SEGMENTS); |
||||||
|
|
||||||
|
|
||||||
|
export type SegmentId = string; |
||||||
|
|
||||||
|
export class Segment { |
||||||
|
constructor( |
||||||
|
public analyticUnitId: AnalyticUnitId, |
||||||
|
public from: number, |
||||||
|
public to: number, |
||||||
|
public labeled: boolean, |
||||||
|
public id?: SegmentId |
||||||
|
) { |
||||||
|
if(analyticUnitId === undefined) { |
||||||
|
throw new Error('AnalyticUnitId is undefined'); |
||||||
|
} |
||||||
|
if(from === undefined) { |
||||||
|
throw new Error('from is undefined'); |
||||||
|
} |
||||||
|
if(isNaN(from)) { |
||||||
|
throw new Error('from is NaN'); |
||||||
|
} |
||||||
|
if(to === undefined) { |
||||||
|
throw new Error('to is undefined'); |
||||||
|
} |
||||||
|
if(isNaN(to)) { |
||||||
|
throw new Error('to is NaN'); |
||||||
|
} |
||||||
|
if(labeled === undefined) { |
||||||
|
throw new Error('labeled is undefined'); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public toObject() { |
||||||
|
return { |
||||||
|
_id: this.id, |
||||||
|
analyticUnitId: this.analyticUnitId, |
||||||
|
from: this.from, |
||||||
|
to: this.to, |
||||||
|
labeled: this.labeled |
||||||
|
}; |
||||||
|
} |
||||||
|
|
||||||
|
static fromObject(obj: any): Segment { |
||||||
|
if(obj === undefined) { |
||||||
|
throw new Error('obj is undefined'); |
||||||
|
} |
||||||
|
return new Segment( |
||||||
|
obj.analyticUnitId,
|
||||||
|
+obj.from, +obj.to, |
||||||
|
obj.labeled, obj._id |
||||||
|
); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
export type FindManyQuery = { |
||||||
|
timeFromGTE?: number, |
||||||
|
timeToLTE?: number, |
||||||
|
intexGT?: number |
||||||
|
} |
||||||
|
|
||||||
|
export async function findMany(id: AnalyticUnitId, query: FindManyQuery): Promise<Segment[]> { |
||||||
|
var dbQuery: any = { analyticUnitId: id }; |
||||||
|
if(query.timeFromGTE !== undefined) { |
||||||
|
dbQuery.from = { $gte: query.timeFromGTE }; |
||||||
|
} |
||||||
|
if(query.timeToLTE !== undefined) { |
||||||
|
dbQuery.to = { $lte: query.timeToLTE }; |
||||||
|
} |
||||||
|
let segs = await db.findMany(dbQuery); |
||||||
|
if(segs === null) { |
||||||
|
return []; |
||||||
|
} |
||||||
|
return segs.map(Segment.fromObject); |
||||||
|
} |
||||||
|
|
||||||
|
export async function insertSegments(segments: Segment[]) { |
||||||
|
return db.insertMany(segments.map(s => s.toObject())); |
||||||
|
} |
||||||
|
|
||||||
|
export function removeSegments(idsToRemove: SegmentId[]) { |
||||||
|
return db.removeMany(idsToRemove); |
||||||
|
} |
||||||
@ -0,0 +1,32 @@ |
|||||||
|
import { AnalyticUnitId } from "./analytic_unit_model"; |
||||||
|
|
||||||
|
|
||||||
|
export type TaskId = string; |
||||||
|
|
||||||
|
export class Task { |
||||||
|
constructor( |
||||||
|
public analyticUnitId: AnalyticUnitId, |
||||||
|
public id?: TaskId |
||||||
|
) { |
||||||
|
if(analyticUnitId === undefined) { |
||||||
|
throw new Error('analyticUnitId is undefined'); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
public toObject() { |
||||||
|
return { |
||||||
|
_id: this.id, |
||||||
|
analyticUnitId: this.analyticUnitId |
||||||
|
}; |
||||||
|
} |
||||||
|
|
||||||
|
static fromObject(obj: any): Task { |
||||||
|
if(obj === undefined) { |
||||||
|
throw new Error('obj is undefined'); |
||||||
|
} |
||||||
|
return new Task( |
||||||
|
obj.analyticUnitId, |
||||||
|
obj._id, |
||||||
|
); |
||||||
|
} |
||||||
|
} |
||||||
@ -1,232 +1,232 @@ |
|||||||
import * as config from '../config'; |
import * as config from '../config'; |
||||||
|
|
||||||
const zmq = require('zeromq'); |
const zmq = require('zeromq'); |
||||||
|
|
||||||
import * as childProcess from 'child_process' |
import * as childProcess from 'child_process' |
||||||
import * as fs from 'fs'; |
import * as fs from 'fs'; |
||||||
import * as path from 'path'; |
import * as path from 'path'; |
||||||
|
|
||||||
|
|
||||||
export class AnalyticsMessage { |
export class AnalyticsMessage { |
||||||
public constructor(public method: string, public payload?: string, public requestId?: number) { |
public constructor(public method: string, public payload?: string, public requestId?: number) { |
||||||
|
|
||||||
} |
} |
||||||
|
|
||||||
static fromJSON(obj: any): AnalyticsMessage { |
static fromJSON(obj: any): AnalyticsMessage { |
||||||
if(obj.method === undefined) { |
if(obj.method === undefined) { |
||||||
throw new Error('No method in obj:' + obj); |
throw new Error('No method in obj:' + obj); |
||||||
} |
} |
||||||
return new AnalyticsMessage(obj.method, obj.payload, obj.requestId); |
return new AnalyticsMessage(obj.method, obj.payload, obj.requestId); |
||||||
} |
} |
||||||
} |
} |
||||||
|
|
||||||
function analyticsMessageFromJson(obj: any): AnalyticsMessage { |
function analyticsMessageFromJson(obj: any): AnalyticsMessage { |
||||||
return new AnalyticsMessage(obj); |
return new AnalyticsMessage(obj); |
||||||
} |
} |
||||||
|
|
||||||
export class AnalyticsService { |
export class AnalyticsService { |
||||||
|
|
||||||
private _requester: any; |
private _requester: any; |
||||||
private _ready: boolean = false; |
private _ready: boolean = false; |
||||||
private _pingResponded = false; |
private _pingResponded = false; |
||||||
private _zmqConnectionString = null; |
private _zmqConnectionString: string = null; |
||||||
private _ipcPath = null; |
private _ipcPath: string = null; |
||||||
private _analyticsPinger: NodeJS.Timer = null; |
private _analyticsPinger: NodeJS.Timer = null; |
||||||
private _isClosed = false; |
private _isClosed = false; |
||||||
|
|
||||||
constructor(private _onMessage: (message: AnalyticsMessage) => void) { |
constructor(private _onMessage: (message: AnalyticsMessage) => void) { |
||||||
this._init(); |
this._init(); |
||||||
} |
} |
||||||
|
|
||||||
public async sendTask(taskObj: any): Promise<void> { |
public async sendTask(taskObj: any): Promise<void> { |
||||||
if(!this._ready) { |
if(!this._ready) { |
||||||
return Promise.reject("Analytics is not ready"); |
return Promise.reject("Analytics is not ready"); |
||||||
} |
} |
||||||
let message = { |
let message = { |
||||||
method: 'TASK', |
method: 'TASK', |
||||||
payload: taskObj |
payload: taskObj |
||||||
} |
} |
||||||
return this.sendMessage(message); |
return this.sendMessage(message); |
||||||
} |
} |
||||||
|
|
||||||
public async sendMessage(message: AnalyticsMessage): Promise<void> { |
public async sendMessage(message: AnalyticsMessage): Promise<void> { |
||||||
let strMessage = JSON.stringify(message); |
let strMessage = JSON.stringify(message); |
||||||
if(message.method === 'PING') { |
if(message.method === 'PING') { |
||||||
strMessage = 'PING'; |
strMessage = 'PING'; |
||||||
} |
} |
||||||
return new Promise<void>((resolve, reject) => { |
return new Promise<void>((resolve, reject) => { |
||||||
this._requester.send(strMessage, undefined, (err) => { |
this._requester.send(strMessage, undefined, (err: any) => { |
||||||
if(err) { |
if(err) { |
||||||
reject(err); |
reject(err); |
||||||
} else { |
} else { |
||||||
resolve(); |
resolve(); |
||||||
} |
} |
||||||
}); |
}); |
||||||
}); |
}); |
||||||
} |
} |
||||||
|
|
||||||
public close() { |
public close() { |
||||||
this._isClosed = true; |
this._isClosed = true; |
||||||
console.log('Terminating analytics service...'); |
console.log('Terminating analytics service...'); |
||||||
clearInterval(this._analyticsPinger); |
clearInterval(this._analyticsPinger); |
||||||
if(this._ipcPath !== null) { |
if(this._ipcPath !== null) { |
||||||
console.log('Remove ipc path: ' + this._ipcPath); |
console.log('Remove ipc path: ' + this._ipcPath); |
||||||
fs.unlinkSync(this._ipcPath); |
fs.unlinkSync(this._ipcPath); |
||||||
} |
} |
||||||
this._requester.close(); |
this._requester.close(); |
||||||
console.log('Ok'); |
console.log('Ok'); |
||||||
} |
} |
||||||
|
|
||||||
public get ready(): boolean { return this._ready; } |
public get ready(): boolean { return this._ready; } |
||||||
|
|
||||||
private async _init() { |
private async _init() { |
||||||
this._requester = zmq.socket('pair'); |
this._requester = zmq.socket('pair'); |
||||||
let productionMode = process.env.NODE_ENV !== 'development'; |
let productionMode = process.env.NODE_ENV !== 'development'; |
||||||
|
|
||||||
this._zmqConnectionString = `tcp://127.0.0.1:${config.ZMQ_DEV_PORT}`; // debug mode
|
this._zmqConnectionString = `tcp://127.0.0.1:${config.ZMQ_DEV_PORT}`; // debug mode
|
||||||
if(productionMode) { |
if(productionMode) { |
||||||
this._zmqConnectionString = config.ZMQ_CONNECTION_STRING; |
this._zmqConnectionString = config.ZMQ_CONNECTION_STRING; |
||||||
if(this._zmqConnectionString === null) { |
if(this._zmqConnectionString === null) { |
||||||
var createResult = await AnalyticsService.createIPCAddress(); |
var createResult = await AnalyticsService.createIPCAddress(); |
||||||
this._zmqConnectionString = createResult.address; |
this._zmqConnectionString = createResult.address; |
||||||
this._ipcPath = createResult.file; |
this._ipcPath = createResult.file; |
||||||
} |
} |
||||||
} |
} |
||||||
|
|
||||||
console.log("Binding to zmq... %s", this._zmqConnectionString); |
console.log("Binding to zmq... %s", this._zmqConnectionString); |
||||||
this._requester.connect(this._zmqConnectionString); |
this._requester.connect(this._zmqConnectionString); |
||||||
this._requester.on("message", this._onAnalyticsMessage.bind(this)); |
this._requester.on("message", this._onAnalyticsMessage.bind(this)); |
||||||
console.log('Ok'); |
console.log('Ok'); |
||||||
|
|
||||||
if(productionMode) { |
if(productionMode) { |
||||||
console.log('Creating analytics process...'); |
console.log('Creating analytics process...'); |
||||||
try { |
try { |
||||||
var cp = await AnalyticsService._runAnalyticsProcess(this._zmqConnectionString); |
var cp = await AnalyticsService._runAnalyticsProcess(this._zmqConnectionString); |
||||||
} catch(error) { |
} catch(error) { |
||||||
console.error('Can`t run analytics process: %s', error); |
console.error('Can`t run analytics process: %s', error); |
||||||
return; |
return; |
||||||
} |
} |
||||||
console.log('Ok, pid: %s', cp.pid); |
console.log('Ok, pid: %s', cp.pid); |
||||||
} |
} |
||||||
|
|
||||||
console.log('Start analytics pinger...'); |
console.log('Start analytics pinger...'); |
||||||
this._runAlalyticsPinger(); |
this._runAlalyticsPinger(); |
||||||
console.log('Ok'); |
console.log('Ok'); |
||||||
|
|
||||||
} |
} |
||||||
|
|
||||||
/** |
/** |
||||||
* Spawns analytics process. Reads process stderr and fails if it isn`t empty.
|
* Spawns analytics process. Reads process stderr and fails if it isn`t empty.
|
||||||
* No need to stop the process later. |
* No need to stop the process later. |
||||||
* |
* |
||||||
* @returns Creaded child process |
* @returns Creaded child process |
||||||
* @throws Process start error or first exception during python start |
* @throws Process start error or first exception during python start |
||||||
*/ |
*/ |
||||||
private static async _runAnalyticsProcess(zmqConnectionString: string): Promise<childProcess.ChildProcess> { |
private static async _runAnalyticsProcess(zmqConnectionString: string): Promise<childProcess.ChildProcess> { |
||||||
let cp: childProcess.ChildProcess; |
let cp: childProcess.ChildProcess; |
||||||
let cpOptions = { |
let cpOptions = { |
||||||
cwd: config.ANALYTICS_PATH, |
cwd: config.ANALYTICS_PATH, |
||||||
env: { |
env: { |
||||||
...process.env, |
...process.env, |
||||||
ZMQ_CONNECTION_STRING: zmqConnectionString |
ZMQ_CONNECTION_STRING: zmqConnectionString |
||||||
} |
} |
||||||
}; |
}; |
||||||
|
|
||||||
if(fs.existsSync(path.join(config.ANALYTICS_PATH, 'dist/worker/worker'))) { |
if(fs.existsSync(path.join(config.ANALYTICS_PATH, 'dist/worker/worker'))) { |
||||||
console.log('dist/worker/worker'); |
console.log('dist/worker/worker'); |
||||||
cp = childProcess.spawn('dist/worker/worker', [], cpOptions); |
cp = childProcess.spawn('dist/worker/worker', [], cpOptions); |
||||||
} else { |
} else { |
||||||
console.log('python3 server.py'); |
console.log('python3 server.py'); |
||||||
// If compiled analytics script doesn't exist - fallback to regular python
|
// If compiled analytics script doesn't exist - fallback to regular python
|
||||||
console.log(config.ANALYTICS_PATH); |
console.log(config.ANALYTICS_PATH); |
||||||
cp = childProcess.spawn('python3', ['server.py'], cpOptions); |
cp = childProcess.spawn('python3', ['server.py'], cpOptions); |
||||||
} |
} |
||||||
|
|
||||||
if(cp.pid === undefined) { |
if(cp.pid === undefined) { |
||||||
return new Promise<childProcess.ChildProcess>((resolve, reject) => { |
return new Promise<childProcess.ChildProcess>((resolve, reject) => { |
||||||
cp.on('error', reject); |
cp.on('error', reject); |
||||||
}); |
}); |
||||||
} |
} |
||||||
|
|
||||||
return new Promise<childProcess.ChildProcess>((resolve, reject) => { |
return new Promise<childProcess.ChildProcess>((resolve, reject) => { |
||||||
var resolved = false; |
var resolved = false; |
||||||
|
|
||||||
cp.stdout.on('data', () => { |
cp.stdout.on('data', () => { |
||||||
if(resolved) { |
if(resolved) { |
||||||
return; |
return; |
||||||
} else { |
} else { |
||||||
resolved = true; |
resolved = true; |
||||||
} |
} |
||||||
resolve(cp); |
resolve(cp); |
||||||
}); |
}); |
||||||
|
|
||||||
cp.stderr.on('data', function(data) { |
cp.stderr.on('data', function(data) { |
||||||
if(resolved) { |
if(resolved) { |
||||||
return; |
return; |
||||||
} else { |
} else { |
||||||
resolved = true; |
resolved = true; |
||||||
} |
} |
||||||
reject(data); |
reject(data); |
||||||
}); |
}); |
||||||
|
|
||||||
}); |
}); |
||||||
|
|
||||||
} |
} |
||||||
|
|
||||||
private _onAnalyticsUp() { |
private _onAnalyticsUp() { |
||||||
console.log('Analytics is up'); |
console.log('Analytics is up'); |
||||||
} |
} |
||||||
|
|
||||||
private async _onAnalyticsDown() { |
private async _onAnalyticsDown() { |
||||||
console.log('Analytics is down'); |
console.log('Analytics is down'); |
||||||
if(process.env.NODE_ENV !== 'development') { |
if(process.env.NODE_ENV !== 'development') { |
||||||
await AnalyticsService._runAnalyticsProcess(this._zmqConnectionString); |
await AnalyticsService._runAnalyticsProcess(this._zmqConnectionString); |
||||||
} |
} |
||||||
} |
} |
||||||
|
|
||||||
private _onAnalyticsMessage(data: any, error) { |
private _onAnalyticsMessage(data: any) { |
||||||
if(data.toString() === 'PONG') { |
if(data.toString() === 'PONG') { |
||||||
this._pingResponded = true; |
this._pingResponded = true; |
||||||
if(!this._ready) { |
if(!this._ready) { |
||||||
this._ready = true; |
this._ready = true; |
||||||
this._onAnalyticsUp(); |
this._onAnalyticsUp(); |
||||||
} |
} |
||||||
return; |
return; |
||||||
} |
} |
||||||
|
|
||||||
|
|
||||||
let text = data.toString(); |
let text = data.toString(); |
||||||
let response; |
let response; |
||||||
try { |
try { |
||||||
response = JSON.parse(text); |
response = JSON.parse(text); |
||||||
} catch (e) { |
} catch (e) { |
||||||
console.error("Can`t parse response from analytics as json:"); |
console.error("Can`t parse response from analytics as json:"); |
||||||
console.error(text); |
console.error(text); |
||||||
throw new Error('Unexpected response'); |
throw new Error('Unexpected response'); |
||||||
} |
} |
||||||
this._onMessage(AnalyticsMessage.fromJSON(response)); |
this._onMessage(AnalyticsMessage.fromJSON(response)); |
||||||
} |
} |
||||||
|
|
||||||
private async _runAlalyticsPinger() { |
private async _runAlalyticsPinger() { |
||||||
this._analyticsPinger = setInterval(() => { |
this._analyticsPinger = setInterval(() => { |
||||||
if(this._isClosed) { |
if(this._isClosed) { |
||||||
return; |
return; |
||||||
} |
} |
||||||
if(!this._pingResponded && this._ready) { |
if(!this._pingResponded && this._ready) { |
||||||
this._ready = false; |
this._ready = false; |
||||||
this._onAnalyticsDown(); |
this._onAnalyticsDown(); |
||||||
} |
} |
||||||
this._pingResponded = false; |
this._pingResponded = false; |
||||||
// TODO: set life limit for this ping
|
// TODO: set life limit for this ping
|
||||||
this.sendMessage({ method: 'PING' }); |
this.sendMessage({ method: 'PING' }); |
||||||
}, config.ANLYTICS_PING_INTERVAL); |
}, config.ANLYTICS_PING_INTERVAL); |
||||||
} |
} |
||||||
|
|
||||||
private static async createIPCAddress(): Promise<{ address: string, file: string }> { |
private static async createIPCAddress(): Promise<{ address: string, file: string }> { |
||||||
let filename = `${process.pid}.ipc` |
let filename = `${process.pid}.ipc` |
||||||
let p = path.join(config.ZMQ_IPC_PATH, filename); |
let p = path.join(config.ZMQ_IPC_PATH, filename); |
||||||
fs.writeFileSync(p, ''); |
fs.writeFileSync(p, ''); |
||||||
return Promise.resolve({ address: 'ipc://' + p, file: p }); |
return Promise.resolve({ address: 'ipc://' + p, file: p }); |
||||||
} |
} |
||||||
|
|
||||||
} |
} |
||||||
|
|||||||
@ -1,77 +1,180 @@ |
|||||||
import * as config from '../config'; |
import * as config from '../config'; |
||||||
|
|
||||||
import * as nedb from 'nedb'; |
import * as nedb from 'nedb'; |
||||||
import * as fs from 'fs'; |
import * as fs from 'fs'; |
||||||
|
|
||||||
export const db = { |
|
||||||
analyticUnits: new nedb({ filename: config.ANALYTIC_UNITS_DATABASE_PATH, autoload: true }), |
export enum Collection { ANALYTIC_UNITS, SEGMENTS }; |
||||||
metrics: new nedb({ filename: config.METRICS_DATABASE_PATH, autoload: true }), |
|
||||||
segments: new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true }), |
|
||||||
files: new nedb({ filename: config.FILES_DATABASE_PATH, autoload: true }) |
/** |
||||||
}; |
* Class which helps to make queries to your collection |
||||||
|
* |
||||||
|
* @param { string | object } query: a key as a string or mongodb-style query |
||||||
let dbUpsertFile = (query: any, updateQuery: any) => { |
*/ |
||||||
return new Promise<void>((resolve, reject) => { |
export type DBQ = { |
||||||
db.files.update(query, updateQuery, { upsert: true }, (err: Error) => { |
findOne: (query: string | object) => Promise<any>, |
||||||
if(err) { |
findMany: (query: string[] | object) => Promise<any[]>, |
||||||
reject(err); |
insertOne: (document: object) => Promise<string>, |
||||||
} else { |
insertMany: (documents: object[]) => Promise<string[]>, |
||||||
console.log('saved shit with query '); |
updateOne: (query: string | object, updateQuery: any) => Promise<void>, |
||||||
console.log(query); |
removeOne: (query: string) => Promise<boolean> |
||||||
console.log('saved shit with updateQuery '); |
removeMany: (query: string[] | object) => Promise<number> |
||||||
console.log(updateQuery); |
} |
||||||
resolve(); |
|
||||||
} |
export function makeDBQ(collection: Collection): DBQ { |
||||||
}); |
return { |
||||||
}); |
findOne: dbFindOne.bind(null, collection), |
||||||
} |
findMany: dbFindMany.bind(null, collection), |
||||||
|
insertOne: dbInsertOne.bind(null, collection), |
||||||
let dbLoadFile = (query: any) => { |
insertMany: dbInsertMany.bind(null, collection), |
||||||
return new Promise<any>((resolve, reject) => { |
updateOne: dbUpdateOne.bind(null, collection), |
||||||
db.files.findOne(query, (err, doc) => { |
removeOne: dbRemoveOne.bind(null, collection), |
||||||
if(err) { |
removeMany: dbRemoveMany.bind(null, collection) |
||||||
reject(err); |
} |
||||||
} else { |
} |
||||||
console.log('got shit with query'); |
|
||||||
console.log(query); |
function wrapIdToQuery(query: string | object): any { |
||||||
console.log('doc'); |
if(typeof query === 'string') { |
||||||
console.log(doc); |
return { _id: query }; |
||||||
resolve(doc); |
} |
||||||
} |
return query; |
||||||
}); |
} |
||||||
}); |
|
||||||
} |
function wrapIdsToQuery(query: string[] | object): any { |
||||||
|
if(Array.isArray(query)) { |
||||||
function maybeCreate(path: string): void { |
return { _id: { $in: query } }; |
||||||
if(fs.existsSync(path)) { |
} |
||||||
return; |
return query; |
||||||
} |
} |
||||||
console.log('mkdir: ' + path); |
|
||||||
fs.mkdirSync(path); |
function isEmptyArray(obj: any): boolean { |
||||||
console.log('exists: ' + fs.existsSync(path)); |
if(!Array.isArray(obj)) { |
||||||
} |
return false; |
||||||
|
} |
||||||
export async function saveFile(filename: string, content: string): Promise<void> { |
return obj.length == 0; |
||||||
return dbUpsertFile({ filename } , { filename, content }); |
} |
||||||
} |
|
||||||
|
const db = new Map<Collection, nedb>(); |
||||||
export async function loadFile(filename: string): Promise<string> { |
|
||||||
let doc = await dbLoadFile({ filename }); |
let dbInsertOne = (collection: Collection, doc: object) => { |
||||||
if(doc === null) { |
return new Promise<string>((resolve, reject) => { |
||||||
return null; |
db.get(collection).insert(doc, (err, newDoc: any) => { |
||||||
} |
if(err) { |
||||||
return doc.content; |
reject(err); |
||||||
} |
} else { |
||||||
|
resolve(newDoc._id); |
||||||
export function checkDataFolders(): void { |
} |
||||||
[ |
}); |
||||||
config.DATA_PATH, |
}); |
||||||
config.DATASETS_PATH, |
} |
||||||
config.ANALYTIC_UNITS_PATH, |
|
||||||
config.MODELS_PATH, |
let dbInsertMany = (collection: Collection, docs: object[]) => { |
||||||
config.METRICS_PATH, |
if(docs.length === 0) { |
||||||
config.SEGMENTS_PATH, |
return Promise.resolve([]); |
||||||
config.ZMQ_IPC_PATH |
} |
||||||
].forEach(maybeCreate); |
return new Promise<string[]>((resolve, reject) => { |
||||||
} |
db.get(collection).insert(docs, (err, newDocs: any[]) => { |
||||||
|
if(err) { |
||||||
|
reject(err); |
||||||
|
} else { |
||||||
|
resolve(newDocs.map(d => d._id)); |
||||||
|
} |
||||||
|
}); |
||||||
|
}); |
||||||
|
} |
||||||
|
|
||||||
|
let dbUpdateOne = (collection: Collection, query: string | object, updateQuery: object) => { |
||||||
|
query = wrapIdToQuery(query); |
||||||
|
return new Promise<void>((resolve, reject) => { |
||||||
|
db.get(collection).update(query, updateQuery, { /* options */ }, (err: Error) => { |
||||||
|
if(err) { |
||||||
|
reject(err); |
||||||
|
} else { |
||||||
|
resolve(); |
||||||
|
} |
||||||
|
}); |
||||||
|
}); |
||||||
|
} |
||||||
|
|
||||||
|
let dbFindOne = (collection: Collection, query: string | object) => { |
||||||
|
query = wrapIdToQuery(query); |
||||||
|
return new Promise<any>((resolve, reject) => { |
||||||
|
db.get(collection).findOne(query, (err, doc) => { |
||||||
|
if(err) { |
||||||
|
reject(err); |
||||||
|
} else { |
||||||
|
resolve(doc); |
||||||
|
} |
||||||
|
}); |
||||||
|
}); |
||||||
|
} |
||||||
|
|
||||||
|
let dbFindMany = (collection: Collection, query: string[] | object) => { |
||||||
|
if(isEmptyArray(query)) { |
||||||
|
return Promise.resolve([]); |
||||||
|
} |
||||||
|
query = wrapIdsToQuery(query); |
||||||
|
return new Promise<any[]>((resolve, reject) => { |
||||||
|
db.get(collection).find(query, (err, docs: any[]) => { |
||||||
|
if(err) { |
||||||
|
reject(err); |
||||||
|
} else { |
||||||
|
resolve(docs); |
||||||
|
} |
||||||
|
}); |
||||||
|
}); |
||||||
|
} |
||||||
|
|
||||||
|
let dbRemoveOne = (collection: Collection, query: string | object) => { |
||||||
|
query = wrapIdToQuery(query); |
||||||
|
return new Promise<boolean>((resolve, reject) => { |
||||||
|
db.get(collection).remove(query, { /* options */ }, (err, numRemoved) => { |
||||||
|
if(err) { |
||||||
|
reject(err); |
||||||
|
} else { |
||||||
|
if(numRemoved > 1) { |
||||||
|
throw new Error(`Removed ${numRemoved} elements with query: ${JSON.stringify(query)}. Only one is Ok.`); |
||||||
|
} else { |
||||||
|
resolve(numRemoved == 1); |
||||||
|
} |
||||||
|
} |
||||||
|
}); |
||||||
|
}); |
||||||
|
} |
||||||
|
|
||||||
|
let dbRemoveMany = (collection: Collection, query: string[] | object) => { |
||||||
|
if(isEmptyArray(query)) { |
||||||
|
return Promise.resolve([]); |
||||||
|
} |
||||||
|
query = wrapIdsToQuery(query); |
||||||
|
return new Promise<number>((resolve, reject) => { |
||||||
|
db.get(collection).remove(query, { multi: true }, (err, numRemoved) => { |
||||||
|
if(err) { |
||||||
|
reject(err); |
||||||
|
} else { |
||||||
|
resolve(numRemoved); |
||||||
|
} |
||||||
|
}); |
||||||
|
}); |
||||||
|
} |
||||||
|
|
||||||
|
function maybeCreateDir(path: string): void { |
||||||
|
if(fs.existsSync(path)) { |
||||||
|
return; |
||||||
|
} |
||||||
|
console.log('mkdir: ' + path); |
||||||
|
fs.mkdirSync(path); |
||||||
|
} |
||||||
|
|
||||||
|
function checkDataFolders(): void { |
||||||
|
[ |
||||||
|
config.DATA_PATH, |
||||||
|
config.ZMQ_IPC_PATH |
||||||
|
].forEach(maybeCreateDir); |
||||||
|
} |
||||||
|
checkDataFolders(); |
||||||
|
|
||||||
|
// TODO: it's better if models request db which we create if it`s needed
|
||||||
|
db.set(Collection.ANALYTIC_UNITS, new nedb({ filename: config.ANALYTIC_UNITS_DATABASE_PATH, autoload: true })); |
||||||
|
db.set(Collection.SEGMENTS, new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true })); |
||||||
|
|||||||
@ -1,38 +1,44 @@ |
|||||||
|
|
||||||
|
|
||||||
var exitHandlers = [] |
var exitHandlers: (() => void)[] = []; |
||||||
var exitHandled = false; |
var exitHandled = false; |
||||||
|
|
||||||
/** |
/** |
||||||
* Add a callback for closing programm bacause of any reason |
* Add a callback for closing programm bacause of any reason |
||||||
*
|
*
|
||||||
* @param callback a sync function |
* @param callback a sync function |
||||||
*/ |
*/ |
||||||
export function registerExitHandler(callback: () => void) { |
export function registerExitHandler(callback: () => void) { |
||||||
exitHandlers.push(callback); |
exitHandlers.push(callback); |
||||||
} |
} |
||||||
|
|
||||||
function exitHandler(options, err) { |
function exitHandler(options: any, err?: any) { |
||||||
if(exitHandled) { |
if(exitHandled) { |
||||||
return; |
return; |
||||||
} |
} |
||||||
exitHandled = true; |
exitHandled = true; |
||||||
for(let i = 0; i < exitHandlers.length; i++) { |
for(let i = 0; i < exitHandlers.length; i++) { |
||||||
exitHandlers[i](); |
exitHandlers[i](); |
||||||
} |
} |
||||||
console.log('process exit'); |
console.log('process exit'); |
||||||
process.exit(); |
process.exit(); |
||||||
} |
} |
||||||
|
|
||||||
//do something when app is closing
|
function catchException(options: any, err: any) { |
||||||
process.on('exit', exitHandler.bind(null,{cleanup:true})); |
console.log('Server exception:'); |
||||||
|
console.log(err); |
||||||
//catches ctrl+c event
|
exitHandler({ exit: true }); |
||||||
process.on('SIGINT', exitHandler.bind(null, {exit:true})); |
} |
||||||
|
|
||||||
// catches "kill pid" (for example: nodemon restart)
|
//do something when app is closing
|
||||||
process.on('SIGUSR1', exitHandler.bind(null, {exit:true})); |
process.on('exit', exitHandler.bind(null, { cleanup:true })); |
||||||
process.on('SIGUSR2', exitHandler.bind(null, {exit:true})); |
|
||||||
|
//catches ctrl+c event
|
||||||
//catches uncaught exceptions
|
process.on('SIGINT', exitHandler.bind(null, { exit:true })); |
||||||
process.on('uncaughtException', exitHandler.bind(null, {exit:true})); |
|
||||||
|
// catches "kill pid" (for example: nodemon restart)
|
||||||
|
process.on('SIGUSR1', exitHandler.bind(null, { exit:true })); |
||||||
|
process.on('SIGUSR2', exitHandler.bind(null, { exit:true })); |
||||||
|
|
||||||
|
//catches uncaught exceptions
|
||||||
|
process.on('uncaughtException', catchException.bind(null, { exit:true })); |
||||||
Loading…
Reference in new issue