Alexey Velikiy
6 years ago
committed by
GitHub
19 changed files with 846 additions and 753 deletions
@ -1,30 +0,0 @@
|
||||
import { getJsonDataSync, writeJsonDataSync } from '../services/json_service'; |
||||
import { METRICS_PATH } from '../config'; |
||||
|
||||
import * as crypto from 'crypto'; |
||||
|
||||
import * as path from 'path'; |
||||
|
||||
|
||||
function saveTargets(targets) { |
||||
let metrics = []; |
||||
for (let target of targets) { |
||||
metrics.push(saveTarget(target)); |
||||
} |
||||
return metrics; |
||||
} |
||||
|
||||
function saveTarget(target) { |
||||
//const md5 = crypto.createHash('md5')
|
||||
const targetId = crypto.createHash('md5').update(JSON.stringify(target)).digest('hex'); |
||||
let filename = path.join(METRICS_PATH, `${targetId}.json`); |
||||
writeJsonDataSync(filename, target); |
||||
return targetId; |
||||
} |
||||
|
||||
function getTarget(targetId) { |
||||
let filename = path.join(METRICS_PATH, `${targetId}.json`); |
||||
return getJsonDataSync(filename); |
||||
} |
||||
|
||||
export { saveTargets, getTarget } |
||||
@ -1,77 +0,0 @@
|
||||
import { getJsonDataSync, writeJsonDataSync } from '../services/json_service'; |
||||
import { AnalyticUnitId, findById, save } from '../models/analytic_unit'; |
||||
import { SEGMENTS_PATH } from '../config'; |
||||
|
||||
import * as _ from 'lodash'; |
||||
|
||||
import * as path from 'path'; |
||||
import * as fs from 'fs'; |
||||
|
||||
|
||||
export function getLabeledSegments(id: AnalyticUnitId) { |
||||
let filename = path.join(SEGMENTS_PATH, `${id}_labeled.json`); |
||||
|
||||
if(!fs.existsSync(filename)) { |
||||
return []; |
||||
} else { |
||||
let segments = getJsonDataSync(filename); |
||||
for(let segment of segments) { |
||||
if(segment.labeled === undefined) { |
||||
segment.labeled = false; |
||||
} |
||||
} |
||||
return segments; |
||||
} |
||||
} |
||||
|
||||
export function getPredictedSegments(id: AnalyticUnitId) { |
||||
let filename = path.join(SEGMENTS_PATH, `${id}_segments.json`); |
||||
|
||||
let jsonData; |
||||
try { |
||||
jsonData = getJsonDataSync(filename); |
||||
} catch(e) { |
||||
console.error(e.message); |
||||
jsonData = []; |
||||
} |
||||
return jsonData; |
||||
} |
||||
|
||||
export function saveSegments(id: AnalyticUnitId, segments) { |
||||
let filename = path.join(SEGMENTS_PATH, `${id}_labeled.json`); |
||||
|
||||
try { |
||||
return writeJsonDataSync(filename, _.uniqBy(segments, 'start')); |
||||
} catch(e) { |
||||
console.error(e.message); |
||||
throw new Error('Can`t write to db'); |
||||
} |
||||
} |
||||
|
||||
export function insertSegments(id: AnalyticUnitId, addedSegments, labeled: boolean) { |
||||
// Set status
|
||||
let info = findById(id); |
||||
let segments = getLabeledSegments(id); |
||||
|
||||
let nextId = info.nextId; |
||||
let addedIds = [] |
||||
for (let segment of addedSegments) { |
||||
segment.id = nextId; |
||||
segment.labeled = labeled; |
||||
addedIds.push(nextId); |
||||
nextId++; |
||||
segments.push(segment); |
||||
} |
||||
info.nextId = nextId; |
||||
saveSegments(id, segments); |
||||
save(id, info); |
||||
return addedIds; |
||||
} |
||||
|
||||
export function removeSegments(id: AnalyticUnitId, removedSegments) { |
||||
let segments = getLabeledSegments(id); |
||||
for (let segmentId of removedSegments) { |
||||
segments = segments.filter(el => el.id !== segmentId); |
||||
} |
||||
saveSegments(id, segments); |
||||
} |
||||
@ -1,84 +0,0 @@
|
||||
import { getJsonDataSync, writeJsonDataSync } from '../services/json_service' |
||||
import { ANALYTIC_UNITS_PATH } from '../config' |
||||
|
||||
import * as crypto from 'crypto'; |
||||
|
||||
import * as path from 'path' |
||||
import * as fs from 'fs' |
||||
|
||||
|
||||
export type Datasource = { |
||||
method: string, |
||||
data: Object, |
||||
params: Object, |
||||
type: string, |
||||
url: string |
||||
} |
||||
|
||||
export type Metric = { |
||||
datasource: string, |
||||
targets: string[] |
||||
} |
||||
|
||||
export type AnalyticUnitId = string; |
||||
|
||||
export type AnalyticUnit = { |
||||
id?: AnalyticUnitId, |
||||
name: string, |
||||
panelUrl: string, |
||||
type: string, |
||||
metric: Metric, |
||||
datasource: Datasource |
||||
status: string, |
||||
error?: string, |
||||
lastPredictionTime: number, |
||||
nextId: number |
||||
} |
||||
|
||||
export function createItem(item: AnalyticUnit): AnalyticUnitId { |
||||
const hashString = item.name + (new Date()).toString(); |
||||
const newId: AnalyticUnitId = crypto.createHash('md5').update(hashString).digest('hex'); |
||||
let filename = path.join(ANALYTIC_UNITS_PATH, `${newId}.json`); |
||||
if(fs.existsSync(filename)) { |
||||
throw new Error(`Can't create item with id ${newId}`); |
||||
} |
||||
save(newId, item); |
||||
item.id = newId; |
||||
return newId; |
||||
} |
||||
|
||||
export function remove(id: AnalyticUnitId) { |
||||
let filename = path.join(ANALYTIC_UNITS_PATH, `${id}.json`); |
||||
fs.unlinkSync(filename); |
||||
} |
||||
|
||||
export function save(id: AnalyticUnitId, unit: AnalyticUnit) { |
||||
let filename = path.join(ANALYTIC_UNITS_PATH, `${id}.json`); |
||||
return writeJsonDataSync(filename, unit); |
||||
} |
||||
|
||||
// TODO: make async
|
||||
export function findById(id: AnalyticUnitId): AnalyticUnit { |
||||
let filename = path.join(ANALYTIC_UNITS_PATH, `${id}.json`); |
||||
if(!fs.existsSync(filename)) { |
||||
throw new Error(`Can't find Analytic Unit with id ${id}`); |
||||
} |
||||
return getJsonDataSync(filename); |
||||
} |
||||
|
||||
export function setStatus(predictorId: AnalyticUnitId, status: string, error?: string) { |
||||
let info = findById(predictorId); |
||||
info.status = status; |
||||
if(error !== undefined) { |
||||
info.error = error; |
||||
} else { |
||||
info.error = ''; |
||||
} |
||||
save(predictorId, info); |
||||
} |
||||
|
||||
export function setPredictionTime(id: AnalyticUnitId, time: number) { |
||||
let info = findById(id); |
||||
info.lastPredictionTime = time; |
||||
save(id, info); |
||||
} |
||||
@ -0,0 +1,98 @@
|
||||
import { Metric } from './metric_model'; |
||||
import { Collection, makeDBQ } from '../services/data_service'; |
||||
|
||||
|
||||
let db = makeDBQ(Collection.ANALYTIC_UNITS); |
||||
|
||||
|
||||
export type AnalyticUnitId = string; |
||||
|
||||
export class AnalyticUnit { |
||||
constructor( |
||||
public name: string, |
||||
public panelUrl: string, |
||||
public type: string, |
||||
public metric: Metric, |
||||
public id?: AnalyticUnitId, |
||||
public lastPredictionTime?: number, |
||||
public status?: string, |
||||
public error?: string, |
||||
) { |
||||
if(name === undefined) { |
||||
throw new Error(`Missing field "name"`); |
||||
} |
||||
if(panelUrl === undefined) { |
||||
throw new Error(`Missing field "panelUrl"`); |
||||
} |
||||
if(type === undefined) { |
||||
throw new Error(`Missing field "type"`); |
||||
} |
||||
if(metric === undefined) { |
||||
throw new Error(`Missing field "metric"`); |
||||
} |
||||
} |
||||
|
||||
public toObject() { |
||||
return { |
||||
_id: this.id, |
||||
name: this.name, |
||||
panelUrl: this.panelUrl, |
||||
type: this.type, |
||||
metric: this.metric.toObject(), |
||||
lastPredictionTime: this.lastPredictionTime, |
||||
status: this.status, |
||||
error: this.error |
||||
}; |
||||
} |
||||
|
||||
static fromObject(obj: any): AnalyticUnit { |
||||
if(obj === undefined) { |
||||
throw new Error('obj is undefined'); |
||||
} |
||||
return new AnalyticUnit( |
||||
obj.name, |
||||
obj.panelUrl, |
||||
obj.type, |
||||
Metric.fromObject(obj.metric), |
||||
obj.status, |
||||
obj.lastPredictionTime, |
||||
obj._id, |
||||
obj.error, |
||||
); |
||||
} |
||||
|
||||
} |
||||
|
||||
|
||||
export async function findById(id: AnalyticUnitId): Promise<AnalyticUnit> { |
||||
return AnalyticUnit.fromObject(await db.findOne(id)); |
||||
} |
||||
|
||||
/** |
||||
* Creates and updates new unit.id |
||||
* |
||||
* @param unit to create |
||||
* @returns unit.id |
||||
*/ |
||||
export async function create(unit: AnalyticUnit): Promise<AnalyticUnitId> { |
||||
var obj = unit.toObject(); |
||||
var r = await db.insertOne(obj); |
||||
return r; |
||||
} |
||||
|
||||
export async function remove(id: AnalyticUnitId): Promise<void> { |
||||
await db.removeOne(id); |
||||
return; |
||||
} |
||||
|
||||
export async function update(id: AnalyticUnitId, unit: AnalyticUnit) { |
||||
return db.updateOne(id, unit); |
||||
} |
||||
|
||||
export async function setStatus(id: AnalyticUnitId, status: string, error?: string) { |
||||
return db.updateOne(id, { status, error }); |
||||
} |
||||
|
||||
export async function setPredictionTime(id: AnalyticUnitId, lastPredictionTime: number) { |
||||
return db.updateOne(id, { lastPredictionTime }); |
||||
} |
||||
@ -0,0 +1,32 @@
|
||||
export class Metric { |
||||
constructor(public datasource: string, public targets: any[]) { |
||||
if(datasource === undefined) { |
||||
throw new Error('datasource is undefined'); |
||||
} |
||||
if(targets === undefined) { |
||||
throw new Error('targets is undefined'); |
||||
} |
||||
if(targets.length === 0) { |
||||
throw new Error('targets is empty'); |
||||
} |
||||
} |
||||
|
||||
public toObject() { |
||||
return { |
||||
datasource: this.datasource, |
||||
targets: this.targets |
||||
}; |
||||
} |
||||
|
||||
static fromObject(obj: any): Metric { |
||||
if(obj === undefined) { |
||||
throw new Error('obj is undefined'); |
||||
} |
||||
return new Metric( |
||||
obj.datasource, |
||||
obj.targets |
||||
); |
||||
} |
||||
} |
||||
|
||||
|
||||
@ -0,0 +1,87 @@
|
||||
import { AnalyticUnitId } from './analytic_unit_model'; |
||||
|
||||
import { Collection, makeDBQ } from '../services/data_service'; |
||||
|
||||
let db = makeDBQ(Collection.SEGMENTS); |
||||
|
||||
|
||||
export type SegmentId = string; |
||||
|
||||
export class Segment { |
||||
constructor( |
||||
public analyticUnitId: AnalyticUnitId, |
||||
public from: number, |
||||
public to: number, |
||||
public labeled: boolean, |
||||
public id?: SegmentId |
||||
) { |
||||
if(analyticUnitId === undefined) { |
||||
throw new Error('AnalyticUnitId is undefined'); |
||||
} |
||||
if(from === undefined) { |
||||
throw new Error('from is undefined'); |
||||
} |
||||
if(isNaN(from)) { |
||||
throw new Error('from is NaN'); |
||||
} |
||||
if(to === undefined) { |
||||
throw new Error('to is undefined'); |
||||
} |
||||
if(isNaN(to)) { |
||||
throw new Error('to is NaN'); |
||||
} |
||||
if(labeled === undefined) { |
||||
throw new Error('labeled is undefined'); |
||||
} |
||||
} |
||||
|
||||
public toObject() { |
||||
return { |
||||
_id: this.id, |
||||
analyticUnitId: this.analyticUnitId, |
||||
from: this.from, |
||||
to: this.to, |
||||
labeled: this.labeled |
||||
}; |
||||
} |
||||
|
||||
static fromObject(obj: any): Segment { |
||||
if(obj === undefined) { |
||||
throw new Error('obj is undefined'); |
||||
} |
||||
return new Segment( |
||||
obj.analyticUnitId,
|
||||
+obj.from, +obj.to, |
||||
obj.labeled, obj._id |
||||
); |
||||
} |
||||
} |
||||
|
||||
export type FindManyQuery = { |
||||
timeFromGTE?: number, |
||||
timeToLTE?: number, |
||||
intexGT?: number |
||||
} |
||||
|
||||
export async function findMany(id: AnalyticUnitId, query: FindManyQuery): Promise<Segment[]> { |
||||
var dbQuery: any = { analyticUnitId: id }; |
||||
if(query.timeFromGTE !== undefined) { |
||||
dbQuery.from = { $gte: query.timeFromGTE }; |
||||
} |
||||
if(query.timeToLTE !== undefined) { |
||||
dbQuery.to = { $lte: query.timeToLTE }; |
||||
} |
||||
let segs = await db.findMany(dbQuery); |
||||
if(segs === null) { |
||||
return []; |
||||
} |
||||
return segs.map(Segment.fromObject); |
||||
} |
||||
|
||||
export async function insertSegments(segments: Segment[]) { |
||||
return db.insertMany(segments.map(s => s.toObject())); |
||||
} |
||||
|
||||
export function removeSegments(idsToRemove: SegmentId[]) { |
||||
return db.removeMany(idsToRemove); |
||||
} |
||||
@ -0,0 +1,32 @@
|
||||
import { AnalyticUnitId } from "./analytic_unit_model"; |
||||
|
||||
|
||||
export type TaskId = string; |
||||
|
||||
export class Task { |
||||
constructor( |
||||
public analyticUnitId: AnalyticUnitId, |
||||
public id?: TaskId |
||||
) { |
||||
if(analyticUnitId === undefined) { |
||||
throw new Error('analyticUnitId is undefined'); |
||||
} |
||||
} |
||||
|
||||
public toObject() { |
||||
return { |
||||
_id: this.id, |
||||
analyticUnitId: this.analyticUnitId |
||||
}; |
||||
} |
||||
|
||||
static fromObject(obj: any): Task { |
||||
if(obj === undefined) { |
||||
throw new Error('obj is undefined'); |
||||
} |
||||
return new Task( |
||||
obj.analyticUnitId, |
||||
obj._id, |
||||
); |
||||
} |
||||
} |
||||
@ -1,232 +1,232 @@
|
||||
import * as config from '../config'; |
||||
|
||||
const zmq = require('zeromq'); |
||||
|
||||
import * as childProcess from 'child_process' |
||||
import * as fs from 'fs'; |
||||
import * as path from 'path'; |
||||
|
||||
|
||||
export class AnalyticsMessage { |
||||
public constructor(public method: string, public payload?: string, public requestId?: number) { |
||||
|
||||
} |
||||
|
||||
static fromJSON(obj: any): AnalyticsMessage { |
||||
if(obj.method === undefined) { |
||||
throw new Error('No method in obj:' + obj); |
||||
} |
||||
return new AnalyticsMessage(obj.method, obj.payload, obj.requestId); |
||||
} |
||||
} |
||||
|
||||
function analyticsMessageFromJson(obj: any): AnalyticsMessage { |
||||
return new AnalyticsMessage(obj); |
||||
} |
||||
|
||||
export class AnalyticsService { |
||||
|
||||
private _requester: any; |
||||
private _ready: boolean = false; |
||||
private _pingResponded = false; |
||||
private _zmqConnectionString = null; |
||||
private _ipcPath = null; |
||||
private _analyticsPinger: NodeJS.Timer = null; |
||||
private _isClosed = false; |
||||
|
||||
constructor(private _onMessage: (message: AnalyticsMessage) => void) { |
||||
this._init(); |
||||
} |
||||
|
||||
public async sendTask(taskObj: any): Promise<void> { |
||||
if(!this._ready) { |
||||
return Promise.reject("Analytics is not ready"); |
||||
} |
||||
let message = { |
||||
method: 'TASK', |
||||
payload: taskObj |
||||
} |
||||
return this.sendMessage(message); |
||||
} |
||||
|
||||
public async sendMessage(message: AnalyticsMessage): Promise<void> { |
||||
let strMessage = JSON.stringify(message); |
||||
if(message.method === 'PING') { |
||||
strMessage = 'PING'; |
||||
} |
||||
return new Promise<void>((resolve, reject) => { |
||||
this._requester.send(strMessage, undefined, (err) => { |
||||
if(err) { |
||||
reject(err); |
||||
} else { |
||||
resolve(); |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
public close() { |
||||
this._isClosed = true; |
||||
console.log('Terminating analytics service...'); |
||||
clearInterval(this._analyticsPinger); |
||||
if(this._ipcPath !== null) { |
||||
console.log('Remove ipc path: ' + this._ipcPath); |
||||
fs.unlinkSync(this._ipcPath); |
||||
} |
||||
this._requester.close(); |
||||
console.log('Ok'); |
||||
} |
||||
|
||||
public get ready(): boolean { return this._ready; } |
||||
|
||||
private async _init() { |
||||
this._requester = zmq.socket('pair'); |
||||
let productionMode = process.env.NODE_ENV !== 'development'; |
||||
|
||||
this._zmqConnectionString = `tcp://127.0.0.1:${config.ZMQ_DEV_PORT}`; // debug mode
|
||||
if(productionMode) { |
||||
this._zmqConnectionString = config.ZMQ_CONNECTION_STRING; |
||||
if(this._zmqConnectionString === null) { |
||||
var createResult = await AnalyticsService.createIPCAddress(); |
||||
this._zmqConnectionString = createResult.address; |
||||
this._ipcPath = createResult.file; |
||||
} |
||||
} |
||||
|
||||
console.log("Binding to zmq... %s", this._zmqConnectionString); |
||||
this._requester.connect(this._zmqConnectionString); |
||||
this._requester.on("message", this._onAnalyticsMessage.bind(this)); |
||||
console.log('Ok'); |
||||
|
||||
if(productionMode) { |
||||
console.log('Creating analytics process...'); |
||||
try { |
||||
var cp = await AnalyticsService._runAnalyticsProcess(this._zmqConnectionString); |
||||
} catch(error) { |
||||
console.error('Can`t run analytics process: %s', error); |
||||
return; |
||||
} |
||||
console.log('Ok, pid: %s', cp.pid); |
||||
} |
||||
|
||||
console.log('Start analytics pinger...'); |
||||
this._runAlalyticsPinger(); |
||||
console.log('Ok'); |
||||
|
||||
} |
||||
|
||||
/** |
||||
* Spawns analytics process. Reads process stderr and fails if it isn`t empty.
|
||||
* No need to stop the process later. |
||||
* |
||||
* @returns Creaded child process |
||||
* @throws Process start error or first exception during python start |
||||
*/ |
||||
private static async _runAnalyticsProcess(zmqConnectionString: string): Promise<childProcess.ChildProcess> { |
||||
let cp: childProcess.ChildProcess; |
||||
let cpOptions = { |
||||
cwd: config.ANALYTICS_PATH, |
||||
env: { |
||||
...process.env, |
||||
ZMQ_CONNECTION_STRING: zmqConnectionString |
||||
} |
||||
}; |
||||
|
||||
if(fs.existsSync(path.join(config.ANALYTICS_PATH, 'dist/worker/worker'))) { |
||||
console.log('dist/worker/worker'); |
||||
cp = childProcess.spawn('dist/worker/worker', [], cpOptions); |
||||
} else { |
||||
console.log('python3 server.py'); |
||||
// If compiled analytics script doesn't exist - fallback to regular python
|
||||
console.log(config.ANALYTICS_PATH); |
||||
cp = childProcess.spawn('python3', ['server.py'], cpOptions); |
||||
} |
||||
|
||||
if(cp.pid === undefined) { |
||||
return new Promise<childProcess.ChildProcess>((resolve, reject) => { |
||||
cp.on('error', reject); |
||||
}); |
||||
} |
||||
|
||||
return new Promise<childProcess.ChildProcess>((resolve, reject) => { |
||||
var resolved = false; |
||||
|
||||
cp.stdout.on('data', () => { |
||||
if(resolved) { |
||||
return; |
||||
} else { |
||||
resolved = true; |
||||
} |
||||
resolve(cp); |
||||
}); |
||||
|
||||
cp.stderr.on('data', function(data) { |
||||
if(resolved) { |
||||
return; |
||||
} else { |
||||
resolved = true; |
||||
} |
||||
reject(data); |
||||
}); |
||||
|
||||
}); |
||||
|
||||
} |
||||
|
||||
private _onAnalyticsUp() { |
||||
console.log('Analytics is up'); |
||||
} |
||||
|
||||
private async _onAnalyticsDown() { |
||||
console.log('Analytics is down'); |
||||
if(process.env.NODE_ENV !== 'development') { |
||||
await AnalyticsService._runAnalyticsProcess(this._zmqConnectionString); |
||||
} |
||||
} |
||||
|
||||
private _onAnalyticsMessage(data: any, error) { |
||||
if(data.toString() === 'PONG') { |
||||
this._pingResponded = true; |
||||
if(!this._ready) { |
||||
this._ready = true; |
||||
this._onAnalyticsUp(); |
||||
} |
||||
return; |
||||
} |
||||
|
||||
|
||||
let text = data.toString(); |
||||
let response; |
||||
try { |
||||
response = JSON.parse(text); |
||||
} catch (e) { |
||||
console.error("Can`t parse response from analytics as json:"); |
||||
console.error(text); |
||||
throw new Error('Unexpected response'); |
||||
} |
||||
this._onMessage(AnalyticsMessage.fromJSON(response)); |
||||
} |
||||
|
||||
private async _runAlalyticsPinger() { |
||||
this._analyticsPinger = setInterval(() => { |
||||
if(this._isClosed) { |
||||
return; |
||||
} |
||||
if(!this._pingResponded && this._ready) { |
||||
this._ready = false; |
||||
this._onAnalyticsDown(); |
||||
} |
||||
this._pingResponded = false; |
||||
// TODO: set life limit for this ping
|
||||
this.sendMessage({ method: 'PING' }); |
||||
}, config.ANLYTICS_PING_INTERVAL); |
||||
} |
||||
|
||||
private static async createIPCAddress(): Promise<{ address: string, file: string }> { |
||||
let filename = `${process.pid}.ipc` |
||||
let p = path.join(config.ZMQ_IPC_PATH, filename); |
||||
fs.writeFileSync(p, ''); |
||||
return Promise.resolve({ address: 'ipc://' + p, file: p }); |
||||
} |
||||
|
||||
} |
||||
import * as config from '../config'; |
||||
|
||||
const zmq = require('zeromq'); |
||||
|
||||
import * as childProcess from 'child_process' |
||||
import * as fs from 'fs'; |
||||
import * as path from 'path'; |
||||
|
||||
|
||||
export class AnalyticsMessage { |
||||
public constructor(public method: string, public payload?: string, public requestId?: number) { |
||||
|
||||
} |
||||
|
||||
static fromJSON(obj: any): AnalyticsMessage { |
||||
if(obj.method === undefined) { |
||||
throw new Error('No method in obj:' + obj); |
||||
} |
||||
return new AnalyticsMessage(obj.method, obj.payload, obj.requestId); |
||||
} |
||||
} |
||||
|
||||
function analyticsMessageFromJson(obj: any): AnalyticsMessage { |
||||
return new AnalyticsMessage(obj); |
||||
} |
||||
|
||||
export class AnalyticsService { |
||||
|
||||
private _requester: any; |
||||
private _ready: boolean = false; |
||||
private _pingResponded = false; |
||||
private _zmqConnectionString: string = null; |
||||
private _ipcPath: string = null; |
||||
private _analyticsPinger: NodeJS.Timer = null; |
||||
private _isClosed = false; |
||||
|
||||
constructor(private _onMessage: (message: AnalyticsMessage) => void) { |
||||
this._init(); |
||||
} |
||||
|
||||
public async sendTask(taskObj: any): Promise<void> { |
||||
if(!this._ready) { |
||||
return Promise.reject("Analytics is not ready"); |
||||
} |
||||
let message = { |
||||
method: 'TASK', |
||||
payload: taskObj |
||||
} |
||||
return this.sendMessage(message); |
||||
} |
||||
|
||||
public async sendMessage(message: AnalyticsMessage): Promise<void> { |
||||
let strMessage = JSON.stringify(message); |
||||
if(message.method === 'PING') { |
||||
strMessage = 'PING'; |
||||
} |
||||
return new Promise<void>((resolve, reject) => { |
||||
this._requester.send(strMessage, undefined, (err: any) => { |
||||
if(err) { |
||||
reject(err); |
||||
} else { |
||||
resolve(); |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
public close() { |
||||
this._isClosed = true; |
||||
console.log('Terminating analytics service...'); |
||||
clearInterval(this._analyticsPinger); |
||||
if(this._ipcPath !== null) { |
||||
console.log('Remove ipc path: ' + this._ipcPath); |
||||
fs.unlinkSync(this._ipcPath); |
||||
} |
||||
this._requester.close(); |
||||
console.log('Ok'); |
||||
} |
||||
|
||||
public get ready(): boolean { return this._ready; } |
||||
|
||||
private async _init() { |
||||
this._requester = zmq.socket('pair'); |
||||
let productionMode = process.env.NODE_ENV !== 'development'; |
||||
|
||||
this._zmqConnectionString = `tcp://127.0.0.1:${config.ZMQ_DEV_PORT}`; // debug mode
|
||||
if(productionMode) { |
||||
this._zmqConnectionString = config.ZMQ_CONNECTION_STRING; |
||||
if(this._zmqConnectionString === null) { |
||||
var createResult = await AnalyticsService.createIPCAddress(); |
||||
this._zmqConnectionString = createResult.address; |
||||
this._ipcPath = createResult.file; |
||||
} |
||||
} |
||||
|
||||
console.log("Binding to zmq... %s", this._zmqConnectionString); |
||||
this._requester.connect(this._zmqConnectionString); |
||||
this._requester.on("message", this._onAnalyticsMessage.bind(this)); |
||||
console.log('Ok'); |
||||
|
||||
if(productionMode) { |
||||
console.log('Creating analytics process...'); |
||||
try { |
||||
var cp = await AnalyticsService._runAnalyticsProcess(this._zmqConnectionString); |
||||
} catch(error) { |
||||
console.error('Can`t run analytics process: %s', error); |
||||
return; |
||||
} |
||||
console.log('Ok, pid: %s', cp.pid); |
||||
} |
||||
|
||||
console.log('Start analytics pinger...'); |
||||
this._runAlalyticsPinger(); |
||||
console.log('Ok'); |
||||
|
||||
} |
||||
|
||||
/** |
||||
* Spawns analytics process. Reads process stderr and fails if it isn`t empty.
|
||||
* No need to stop the process later. |
||||
* |
||||
* @returns Creaded child process |
||||
* @throws Process start error or first exception during python start |
||||
*/ |
||||
private static async _runAnalyticsProcess(zmqConnectionString: string): Promise<childProcess.ChildProcess> { |
||||
let cp: childProcess.ChildProcess; |
||||
let cpOptions = { |
||||
cwd: config.ANALYTICS_PATH, |
||||
env: { |
||||
...process.env, |
||||
ZMQ_CONNECTION_STRING: zmqConnectionString |
||||
} |
||||
}; |
||||
|
||||
if(fs.existsSync(path.join(config.ANALYTICS_PATH, 'dist/worker/worker'))) { |
||||
console.log('dist/worker/worker'); |
||||
cp = childProcess.spawn('dist/worker/worker', [], cpOptions); |
||||
} else { |
||||
console.log('python3 server.py'); |
||||
// If compiled analytics script doesn't exist - fallback to regular python
|
||||
console.log(config.ANALYTICS_PATH); |
||||
cp = childProcess.spawn('python3', ['server.py'], cpOptions); |
||||
} |
||||
|
||||
if(cp.pid === undefined) { |
||||
return new Promise<childProcess.ChildProcess>((resolve, reject) => { |
||||
cp.on('error', reject); |
||||
}); |
||||
} |
||||
|
||||
return new Promise<childProcess.ChildProcess>((resolve, reject) => { |
||||
var resolved = false; |
||||
|
||||
cp.stdout.on('data', () => { |
||||
if(resolved) { |
||||
return; |
||||
} else { |
||||
resolved = true; |
||||
} |
||||
resolve(cp); |
||||
}); |
||||
|
||||
cp.stderr.on('data', function(data) { |
||||
if(resolved) { |
||||
return; |
||||
} else { |
||||
resolved = true; |
||||
} |
||||
reject(data); |
||||
}); |
||||
|
||||
}); |
||||
|
||||
} |
||||
|
||||
private _onAnalyticsUp() { |
||||
console.log('Analytics is up'); |
||||
} |
||||
|
||||
private async _onAnalyticsDown() { |
||||
console.log('Analytics is down'); |
||||
if(process.env.NODE_ENV !== 'development') { |
||||
await AnalyticsService._runAnalyticsProcess(this._zmqConnectionString); |
||||
} |
||||
} |
||||
|
||||
private _onAnalyticsMessage(data: any) { |
||||
if(data.toString() === 'PONG') { |
||||
this._pingResponded = true; |
||||
if(!this._ready) { |
||||
this._ready = true; |
||||
this._onAnalyticsUp(); |
||||
} |
||||
return; |
||||
} |
||||
|
||||
|
||||
let text = data.toString(); |
||||
let response; |
||||
try { |
||||
response = JSON.parse(text); |
||||
} catch (e) { |
||||
console.error("Can`t parse response from analytics as json:"); |
||||
console.error(text); |
||||
throw new Error('Unexpected response'); |
||||
} |
||||
this._onMessage(AnalyticsMessage.fromJSON(response)); |
||||
} |
||||
|
||||
private async _runAlalyticsPinger() { |
||||
this._analyticsPinger = setInterval(() => { |
||||
if(this._isClosed) { |
||||
return; |
||||
} |
||||
if(!this._pingResponded && this._ready) { |
||||
this._ready = false; |
||||
this._onAnalyticsDown(); |
||||
} |
||||
this._pingResponded = false; |
||||
// TODO: set life limit for this ping
|
||||
this.sendMessage({ method: 'PING' }); |
||||
}, config.ANLYTICS_PING_INTERVAL); |
||||
} |
||||
|
||||
private static async createIPCAddress(): Promise<{ address: string, file: string }> { |
||||
let filename = `${process.pid}.ipc` |
||||
let p = path.join(config.ZMQ_IPC_PATH, filename); |
||||
fs.writeFileSync(p, ''); |
||||
return Promise.resolve({ address: 'ipc://' + p, file: p }); |
||||
} |
||||
|
||||
} |
||||
|
||||
@ -1,77 +1,180 @@
|
||||
import * as config from '../config'; |
||||
|
||||
import * as nedb from 'nedb'; |
||||
import * as fs from 'fs'; |
||||
|
||||
export const db = { |
||||
analyticUnits: new nedb({ filename: config.ANALYTIC_UNITS_DATABASE_PATH, autoload: true }), |
||||
metrics: new nedb({ filename: config.METRICS_DATABASE_PATH, autoload: true }), |
||||
segments: new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true }), |
||||
files: new nedb({ filename: config.FILES_DATABASE_PATH, autoload: true }) |
||||
}; |
||||
|
||||
|
||||
let dbUpsertFile = (query: any, updateQuery: any) => { |
||||
return new Promise<void>((resolve, reject) => { |
||||
db.files.update(query, updateQuery, { upsert: true }, (err: Error) => { |
||||
if(err) { |
||||
reject(err); |
||||
} else { |
||||
console.log('saved shit with query '); |
||||
console.log(query); |
||||
console.log('saved shit with updateQuery '); |
||||
console.log(updateQuery); |
||||
resolve(); |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
let dbLoadFile = (query: any) => { |
||||
return new Promise<any>((resolve, reject) => { |
||||
db.files.findOne(query, (err, doc) => { |
||||
if(err) { |
||||
reject(err); |
||||
} else { |
||||
console.log('got shit with query'); |
||||
console.log(query); |
||||
console.log('doc'); |
||||
console.log(doc); |
||||
resolve(doc); |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
function maybeCreate(path: string): void { |
||||
if(fs.existsSync(path)) { |
||||
return; |
||||
} |
||||
console.log('mkdir: ' + path); |
||||
fs.mkdirSync(path); |
||||
console.log('exists: ' + fs.existsSync(path)); |
||||
} |
||||
|
||||
export async function saveFile(filename: string, content: string): Promise<void> { |
||||
return dbUpsertFile({ filename } , { filename, content }); |
||||
} |
||||
|
||||
export async function loadFile(filename: string): Promise<string> { |
||||
let doc = await dbLoadFile({ filename }); |
||||
if(doc === null) { |
||||
return null; |
||||
} |
||||
return doc.content; |
||||
} |
||||
|
||||
export function checkDataFolders(): void { |
||||
[ |
||||
config.DATA_PATH, |
||||
config.DATASETS_PATH, |
||||
config.ANALYTIC_UNITS_PATH, |
||||
config.MODELS_PATH, |
||||
config.METRICS_PATH, |
||||
config.SEGMENTS_PATH, |
||||
config.ZMQ_IPC_PATH |
||||
].forEach(maybeCreate); |
||||
} |
||||
import * as config from '../config'; |
||||
|
||||
import * as nedb from 'nedb'; |
||||
import * as fs from 'fs'; |
||||
|
||||
|
||||
export enum Collection { ANALYTIC_UNITS, SEGMENTS }; |
||||
|
||||
|
||||
/** |
||||
* Class which helps to make queries to your collection |
||||
* |
||||
* @param { string | object } query: a key as a string or mongodb-style query |
||||
*/ |
||||
export type DBQ = { |
||||
findOne: (query: string | object) => Promise<any>, |
||||
findMany: (query: string[] | object) => Promise<any[]>, |
||||
insertOne: (document: object) => Promise<string>, |
||||
insertMany: (documents: object[]) => Promise<string[]>, |
||||
updateOne: (query: string | object, updateQuery: any) => Promise<void>, |
||||
removeOne: (query: string) => Promise<boolean> |
||||
removeMany: (query: string[] | object) => Promise<number> |
||||
} |
||||
|
||||
export function makeDBQ(collection: Collection): DBQ { |
||||
return { |
||||
findOne: dbFindOne.bind(null, collection), |
||||
findMany: dbFindMany.bind(null, collection), |
||||
insertOne: dbInsertOne.bind(null, collection), |
||||
insertMany: dbInsertMany.bind(null, collection), |
||||
updateOne: dbUpdateOne.bind(null, collection), |
||||
removeOne: dbRemoveOne.bind(null, collection), |
||||
removeMany: dbRemoveMany.bind(null, collection) |
||||
} |
||||
} |
||||
|
||||
function wrapIdToQuery(query: string | object): any { |
||||
if(typeof query === 'string') { |
||||
return { _id: query }; |
||||
} |
||||
return query; |
||||
} |
||||
|
||||
function wrapIdsToQuery(query: string[] | object): any { |
||||
if(Array.isArray(query)) { |
||||
return { _id: { $in: query } }; |
||||
} |
||||
return query; |
||||
} |
||||
|
||||
function isEmptyArray(obj: any): boolean { |
||||
if(!Array.isArray(obj)) { |
||||
return false; |
||||
} |
||||
return obj.length == 0; |
||||
} |
||||
|
||||
const db = new Map<Collection, nedb>(); |
||||
|
||||
let dbInsertOne = (collection: Collection, doc: object) => { |
||||
return new Promise<string>((resolve, reject) => { |
||||
db.get(collection).insert(doc, (err, newDoc: any) => { |
||||
if(err) { |
||||
reject(err); |
||||
} else { |
||||
resolve(newDoc._id); |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
let dbInsertMany = (collection: Collection, docs: object[]) => { |
||||
if(docs.length === 0) { |
||||
return Promise.resolve([]); |
||||
} |
||||
return new Promise<string[]>((resolve, reject) => { |
||||
db.get(collection).insert(docs, (err, newDocs: any[]) => { |
||||
if(err) { |
||||
reject(err); |
||||
} else { |
||||
resolve(newDocs.map(d => d._id)); |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
let dbUpdateOne = (collection: Collection, query: string | object, updateQuery: object) => { |
||||
query = wrapIdToQuery(query); |
||||
return new Promise<void>((resolve, reject) => { |
||||
db.get(collection).update(query, updateQuery, { /* options */ }, (err: Error) => { |
||||
if(err) { |
||||
reject(err); |
||||
} else { |
||||
resolve(); |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
let dbFindOne = (collection: Collection, query: string | object) => { |
||||
query = wrapIdToQuery(query); |
||||
return new Promise<any>((resolve, reject) => { |
||||
db.get(collection).findOne(query, (err, doc) => { |
||||
if(err) { |
||||
reject(err); |
||||
} else { |
||||
resolve(doc); |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
let dbFindMany = (collection: Collection, query: string[] | object) => { |
||||
if(isEmptyArray(query)) { |
||||
return Promise.resolve([]); |
||||
} |
||||
query = wrapIdsToQuery(query); |
||||
return new Promise<any[]>((resolve, reject) => { |
||||
db.get(collection).find(query, (err, docs: any[]) => { |
||||
if(err) { |
||||
reject(err); |
||||
} else { |
||||
resolve(docs); |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
let dbRemoveOne = (collection: Collection, query: string | object) => { |
||||
query = wrapIdToQuery(query); |
||||
return new Promise<boolean>((resolve, reject) => { |
||||
db.get(collection).remove(query, { /* options */ }, (err, numRemoved) => { |
||||
if(err) { |
||||
reject(err); |
||||
} else { |
||||
if(numRemoved > 1) { |
||||
throw new Error(`Removed ${numRemoved} elements with query: ${JSON.stringify(query)}. Only one is Ok.`); |
||||
} else { |
||||
resolve(numRemoved == 1); |
||||
} |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
let dbRemoveMany = (collection: Collection, query: string[] | object) => { |
||||
if(isEmptyArray(query)) { |
||||
return Promise.resolve([]); |
||||
} |
||||
query = wrapIdsToQuery(query); |
||||
return new Promise<number>((resolve, reject) => { |
||||
db.get(collection).remove(query, { multi: true }, (err, numRemoved) => { |
||||
if(err) { |
||||
reject(err); |
||||
} else { |
||||
resolve(numRemoved); |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
function maybeCreateDir(path: string): void { |
||||
if(fs.existsSync(path)) { |
||||
return; |
||||
} |
||||
console.log('mkdir: ' + path); |
||||
fs.mkdirSync(path); |
||||
} |
||||
|
||||
function checkDataFolders(): void { |
||||
[ |
||||
config.DATA_PATH, |
||||
config.ZMQ_IPC_PATH |
||||
].forEach(maybeCreateDir); |
||||
} |
||||
checkDataFolders(); |
||||
|
||||
// TODO: it's better if models request db which we create if it`s needed
|
||||
db.set(Collection.ANALYTIC_UNITS, new nedb({ filename: config.ANALYTIC_UNITS_DATABASE_PATH, autoload: true })); |
||||
db.set(Collection.SEGMENTS, new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true })); |
||||
|
||||
@ -1,38 +1,44 @@
|
||||
|
||||
|
||||
var exitHandlers = [] |
||||
var exitHandled = false; |
||||
|
||||
/** |
||||
* Add a callback for closing programm bacause of any reason |
||||
*
|
||||
* @param callback a sync function |
||||
*/ |
||||
export function registerExitHandler(callback: () => void) { |
||||
exitHandlers.push(callback); |
||||
} |
||||
|
||||
function exitHandler(options, err) { |
||||
if(exitHandled) { |
||||
return; |
||||
} |
||||
exitHandled = true; |
||||
for(let i = 0; i < exitHandlers.length; i++) { |
||||
exitHandlers[i](); |
||||
} |
||||
console.log('process exit'); |
||||
process.exit(); |
||||
} |
||||
|
||||
//do something when app is closing
|
||||
process.on('exit', exitHandler.bind(null,{cleanup:true})); |
||||
|
||||
//catches ctrl+c event
|
||||
process.on('SIGINT', exitHandler.bind(null, {exit:true})); |
||||
|
||||
// catches "kill pid" (for example: nodemon restart)
|
||||
process.on('SIGUSR1', exitHandler.bind(null, {exit:true})); |
||||
process.on('SIGUSR2', exitHandler.bind(null, {exit:true})); |
||||
|
||||
//catches uncaught exceptions
|
||||
process.on('uncaughtException', exitHandler.bind(null, {exit:true})); |
||||
|
||||
|
||||
var exitHandlers: (() => void)[] = []; |
||||
var exitHandled = false; |
||||
|
||||
/** |
||||
* Add a callback for closing programm bacause of any reason |
||||
*
|
||||
* @param callback a sync function |
||||
*/ |
||||
export function registerExitHandler(callback: () => void) { |
||||
exitHandlers.push(callback); |
||||
} |
||||
|
||||
function exitHandler(options: any, err?: any) { |
||||
if(exitHandled) { |
||||
return; |
||||
} |
||||
exitHandled = true; |
||||
for(let i = 0; i < exitHandlers.length; i++) { |
||||
exitHandlers[i](); |
||||
} |
||||
console.log('process exit'); |
||||
process.exit(); |
||||
} |
||||
|
||||
function catchException(options: any, err: any) { |
||||
console.log('Server exception:'); |
||||
console.log(err); |
||||
exitHandler({ exit: true }); |
||||
} |
||||
|
||||
//do something when app is closing
|
||||
process.on('exit', exitHandler.bind(null, { cleanup:true })); |
||||
|
||||
//catches ctrl+c event
|
||||
process.on('SIGINT', exitHandler.bind(null, { exit:true })); |
||||
|
||||
// catches "kill pid" (for example: nodemon restart)
|
||||
process.on('SIGUSR1', exitHandler.bind(null, { exit:true })); |
||||
process.on('SIGUSR2', exitHandler.bind(null, { exit:true })); |
||||
|
||||
//catches uncaught exceptions
|
||||
process.on('uncaughtException', catchException.bind(null, { exit:true })); |
||||
Loading…
Reference in new issue