Browse Source
* add zmq to deps * basic zmq usage & build system fxs * continue zmq integration & refactorings * server.py + logging * some commit * ping-pong server-analytics & pair type * packing zmq.node for productionpull/1/head
Alexey Velikiy
15 changed files with 212 additions and 76 deletions
@ -0,0 +1,14 @@
|
||||
{ |
||||
// Use IntelliSense to learn about possible attributes. |
||||
// Hover to view descriptions of existing attributes. |
||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 |
||||
"version": "0.2.0", |
||||
"configurations": [ |
||||
{ |
||||
"name": "Python: Current File", |
||||
"type": "python", |
||||
"request": "launch", |
||||
"program": "${workspaceFolder}\\server.py" |
||||
} |
||||
] |
||||
} |
||||
@ -1,5 +1,6 @@
|
||||
{ |
||||
"python.pythonPath": "python", |
||||
"python.linting.enabled": false, |
||||
"terminal.integrated.shell.windows": "C:\\WINDOWS\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", |
||||
"editor.tabSize": 4, |
||||
"editor.insertSpaces": true |
||||
|
||||
@ -0,0 +1,63 @@
|
||||
import config |
||||
import json |
||||
import logging |
||||
import zmq |
||||
import sys |
||||
|
||||
from worker import Worker |
||||
|
||||
root = logging.getLogger() |
||||
root.setLevel(logging.DEBUG) |
||||
|
||||
ch = logging.StreamHandler(sys.stdout) |
||||
ch.setLevel(logging.DEBUG) |
||||
formatter = logging.Formatter("%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s") |
||||
ch.setFormatter(formatter) |
||||
root.addHandler(ch) |
||||
|
||||
logger = logging.getLogger('SERVER') |
||||
socket = None |
||||
|
||||
def handlePing(): |
||||
socket.send(b'pong') |
||||
|
||||
def handleTask(text): |
||||
try: |
||||
task = json.loads(text) |
||||
logger.info("Command is OK") |
||||
|
||||
socket.send_string(json.dumps({ |
||||
'task': task['type'], |
||||
'anomaly_id': task['anomaly_id'], |
||||
'__task_id': task['__task_id'], |
||||
'status': "in progress" |
||||
})) |
||||
|
||||
res = w.do_task(task) |
||||
res['__task_id'] = task['__task_id'] |
||||
socket.send_string(json.dumps(res)) |
||||
|
||||
except Exception as e: |
||||
logger.error("Exception: '%s'" % str(e)) |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
w = Worker() |
||||
logger.info("Worker was started") |
||||
|
||||
logger.info("Binding to %s ..." % config.ZEROMQ_CONNECTION_STRING) |
||||
context = zmq.Context() |
||||
socket = context.socket(zmq.PAIR) |
||||
socket.bind(config.ZEROMQ_CONNECTION_STRING) |
||||
logger.info("Ok") |
||||
|
||||
while True: |
||||
text = socket.recv() |
||||
logger.info('Got message %s' % text) |
||||
if text == b'ping': |
||||
handlePing() |
||||
logger.info('Sent pong') |
||||
else: |
||||
handleTask(text) |
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
{ |
||||
"HASTIC_PORT": 8000, |
||||
"HASTIC_API_KEY": "eyJrIjoiVjZqMHY0dHk4UEE3eEN4MzgzRnd2aURlMWlIdXdHNW4iLCJuIjoiaGFzdGljIiwiaWQiOjF9" |
||||
"HASTIC_API_KEY": "eyJrIjoiVjZqMHY0dHk4UEE3eEN4MzgzRnd2aURlMWlIdXdHNW4iLCJuIjoiaGFzdGljIiwiaWQiOjF9", |
||||
"ZEROMQ_CONNECTION_STRING": "ipc:///tmp/hastic/8000" |
||||
} |
||||
|
||||
@ -0,0 +1,13 @@
|
||||
var path = require('path'); |
||||
|
||||
// based on: https://github.com/webpack-contrib/node-loader/blob/master/index.js
|
||||
module.exports = function nodeLoader(m, q) { |
||||
return (` |
||||
var modulePath = __dirname + '/${path.basename(this.resourcePath)}'; |
||||
try { |
||||
global.process.dlopen(module, modulePath);
|
||||
} catch(e) { |
||||
throw new Error('dlopen: Cannot open ' + modulePath + ': ' + e); |
||||
} |
||||
`);
|
||||
} |
||||
@ -1,4 +1,15 @@
|
||||
var base = require('./webpack.base.conf'); |
||||
|
||||
|
||||
base.mode = 'production'; |
||||
|
||||
base.module.rules.push({ |
||||
test: /\.node$/, |
||||
use: [ |
||||
{ loader: './build/node-loader' }, |
||||
{ loader: 'file-loader', options: { name: '[name].[ext]' } }, |
||||
] |
||||
}); |
||||
|
||||
module.exports = base; |
||||
|
||||
|
||||
@ -1,39 +1,89 @@
|
||||
import { ANALYTICS_PATH } from '../config' |
||||
const zmq = require('zeromq'); |
||||
|
||||
import { spawn, ChildProcess } from 'child_process' |
||||
import { split, mapSync } from 'event-stream'; |
||||
import { spawn } from 'child_process' |
||||
import { ANALYTICS_PATH, ZEROMQ_CONNECTION_STRING } from '../config' |
||||
|
||||
import * as fs from 'fs'; |
||||
import * as path from 'path'; |
||||
|
||||
|
||||
|
||||
export class AnalyticsConnection { |
||||
|
||||
private _learnWorker: ChildProcess; |
||||
private _requester: any; |
||||
|
||||
constructor(private _onResponse: (response: any) => void) { |
||||
this._initConnection(); |
||||
} |
||||
|
||||
public async sendTask(msgObj: any): Promise<void> { |
||||
let message = JSON.stringify(msgObj); |
||||
return this.sendMessage(message); |
||||
} |
||||
|
||||
public async sendMessage(message: string): Promise<void> { |
||||
return new Promise<void>((resolve, reject) => { |
||||
this._requester.send(message, undefined, (err) => { |
||||
if(err) { |
||||
reject(err); |
||||
} else { |
||||
resolve(); |
||||
} |
||||
}); |
||||
}); |
||||
} |
||||
|
||||
public close() { |
||||
// TODO: close socket & terminate process if you have any
|
||||
this._requester.close(); |
||||
} |
||||
|
||||
private async _initConnection() { |
||||
this._requester = zmq.socket('pair'); |
||||
|
||||
if(process.env.NODE_ENV !== 'development') { |
||||
this._runAnalyticsProcess(); |
||||
} |
||||
|
||||
console.log("Binding to zmq...: %s", ZEROMQ_CONNECTION_STRING); |
||||
this._requester.connect(ZEROMQ_CONNECTION_STRING); |
||||
console.log('Ok'); |
||||
|
||||
console.log('Sending ping to analytics...'); |
||||
await this._connectToAnalytics(); |
||||
console.log('Ok') |
||||
|
||||
this._requester.on("message", this._onAnalyticsMessage.bind(this)); |
||||
|
||||
} |
||||
|
||||
private async _connectToAnalytics() { |
||||
this.sendMessage('ping'); // we don`t await here
|
||||
return new Promise(resolve => { |
||||
this._requester.once('message', (message) => { |
||||
console.log('Got message from analytics: ' + message); |
||||
resolve(); |
||||
}) |
||||
}); |
||||
} |
||||
|
||||
private _runAnalyticsProcess() { |
||||
console.log('Creating analytics process...'); |
||||
if(fs.existsSync(path.join(ANALYTICS_PATH, 'dist/worker/worker'))) { |
||||
this._learnWorker = spawn('dist/worker/worker', [], { cwd: ANALYTICS_PATH }) |
||||
console.log('dist/worker/worker'); |
||||
spawn('dist/worker/worker', [], { cwd: ANALYTICS_PATH }) |
||||
} else { |
||||
console.log('python3 server.py'); |
||||
// If compiled analytics script doesn't exist - fallback to regular python
|
||||
this._learnWorker = spawn('python3', ['worker.py'], { cwd: ANALYTICS_PATH }) |
||||
spawn('python3', ['server.py'], { cwd: ANALYTICS_PATH }) |
||||
} |
||||
|
||||
this._learnWorker.stdout.pipe( |
||||
split()).pipe(mapSync(this._onPipeMessage.bind(this)) |
||||
); |
||||
this._learnWorker.stderr.on('data', data => console.error(`worker stderr: ${data}`)); |
||||
console.log('ok'); |
||||
} |
||||
|
||||
private _onPipeMessage(data) { |
||||
console.log(`worker stdout: ${data}`); |
||||
private _onAnalyticsMessage(data) { |
||||
console.log(`analytics message: ${data}`); |
||||
let response = JSON.parse(data); |
||||
this._onResponse(response); |
||||
} |
||||
|
||||
public async sendMessage(task: any): Promise<void> { |
||||
let command = JSON.stringify(task); |
||||
this._learnWorker.stdin.write(`${command}\n`); |
||||
} |
||||
|
||||
} |
||||
|
||||
Loading…
Reference in new issue