From df4866ce298bb19b590d519eaeccb869188da650 Mon Sep 17 00:00:00 2001 From: Evgeny Smyshlyaev Date: Wed, 10 Oct 2018 21:10:24 +0300 Subject: [PATCH 1/3] Anti-segments do not fall into analytics #181 (#190) --- server/src/controllers/analytics_controller.ts | 5 +++++ server/src/models/segment_model.ts | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/server/src/controllers/analytics_controller.ts b/server/src/controllers/analytics_controller.ts index 47b0f62..c74844d 100644 --- a/server/src/controllers/analytics_controller.ts +++ b/server/src/controllers/analytics_controller.ts @@ -120,6 +120,11 @@ export async function runLearning(id: AnalyticUnit.AnalyticUnitId) { } else { await AnalyticUnitCache.create(id); } + + let deletedSegments = await Segment.findMany(id, { deleted: true }); + let deletedSegmentsObjs = deletedSegments.map(s => s.toObject()); + segmentObjs = _.concat(segmentObjs, deletedSegmentsObjs); + let task = new AnalyticsTask( id, AnalyticsTaskType.LEARN, { pattern, segments: segmentObjs, data, cache: oldCache } ); diff --git a/server/src/models/segment_model.ts b/server/src/models/segment_model.ts index cb7e67b..9a397c3 100644 --- a/server/src/models/segment_model.ts +++ b/server/src/models/segment_model.ts @@ -62,7 +62,8 @@ export type FindManyQuery = { timeFromGTE?: number, timeToLTE?: number, intexGT?: number, - labeled?: boolean + labeled?: boolean, + deleted?: boolean } export async function findMany(id: AnalyticUnitId, query: FindManyQuery): Promise { @@ -76,6 +77,9 @@ export async function findMany(id: AnalyticUnitId, query: FindManyQuery): Promis if(query.labeled !== undefined) { dbQuery.labeled = query.labeled; } + if(query.deleted !== undefined) { + dbQuery.deleted = query.deleted; + } let segs = await db.findMany(dbQuery); if(segs === null) { return []; @@ -113,7 +117,7 @@ export async function insertSegments(segments: Segment[]) { } export async function setSegmentsDeleted(ids: SegmentId[]) { - return db.updateMany(ids, { deleted: true }); + return db.updateMany(ids, { deleted: true, labeled: false }); } export function removeSegments(idsToRemove: SegmentId[]) { From de3b64ce95e2ebbfb2621616d1ac3be8171ce53d Mon Sep 17 00:00:00 2001 From: Evgeny Smyshlyaev Date: Thu, 11 Oct 2018 14:34:23 +0300 Subject: [PATCH 2/3] Decouple processes to different docker containers#187(WIP) (#188) analytics and server moved to separate containers, added docker-compose file --- Dockerfile | 29 ------------------------ analytics/.dockerignore | 5 ++++ analytics/Dockerfile | 20 ++++++++++++++++ analytics/config.py | 2 +- docker-compose.yml | 19 ++++++++++++++++ server/.dockerignore | 4 ++++ server/Dockerfile | 26 +++++++++++++++++++++ server/src/config.ts | 1 + server/src/services/analytics_service.ts | 17 ++++++++++---- 9 files changed, 88 insertions(+), 35 deletions(-) delete mode 100644 Dockerfile create mode 100644 analytics/.dockerignore create mode 100644 analytics/Dockerfile create mode 100644 docker-compose.yml create mode 100644 server/.dockerignore create mode 100644 server/Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 4bf865f..0000000 --- a/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -FROM python:3.6.6 - -EXPOSE 8000 - -VOLUME [ "/var/www/data" ] - -COPY . /var/www - -WORKDIR /var/www/analytics - -RUN pip install -r requirements.txt - -RUN apt-get update && apt-get install -y \ - apt-utils \ - gnupg \ - curl \ - python \ - make \ - g++ \ - git -RUN curl -sL https://deb.nodesource.com/setup_8.x | bash - - -RUN apt-get update && apt-get install -y nodejs - -WORKDIR /var/www/server - -RUN npm install && npm run build - -CMD ["npm", "start"] diff --git a/analytics/.dockerignore b/analytics/.dockerignore new file mode 100644 index 0000000..ce5fca2 --- /dev/null +++ b/analytics/.dockerignore @@ -0,0 +1,5 @@ +.git +npm-debug +node_modules +__pycache__ +.vscode diff --git a/analytics/Dockerfile b/analytics/Dockerfile new file mode 100644 index 0000000..7b3e23a --- /dev/null +++ b/analytics/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.6.6 + +WORKDIR /var/www/analytics + +COPY ./requirements.txt /var/www/analytics + +RUN pip install -r requirements.txt \ + && apt-get update && apt-get install -y \ + apt-utils \ + gnupg \ + curl \ + make \ + g++ \ + git + +VOLUME [ "/var/www/data" ] + +COPY . /var/www/analytics/ + +CMD ["python", "server.py"] diff --git a/analytics/config.py b/analytics/config.py index c4b2b2e..d71d955 100644 --- a/analytics/config.py +++ b/analytics/config.py @@ -25,4 +25,4 @@ def get_config_field(field, default_val = None): raise Exception('Please configure {}'.format(field)) ZMQ_DEV_PORT = get_config_field('ZMQ_DEV_PORT', '8002') -ZMQ_CONNECTION_STRING = get_config_field('ZMQ_CONNECTION_STRING', 'tcp://*:%s' % ZMQ_DEV_PORT) +ZMQ_CONNECTION_STRING = get_config_field('ZMQ_CONNECTION_STRING', 'tcp://0.0.0.0:%s' % ZMQ_DEV_PORT) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1dde626 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +version: '2' +services: + server: + image: hastic/server:latest + build: server + environment: + HASTIC_API_KEY: ${HASTIC_API_KEY} + ZMQ_CONNECTION_STRING: tcp://analytics:8002 + ports: + - 8000:8000 + volumes: + - data-volume:/var/www/data + + analytics: + image: hastic/analytics:latest + build: analytics + +volumes: + data-volume: diff --git a/server/.dockerignore b/server/.dockerignore new file mode 100644 index 0000000..cf23c6e --- /dev/null +++ b/server/.dockerignore @@ -0,0 +1,4 @@ +.git +node_modules +npm-debug +.vscode diff --git a/server/Dockerfile b/server/Dockerfile new file mode 100644 index 0000000..ff47b00 --- /dev/null +++ b/server/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.6.6 + +RUN apt-get install curl \ + bash \ + gnupg \ + make \ + g++ \ + && curl -sL https://deb.nodesource.com/setup_8.x | bash - \ + && apt-get update \ + && apt-get install nodejs + +VOLUME [ "/var/www/data" ] + +WORKDIR /var/www/server + +COPY package.json /var/www/server + +RUN npm install + +COPY . /var/www/server + +RUN npm run build + +ENV INSIDE_DOCKER true + +CMD ["npm", "start"] diff --git a/server/src/config.ts b/server/src/config.ts index 46349b9..ced4f12 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -20,6 +20,7 @@ export const HASTIC_PORT = getConfigField('HASTIC_PORT', '8000'); export const ZMQ_CONNECTION_STRING = getConfigField('ZMQ_CONNECTION_STRING', null); export const ZMQ_IPC_PATH = getConfigField('ZMQ_IPC_PATH', path.join(os.tmpdir(), 'hastic')); export const ZMQ_DEV_PORT = getConfigField('ZMQ_DEV_PORT', '8002'); +export const ZMQ_HOST = getConfigField('ZMQ_HOST', '127.0.0.1'); export const HASTIC_API_KEY = getConfigField('HASTIC_API_KEY'); export const ANLYTICS_PING_INTERVAL = 500; // ms diff --git a/server/src/services/analytics_service.ts b/server/src/services/analytics_service.ts index 4c7ab22..38c4954 100644 --- a/server/src/services/analytics_service.ts +++ b/server/src/services/analytics_service.ts @@ -18,8 +18,12 @@ export class AnalyticsService { private _ipcPath: string = null; private _analyticsPinger: NodeJS.Timer = null; private _isClosed = false; + private _productionMode = false; + private _inDocker = false; constructor(private _onMessage: (message: AnalyticsMessage) => void) { + this._productionMode = process.env.NODE_ENV !== 'development'; + this._inDocker = process.env.INSIDE_DOCKER !== undefined; this._init(); } @@ -67,10 +71,12 @@ export class AnalyticsService { private async _init() { this._requester = zmq.socket('pair'); - let productionMode = process.env.NODE_ENV !== 'development'; - this._zmqConnectionString = `tcp://127.0.0.1:${config.ZMQ_DEV_PORT}`; // debug mode - if(productionMode) { + this._zmqConnectionString = `tcp://${config.ZMQ_HOST}:${config.ZMQ_DEV_PORT}`; // debug mode + + if(this._inDocker) { + this._zmqConnectionString = config.ZMQ_CONNECTION_STRING; + } else if(this._productionMode && !this._inDocker) { this._zmqConnectionString = config.ZMQ_CONNECTION_STRING; if(this._zmqConnectionString === null) { var createResult = await AnalyticsService.createIPCAddress(); @@ -84,7 +90,7 @@ export class AnalyticsService { this._requester.on("message", this._onAnalyticsMessage.bind(this)); console.log('Ok'); - if(productionMode) { + if(this._productionMode && !this._inDocker) { console.log('Creating analytics process...'); try { var cp = await AnalyticsService._runAnalyticsProcess(this._zmqConnectionString); @@ -165,12 +171,13 @@ export class AnalyticsService { private async _onAnalyticsDown() { console.log('Analytics is down'); - if(process.env.NODE_ENV !== 'development') { + if(this._productionMode && !this._inDocker) { await AnalyticsService._runAnalyticsProcess(this._zmqConnectionString); } } private _onAnalyticsMessage(data: any) { + let text = data.toString(); if(text === 'PONG') { this._pingResponded = true; From b79abf52991a559f146827d3fbc3fcc867a7cc82 Mon Sep 17 00:00:00 2001 From: Alexandr Velikiy <39257464+VargBurz@users.noreply.github.com> Date: Thu, 11 Oct 2018 16:39:35 +0300 Subject: [PATCH 3/3] Anti-segments in general model #142 (#185) --- analytics/models/drop_model.py | 2 +- analytics/models/general_model.py | 30 +++++++++++++++++++++++++++++- analytics/models/jump_model.py | 2 +- analytics/models/peak_model.py | 2 +- analytics/models/trough_model.py | 2 +- 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/analytics/models/drop_model.py b/analytics/models/drop_model.py index f5eae71..0c09858 100644 --- a/analytics/models/drop_model.py +++ b/analytics/models/drop_model.py @@ -172,7 +172,7 @@ class DropModel(Model): conv = scipy.signal.fftconvolve(convol_data, pattern_data) if conv[self.state['WINDOW_SIZE']*2] > self.state['convolve_max'] * 1.2 or conv[self.state['WINDOW_SIZE']*2] < self.state['convolve_min'] * 0.8: delete_list.append(segment) - if max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: + elif max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: delete_list.append(segment) else: delete_list.append(segment) diff --git a/analytics/models/general_model.py b/analytics/models/general_model.py index 0a1e9f4..dc74e3e 100644 --- a/analytics/models/general_model.py +++ b/analytics/models/general_model.py @@ -23,6 +23,8 @@ class GeneralModel(Model): 'convolve_max': 240, 'convolve_min': 200, 'WINDOW_SIZE': 240, + 'conv_del_min': 100, + 'conv_del_max': 120, } self.all_conv = [] @@ -38,7 +40,7 @@ class GeneralModel(Model): segment_data = data[segment_from_index: segment_to_index + 1] if len(segment_data) == 0: continue - x = segment_from_index + int((segment_to_index - segment_from_index) / 2) + x = segment_from_index + math.ceil((segment_to_index - segment_from_index) / 2) self.ipats.append(x) segment_data = data[x - self.state['WINDOW_SIZE'] : x + self.state['WINDOW_SIZE']] segment_min = min(segment_data) @@ -53,6 +55,20 @@ class GeneralModel(Model): convolve_data = scipy.signal.fftconvolve(labeled_data, self.model_gen) convolve_list.append(max(auto_convolve)) convolve_list.append(max(convolve_data)) + + del_conv_list = [] + for segment in segments: + if segment['deleted']: + segment_from_index = utils.timestamp_to_index(dataframe, pd.to_datetime(segment['from'], unit='ms')) + segment_to_index = utils.timestamp_to_index(dataframe, pd.to_datetime(segment['to'], unit='ms')) + segment_data = data[segment_from_index: segment_to_index + 1] + if len(segment_data) == 0: + continue + del_mid_index = segment_from_index + math.ceil((segment_to_index - segment_from_index) / 2) + deleted_pat = data[del_mid_index - self.state['WINDOW_SIZE']: del_mid_index + self.state['WINDOW_SIZE'] + 1] + deleted_pat = deleted_pat - min(deleted_pat) + del_conv_pat = scipy.signal.fftconvolve(deleted_pat, self.model_gen) + del_conv_list.append(max(del_conv_pat)) if len(convolve_list) > 0: self.state['convolve_max'] = float(max(convolve_list)) @@ -63,6 +79,16 @@ class GeneralModel(Model): self.state['convolve_min'] = float(min(convolve_list)) else: self.state['convolve_min'] = self.state['WINDOW_SIZE'] / 3 + + if len(del_conv_list) > 0: + self.state['conv_del_min'] = float(min(del_conv_list)) + else: + self.state['conv_del_min'] = self.state['WINDOW_SIZE'] + + if len(del_conv_list) > 0: + self.state['conv_del_max'] = float(max(del_conv_list)) + else: + self.state['conv_del_max'] = self.state['WINDOW_SIZE'] def do_predict(self, dataframe: pd.DataFrame) -> list: data = dataframe['value'] @@ -88,6 +114,8 @@ class GeneralModel(Model): for val in segments: if self.all_conv[val] < self.state['convolve_min'] * 0.8: delete_list.append(val) + elif (self.all_conv[val] < self.state['conv_del_max'] * 1.02 and self.all_conv[val] > self.state['conv_del_min'] * 0.98): + delete_list.append(val) for item in delete_list: segments.remove(item) diff --git a/analytics/models/jump_model.py b/analytics/models/jump_model.py index 8392d47..7f43b19 100644 --- a/analytics/models/jump_model.py +++ b/analytics/models/jump_model.py @@ -173,7 +173,7 @@ class JumpModel(Model): conv = scipy.signal.fftconvolve(convol_data, pattern_data) if max(conv) > self.state['convolve_max'] * 1.2 or max(conv) < self.state['convolve_min'] * 0.8: delete_list.append(segment) - if max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: + elif max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: delete_list.append(segment) else: delete_list.append(segment) diff --git a/analytics/models/peak_model.py b/analytics/models/peak_model.py index 9f93d8e..a924ee6 100644 --- a/analytics/models/peak_model.py +++ b/analytics/models/peak_model.py @@ -134,7 +134,7 @@ class PeakModel(Model): conv = scipy.signal.fftconvolve(convol_data, pattern_data) if max(conv) > self.state['convolve_max'] * 1.05 or max(conv) < self.state['convolve_min'] * 0.95: delete_list.append(segment) - if max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: + elif max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: delete_list.append(segment) else: delete_list.append(segment) diff --git a/analytics/models/trough_model.py b/analytics/models/trough_model.py index 03ee6a9..8baec97 100644 --- a/analytics/models/trough_model.py +++ b/analytics/models/trough_model.py @@ -136,7 +136,7 @@ class TroughModel(Model): conv = scipy.signal.fftconvolve(convol_data, pattern_data) if max(conv) > self.state['convolve_max'] * 1.1 or max(conv) < self.state['convolve_min'] * 0.9: delete_list.append(segment) - if max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: + elif max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: delete_list.append(segment) else: delete_list.append(segment)