from typing import Dict import logging as log import traceback from concurrent.futures import Executor, ThreadPoolExecutor from analytic_unit_worker import AnalyticUnitWorker from analytic_types import AnalyticUnitId, ModelCache from analytic_types.segment import Segment import detectors logger = log.getLogger('AnalyticUnitManager') def get_detector_by_type( detector_type: str, analytic_unit_type: str, analytic_unit_id: AnalyticUnitId ) -> detectors.Detector: if detector_type == 'pattern': return detectors.PatternDetector(analytic_unit_type, analytic_unit_id) elif detector_type == 'threshold': return detectors.ThresholdDetector(analytic_unit_id) elif detector_type == 'anomaly': return detectors.AnomalyDetector(analytic_unit_id) raise ValueError('Unknown detector type "%s"' % detector_type) class AnalyticUnitManager: def __init__(self): self.analytic_workers: Dict[AnalyticUnitId, AnalyticUnitWorker] = dict() self.workers_executor = ThreadPoolExecutor() def __ensure_worker( self, analytic_unit_id: AnalyticUnitId, detector_type: str, analytic_unit_type: str ) -> AnalyticUnitWorker: if analytic_unit_id in self.analytic_workers: # TODO: check that type is the same return self.analytic_workers[analytic_unit_id] detector = get_detector_by_type(detector_type, analytic_unit_type, analytic_unit_id) worker = AnalyticUnitWorker(analytic_unit_id, detector, self.workers_executor) self.analytic_workers[analytic_unit_id] = worker return worker async def __handle_analytic_task(self, task: object) -> dict: """ returns payload or None """ analytic_unit_id: AnalyticUnitId = task['analyticUnitId'] log.debug('Analytics get task with type: {} for unit: {}'.format(task['type'], analytic_unit_id)) if task['type'] == 'CANCEL': if analytic_unit_id in self.analytic_workers: self.analytic_workers[analytic_unit_id].cancel() return payload = task['payload'] worker = self.__ensure_worker(analytic_unit_id, payload['detector'], payload['analyticUnitType']) data = payload.get('data') if task['type'] == 'PUSH': # TODO: do it a better way res = await worker.consume_data(data, payload['cache']) if res: res.update({ 'analyticUnitId': analytic_unit_id }) return res elif task['type'] == 'LEARN': if 'segments' in payload: segments = payload['segments'] segments = [Segment.from_json(segment) for segment in segments] return await worker.do_train(segments, data, payload['cache']) elif 'threshold' in payload: return await worker.do_train(payload['threshold'], data, payload['cache']) elif 'anomaly' in payload: return await worker.do_train(payload['anomaly'], data, payload['cache']) else: raise ValueError('No segments or threshold in LEARN payload') elif task['type'] == 'DETECT': return await worker.do_detect(data, payload['cache']) elif task['type'] == 'PROCESS': return await worker.process_data(data, payload['cache']) raise ValueError('Unknown task type "%s"' % task['type']) async def handle_analytic_task(self, task: object): try: log.debug('Start handle_analytic_task with analytic unit: {}'.format(task['analyticUnitId'])) result_payload = await self.__handle_analytic_task(task) result_message = { 'status': 'SUCCESS', 'payload': result_payload } log.debug('End correctly handle_analytic_task with anatytic unit: {}'.format(task['analyticUnitId'])) return result_message except Exception as e: error_text = traceback.format_exc() logger.error("handle_analytic_task Exception: '%s'" % error_text) # TODO: move result to a class which renders to json for messaging to analytics return { 'status': 'FAILED', 'error': repr(e) }