Browse Source

jsonclass usage in models + fixes in meta (#583)

* jsonclass usage in models + fixes in meta

* remove some empty lines
pull/1/head
Alexey Velikiy 6 years ago committed by GitHub
parent
commit
30108ae643
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 17
      analytics/analytics/models/drop_model.py
  2. 20
      analytics/analytics/models/general_model.py
  3. 18
      analytics/analytics/models/jump_model.py
  4. 23
      analytics/analytics/models/model.py
  5. 17
      analytics/analytics/models/peak_model.py
  6. 17
      analytics/analytics/models/trough_model.py
  7. 28
      analytics/analytics/services/server_service.py
  8. 22
      analytics/analytics/utils/meta.py

17
analytics/analytics/models/drop_model.py

@ -6,9 +6,12 @@ from scipy.signal import argrelextrema
from scipy.stats import gaussian_kde from scipy.stats import gaussian_kde
from typing import Optional from typing import Optional
import utils import utils
import utils.meta
import numpy as np import numpy as np
import pandas as pd import pandas as pd
@utils.meta.JSONClass
class DropModelState(ModelState): class DropModelState(ModelState):
def __init__( def __init__(
@ -23,20 +26,6 @@ class DropModelState(ModelState):
self.drop_height = drop_height self.drop_height = drop_height
self.drop_length = drop_length self.drop_length = drop_length
def to_json(self) -> dict:
json = super().to_json()
json.update({
'confidence': self.confidence,
'drop_height': self.drop_height,
'drop_length': self.drop_length,
})
return json
@staticmethod
def from_json(json: Optional[dict] = None):
if json is None:
json = {}
return DropModelState(**json)
class DropModel(Model): class DropModel(Model):
def __init__(self): def __init__(self):

20
analytics/analytics/models/general_model.py

@ -1,40 +1,34 @@
from analytic_types import AnalyticUnitId
from models import Model, ModelState from models import Model, ModelState
from typing import Union, List, Generator from typing import Union, List, Generator
import utils import utils
import utils.meta
import numpy as np import numpy as np
import pandas as pd import pandas as pd
import scipy.signal import scipy.signal
from scipy.fftpack import fft from scipy.fftpack import fft
from scipy.signal import argrelextrema from scipy.signal import argrelextrema
from scipy.stats.stats import pearsonr from scipy.stats.stats import pearsonr
from typing import Optional
import math
from scipy.stats import gaussian_kde from scipy.stats import gaussian_kde
from scipy.stats import norm from scipy.stats import norm
import logging import logging
from analytic_types import AnalyticUnitId
from typing import Optional
import math
PEARSON_FACTOR = 0.7 PEARSON_FACTOR = 0.7
class GeneralModelState(ModelState):
@utils.meta.JSONClass
class GeneralModelState(ModelState):
def __init__(self, **kwargs): def __init__(self, **kwargs):
super().__init__(**kwargs) super().__init__(**kwargs)
def to_json(self) -> dict:
return super().to_json()
@staticmethod
def from_json(json: Optional[dict] = None):
if json is None:
json = {}
return GeneralModelState(**json)
class GeneralModel(Model): class GeneralModel(Model):
def __init__(self): def __init__(self):
super()
self.state = { self.state = {
'pattern_center': [], 'pattern_center': [],
'pattern_model': [], 'pattern_model': [],

18
analytics/analytics/models/jump_model.py

@ -1,6 +1,7 @@
from models import Model, ModelState from models import Model, ModelState
import utils import utils
import utils.meta
import numpy as np import numpy as np
import pandas as pd import pandas as pd
import scipy.signal import scipy.signal
@ -10,8 +11,9 @@ import math
from scipy.signal import argrelextrema from scipy.signal import argrelextrema
from scipy.stats import gaussian_kde from scipy.stats import gaussian_kde
class JumpModelState(ModelState):
@utils.meta.JSONClass
class JumpModelState(ModelState):
def __init__( def __init__(
self, self,
confidence: float = 0, confidence: float = 0,
@ -24,20 +26,6 @@ class JumpModelState(ModelState):
self.jump_height = jump_height self.jump_height = jump_height
self.jump_length = jump_length self.jump_length = jump_length
def to_json(self) -> dict:
json = super().to_json()
json.update({
'confidence': self.confidence,
'jump_height': self.jump_height,
'jump_length': self.jump_length,
})
return json
@staticmethod
def from_json(json: Optional[dict] = None):
if json is None:
json = {}
return JumpModelState(**json)
class JumpModel(Model): class JumpModel(Model):

23
analytics/analytics/models/model.py

@ -8,17 +8,18 @@ import math
import logging import logging
from analytic_types import AnalyticUnitId from analytic_types import AnalyticUnitId
import utils.meta
ModelCache = dict ModelCache = dict
class Segment(AttrDict): class Segment(AttrDict):
__percent_of_nans = 0
def __init__(self, dataframe: pd.DataFrame, segment_map: dict, center_finder = None): def __init__(self, dataframe: pd.DataFrame, segment_map: dict, center_finder = None):
self.update(segment_map) self.update(segment_map)
self.start = utils.timestamp_to_index(dataframe, pd.to_datetime(self['from'], unit='ms')) self.start = utils.timestamp_to_index(dataframe, pd.to_datetime(self['from'], unit='ms'))
self.end = utils.timestamp_to_index(dataframe, pd.to_datetime(self['to'], unit='ms')) self.end = utils.timestamp_to_index(dataframe, pd.to_datetime(self['to'], unit='ms'))
self.length = abs(self.end - self.start) self.length = abs(self.end - self.start)
self.__percent_of_nans = 0
if callable(center_finder): if callable(center_finder):
self.center_index = center_finder(dataframe, self.start, self.end) self.center_index = center_finder(dataframe, self.start, self.end)
@ -42,6 +43,8 @@ class Segment(AttrDict):
nan_list = utils.find_nan_indexes(self.data) nan_list = utils.find_nan_indexes(self.data)
self.data = utils.nan_to_zero(self.data, nan_list) self.data = utils.nan_to_zero(self.data, nan_list)
@utils.meta.JSONClass
class ModelState(): class ModelState():
def __init__( def __init__(
@ -62,22 +65,6 @@ class ModelState():
self.conv_del_min = conv_del_min self.conv_del_min = conv_del_min
self.conv_del_max = conv_del_max self.conv_del_max = conv_del_max
def to_json(self) -> dict:
return {
'pattern_center': self.pattern_center,
'pattern_model': self.pattern_model,
'convolve_max': self.convolve_max,
'convolve_min': self.convolve_min,
'window_size': self.window_size,
'conv_del_min': self.conv_del_min,
'conv_del_max': self.conv_del_max,
}
@staticmethod
def from_json(json: Optional[dict] = None):
if json is None:
json = {}
return ModelState(**json)
class Model(ABC): class Model(ABC):

17
analytics/analytics/models/peak_model.py

@ -5,12 +5,15 @@ from scipy.fftpack import fft
from scipy.signal import argrelextrema from scipy.signal import argrelextrema
from typing import Optional, List from typing import Optional, List
import utils import utils
import utils.meta
import numpy as np import numpy as np
import pandas as pd import pandas as pd
SMOOTHING_COEFF = 2400 SMOOTHING_COEFF = 2400
EXP_SMOOTHING_FACTOR = 0.01 EXP_SMOOTHING_FACTOR = 0.01
@utils.meta.JSONClass
class PeakModelState(ModelState): class PeakModelState(ModelState):
def __init__( def __init__(
@ -25,20 +28,6 @@ class PeakModelState(ModelState):
self.height_max = height_max self.height_max = height_max
self.height_min = height_min self.height_min = height_min
def to_json(self) -> dict:
json = super().to_json()
json.update({
'confidence': self.confidence,
'height_max': self.height_max,
'height_min': self.height_min,
})
return json
@staticmethod
def from_json(json: Optional[dict] = None):
if json is None:
json = {}
return PeakModelState(**json)
class PeakModel(Model): class PeakModel(Model):

17
analytics/analytics/models/trough_model.py

@ -5,12 +5,15 @@ from scipy.fftpack import fft
from scipy.signal import argrelextrema from scipy.signal import argrelextrema
from typing import Optional from typing import Optional
import utils import utils
import utils.meta
import numpy as np import numpy as np
import pandas as pd import pandas as pd
SMOOTHING_COEFF = 2400 SMOOTHING_COEFF = 2400
EXP_SMOOTHING_FACTOR = 0.01 EXP_SMOOTHING_FACTOR = 0.01
@utils.meta.JSONClass
class TroughModelState(ModelState): class TroughModelState(ModelState):
def __init__( def __init__(
@ -25,20 +28,6 @@ class TroughModelState(ModelState):
self.height_max = height_max self.height_max = height_max
self.height_min = height_min self.height_min = height_min
def to_json(self) -> dict:
json = super().to_json()
json.update({
'confidence': self.confidence,
'height_max': self.height_max,
'height_min': self.height_min,
})
return json
@staticmethod
def from_json(json: Optional[dict] = None):
if json is None:
json = {}
return TroughModelState(**json)
class TroughModel(Model): class TroughModel(Model):

28
analytics/analytics/services/server_service.py

@ -9,6 +9,7 @@ import asyncio
import traceback import traceback
import utils.concurrent import utils.concurrent
import utils.meta
from typing import Optional from typing import Optional
@ -19,32 +20,13 @@ PARSE_MESSAGE_OR_SAVE_LOOP_INTERRUPTED = False
SERVER_SOCKET_RECV_LOOP_INTERRUPTED = False SERVER_SOCKET_RECV_LOOP_INTERRUPTED = False
@utils.meta.JSONClass
class ServerMessage: class ServerMessage:
def __init__(self, method: str, payload: object = None, request_id: int = None): def __init__(self, method: str, payload: object = None, request_id: int = None):
self.method = method self.method = method
self.payload = payload self.payload = payload
self.request_id = request_id self.request_id = request_id
def toJSON(self) -> dict:
result = {
'method': self.method
}
if self.payload is not None:
result['payload'] = self.payload
if self.request_id is not None:
result['requestId'] = self.request_id
return result
@staticmethod
def fromJSON(json: dict):
method = json['method']
payload = None
request_id = None
if 'payload' in json:
payload = json['payload']
if 'requestId' in json:
request_id = json['requestId']
return ServerMessage(method, payload, request_id)
class ServerService(utils.concurrent.AsyncZmqActor): class ServerService(utils.concurrent.AsyncZmqActor):
@ -63,8 +45,8 @@ class ServerService(utils.concurrent.AsyncZmqActor):
# in theory, we can try to use zmq.proxy: # in theory, we can try to use zmq.proxy:
# zmq.proxy(self.__actor_socket, self.__server_socket) # zmq.proxy(self.__actor_socket, self.__server_socket)
# and do here something like: # and do here something like:
# self.__actor_socket.send_string(json.dumps(message.toJSON())) # self.__actor_socket.send_string(json.dumps(message.to_json()))
await self._put_message_to_thread(json.dumps(message.toJSON())) await self._put_message_to_thread(json.dumps(message.to_json()))
async def send_request_to_server(self, message: ServerMessage) -> object: async def send_request_to_server(self, message: ServerMessage) -> object:
if message.request_id is not None: if message.request_id is not None:
@ -118,7 +100,7 @@ class ServerService(utils.concurrent.AsyncZmqActor):
def __parse_message_or_save(self, text: str) -> Optional[ServerMessage]: def __parse_message_or_save(self, text: str) -> Optional[ServerMessage]:
try: try:
message_object = json.loads(text) message_object = json.loads(text)
message = ServerMessage.fromJSON(message_object) message = ServerMessage.from_json(message_object)
if message.request_id is not None: if message.request_id is not None:
self.__responses[message_object['requestId']] = message.payload self.__responses[message_object['requestId']] = message.payload
return None return None

22
analytics/analytics/utils/meta.py

@ -1,11 +1,20 @@
from inspect import signature, Parameter from inspect import signature, Parameter
from functools import wraps from functools import wraps
from typing import Optional from typing import Optional
from re import match import re
CAMEL_REGEX = re.compile(r'([A-Z])')
UNDERSCORE_REGEX = re.compile(r'_([a-z])')
def camel_to_underscore(name):
return CAMEL_REGEX.sub(lambda x: '_' + x.group(1).lower(), name)
def underscore_to_camel(name):
return UNDERSCORE_REGEX.sub(lambda x: x.group(1).upper(), name)
def is_field_private(field_name: str) -> Optional[str]: def is_field_private(field_name: str) -> Optional[str]:
m = match(r'_[^(__)]+__', field_name) m = re.match(r'_[^(__)]+__', field_name)
return m is not None return m is not None
def inited_params(target_init): def inited_params(target_init):
@ -41,12 +50,15 @@ def JSONClass(target_class):
where all None - values and private fileds are skipped where all None - values and private fileds are skipped
""" """
return { return {
k: v for k, v in self.__dict__.items() underscore_to_camel(k): v for k, v in self.__dict__.items()
if v is not None and not is_field_private(k) if v is not None and not is_field_private(k)
} }
def from_json(json_object: dict) -> target_class: def from_json(json_object: Optional[dict]) -> target_class:
return target_class(**json_object) if json_object is None:
json_object = {}
init_object = { camel_to_underscore(k): v for k, v in json_object.items() }
return target_class(**init_object)
# target_class.__init__ = inited_params(target_class.__init__) # target_class.__init__ = inited_params(target_class.__init__)
target_class.to_json = to_json target_class.to_json = to_json

Loading…
Cancel
Save