Browse Source

Anomaly detector bounds switch: disabling to enabling #709 (#710)

pull/1/head
Evgeny Smyshlyaev 5 years ago committed by rozetko
parent
commit
a0ab7854ff
  1. 2
      analytics/.vscode/settings.json
  2. 25
      analytics/analytics/detectors/anomaly_detector.py
  3. 2
      server/src/controllers/analytics_controller.ts
  4. 22
      server/src/migrations.ts
  5. 10
      server/src/models/analytic_units/anomaly_analytic_unit_model.ts

2
analytics/.vscode/settings.json vendored

@ -14,7 +14,7 @@
"python.pythonPath": "python", "python.pythonPath": "python",
"python.linting.enabled": true, "python.linting.enabled": true,
"python.testing.unittestArgs": [ "-v" ], "python.testing.unittestArgs": [ "-v" ],
"python.testing.pyTestEnabled": false, "python.testing.pytestEnabled": false,
"python.testing.nosetestsEnabled": false, "python.testing.nosetestsEnabled": false,
"python.testing.unittestEnabled": true, "python.testing.unittestEnabled": true,
"python.linting.pylintEnabled": true, "python.linting.pylintEnabled": true,

25
analytics/analytics/detectors/anomaly_detector.py

@ -20,7 +20,7 @@ BASIC_ALPHA = 0.5
logger = logging.getLogger('ANOMALY_DETECTOR') logger = logging.getLogger('ANOMALY_DETECTOR')
class Bound(Enum): class Bound(Enum):
NONE = 'NONE' ALL = 'ALL'
UPPER = 'UPPER' UPPER = 'UPPER'
LOWER = 'LOWER' LOWER = 'LOWER'
@ -32,7 +32,7 @@ class AnomalyDetector(ProcessingDetector):
def train(self, dataframe: pd.DataFrame, payload: Union[list, dict], cache: Optional[ModelCache]) -> ModelCache: def train(self, dataframe: pd.DataFrame, payload: Union[list, dict], cache: Optional[ModelCache]) -> ModelCache:
segments = payload.get('segments') segments = payload.get('segments')
disable_bound: str = payload.get('disableBound') or 'NONE' enable_bounds: str = payload.get('enableBounds') or 'ALL'
prepared_segments = [] prepared_segments = []
time_step = utils.find_interval(dataframe) time_step = utils.find_interval(dataframe)
@ -40,7 +40,7 @@ class AnomalyDetector(ProcessingDetector):
'confidence': payload['confidence'], 'confidence': payload['confidence'],
'alpha': payload['alpha'], 'alpha': payload['alpha'],
'timeStep': time_step, 'timeStep': time_step,
'disableBound': disable_bound 'enableBounds': enable_bounds
} }
if segments is not None: if segments is not None:
@ -70,7 +70,7 @@ class AnomalyDetector(ProcessingDetector):
data = dataframe['value'] data = dataframe['value']
time_step = cache['timeStep'] time_step = cache['timeStep']
segments = cache.get('segments') segments = cache.get('segments')
disable_bound: str = cache.get('disableBound') or 'NONE' enable_bounds: str = cache.get('enableBounds') or 'ALL'
smoothed_data = utils.exponential_smoothing(data, cache['alpha']) smoothed_data = utils.exponential_smoothing(data, cache['alpha'])
@ -79,8 +79,12 @@ class AnomalyDetector(ProcessingDetector):
bounds[Bound.LOWER.value] = ( smoothed_data - cache['confidence'], operator.lt ) bounds[Bound.LOWER.value] = ( smoothed_data - cache['confidence'], operator.lt )
bounds[Bound.UPPER.value] = ( smoothed_data + cache['confidence'], operator.gt ) bounds[Bound.UPPER.value] = ( smoothed_data + cache['confidence'], operator.gt )
if disable_bound != Bound.NONE.value: if enable_bounds == Bound.LOWER.value:
del bounds[disable_bound] del bounds[Bound.UPPER.value]
if enable_bounds == Bound.UPPER.value:
del bounds[Bound.LOWER.value]
if segments is not None: if segments is not None:
@ -175,7 +179,7 @@ class AnomalyDetector(ProcessingDetector):
# TODO: ModelCache -> ModelState (don't use string literals) # TODO: ModelCache -> ModelState (don't use string literals)
def process_data(self, dataframe: pd.DataFrame, cache: ModelCache) -> AnomalyProcessingResult: def process_data(self, dataframe: pd.DataFrame, cache: ModelCache) -> AnomalyProcessingResult:
segments = cache.get('segments') segments = cache.get('segments')
disable_bound: str = cache.get('disableBound') or 'NONE' enable_bounds: str = cache.get('enableBounds') or 'ALL'
# TODO: exponential_smoothing should return dataframe with related timestamps # TODO: exponential_smoothing should return dataframe with related timestamps
smoothed_data = utils.exponential_smoothing(dataframe['value'], cache['alpha']) smoothed_data = utils.exponential_smoothing(dataframe['value'], cache['alpha'])
@ -184,8 +188,11 @@ class AnomalyDetector(ProcessingDetector):
bounds[Bound.LOWER.value] = smoothed_data - cache['confidence'] bounds[Bound.LOWER.value] = smoothed_data - cache['confidence']
bounds[Bound.UPPER.value] = smoothed_data + cache['confidence'] bounds[Bound.UPPER.value] = smoothed_data + cache['confidence']
if disable_bound != Bound.NONE.value: if enable_bounds == Bound.LOWER.value:
del bounds[disable_bound] del bounds[Bound.UPPER.value]
if enable_bounds == Bound.UPPER.value:
del bounds[Bound.LOWER.value]
# TODO: remove duplication with detect() # TODO: remove duplication with detect()

2
server/src/controllers/analytics_controller.ts

@ -287,7 +287,7 @@ export async function runLearning(id: AnalyticUnit.AnalyticUnitId, from?: number
taskPayload.anomaly = { taskPayload.anomaly = {
alpha: (analyticUnit as AnomalyAnalyticUnit).alpha, alpha: (analyticUnit as AnomalyAnalyticUnit).alpha,
confidence: (analyticUnit as AnomalyAnalyticUnit).confidence, confidence: (analyticUnit as AnomalyAnalyticUnit).confidence,
disableBound: (analyticUnit as AnomalyAnalyticUnit).disableBound enableBounds: (analyticUnit as AnomalyAnalyticUnit).enableBounds
}; };
taskPayload.data = await getPayloadData(analyticUnit, from, to); taskPayload.data = await getPayloadData(analyticUnit, from, to);

22
server/src/migrations.ts

@ -27,7 +27,8 @@ const REVISIONS = new Map<number, Function>([
[1, convertPanelUrlToPanelId], [1, convertPanelUrlToPanelId],
[2, convertUnderscoreToCamelCase], [2, convertUnderscoreToCamelCase],
[3, integrateThresholdsIntoAnalyticUnits], [3, integrateThresholdsIntoAnalyticUnits],
[4, addDetectorTypes] [4, addDetectorTypes],
[5, switchBoundsDisabling]
]); ]);
export async function applyDBMigrations() { export async function applyDBMigrations() {
@ -129,6 +130,25 @@ async function addDetectorTypes() {
await Promise.all(promises); await Promise.all(promises);
} }
async function switchBoundsDisabling() {
const analyticUnits = await analyticUnitsDB.findMany({ disableBound: { $exists: true } });
const promises = analyticUnits.map(analyticUnit => {
let enableBounds;
if(analyticUnit.disableBound === 'NONE') {
enableBounds = 'ALL';
}
if(analyticUnit.disableBound === 'UPPER') {
enableBounds = 'LOWER';
} else {
enableBounds = 'UPPER';
}
analyticUnitsDB.updateOne(analyticUnit._id, { enableBounds })
});
await Promise.all(promises);
}
function getDetectorByType(analyticUnitType: string): string { function getDetectorByType(analyticUnitType: string): string {
const analyticUnitTypesMapping = { const analyticUnitTypesMapping = {
pattern: [ 'GENERAL', 'PEAK', 'TROUGH', 'JUMP', 'DROP' ], pattern: [ 'GENERAL', 'PEAK', 'TROUGH', 'JUMP', 'DROP' ],

10
server/src/models/analytic_units/anomaly_analytic_unit_model.ts

@ -9,7 +9,7 @@ type SeasonalityPeriod = {
} }
enum Bound { enum Bound {
NONE = 'NONE', ALL = 'ALL',
UPPER = 'UPPER', UPPER = 'UPPER',
LOWER = 'LOWER' LOWER = 'LOWER'
}; };
@ -26,7 +26,7 @@ export class AnomalyAnalyticUnit extends AnalyticUnit {
public confidence: number, public confidence: number,
public seasonality: number, //seasonality in ms public seasonality: number, //seasonality in ms
private seasonalityPeriod: SeasonalityPeriod, private seasonalityPeriod: SeasonalityPeriod,
public disableBound: Bound, public enableBounds: Bound,
metric?: Metric, metric?: Metric,
alert?: boolean, alert?: boolean,
id?: AnalyticUnitId, id?: AnalyticUnitId,
@ -65,7 +65,7 @@ export class AnomalyAnalyticUnit extends AnalyticUnit {
confidence: this.confidence, confidence: this.confidence,
seasonality: this.seasonality, seasonality: this.seasonality,
seasonalityPeriod: this.seasonalityPeriod, seasonalityPeriod: this.seasonalityPeriod,
disableBound: this.disableBound enableBounds: this.enableBounds
}; };
} }
@ -77,7 +77,7 @@ export class AnomalyAnalyticUnit extends AnalyticUnit {
confidence: this.confidence, confidence: this.confidence,
seasonality: this.seasonality, seasonality: this.seasonality,
seasonalityPeriod: this.seasonalityPeriod, seasonalityPeriod: this.seasonalityPeriod,
disableBound: this.disableBound enableBounds: this.enableBounds
}; };
} }
@ -97,7 +97,7 @@ export class AnomalyAnalyticUnit extends AnalyticUnit {
obj.confidence, obj.confidence,
obj.seasonality, obj.seasonality,
obj.seasonalityPeriod, obj.seasonalityPeriod,
obj.disableBound, obj.enableBounds,
metric, metric,
obj.alert, obj.alert,
obj._id, obj._id,

Loading…
Cancel
Save