From bfccd99856aa748c772813edea5cef7673b11ef8 Mon Sep 17 00:00:00 2001 From: Alexey Velikiy Date: Sun, 27 May 2018 20:55:34 +0300 Subject: [PATCH] folders config++ --- .gitignore | 11 ++++++++--- analytics/.gitignore | 11 ----------- analytics/anomaly_model.py | 21 +++++++++------------ analytics/config.py | 8 ++++++++ analytics/pattern_detection_model.py | 20 +++++++++----------- server/src/config.ts | 7 ++++++- server/src/services/data.ts | 20 +++++++++++++++----- 7 files changed, 55 insertions(+), 43 deletions(-) delete mode 100644 analytics/.gitignore create mode 100644 analytics/config.py diff --git a/.gitignore b/.gitignore index 3fdaa4d..a384622 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,11 @@ +data +dist + node_modules/ -dist/ + .vscode/ package-lock.json -server/dist -data \ No newline at end of file +__pycache__/ +*.pyc +*.txt +*.log diff --git a/analytics/.gitignore b/analytics/.gitignore deleted file mode 100644 index a62c4bc..0000000 --- a/analytics/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -anomalies/ -segments/ -datasets/ -datasources/ -models/ -metrics/ -__pycache__/ -*.pyc -*.txt -*.log -tasks.csv diff --git a/analytics/anomaly_model.py b/analytics/anomaly_model.py index c582f56..30cf7b3 100644 --- a/analytics/anomaly_model.py +++ b/analytics/anomaly_model.py @@ -5,12 +5,9 @@ import json import pandas as pd import logging from urllib.parse import urlparse +import config + -datasource_folder = "datasources/" -dataset_folder = "datasets/" -anomalies_folder = "anomalies/" -models_folder = "models/" -metrics_folder = "metrics/" logger = logging.getLogger('analytic_toolset') @@ -34,10 +31,10 @@ class AnomalyModel: datasource['origin'] = origin metric_name = self.anomaly_config['metric']['targets'][0] - target_filename = os.path.join(metrics_folder, metric_name + ".json") + target_filename = os.path.join(config.METRICS_FOLDER, metric_name + ".json") - dataset_filename = os.path.join(dataset_folder, metric_name + ".csv") - augmented_path = os.path.join(dataset_folder, metric_name + "_augmented.csv") + dataset_filename = os.path.join(config.DATASET_FOLDER, metric_name + ".csv") + augmented_path = os.path.join(config.DATASET_FOLDER, metric_name + "_augmented.csv") with open(target_filename, 'r') as file: target = json.load(file) @@ -129,11 +126,11 @@ class AnomalyModel: self.preprocessor.synchronize() def load_anomaly_config(self): - with open(os.path.join(anomalies_folder, self.anomaly_name + ".json"), 'r') as config_file: + with open(os.path.join(config.ANOMALIES_FOLDER, self.anomaly_name + ".json"), 'r') as config_file: self.anomaly_config = json.load(config_file) def get_anomalies(self): - labeled_anomalies_file = os.path.join(anomalies_folder, self.anomaly_name + "_labeled.json") + labeled_anomalies_file = os.path.join(config.ANOMALIES_FOLDER, self.anomaly_name + "_labeled.json") if not os.path.exists(labeled_anomalies_file): return [] with open(labeled_anomalies_file) as file: @@ -145,12 +142,12 @@ class AnomalyModel: def __save_model(self): logger.info("Save model '%s'" % self.anomaly_name) - model_filename = os.path.join(models_folder, self.anomaly_name + ".m") + model_filename = os.path.join(config.MODELS_FOLDER, self.anomaly_name + ".m") self.model.save(model_filename) def __load_model(self): logger.info("Load model '%s'" % self.anomaly_name) - model_filename = os.path.join(models_folder, self.anomaly_name + ".m") + model_filename = os.path.join(config.MODELS_FOLDER, self.anomaly_name + ".m") if os.path.exists(model_filename): self.model = self.create_algorithm() self.model.load(model_filename) diff --git a/analytics/config.py b/analytics/config.py new file mode 100644 index 0000000..d713458 --- /dev/null +++ b/analytics/config.py @@ -0,0 +1,8 @@ +import os + +DATA_FOLDER = '../../data' +DATASOURCE_FOLDER = os.path.join(DATA_FOLDER, "datasources/") +DATASET_FOLDER = os.path.join(DATA_FOLDER, "datasets/") +ANOMALIES_FOLDER = os.path.join(DATA_FOLDER, "anomalies/") +MODELS_FOLDER = os.path.join(DATA_FOLDER, "models/") +METRICS_FOLDER = os.path.join(DATA_FOLDER, "metrics/") \ No newline at end of file diff --git a/analytics/pattern_detection_model.py b/analytics/pattern_detection_model.py index 8396578..b268528 100644 --- a/analytics/pattern_detection_model.py +++ b/analytics/pattern_detection_model.py @@ -2,13 +2,11 @@ from data_provider import DataProvider import logging import os.path import json +import config + import pandas as pd -datasource_folder = "datasources/" -dataset_folder = "datasets/" -anomalies_folder = "anomalies/" -models_folder = "models/" -metrics_folder = "metrics/" + logger = logging.getLogger('analytic_toolset') @@ -33,10 +31,10 @@ class PatternDetectionModel: datasource = self.anomaly_config['metric']['datasource'] metric_name = self.anomaly_config['metric']['targets'][0] - dbconfig_filename = os.path.join(datasource_folder, datasource + ".json") - target_filename = os.path.join(metrics_folder, metric_name + ".json") + dbconfig_filename = os.path.join(config.DATASOURCE_FOLDER, datasource + ".json") + target_filename = os.path.join(config.METRICS_FOLDER, metric_name + ".json") - dataset_filename = os.path.join(dataset_folder, metric_name + ".csv") + dataset_filename = os.path.join(config.DATASET_FOLDER, metric_name + ".csv") with open(dbconfig_filename, 'r') as config_file: dbconfig = json.load(config_file) @@ -111,17 +109,17 @@ class PatternDetectionModel: return StepDetector(preset) def __load_anomaly_config(self): - with open(os.path.join(anomalies_folder, self.pattern_name + ".json"), 'r') as config_file: + with open(os.path.join(config.ANOMALIES_FOLDER, self.pattern_name + ".json"), 'r') as config_file: self.anomaly_config = json.load(config_file) def __save_model(self): logger.info("Save model '%s'" % self.pattern_name) - model_filename = os.path.join(models_folder, self.pattern_name + ".m") + model_filename = os.path.join(config.MODELS_FOLDER, self.pattern_name + ".m") self.model.save(model_filename) def __load_model(self, preset): logger.info("Load model '%s'" % self.pattern_name) - model_filename = os.path.join(models_folder, self.pattern_name + ".m") + model_filename = os.path.join(config.MODELS_FOLDER, self.pattern_name + ".m") if os.path.exists(model_filename): self.model = self.__create_model(preset) self.model.load(model_filename) \ No newline at end of file diff --git a/server/src/config.ts b/server/src/config.ts index 027ad0f..ee4d3d4 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -3,6 +3,11 @@ import * as path from 'path'; export const ANALYTICS_PATH = path.join(__dirname, '../../analytics'); export const DATA_PATH = path.join(__dirname, '../../data'); + +export const DATASOURCES_PATH = path.join(DATA_PATH, 'datasources'); +export const DATASETS_PATH = path.join(DATA_PATH, 'datasets'); export const ANOMALIES_PATH = path.join(DATA_PATH, 'anomalies'); -export const SEGMENTS_PATH = path.join(DATA_PATH, 'segments'); +export const MODELS_PATH = path.join(DATA_PATH, 'models'); export const METRICS_PATH = path.join(DATA_PATH, 'metrics'); +export const SEGMENTS_PATH = path.join(DATA_PATH, 'segments'); + diff --git a/server/src/services/data.ts b/server/src/services/data.ts index 6aac1ec..212820d 100644 --- a/server/src/services/data.ts +++ b/server/src/services/data.ts @@ -2,13 +2,23 @@ import * as config from '../config' import * as fs from 'fs'; +// see analytics/pattern_detection_model.py with folders available -export function checkDataFolders() { +function maybeCreate(path: string): void { if(fs.existsSync(config.DATA_PATH)) { return; } - fs.mkdirSync(config.DATA_PATH); - fs.mkdirSync(config.ANOMALIES_PATH); - fs.mkdirSync(config.SEGMENTS_PATH); - fs.mkdirSync(config.METRICS_PATH); + fs.mkdirSync(path); +} + +export function checkDataFolders(): void { + var folders = [ + config.DATA_PATH, + config.DATASOURCES_PATH, + config.DATASETS_PATH, + config.ANOMALIES_PATH, + config.MODELS_PATH, + config.METRICS_PATH, + config.SEGMENTS_PATH + ].forEach(maybeCreate); } \ No newline at end of file