folders config++

.gitignore

@@ -1,6 +1,11 @@
+data
+dist
+
 node_modules/
-dist/
+
 .vscode/
 package-lock.json
-server/dist
+__pycache__/
-data
+*.pyc
+*.txt
+*.log

analytics/.gitignore

@@ -1,11 +0,0 @@
-anomalies/
-segments/
-datasets/
-datasources/
-models/
-metrics/
-__pycache__/
-*.pyc
-*.txt
-*.log
-tasks.csv

analytics/anomaly_model.py

@@ -5,12 +5,9 @@ import json
 import pandas as pd
 import logging
 from urllib.parse import urlparse
+import config
+
 
-datasource_folder = "datasources/"
-dataset_folder = "datasets/"
-anomalies_folder = "anomalies/"
-models_folder = "models/"
-metrics_folder = "metrics/"
 logger = logging.getLogger('analytic_toolset')
 
 
@@ -34,10 +31,10 @@ class AnomalyModel:
         datasource['origin'] = origin
         metric_name = self.anomaly_config['metric']['targets'][0]
 
-        target_filename = os.path.join(metrics_folder, metric_name + ".json")
+        target_filename = os.path.join(config.METRICS_FOLDER, metric_name + ".json")
 
-        dataset_filename = os.path.join(dataset_folder, metric_name + ".csv")
+        dataset_filename = os.path.join(config.DATASET_FOLDER, metric_name + ".csv")
-        augmented_path = os.path.join(dataset_folder, metric_name + "_augmented.csv")
+        augmented_path = os.path.join(config.DATASET_FOLDER, metric_name + "_augmented.csv")
 
         with open(target_filename, 'r') as file:
             target = json.load(file)
@@ -129,11 +126,11 @@ class AnomalyModel:
         self.preprocessor.synchronize()
 
     def load_anomaly_config(self):
-        with open(os.path.join(anomalies_folder, self.anomaly_name + ".json"), 'r') as config_file:
+        with open(os.path.join(config.ANOMALIES_FOLDER, self.anomaly_name + ".json"), 'r') as config_file:
             self.anomaly_config = json.load(config_file)
 
     def get_anomalies(self):
-        labeled_anomalies_file = os.path.join(anomalies_folder, self.anomaly_name + "_labeled.json")
+        labeled_anomalies_file = os.path.join(config.ANOMALIES_FOLDER, self.anomaly_name + "_labeled.json")
         if not os.path.exists(labeled_anomalies_file):
             return []
         with open(labeled_anomalies_file) as file:
@@ -145,12 +142,12 @@ class AnomalyModel:
 
     def __save_model(self):
         logger.info("Save model '%s'" % self.anomaly_name)
-        model_filename = os.path.join(models_folder, self.anomaly_name + ".m")
+        model_filename = os.path.join(config.MODELS_FOLDER, self.anomaly_name + ".m")
         self.model.save(model_filename)
 
     def __load_model(self):
         logger.info("Load model '%s'" % self.anomaly_name)
-        model_filename = os.path.join(models_folder, self.anomaly_name + ".m")
+        model_filename = os.path.join(config.MODELS_FOLDER, self.anomaly_name + ".m")
         if os.path.exists(model_filename):
             self.model = self.create_algorithm()
             self.model.load(model_filename)

analytics/config.py

@@ -0,0 +1,8 @@
+import os
+
+DATA_FOLDER = '../../data'
+DATASOURCE_FOLDER = os.path.join(DATA_FOLDER, "datasources/")
+DATASET_FOLDER = os.path.join(DATA_FOLDER, "datasets/")
+ANOMALIES_FOLDER = os.path.join(DATA_FOLDER, "anomalies/")
+MODELS_FOLDER = os.path.join(DATA_FOLDER, "models/")
+METRICS_FOLDER = os.path.join(DATA_FOLDER, "metrics/")

analytics/pattern_detection_model.py

@@ -2,13 +2,11 @@ from data_provider import DataProvider
 import logging
 import os.path
 import json
+import config
+
 import pandas as pd
 
-datasource_folder = "datasources/"
+
-dataset_folder = "datasets/"
-anomalies_folder = "anomalies/"
-models_folder = "models/"
-metrics_folder = "metrics/"
 logger = logging.getLogger('analytic_toolset')
 
 
@@ -33,10 +31,10 @@ class PatternDetectionModel:
         datasource = self.anomaly_config['metric']['datasource']
         metric_name = self.anomaly_config['metric']['targets'][0]
 
-        dbconfig_filename = os.path.join(datasource_folder, datasource + ".json")
+        dbconfig_filename = os.path.join(config.DATASOURCE_FOLDER, datasource + ".json")
-        target_filename = os.path.join(metrics_folder, metric_name + ".json")
+        target_filename = os.path.join(config.METRICS_FOLDER, metric_name + ".json")
 
-        dataset_filename = os.path.join(dataset_folder, metric_name + ".csv")
+        dataset_filename = os.path.join(config.DATASET_FOLDER, metric_name + ".csv")
 
         with open(dbconfig_filename, 'r') as config_file:
             dbconfig = json.load(config_file)
@@ -111,17 +109,17 @@ class PatternDetectionModel:
             return StepDetector(preset)
 
     def __load_anomaly_config(self):
-        with open(os.path.join(anomalies_folder, self.pattern_name + ".json"), 'r') as config_file:
+        with open(os.path.join(config.ANOMALIES_FOLDER, self.pattern_name + ".json"), 'r') as config_file:
             self.anomaly_config = json.load(config_file)
 
     def __save_model(self):
         logger.info("Save model '%s'" % self.pattern_name)
-        model_filename = os.path.join(models_folder, self.pattern_name + ".m")
+        model_filename = os.path.join(config.MODELS_FOLDER, self.pattern_name + ".m")
         self.model.save(model_filename)
 
     def __load_model(self, preset):
         logger.info("Load model '%s'" % self.pattern_name)
-        model_filename = os.path.join(models_folder, self.pattern_name + ".m")
+        model_filename = os.path.join(config.MODELS_FOLDER, self.pattern_name + ".m")
         if os.path.exists(model_filename):
             self.model = self.__create_model(preset)
             self.model.load(model_filename)

server/src/config.ts

@@ -3,6 +3,11 @@ import * as path from 'path';
 export const ANALYTICS_PATH = path.join(__dirname, '../../analytics');
 
 export const DATA_PATH = path.join(__dirname, '../../data');
+
+export const DATASOURCES_PATH = path.join(DATA_PATH, 'datasources');
+export const DATASETS_PATH = path.join(DATA_PATH, 'datasets');
 export const ANOMALIES_PATH = path.join(DATA_PATH, 'anomalies');
-export const SEGMENTS_PATH = path.join(DATA_PATH, 'segments');
+export const MODELS_PATH = path.join(DATA_PATH, 'models');
 export const METRICS_PATH = path.join(DATA_PATH, 'metrics');
+export const SEGMENTS_PATH = path.join(DATA_PATH, 'segments');
+

server/src/services/data.ts

@@ -2,13 +2,23 @@ import * as config from '../config'
 import * as fs from 'fs';
 
 
+// see analytics/pattern_detection_model.py with folders available
 
-export function checkDataFolders() {
+function maybeCreate(path: string): void {
   if(fs.existsSync(config.DATA_PATH)) {
     return;
   }
-  fs.mkdirSync(config.DATA_PATH);
+  fs.mkdirSync(path);
-  fs.mkdirSync(config.ANOMALIES_PATH);
+}
-  fs.mkdirSync(config.SEGMENTS_PATH);
+
-  fs.mkdirSync(config.METRICS_PATH);
+export function checkDataFolders(): void {
+  var folders = [
+    config.DATA_PATH,
+    config.DATASOURCES_PATH,
+    config.DATASETS_PATH,
+    config.ANOMALIES_PATH,
+    config.MODELS_PATH,
+    config.METRICS_PATH,
+    config.SEGMENTS_PATH
+  ].forEach(maybeCreate);
 }