Browse Source

folders config++

pull/1/head
Alexey Velikiy 6 years ago
parent
commit
bfccd99856
  1. 11
      .gitignore
  2. 11
      analytics/.gitignore
  3. 21
      analytics/anomaly_model.py
  4. 8
      analytics/config.py
  5. 20
      analytics/pattern_detection_model.py
  6. 7
      server/src/config.ts
  7. 20
      server/src/services/data.ts

11
.gitignore vendored

@ -1,6 +1,11 @@
data
dist
node_modules/
dist/
.vscode/
package-lock.json
server/dist
data
__pycache__/
*.pyc
*.txt
*.log

11
analytics/.gitignore vendored

@ -1,11 +0,0 @@
anomalies/
segments/
datasets/
datasources/
models/
metrics/
__pycache__/
*.pyc
*.txt
*.log
tasks.csv

21
analytics/anomaly_model.py

@ -5,12 +5,9 @@ import json
import pandas as pd
import logging
from urllib.parse import urlparse
import config
datasource_folder = "datasources/"
dataset_folder = "datasets/"
anomalies_folder = "anomalies/"
models_folder = "models/"
metrics_folder = "metrics/"
logger = logging.getLogger('analytic_toolset')
@ -34,10 +31,10 @@ class AnomalyModel:
datasource['origin'] = origin
metric_name = self.anomaly_config['metric']['targets'][0]
target_filename = os.path.join(metrics_folder, metric_name + ".json")
target_filename = os.path.join(config.METRICS_FOLDER, metric_name + ".json")
dataset_filename = os.path.join(dataset_folder, metric_name + ".csv")
augmented_path = os.path.join(dataset_folder, metric_name + "_augmented.csv")
dataset_filename = os.path.join(config.DATASET_FOLDER, metric_name + ".csv")
augmented_path = os.path.join(config.DATASET_FOLDER, metric_name + "_augmented.csv")
with open(target_filename, 'r') as file:
target = json.load(file)
@ -129,11 +126,11 @@ class AnomalyModel:
self.preprocessor.synchronize()
def load_anomaly_config(self):
with open(os.path.join(anomalies_folder, self.anomaly_name + ".json"), 'r') as config_file:
with open(os.path.join(config.ANOMALIES_FOLDER, self.anomaly_name + ".json"), 'r') as config_file:
self.anomaly_config = json.load(config_file)
def get_anomalies(self):
labeled_anomalies_file = os.path.join(anomalies_folder, self.anomaly_name + "_labeled.json")
labeled_anomalies_file = os.path.join(config.ANOMALIES_FOLDER, self.anomaly_name + "_labeled.json")
if not os.path.exists(labeled_anomalies_file):
return []
with open(labeled_anomalies_file) as file:
@ -145,12 +142,12 @@ class AnomalyModel:
def __save_model(self):
logger.info("Save model '%s'" % self.anomaly_name)
model_filename = os.path.join(models_folder, self.anomaly_name + ".m")
model_filename = os.path.join(config.MODELS_FOLDER, self.anomaly_name + ".m")
self.model.save(model_filename)
def __load_model(self):
logger.info("Load model '%s'" % self.anomaly_name)
model_filename = os.path.join(models_folder, self.anomaly_name + ".m")
model_filename = os.path.join(config.MODELS_FOLDER, self.anomaly_name + ".m")
if os.path.exists(model_filename):
self.model = self.create_algorithm()
self.model.load(model_filename)

8
analytics/config.py

@ -0,0 +1,8 @@
import os
DATA_FOLDER = '../../data'
DATASOURCE_FOLDER = os.path.join(DATA_FOLDER, "datasources/")
DATASET_FOLDER = os.path.join(DATA_FOLDER, "datasets/")
ANOMALIES_FOLDER = os.path.join(DATA_FOLDER, "anomalies/")
MODELS_FOLDER = os.path.join(DATA_FOLDER, "models/")
METRICS_FOLDER = os.path.join(DATA_FOLDER, "metrics/")

20
analytics/pattern_detection_model.py

@ -2,13 +2,11 @@ from data_provider import DataProvider
import logging
import os.path
import json
import config
import pandas as pd
datasource_folder = "datasources/"
dataset_folder = "datasets/"
anomalies_folder = "anomalies/"
models_folder = "models/"
metrics_folder = "metrics/"
logger = logging.getLogger('analytic_toolset')
@ -33,10 +31,10 @@ class PatternDetectionModel:
datasource = self.anomaly_config['metric']['datasource']
metric_name = self.anomaly_config['metric']['targets'][0]
dbconfig_filename = os.path.join(datasource_folder, datasource + ".json")
target_filename = os.path.join(metrics_folder, metric_name + ".json")
dbconfig_filename = os.path.join(config.DATASOURCE_FOLDER, datasource + ".json")
target_filename = os.path.join(config.METRICS_FOLDER, metric_name + ".json")
dataset_filename = os.path.join(dataset_folder, metric_name + ".csv")
dataset_filename = os.path.join(config.DATASET_FOLDER, metric_name + ".csv")
with open(dbconfig_filename, 'r') as config_file:
dbconfig = json.load(config_file)
@ -111,17 +109,17 @@ class PatternDetectionModel:
return StepDetector(preset)
def __load_anomaly_config(self):
with open(os.path.join(anomalies_folder, self.pattern_name + ".json"), 'r') as config_file:
with open(os.path.join(config.ANOMALIES_FOLDER, self.pattern_name + ".json"), 'r') as config_file:
self.anomaly_config = json.load(config_file)
def __save_model(self):
logger.info("Save model '%s'" % self.pattern_name)
model_filename = os.path.join(models_folder, self.pattern_name + ".m")
model_filename = os.path.join(config.MODELS_FOLDER, self.pattern_name + ".m")
self.model.save(model_filename)
def __load_model(self, preset):
logger.info("Load model '%s'" % self.pattern_name)
model_filename = os.path.join(models_folder, self.pattern_name + ".m")
model_filename = os.path.join(config.MODELS_FOLDER, self.pattern_name + ".m")
if os.path.exists(model_filename):
self.model = self.__create_model(preset)
self.model.load(model_filename)

7
server/src/config.ts

@ -3,6 +3,11 @@ import * as path from 'path';
export const ANALYTICS_PATH = path.join(__dirname, '../../analytics');
export const DATA_PATH = path.join(__dirname, '../../data');
export const DATASOURCES_PATH = path.join(DATA_PATH, 'datasources');
export const DATASETS_PATH = path.join(DATA_PATH, 'datasets');
export const ANOMALIES_PATH = path.join(DATA_PATH, 'anomalies');
export const SEGMENTS_PATH = path.join(DATA_PATH, 'segments');
export const MODELS_PATH = path.join(DATA_PATH, 'models');
export const METRICS_PATH = path.join(DATA_PATH, 'metrics');
export const SEGMENTS_PATH = path.join(DATA_PATH, 'segments');

20
server/src/services/data.ts

@ -2,13 +2,23 @@ import * as config from '../config'
import * as fs from 'fs';
// see analytics/pattern_detection_model.py with folders available
export function checkDataFolders() {
function maybeCreate(path: string): void {
if(fs.existsSync(config.DATA_PATH)) {
return;
}
fs.mkdirSync(config.DATA_PATH);
fs.mkdirSync(config.ANOMALIES_PATH);
fs.mkdirSync(config.SEGMENTS_PATH);
fs.mkdirSync(config.METRICS_PATH);
fs.mkdirSync(path);
}
export function checkDataFolders(): void {
var folders = [
config.DATA_PATH,
config.DATASOURCES_PATH,
config.DATASETS_PATH,
config.ANOMALIES_PATH,
config.MODELS_PATH,
config.METRICS_PATH,
config.SEGMENTS_PATH
].forEach(maybeCreate);
}
Loading…
Cancel
Save