hastic
/
hastic-server
4
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity
Browse Source

renaming++

pull/1/head
Alexey Velikiy 6 years ago
parent
4600a834bd
commit
326987c709
8 changed files with 96 additions and 113 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 46
      server/src/models/analytic_unit.ts
  2. 8
      server/src/routes/alerts.ts
  3. 68
      server/src/routes/analytic_units.ts
  4. 29
      server/src/routes/segments.ts
  5. 6
      server/src/services/alerts.ts
  6. 14
      server/src/services/analytics.ts
  7. 4
      server/src/services/notification.ts
  8. 34
      server/src/services/segments.ts

46
server/src/models/analytic_unit.ts
Unescape View File

@ -1,9 +1,12 @@
import * as path from 'path' import { getJsonDataSync, writeJsonDataSync } from '../services/json'
import { getJsonDataSync, writeJsonDataSync } from './json'
import { ANOMALIES_PATH } from '../config' import { ANOMALIES_PATH } from '../config'
import * as fs from 'fs'
import * as crypto from 'crypto'; import * as crypto from 'crypto';
import * as path from 'path'
import * as fs from 'fs'
export type Datasource = { export type Datasource = {
method: string, method: string,
data: Object, data: Object,
@ -32,34 +35,31 @@ export type AnalyticUnit = {
nextId: number nextId: number
} }
export type AnomalyUnitKey = string; export type AnalyticUnitId = string;
let anomaliesNameToIdMap = {};
function insertAnomaly(item: AnalyticUnit): AnomalyUnitKey { function createItem(item: AnalyticUnit): AnalyticUnitId {
const hashString = item.name + (new Date()).toString(); const hashString = item.name + (new Date()).toString();
const predictorId: AnomalyUnitKey = crypto.createHash('md5').update(hashString).digest('hex'); const newId: AnalyticUnitId = crypto.createHash('md5').update(hashString).digest('hex');
anomaliesNameToIdMap[item.name] = predictorId; let filename = path.join(ANOMALIES_PATH, `${newId}.json`);
let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`);
if(fs.existsSync(filename)) { if(fs.existsSync(filename)) {
return null; return null;
} }
saveAnomaly(predictorId, item); save(newId, item);
return predictorId; return newId;
} }
function removeItem(predictorId: AnomalyUnitKey) { function removeItem(key: AnalyticUnitId) {
let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); let filename = path.join(ANOMALIES_PATH, `${key}.json`);
fs.unlinkSync(filename); fs.unlinkSync(filename);
} }
function saveAnomaly(predictorId: AnomalyUnitKey, anomaly: AnalyticUnit) { function save(predictorId: AnalyticUnitId, anomaly: AnalyticUnit) {
let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`);
return writeJsonDataSync(filename, anomaly); return writeJsonDataSync(filename, anomaly);
} }
function loadPredictorById(predictorId: AnomalyUnitKey): AnalyticUnit { function loadById(predictorId: AnalyticUnitId): AnalyticUnit {
let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`);
if(!fs.existsSync(filename)) { if(!fs.existsSync(filename)) {
return null; return null;
@ -84,24 +84,24 @@ function getAnomalyTypeInfo(name) {
return getJsonDataSync(path.join(ANOMALIES_PATH, `${name}.json`)); return getJsonDataSync(path.join(ANOMALIES_PATH, `${name}.json`));
} }
function setAnomalyStatus(predictorId: AnomalyUnitKey, status: string, error?: string) { function setAnomalyStatus(predictorId: AnalyticUnitId, status: string, error?: string) {
let info = loadPredictorById(predictorId); let info = loadById(predictorId);
info.status = status; info.status = status;
if(error !== undefined) { if(error !== undefined) {
info.error = error; info.error = error;
} else { } else {
info.error = ''; info.error = '';
} }
saveAnomaly(predictorId, info); save(predictorId, info);
} }
function setAnomalyPredictionTime(predictorId: AnomalyUnitKey, lastPredictionTime: number) { function setAnomalyPredictionTime(predictorId: AnalyticUnitId, lastPredictionTime: number) {
let info = loadPredictorById(predictorId); let info = loadById(predictorId);
info.lastPredictionTime = lastPredictionTime; info.lastPredictionTime = lastPredictionTime;
saveAnomaly(predictorId, info); save(predictorId, info);
} }
export { export {
saveAnomaly, loadPredictorById, insertAnomaly, removeItem, saveAnomalyTypeInfo, save, loadById, createItem, removeItem, saveAnomalyTypeInfo,
getAnomalyTypeInfo, setAnomalyStatus, setAnomalyPredictionTime getAnomalyTypeInfo, setAnomalyStatus, setAnomalyPredictionTime
} }

8
server/src/routes/alerts.ts
Unescape View File

@ -1,4 +1,4 @@
import { AnomalyUnitKey, loadPredictorById } from '../models/analytic_unit'; import { AnalyticUnitId, loadById } from '../models/analytic_unit';
import { getAlertsAnomalies, saveAlertsAnomalies } from '../services/alerts'; import { getAlertsAnomalies, saveAlertsAnomalies } from '../services/alerts';
import * as Router from 'koa-router'; import * as Router from 'koa-router';
@ -6,7 +6,7 @@ import * as Router from 'koa-router';
function getAlert(ctx: Router.IRouterContext) { function getAlert(ctx: Router.IRouterContext) {
let predictorId: AnomalyUnitKey = ctx.request.query.predictor_id.toLowerCase(); let predictorId: AnalyticUnitId = ctx.request.query.predictor_id.toLowerCase();
let alertsAnomalies = getAlertsAnomalies(); let alertsAnomalies = getAlertsAnomalies();
let pos = alertsAnomalies.indexOf(predictorId); let pos = alertsAnomalies.indexOf(predictorId);
@ -18,10 +18,10 @@ function getAlert(ctx: Router.IRouterContext) {
function changeAlert(ctx: Router.IRouterContext) { function changeAlert(ctx: Router.IRouterContext) {
let predictorId: AnomalyUnitKey = ctx.request.body.predictor_id.toLowerCase(); let predictorId: AnalyticUnitId = ctx.request.body.predictor_id.toLowerCase();
let enable: boolean = ctx.request.body.enable; let enable: boolean = ctx.request.body.enable;
let predictor = loadPredictorById(predictorId) let predictor = loadById(predictorId)
if(predictor == null) { if(predictor == null) {
throw new Error('Predctor is null'); throw new Error('Predctor is null');
} }

68
server/src/routes/analytic_units.ts
Unescape View File

@ -1,33 +1,28 @@
import * as Router from 'koa-router'; import * as Router from 'koa-router';
import { import * as AnalyticUnit from '../models/analytic_unit';
Datasource,
Metric,
AnalyticUnit,
insertAnomaly, removeItem, loadPredictorById
} from '../models/analytic_unit';
import { runLearning } from '../services/analytics' import { runLearning } from '../services/analytics'
import { saveTargets } from '../services/metrics'; import { saveTargets } from '../services/metrics';
async function sendAnomalyTypeStatus(ctx: Router.IRouterContext) { async function sendStatus(ctx: Router.IRouterContext) {
let id = ctx.request.query.id; let id = ctx.request.query.id;
let name = ctx.request.query.name.toLowerCase(); let name = ctx.request.query.name;
try { try {
let anomaly: AnalyticUnit; let unit: AnalyticUnit.AnalyticUnit;
if(id === undefined) { if(id === undefined) {
throw new Error('Id is undefined'); throw new Error('Id is undefined');
} }
anomaly = loadPredictorById(id); unit = AnalyticUnit.loadById(id);
if(anomaly === null) { if(unit === null) {
ctx.response.status = 404; ctx.response.status = 404;
return; return;
} }
if(anomaly.status === undefined) { if(unit.status === undefined) {
throw new Error('No status for ' + name); throw new Error('No status for ' + name);
} }
ctx.response.body = { status: anomaly.status, errorMessage: anomaly.error }; ctx.response.body = { status: unit.status, errorMessage: unit.error };
} catch(e) { } catch(e) {
console.error(e); console.error(e);
// TODO: better send 404 when we know than isn`t found // TODO: better send 404 when we know than isn`t found
@ -37,20 +32,15 @@ async function sendAnomalyTypeStatus(ctx: Router.IRouterContext) {
} }
async function getAnalyticUnit(ctx: Router.IRouterContext) { async function findItem(ctx: Router.IRouterContext) {
try { try {
let id = ctx.request.query.id; let id = ctx.request.query.id;
let name = ctx.request.query.name.toLowerCase();
if(id === undefined) { if(id === undefined) {
throw new Error('No id param in query'); throw new Error('No id param in query');
} }
if(name === undefined) { let unit: AnalyticUnit.AnalyticUnit = AnalyticUnit.loadById(id);
throw new Error('No name param in query');
}
let unit: AnalyticUnit = loadPredictorById(id);
if(unit === null) { if(unit === null) {
ctx.response.status = 404; ctx.response.status = 404;
@ -67,19 +57,19 @@ async function getAnalyticUnit(ctx: Router.IRouterContext) {
console.error(e); console.error(e);
// TODO: better send 404 when we know than isn`t found // TODO: better send 404 when we know than isn`t found
ctx.response.status = 500; ctx.response.status = 500;
ctx.response.body = 'Can`t get anything'; ctx.response.body = 'Can`t find anything';
} }
} }
async function createAnalyticUnit(ctx: Router.IRouterContext) { async function createItem(ctx: Router.IRouterContext) {
try { try {
let body = ctx.request.body; let body = ctx.request.body;
const metric:Metric = { const metric: AnalyticUnit.Metric = {
datasource: body.metric.datasource, datasource: body.metric.datasource,
targets: saveTargets(body.metric.targets) targets: saveTargets(body.metric.targets)
}; };
const anomaly:AnalyticUnit = { const unit: AnalyticUnit.AnalyticUnit = {
name: body.name.toLowerCase(), name: body.name.toLowerCase(),
panelUrl: body.panelUrl, panelUrl: body.panelUrl,
pattern: body.pattern.toLowerCase(), pattern: body.pattern.toLowerCase(),
@ -89,36 +79,34 @@ async function createAnalyticUnit(ctx: Router.IRouterContext) {
lastPredictionTime: 0, lastPredictionTime: 0,
nextId: 0 nextId: 0
}; };
let predictorId = insertAnomaly(anomaly);
if(predictorId === null) { let newId = AnalyticUnit.createItem(unit);
if(newId === null) {
ctx.response.status = 403; ctx.response.status = 403;
ctx.response.body = { ctx.response.body = {
code: 403, code: 403,
message: 'Already exists' message: 'Item exists'
}; };
} }
ctx.response.body = { predictor_id: predictorId }; ctx.response.body = { id: newId };
runLearning(predictorId); runLearning(newId);
} catch(e) { } catch(e) {
ctx.response.status = 500; ctx.response.status = 500;
ctx.response.body = { ctx.response.body = {
code: 500, code: 500,
message: `Anomaly creation error: ${e.message}` message: `Creation error: ${e.message}`
}; };
} }
} }
function deleteAnomaly(ctx: Router.IRouterContext) { function deleteItem(ctx: Router.IRouterContext) {
try { try {
let id = ctx.request.query.id; let id = ctx.request.query.id;
let name = ctx.request.query.name.toLowerCase();
if(id !== undefined) { if(id !== undefined) {
removeItem(id); AnalyticUnit.removeItem(id);
} else {
removeItem(name);
} }
ctx.response.body = { ctx.response.body = {
@ -129,7 +117,7 @@ function deleteAnomaly(ctx: Router.IRouterContext) {
ctx.response.status = 500; ctx.response.status = 500;
ctx.response.body = { ctx.response.body = {
code: 500, code: 500,
message: `Anomaly deletion error: ${e.message}` message: `Deletion error: ${e.message}`
}; };
} }
} }
@ -137,7 +125,7 @@ function deleteAnomaly(ctx: Router.IRouterContext) {
export var router = new Router(); export var router = new Router();
router.get('/status', sendAnomalyTypeStatus); router.get('/status', sendStatus);
router.get('/', getAnalyticUnit); router.get('/', findItem);
router.post('/', createAnalyticUnit); router.post('/', createItem);
router.delete('/', deleteAnomaly); router.delete('/', deleteItem);

29
server/src/routes/segments.ts
Unescape View File

@ -7,25 +7,25 @@ import {
} from '../services/segments'; } from '../services/segments';
import { import {
AnalyticUnit, AnomalyUnitKey, getPredictorIdByName, loadPredictorById AnalyticUnit, AnalyticUnitId, loadById
} from '../models/analytic_unit'; } from '../models/analytic_unit';
import { runLearning } from '../services/analytics'; import { runLearning } from '../services/analytics';
async function sendSegments(ctx: Router.IRouterContext) { async function sendSegments(ctx: Router.IRouterContext) {
let id: AnalyticUnitId = ctx.request.query.id;
let unit: AnalyticUnit = loadById(id);
let predictorId: AnomalyUnitKey = ctx.request.query.predictor_id.toLowerCase(); if(unit === null) {
let anomaly:AnalyticUnit = loadPredictorById(predictorId); throw new Error(`Can't find Analitic unit with id ${id}`);
if(anomaly === null) {
predictorId = getPredictorIdByName(predictorId);
} }
let lastSegmentId = ctx.request.query.last_segment; let lastSegmentId = ctx.request.query.lastSegmentId;
let timeFrom = ctx.request.query.from; let timeFrom = ctx.request.query.from;
let timeTo = ctx.request.query.to; let timeTo = ctx.request.query.to;
let segments = getLabeledSegments(predictorId); let segments = getLabeledSegments(id);
// Id filtering // Id filtering
if(lastSegmentId !== undefined) { if(lastSegmentId !== undefined) {
@ -49,19 +49,14 @@ async function updateSegments(ctx: Router.IRouterContext) {
try { try {
let segmentsUpdate = ctx.request.body; let segmentsUpdate = ctx.request.body;
let predictorId = segmentsUpdate.predictor_id; let key = segmentsUpdate.analyticUnitKey;
let anomalyName = segmentsUpdate.name.toLowerCase();
if(predictorId === undefined) { let addedIds = insertSegments(key, segmentsUpdate.addedSegments, true);
predictorId = getPredictorIdByName(anomalyName); removeSegments(key, segmentsUpdate.removedSegments);
}
let addedIds = insertSegments(predictorId, segmentsUpdate.added_segments, true);
removeSegments(predictorId, segmentsUpdate.removed_segments);
ctx.response.body = { added_ids: addedIds }; ctx.response.body = { addedIds };
runLearning(predictorId); runLearning(key);
} catch(e) { } catch(e) {
ctx.response.status = 500; ctx.response.status = 500;
ctx.response.body = { ctx.response.body = {

6
server/src/services/alerts.ts
Unescape View File

@ -1,5 +1,5 @@
import { getJsonDataSync, writeJsonDataSync } from './json'; import { getJsonDataSync, writeJsonDataSync } from './json';
import { AnomalyUnitKey } from '../models/analytic_unit'; import { AnalyticUnitId } from '../models/analytic_unit';
import { runPredict } from './analytics'; import { runPredict } from './analytics';
import { sendNotification } from './notification'; import { sendNotification } from './notification';
import { getLabeledSegments } from './segments'; import { getLabeledSegments } from './segments';
@ -13,14 +13,14 @@ import * as fs from 'fs';
const ALERTS_DB_PATH = path.join(ANOMALIES_PATH, `alerts_anomalies.json`); const ALERTS_DB_PATH = path.join(ANOMALIES_PATH, `alerts_anomalies.json`);
function getAlertsAnomalies(): AnomalyUnitKey[] { function getAlertsAnomalies(): AnalyticUnitId[] {
if(!fs.existsSync(ALERTS_DB_PATH)) { if(!fs.existsSync(ALERTS_DB_PATH)) {
saveAlertsAnomalies([]); saveAlertsAnomalies([]);
} }
return getJsonDataSync(ALERTS_DB_PATH); return getJsonDataSync(ALERTS_DB_PATH);
} }
function saveAlertsAnomalies(anomalies: AnomalyUnitKey[]) { function saveAlertsAnomalies(anomalies: AnalyticUnitId[]) {
return writeJsonDataSync(ALERTS_DB_PATH, anomalies); return writeJsonDataSync(ALERTS_DB_PATH, anomalies);
} }

14
server/src/services/analytics.ts
Unescape View File

@ -1,7 +1,7 @@
import { import {
AnalyticUnit, AnalyticUnit,
AnomalyUnitKey, getAnomalyTypeInfo, AnalyticUnitId, getAnomalyTypeInfo,
loadPredictorById, loadById,
setAnomalyPredictionTime, setAnomalyPredictionTime,
setAnomalyStatus setAnomalyStatus
} from '../models/analytic_unit' } from '../models/analytic_unit'
@ -28,7 +28,7 @@ function onResponse(response: any) {
} }
async function runTask(task): Promise<any> { async function runTask(task): Promise<any> {
let anomaly: AnalyticUnit = loadPredictorById(task.predictor_id); let anomaly: AnalyticUnit = loadById(task.predictor_id);
task.metric = { task.metric = {
datasource: anomaly.metric.datasource, datasource: anomaly.metric.datasource,
targets: anomaly.metric.targets.map(t => getTarget(t)) targets: anomaly.metric.targets.map(t => getTarget(t))
@ -42,10 +42,10 @@ async function runTask(task): Promise<any> {
}) })
} }
export async function runLearning(predictorId:AnomalyUnitKey) { export async function runLearning(predictorId:AnalyticUnitId) {
let segments = getLabeledSegments(predictorId); let segments = getLabeledSegments(predictorId);
setAnomalyStatus(predictorId, 'learning'); setAnomalyStatus(predictorId, 'learning');
let anomaly:AnalyticUnit = loadPredictorById(predictorId); let anomaly:AnalyticUnit = loadById(predictorId);
let pattern = anomaly.pattern; let pattern = anomaly.pattern;
let task = { let task = {
type: 'learn', type: 'learn',
@ -65,8 +65,8 @@ export async function runLearning(predictorId:AnomalyUnitKey) {
} }
} }
export async function runPredict(predictorId:AnomalyUnitKey) { export async function runPredict(predictorId:AnalyticUnitId) {
let anomaly:AnalyticUnit = loadPredictorById(predictorId); let anomaly:AnalyticUnit = loadById(predictorId);
let pattern = anomaly.pattern; let pattern = anomaly.pattern;
let task = { let task = {
type: 'predict', type: 'predict',

4
server/src/services/notification.ts
Unescape View File

@ -1,8 +1,8 @@
import axios from 'axios'; import axios from 'axios';
import { loadPredictorById } from '../models/analytic_unit'; import { loadById } from '../models/analytic_unit';
export async function sendNotification(predictorId, active) { export async function sendNotification(predictorId, active) {
let anomalyName = loadPredictorById(predictorId).name; let anomalyName = loadById(predictorId).name;
console.log('Notification ' + anomalyName); console.log('Notification ' + anomalyName);
let notification = { let notification = {

34
server/src/services/segments.ts
Unescape View File

@ -1,12 +1,14 @@
import * as path from 'path';
import * as fs from 'fs';
import { getJsonDataSync, writeJsonDataSync } from './json'; import { getJsonDataSync, writeJsonDataSync } from './json';
import { AnalyticUnitId, loadById, save } from '../models/analytic_unit';
import { SEGMENTS_PATH } from '../config'; import { SEGMENTS_PATH } from '../config';
import { AnomalyUnitKey, loadPredictorById, saveAnomaly } from '../models/analytic_unit';
import * as _ from 'lodash'; import * as _ from 'lodash';
function getLabeledSegments(predictorId: AnomalyUnitKey) { import * as path from 'path';
import * as fs from 'fs';
export function getLabeledSegments(predictorId: AnalyticUnitId) {
let filename = path.join(SEGMENTS_PATH, `${predictorId}_labeled.json`); let filename = path.join(SEGMENTS_PATH, `${predictorId}_labeled.json`);
if(!fs.existsSync(filename)) { if(!fs.existsSync(filename)) {
@ -22,7 +24,7 @@ function getLabeledSegments(predictorId: AnomalyUnitKey) {
} }
} }
function getPredictedSegments(predictorId: AnomalyUnitKey) { export function getPredictedSegments(predictorId: AnalyticUnitId) {
let filename = path.join(SEGMENTS_PATH, `${predictorId}_segments.json`); let filename = path.join(SEGMENTS_PATH, `${predictorId}_segments.json`);
let jsonData; let jsonData;
@ -35,8 +37,8 @@ function getPredictedSegments(predictorId: AnomalyUnitKey) {
return jsonData; return jsonData;
} }
function saveSegments(predictorId: AnomalyUnitKey, segments) { export function saveSegments(id: AnalyticUnitId, segments) {
let filename = path.join(SEGMENTS_PATH, `${predictorId}_labeled.json`); let filename = path.join(SEGMENTS_PATH, `${id}_labeled.json`);
try { try {
return writeJsonDataSync(filename, _.uniqBy(segments, 'start')); return writeJsonDataSync(filename, _.uniqBy(segments, 'start'));
@ -46,12 +48,12 @@ function saveSegments(predictorId: AnomalyUnitKey, segments) {
} }
} }
function insertSegments(predictorId: AnomalyUnitKey, addedSegments, labeled:boolean) { export function insertSegments(id: AnalyticUnitId, addedSegments, labeled:boolean) {
// Set status // Set status
let info = loadPredictorById(predictorId); let info = loadById(id);
let segments = getLabeledSegments(predictorId); let segments = getLabeledSegments(id);
let nextId = info.next_id; let nextId = info.nextId;
let addedIds = [] let addedIds = []
for (let segment of addedSegments) { for (let segment of addedSegments) {
segment.id = nextId; segment.id = nextId;
@ -60,18 +62,16 @@ function insertSegments(predictorId: AnomalyUnitKey, addedSegments, labeled:bool
nextId++; nextId++;
segments.push(segment); segments.push(segment);
} }
info.next_id = nextId; info.nextId = nextId;
saveSegments(predictorId, segments); saveSegments(id, segments);
saveAnomaly(predictorId, info); save(id, info);
return addedIds; return addedIds;
} }
function removeSegments(predictorId: AnomalyUnitKey, removedSegments) { export function removeSegments(predictorId: AnalyticUnitId, removedSegments) {
let segments = getLabeledSegments(predictorId); let segments = getLabeledSegments(predictorId);
for (let segmentId of removedSegments) { for (let segmentId of removedSegments) {
segments = segments.filter(el => el.id !== segmentId); segments = segments.filter(el => el.id !== segmentId);
} }
saveSegments(predictorId, segments); saveSegments(predictorId, segments);
} }
export { getLabeledSegments, getPredictedSegments, saveSegments, insertSegments, removeSegments }

Write Preview
Loading…
Cancel
Save