Migrate analytic unit cache to camel case #609 (#613)

server/src/config.ts

@@ -18,7 +18,7 @@ export const ANALYTIC_UNIT_CACHES_DATABASE_PATH = path.join(DATA_PATH, 'analytic
 export const SEGMENTS_DATABASE_PATH = path.join(DATA_PATH, 'segments.db');
 export const THRESHOLD_DATABASE_PATH = path.join(DATA_PATH, 'treshold.db');
 export const DETECTION_SPANS_DATABASE_PATH = path.join(DATA_PATH, 'detection_spans.db');
-
+export const DB_META_PATH = path.join(DATA_PATH, 'db_meta.db');
 
 export const HASTIC_PORT = getConfigField('HASTIC_PORT', '8000');
 export const ZMQ_IPC_PATH = getConfigField('ZMQ_IPC_PATH', path.join(os.tmpdir(), 'hastic'));

server/src/index.ts

@@ -10,7 +10,7 @@ import * as ProcessService from './services/process_service';
 
 import { HASTIC_PORT, PACKAGE_VERSION, GIT_INFO, ZMQ_CONNECTION_STRING, HASTIC_INSTANCE_NAME } from './config';
 
-import { convertPanelUrlToPanelId } from './migrations/0.3.2-beta';
+import { applyDBMigrations } from './migrations';
 
 import * as Koa from 'koa';
 import * as Router from 'koa-router';
@@ -19,7 +19,7 @@ import * as bodyParser from 'koa-bodyparser';
 init();
 
 async function init() {
-  await convertPanelUrlToPanelId();
+  await applyDBMigrations();
   AnalyticsController.init();
   ProcessService.registerExitHandler(AnalyticsController.terminate);
 
@@ -31,7 +31,7 @@ async function init() {
   });
 
 
-  app.use(bodyParser())
+  app.use(bodyParser());
 
   app.use(async function(ctx, next) {
     ctx.set('Access-Control-Allow-Origin', '*');

server/src/migrations.ts

@@ -0,0 +1,97 @@
+import { Collection, makeDBQ } from './services/data_service';
+
+import * as _ from 'lodash';
+
+
+const metaDB = makeDBQ(Collection.DB_META);
+const analyticUnitsDB = makeDBQ(Collection.ANALYTIC_UNITS);
+const analyticUnitCachesDB = makeDBQ(Collection.ANALYTIC_UNIT_CACHES);
+
+const DB_META_ID = '0';
+
+type DbMeta = {
+  revision: number
+};
+
+const REVISIONS = new Map<number, Function>([
+  [1, convertPanelUrlToPanelId],
+  [2, convertUnderscoreToCamelCase]
+]);
+
+export async function applyDBMigrations() {
+  let meta: DbMeta = await metaDB.findOne(DB_META_ID);
+  if(meta === null) {
+    meta = { revision: 0 };
+    await metaDB.insertOne({ _id: DB_META_ID, ...meta });
+  }
+
+  await REVISIONS.forEach(async (migration, revision) => {
+    if(meta.revision < revision) {
+      console.log(`Applying migration ${revision}`);
+      await migration();
+
+      meta.revision = revision;
+      await metaDB.updateOne(DB_META_ID, meta);
+    }
+  });
+}
+
+async function convertPanelUrlToPanelId() {
+  const analyticUnits = await analyticUnitsDB.findMany({ panelUrl: { $exists: true } });
+  console.log(`Found ${analyticUnits.length} analytic units with panelUrl field`);
+  if(analyticUnits.length === 0) {
+    console.log('Nothing to migrate');
+    return;
+  }
+
+  const PANEL_URL_REGEX = /^(.+)\/d\/([^\/]+)\/.+panelId=(\d+)/;
+  const NEW_PANEL_URL_REGEX = /^(.+)\/dashboard\/(\w+).+panelId=(\d+)/;
+  const updatedAnalyticUnits = analyticUnits
+    .map(analyticUnit => {
+      const parsedPanelUrl = analyticUnit.panelUrl.match(PANEL_URL_REGEX) || analyticUnit.panelUrl.match(NEW_PANEL_URL_REGEX);
+      if(parsedPanelUrl === null) {
+        console.log(`Cannot parse url: ${analyticUnit.panelUrl}`);
+        return null;
+      }
+      const grafanaUrl = parsedPanelUrl[1];
+      const dashboardId = parsedPanelUrl[2];
+      const oldPanelId = parsedPanelUrl[3];
+      const panelId = `${dashboardId}/${oldPanelId}`;
+
+      return {
+        _id: analyticUnit._id,
+        grafanaUrl,
+        panelId
+      };
+    })
+    .filter(analyticUnit => analyticUnit !== null);
+
+  console.log(updatedAnalyticUnits);
+  const promises = updatedAnalyticUnits.map(analyticUnit =>
+    analyticUnitsDB.updateOne(analyticUnit._id, {
+      panelUrl: undefined,
+      ...analyticUnit
+    })
+  );
+
+  await Promise.all(promises);
+}
+
+async function convertUnderscoreToCamelCase() {
+  const analyticUnitCaches = await analyticUnitCachesDB.findMany({});
+
+  const updatedAnalyticUnitCaches = analyticUnitCaches.map(analyticUnitCache => {
+    let data = null;
+    if(analyticUnitCache.data !== null) {
+      data = _.mapKeys(analyticUnitCache.data, (value, key) => _.camelCase(key));
+    }
+
+    return { data, _id: analyticUnitCache._id };
+  });
+
+  const promises = updatedAnalyticUnitCaches.map(analyticUnitCache =>
+    analyticUnitCachesDB.updateOne(analyticUnitCache._id, { data: analyticUnitCache.data })
+  );
+
+  await Promise.all(promises);
+}

server/src/migrations/0.3.2-beta.ts

@@ -1,41 +0,0 @@
-import { Collection, makeDBQ } from '../services/data_service';
-
-const db = makeDBQ(Collection.ANALYTIC_UNITS);
-
-export async function convertPanelUrlToPanelId() {
-  const analyticUnits = await db.findMany({ panelUrl: { $exists: true } });
-  console.log(`Found ${analyticUnits.length} analytic units with panelUrl field`);
-  if(analyticUnits.length === 0) {
-    console.log('Nothing to migrate');
-    return;
-  }
-
-  const panelUrlRegex = /^(.+)\/d\/([^\/]+)\/.+panelId=(\d+)/;
-  const newPanelUrlRegex = /^(.+)\/dashboard\/(\w+).+panelId=(\d+)/;
-  const updatedAnalyticUnits = analyticUnits
-    .map(analyticUnit => {
-      const parsedPanelUrl = analyticUnit.panelUrl.match(panelUrlRegex) || analyticUnit.panelUrl.match(newPanelUrlRegex);
-      if(parsedPanelUrl === null) {
-        console.log(`Cannot parse url: ${analyticUnit.panelUrl}`);
-        return null;
-      }
-      const grafanaUrl = parsedPanelUrl[1];
-      const dashboardId = parsedPanelUrl[2];
-      const oldPanelId = parsedPanelUrl[3];
-      const panelId = `${dashboardId}/${oldPanelId}`;
-
-      return {
-        _id: analyticUnit._id,
-        grafanaUrl,
-        panelId
-      };
-    })
-    .filter(analyticUnit => analyticUnit !== null);
-  
-  console.log(updatedAnalyticUnits);
-  await updatedAnalyticUnits.forEach(analyticUnit => db.updateOne(analyticUnit._id, {
-    panelUrl: undefined,
-    ...analyticUnit
-  }));
-}
-

server/src/services/data_service.ts

@@ -9,7 +9,8 @@ export enum Collection {
   ANALYTIC_UNIT_CACHES,
   SEGMENTS,
   THRESHOLD,
-  DETECTION_SPANS
+  DETECTION_SPANS,
+  DB_META
 };
 
 export enum SortingOrder { ASCENDING = 1, DESCENDING = -1 };
@@ -220,3 +221,4 @@ db.set(Collection.ANALYTIC_UNIT_CACHES, new nedb({ filename: config.ANALYTIC_UNI
 db.set(Collection.SEGMENTS, new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true }));
 db.set(Collection.THRESHOLD, new nedb({ filename: config.THRESHOLD_DATABASE_PATH, autoload: true }));
 db.set(Collection.DETECTION_SPANS, new nedb({ filename: config.DETECTION_SPANS_DATABASE_PATH, autoload: true }));
+db.set(Collection.DB_META, new nedb({ filename: config.DB_META_PATH, autoload: true }));