diff --git a/server/src/config.ts b/server/src/config.ts index 4616142..c385c3b 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -18,7 +18,7 @@ export const ANALYTIC_UNIT_CACHES_DATABASE_PATH = path.join(DATA_PATH, 'analytic export const SEGMENTS_DATABASE_PATH = path.join(DATA_PATH, 'segments.db'); export const THRESHOLD_DATABASE_PATH = path.join(DATA_PATH, 'treshold.db'); export const DETECTION_SPANS_DATABASE_PATH = path.join(DATA_PATH, 'detection_spans.db'); - +export const DB_META_PATH = path.join(DATA_PATH, 'db_meta.db'); export const HASTIC_PORT = getConfigField('HASTIC_PORT', '8000'); export const ZMQ_IPC_PATH = getConfigField('ZMQ_IPC_PATH', path.join(os.tmpdir(), 'hastic')); diff --git a/server/src/index.ts b/server/src/index.ts index a5d6511..c59de28 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -10,7 +10,7 @@ import * as ProcessService from './services/process_service'; import { HASTIC_PORT, PACKAGE_VERSION, GIT_INFO, ZMQ_CONNECTION_STRING, HASTIC_INSTANCE_NAME } from './config'; -import { convertPanelUrlToPanelId } from './migrations/0.3.2-beta'; +import { applyDBMigrations } from './migrations'; import * as Koa from 'koa'; import * as Router from 'koa-router'; @@ -19,7 +19,7 @@ import * as bodyParser from 'koa-bodyparser'; init(); async function init() { - await convertPanelUrlToPanelId(); + await applyDBMigrations(); AnalyticsController.init(); ProcessService.registerExitHandler(AnalyticsController.terminate); @@ -31,7 +31,7 @@ async function init() { }); - app.use(bodyParser()) + app.use(bodyParser()); app.use(async function(ctx, next) { ctx.set('Access-Control-Allow-Origin', '*'); diff --git a/server/src/migrations.ts b/server/src/migrations.ts new file mode 100644 index 0000000..7b2cd50 --- /dev/null +++ b/server/src/migrations.ts @@ -0,0 +1,97 @@ +import { Collection, makeDBQ } from './services/data_service'; + +import * as _ from 'lodash'; + + +const metaDB = makeDBQ(Collection.DB_META); +const analyticUnitsDB = makeDBQ(Collection.ANALYTIC_UNITS); +const analyticUnitCachesDB = makeDBQ(Collection.ANALYTIC_UNIT_CACHES); + +const DB_META_ID = '0'; + +type DbMeta = { + revision: number +}; + +const REVISIONS = new Map([ + [1, convertPanelUrlToPanelId], + [2, convertUnderscoreToCamelCase] +]); + +export async function applyDBMigrations() { + let meta: DbMeta = await metaDB.findOne(DB_META_ID); + if(meta === null) { + meta = { revision: 0 }; + await metaDB.insertOne({ _id: DB_META_ID, ...meta }); + } + + await REVISIONS.forEach(async (migration, revision) => { + if(meta.revision < revision) { + console.log(`Applying migration ${revision}`); + await migration(); + + meta.revision = revision; + await metaDB.updateOne(DB_META_ID, meta); + } + }); +} + +async function convertPanelUrlToPanelId() { + const analyticUnits = await analyticUnitsDB.findMany({ panelUrl: { $exists: true } }); + console.log(`Found ${analyticUnits.length} analytic units with panelUrl field`); + if(analyticUnits.length === 0) { + console.log('Nothing to migrate'); + return; + } + + const PANEL_URL_REGEX = /^(.+)\/d\/([^\/]+)\/.+panelId=(\d+)/; + const NEW_PANEL_URL_REGEX = /^(.+)\/dashboard\/(\w+).+panelId=(\d+)/; + const updatedAnalyticUnits = analyticUnits + .map(analyticUnit => { + const parsedPanelUrl = analyticUnit.panelUrl.match(PANEL_URL_REGEX) || analyticUnit.panelUrl.match(NEW_PANEL_URL_REGEX); + if(parsedPanelUrl === null) { + console.log(`Cannot parse url: ${analyticUnit.panelUrl}`); + return null; + } + const grafanaUrl = parsedPanelUrl[1]; + const dashboardId = parsedPanelUrl[2]; + const oldPanelId = parsedPanelUrl[3]; + const panelId = `${dashboardId}/${oldPanelId}`; + + return { + _id: analyticUnit._id, + grafanaUrl, + panelId + }; + }) + .filter(analyticUnit => analyticUnit !== null); + + console.log(updatedAnalyticUnits); + const promises = updatedAnalyticUnits.map(analyticUnit => + analyticUnitsDB.updateOne(analyticUnit._id, { + panelUrl: undefined, + ...analyticUnit + }) + ); + + await Promise.all(promises); +} + +async function convertUnderscoreToCamelCase() { + const analyticUnitCaches = await analyticUnitCachesDB.findMany({}); + + const updatedAnalyticUnitCaches = analyticUnitCaches.map(analyticUnitCache => { + let data = null; + if(analyticUnitCache.data !== null) { + data = _.mapKeys(analyticUnitCache.data, (value, key) => _.camelCase(key)); + } + + return { data, _id: analyticUnitCache._id }; + }); + + const promises = updatedAnalyticUnitCaches.map(analyticUnitCache => + analyticUnitCachesDB.updateOne(analyticUnitCache._id, { data: analyticUnitCache.data }) + ); + + await Promise.all(promises); +} diff --git a/server/src/migrations/0.3.2-beta.ts b/server/src/migrations/0.3.2-beta.ts deleted file mode 100644 index 7611d6c..0000000 --- a/server/src/migrations/0.3.2-beta.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Collection, makeDBQ } from '../services/data_service'; - -const db = makeDBQ(Collection.ANALYTIC_UNITS); - -export async function convertPanelUrlToPanelId() { - const analyticUnits = await db.findMany({ panelUrl: { $exists: true } }); - console.log(`Found ${analyticUnits.length} analytic units with panelUrl field`); - if(analyticUnits.length === 0) { - console.log('Nothing to migrate'); - return; - } - - const panelUrlRegex = /^(.+)\/d\/([^\/]+)\/.+panelId=(\d+)/; - const newPanelUrlRegex = /^(.+)\/dashboard\/(\w+).+panelId=(\d+)/; - const updatedAnalyticUnits = analyticUnits - .map(analyticUnit => { - const parsedPanelUrl = analyticUnit.panelUrl.match(panelUrlRegex) || analyticUnit.panelUrl.match(newPanelUrlRegex); - if(parsedPanelUrl === null) { - console.log(`Cannot parse url: ${analyticUnit.panelUrl}`); - return null; - } - const grafanaUrl = parsedPanelUrl[1]; - const dashboardId = parsedPanelUrl[2]; - const oldPanelId = parsedPanelUrl[3]; - const panelId = `${dashboardId}/${oldPanelId}`; - - return { - _id: analyticUnit._id, - grafanaUrl, - panelId - }; - }) - .filter(analyticUnit => analyticUnit !== null); - - console.log(updatedAnalyticUnits); - await updatedAnalyticUnits.forEach(analyticUnit => db.updateOne(analyticUnit._id, { - panelUrl: undefined, - ...analyticUnit - })); -} - diff --git a/server/src/services/data_service.ts b/server/src/services/data_service.ts index ddb2c34..da96880 100644 --- a/server/src/services/data_service.ts +++ b/server/src/services/data_service.ts @@ -9,7 +9,8 @@ export enum Collection { ANALYTIC_UNIT_CACHES, SEGMENTS, THRESHOLD, - DETECTION_SPANS + DETECTION_SPANS, + DB_META }; export enum SortingOrder { ASCENDING = 1, DESCENDING = -1 }; @@ -220,3 +221,4 @@ db.set(Collection.ANALYTIC_UNIT_CACHES, new nedb({ filename: config.ANALYTIC_UNI db.set(Collection.SEGMENTS, new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true })); db.set(Collection.THRESHOLD, new nedb({ filename: config.THRESHOLD_DATABASE_PATH, autoload: true })); db.set(Collection.DETECTION_SPANS, new nedb({ filename: config.DETECTION_SPANS_DATABASE_PATH, autoload: true })); +db.set(Collection.DB_META, new nedb({ filename: config.DB_META_PATH, autoload: true }));