9 var Emitter = require(
"events").EventEmitter;
10 var fs = require(
"fs");
11 var stream = require(
"stream");
12 var child_process = require(
"child_process");
13 var Utils = require(
"./Utils");
14 var Connector = require(
"./DBConnector");
15 var path_module = require(
"path");
16 var db =
new Emitter();
20 var DefaultColumns = [
35 title:
"User Comment",
65 var MakeDbConfig =
function (configFile) {
67 dbprovider:
"filesystem"
68 , configNameFilter:
"*"
69 , baseDir: process.env.ARTDAQ_DATABASE_URI ? process.env[
"ARTDAQ_DATABASE_URI"] :
""
75 , instanceName: configFile.instanceName ? configFile.instanceName :
"test_db"
78 if (defaultConfig.baseDir.indexOf(
"filesystemdb://") === 0) {
79 console.log(
"Parsing filesystemdb URI: " + defaultConfig.baseDir);
80 defaultConfig.baseDir = defaultConfig.baseDir.replace(
"filesystemdb://",
"").replace(/\/$/,
"");
81 var tempArr = defaultConfig.baseDir.split(
'/');
82 defaultConfig.instanceName = tempArr.splice(-1);
83 defaultConfig.baseDir = tempArr.slice(0,-1).join(
'/');
84 }
else if (defaultConfig.baseDir.indexOf(
"mongodb://") === 0) {
85 console.log(
"Parsing mongodb URI: " + defaultConfig.baseDir);
86 var matches = defaultConfig.baseDir.match(/mongodb:\/\/(?:([^@:]*)(?::([^@]*))?@)?([^:]*):([0-9]+)\/([^/]*)\/?/);
87 defaultConfig.baseDir =
"";
88 defaultConfig.dbprovider =
"mongo";
89 console.log(JSON.stringify(matches, null, 4));
90 defaultConfig.mongoConfig.dbUser = matches[1] ? matches[1] :
"";
91 var pass = matches[2];
92 if (pass !== null && pass !== undefined) {
93 fs.writeFileSync(
".dbpasswd", pass);
95 defaultConfig.mongoConfig.dbHost = matches[3];
96 defaultConfig.mongoConfig.dbPort = matches[4];
97 defaultConfig.instanceName = matches[5];
100 if (defaultConfig.baseDir ===
"") {
101 defaultConfig.baseDir = process.env[
"ARTDAQ_DATABASE_DIR"];
104 console.log(
"DBConfig: " + JSON.stringify(defaultConfig, null, 4));
105 return defaultConfig;
108 function GetFileBase(configName, collection, entity) {
109 return path_module.join(configName, collection, entity +
".gui.json");
112 function GetTempFilePath(configName, collection, entity, dirs) {
113 var filebase = GetFileBase(configName, collection, entity);
115 if (!fs.existsSync(path_module.join(dirs.tmp, configName))) fs.mkdirSync(path_module.join(dirs.tmp, configName));
116 if (!fs.existsSync(path_module.join(dirs.tmp, configName, collection))) fs.mkdirSync(path_module.join(dirs.tmp, configName, collection));
117 return path_module.join(dirs.tmp, filebase);
120 function GetFilePath(configName, collection, entity, dirs, dbOnly) {
121 var filebase = GetFileBase(configName, collection, entity);
122 console.log(
"GetFilePath: filebase is " + filebase +
", dirs: " + JSON.stringify(dirs, null, 4));
123 if (!dbOnly && fs.existsSync(path_module.join(dirs.tmp, filebase))) {
124 return path_module.join(dirs.tmp, filebase);
127 if (!fs.existsSync(path_module.join(dirs.db, configName))) fs.mkdirSync(path_module.join(dirs.db, configName));
128 if (!fs.existsSync(path_module.join(dirs.db, configName, collection))) fs.mkdirSync(path_module.join(dirs.db, configName, collection));
129 return path_module.join(dirs.db, filebase);
137 function LoadFile(fileInfo, dirs, dbConfig) {
138 console.log(
"LoadFile: fileInfo: " + JSON.stringify(fileInfo, null, 4) +
", dirs: " + JSON.stringify(dirs, null, 4) +
", dbConfig: " + JSON.stringify(dbConfig, null, 4));
147 output.entity = fileInfo.query.filter[
"entities.name"];
148 output.collection = fileInfo.query.collection;
149 output.configName = fileInfo.query.filter[
"configurations.name"];
151 console.log(
"File info: " + JSON.stringify(fileInfo, null, 4));
152 output.filePath = GetFilePath(output.configName, output.collection, output.entity, dirs,
true);
154 if (!fs.existsSync(output.filePath)) {
155 console.log(
"Loading file from database");
156 output.data = Connector.RunLoadQuery(dbConfig, fileInfo.query);
157 console.log(
"Writing file " + output.filePath);
158 fs.writeFileSync(output.filePath, JSON.stringify(output.data, null, 4));
160 output.data = JSON.parse(
"" + fs.readFileSync(output.filePath));
162 console.log(
"File Data: " + JSON.stringify(output, null, 4));
163 console.log(
"Done Loading file");
178 function FetchFile(fileInfo, dataFormat, dbdirectory, dbConfig) {
179 console.log(
"FetchFile: fileInfo: " + JSON.stringify(fileInfo, null, 4) +
", dataFormat: " + dataFormat +
", dbDirectory: " + dbdirectory);
180 var fileName = path_module.join(fileInfo.collection +
"_" + fileInfo.name +
"_" + fileInfo.version);
181 if (dataFormat ===
"fhicl") {
183 }
else if (dataFormat ===
"gui") {
184 fileName +=
".gui.json";
188 var filePath = path_module.join(dbdirectory, fileName);
191 "entities.name": fileInfo.name,
192 version: fileInfo.version
194 collection: fileInfo.collection,
195 configurable_entity: fileInfo.name,
196 dbprovider: dbConfig.dbprovider,
198 dataformat: dataFormat
200 var fhiclData = Connector.RunLoadQuery(dbConfig, query,
true);
201 fs.writeFileSync(filePath, JSON.stringify(fhiclData, null, 4));
203 var stat = fs.statSync(filePath);
204 return { fileName: fileName, filePath: filePath, size: stat.size }
207 function MakeColumns(rowIn, index, columns, columnGroups, group) {
208 console.log(
"MakeColumns: rowIn: " + JSON.stringify(rowIn,null,4) +
", index: " + index +
", columns: " + JSON.stringify(columns,null,4) +
", columnGroups: " + JSON.stringify(columnGroups,null,4) +
", group: " + group);
213 for (var p in rowIn) {
214 if (rowIn.hasOwnProperty(p)) {
215 var value = rowIn[p];
216 if (value !== Object(value) && Utils.ContainsName(columns, p,
"name") < 0) {
217 columns.push({ name: p, type:
"string", editable:
true, display:
true, columnGroup: group });
219 }
else if (value === Object(value)) {
221 if (Utils.ContainsName(columnGroups, p,
"name") < 0) {
222 columnGroups.push({ name:
"" + p, text:
"" + p, parentGroup: group });
224 if (value.constructor === Array) {
225 for (var i in value) {
226 if (value.hasOwnProperty(i)) {
227 res = MakeColumns(value[i], index +
"." + i, columns, columnGroups, p);
228 columns = res.columns;
229 columnGroups = res.columnGroups;
230 res.row.name = rowIn.name +
"/" + p +
"___" + i;
231 res.row.index = index +
"." + i;
232 rowOut.children.push(res.row);
236 res = MakeColumns(value, index, columns, columnGroups, p);
237 columns = res.columns;
238 columnGroups = res.columnGroups;
239 for (var pp in res.row) {
240 if (res.row.hasOwnProperty(pp)) {
241 rowOut[pp] = res.row[pp];
251 if (rowOut.children.length === 0)
delete rowOut.children;
252 else if (Utils.ContainsName(columns,
"children",
"name") < 0) { columns.push({ name:
"children", type:
"array", editable:
false, display:
false }); }
254 return { row: rowOut, columns: columns, columnGroups: columnGroups };
263 function ParseSequence(sequence, name) {
264 console.log(
"ParseSequence: sequence: " + JSON.stringify(sequence, null, 4) +
", name: " + name);
266 var hasTable =
false;
269 var columnGroups = [];
270 columns.push({ name:
"index", type:
"number", editable:
false, display:
true, columnGroup: name });
271 columns.push({ name:
"name", type:
"string", editable:
false, display:
true, columnGroup: name });
272 columns.push({ name:
"type", type:
"string", editable:
false, display:
false, columnGroup: name });
273 columns.push({ name:
"comment", type:
"string", editable:
false, display:
false, columnGroup: name });
274 console.log(
"SEQUENCE BEFORE: " + JSON.stringify(sequence,null,4));
275 for (var i = 0; i < sequence.length; ++i) {
276 if (sequence[i] === Object(sequence[i])) {
278 if (sequence[i].constructor === Array) {
280 var arr = ParseSequence(sequence[i], name +
"___" + i);
281 if (arr.children !== undefined && arr.children.length > 0) {
282 children.push({ name: name +
"___" + i, children: arr.children });
283 rows.push({ index: i, name: name +
"___" + i, children: arr.children });
284 if (Utils.ContainsName(columns,
"children",
"name") < 0) {
285 columns.push({ name:
"children", type:
"array", editable:
false, display:
false });
288 if (arr.rows !== undefined && arr.rows.length > 0) {
292 for (var r in arr.rows) {
293 if (arr.rows.hasOwnProperty(r)) {
294 rows.push(arr.rows[r]);
297 for (var c in arr.columns) {
298 if (arr.columns.hasOwnProperty(c) && Utils.ContainsName(columns, arr.columns[c].name,
"name") < 0) {
299 columns.push(arr.columns[c]);
306 console.log(
"Parsing as table: " + JSON.stringify(sequence[i], null, 4));
307 if (!sequence[i].hasOwnProperty(
"name") || sequence[i].name.length === 0 || sequence[i].name == i) {
308 sequence[i].name = name +
"___" + i;
310 if (Utils.ContainsName(columnGroups, name,
"name") < 0) {
311 columnGroups.push({ name:
"" + name, text:
"" + name });
313 var res = MakeColumns(sequence[i], i, columns, columnGroups, name);
316 columns = res.columns;
317 columnGroups = res.columnGroups;
320 var vname = name +
"___" + i;
321 var value = sequence[i];
322 children.push({ name: vname, value: value });
323 rows.push({ index: i, name: vname, value: value });
324 if (Utils.ContainsName(columns,
"value",
"name") < 0) {
325 columns.push({ name:
"value", type:
"string", editable:
true, display:
true, columnGroup: name });
329 var comment = sequence.comment ? sequence.comment :
" ";
340 , columnGroups: columnGroups
343 console.log(
"\nParseSequence returning table: " + JSON.stringify(table, null, 4));
346 var output = { name: name, comment: comment, children: children };
347 console.log(
"\nParseSequence returning: " + JSON.stringify(output, null, 4));
358 function ParseFhiclTable(table, sub) {
359 console.log(
"ParseFhiclTable: table: " + JSON.stringify(table, null, 4) +
", sub: " + sub);
362 var hasSubtables =
false;
363 var comment = table.comment ? table.comment :
" ";
366 for (var e in table.children) {
367 if (table.children.hasOwnProperty(e)) {
368 var element = table.children[e];
369 if (element.type ===
"table" && element.isSequence) {
370 element.type =
"sequence";
373 switch (element.type) {
377 children.push(ParseFhiclTable(element, 1));
380 children.push(ParseSequence(element.children, element.name));
384 case "string_unquoted":
385 case "string_doublequoted":
386 case "string_singlequoted":
388 children.push(element);
391 console.log(
"Unknown type " + element.type +
" encountered!");
397 var obj = { name: table.name, hasSubtables: hasSubtables, children: children, subtables: subtables, type:
"table", comment: comment, columns: DefaultColumns };
398 if (sub === 0 || sub === undefined) {
404 function FindConfigs(configNameFilter, dbConfig) {
405 console.log(
"\nFindConfigs: Request for Named Configurations received");
406 var configsOutput = { Success:
false, data: {}, configs: {} };
408 var configs = Connector.RunGetConfigsQuery(dbConfig).search;
409 console.log(
"CONFIGS: " + JSON.stringify(configs));
410 for (var conf in configs) {
411 if (configs.hasOwnProperty(conf)) {
412 var config = configs[conf];
413 if (config.name.search(configNameFilter) >= 0) {
414 var prefix = config.name;
415 var configNumber = 0;
416 var matches = config.name.match(/(.*?)[_\-]*(\d*)$/);
418 if(matches.length > 1) prefix = matches[1];
419 if(matches.length > 2) configNumber = parseInt(matches[2]);
422 if (!configsOutput.data.hasOwnProperty(prefix)) {
423 configsOutput.data[prefix] = [];
426 configsOutput.data[prefix].push({ version: configNumber, data: JSON.stringify(config.query), name: config.name });
427 configsOutput.configs[config.name] = config.query;
431 if (configsOutput.data.length === 0) {
432 configsOutput.data[
"null"] = {version: 0, data:
""};
433 configsOutput.configs[configNameFilter] = {};
435 configsOutput.Success =
true;
437 console.log(
"Exception caught: " + e.name +
": " + e.message);
439 console.log(
"NamedConfigs complete");
440 return configsOutput;
443 function FindInstances(dbConfig) {
444 console.log(
"FindInstances: Request for Database Instances received");
445 var instancesOutput = { Success:
false, data: [] };
446 instancesOutput.data.push(
"<option value=\"NONE\">Create New Database Instance</option>");
448 var dbs = Connector.RunListDatabasesQuery(dbConfig, {}).search;
449 for (var db in dbs) {
450 if (dbs.hasOwnProperty(db)) {
451 instancesOutput.data.push(
"<option value=" + dbs[db].name +
">" + dbs[db].name +
"</option>");
454 instancesOutput.Success =
true;
456 console.log(
"Exception caught: " + e.name +
": " + e.message);
458 console.log(
"FindInstances complete");
459 return instancesOutput;
469 function LoadConfigFiles(configName, dirs, query, dbConfig) {
470 console.log(
"LoadConfigFiles: configName: " + configName +
", dirs: " + JSON.stringify(dirs, null, 4) +
", query: " + JSON.stringify(query, null, 4));
476 var configFiles = [];
479 configFiles = Connector.RunBuildFilterQuery(dbConfig, configName, query).search;
482 console.log(
"Exception occurred: " + e.name +
": " + e.message);
486 for (var file in configFiles) {
487 if (configFiles.hasOwnProperty(file)) {
488 console.log(
"File info: " + JSON.stringify(configFiles[file], null, 4));
489 var entity = LoadFile(configFiles[file], dirs, dbConfig).entity;
490 var collection = configFiles[file].query.collection;
491 if (!retval.collections.hasOwnProperty(collection)) {
492 retval.collections[collection] = {
497 console.log(
"Adding " + entity +
" to output list");
498 retval.collections[collection].files.push(entity);
501 retval.Success =
true;
503 console.log(
"Exception occurred: " + e.name +
": " + e.message);
509 function SearchFile(fileData, searchKey) {
510 console.log(
"\nSearching for " + searchKey +
" in object " + JSON.stringify(fileData, null, 4));
511 var retval = { name: fileData.name, children: [], columns: fileData.columns };
512 for (var ii in fileData.children) {
513 if (fileData.children.hasOwnProperty(ii)) {
514 var child = fileData.children[ii];
515 console.log(
"Key name: " + child.name);
516 if (child.name.indexOf(searchKey) !== -1) {
517 console.log(
"Adding child to output");
518 retval.children.push(child);
519 }
else if (child.children && child.children.length > 0) {
520 console.log(
"Searching child's children");
521 var table = SearchFile(child, searchKey);
522 if (table.children.length > 0) {
523 retval.children.push(table);
528 console.log(
"\nSearchFile returning " + JSON.stringify(retval, null, 4));
532 function SearchConfigFiles(searchKey, configName, dirs, dbConfig) {
533 console.log(
"\nSearchConfigFiles: keySearch: " + searchKey +
", configName: " + configName +
", dirs: " + JSON.stringify(dirs, null, 4));
537 columns: DefaultColumns,
541 var configFiles = [];
542 var query = FindConfigs(configName, dbConfig).configs[configName];
545 configFiles = Connector.RunBuildFilterQuery(dbConfig, configName, query).search;
548 console.log(
"Exception occurred: " + e.name +
": " + e.message);
552 for (var file in configFiles) {
553 if (configFiles.hasOwnProperty(file)) {
554 console.log(
"File info: " + JSON.stringify(configFiles[file], null, 4));
555 var entity = LoadFile(configFiles[file], dirs, dbConfig).entity;
556 var collection = configFiles[file].query.collection;
557 if (!retval.collectionsTemp.hasOwnProperty(collection)) {
558 retval.collectionsTemp[collection] = {
563 var fileData = GetData(configName, collection, entity, dirs);
564 fileData.name = entity;
565 var fileMatches = SearchFile(fileData, searchKey);
566 if (fileMatches.children.length > 0) {
567 console.log(
"Adding " + entity +
" to output list");
568 retval.collectionsTemp[collection].children.push(fileMatches);
572 retval.Success =
true;
574 console.log(
"Exception occurred: " + e.name +
": " + e.message);
578 for (var ii in retval.collectionsTemp) {
579 if (retval.collectionsTemp.hasOwnProperty(ii)) {
580 retval.collections.push(retval.collectionsTemp[ii]);
583 delete retval.collectionsTemp;
584 console.log(
"\nSearchConfigFiles returning: " + JSON.stringify(retval, null, 4));
600 function GetData(configName, collectionName, entity, dirs) {
601 console.log(
"GetData: configName: " + configName +
", collectionName: " + collectionName +
", entity: " + entity +
", dirs: " + JSON.stringify(dirs, null, 4));
602 var fileName = GetFilePath(configName, collectionName, entity, dirs,
false);
603 if (!fs.existsSync(fileName)) {
throw { name:
"FileNotFoundException", message:
"The requested file was not found" }; }
604 var jsonFile = JSON.parse(
"" + fs.readFileSync(fileName));
605 var jsonBase = ParseFhiclTable({ children: jsonFile.document.converted.guidata, name: entity }, 0);
610 function ReplaceByPath(obj, pathArr, data) {
611 console.log(
"ReplaceByPath: obj: " + JSON.stringify(obj, null, 4) +
", pathArr: " + JSON.stringify(pathArr, null, 4) +
", data: " + JSON.stringify(data, null, 4));
612 if (pathArr.length === 0 && (!obj.name || obj.name === data.name || obj.name === data.name.slice(data.name.indexOf(
"___") + 3))) {
613 if (data.column && data.value) {
614 obj[data.column] = data.value;
615 }
else if (data.type) {
616 for (var i in data) {
617 if (data.hasOwnProperty(i)) {
625 var thisName = pathArr.shift();
627 if (obj.type ===
"table") {
628 index = Utils.ContainsName(obj.children, thisName,
"name");
629 }
else if (obj.type ===
"sequence") {
630 index = thisName.slice(thisName.indexOf(
"___") + 3);
632 console.log(
"Index is " + index +
" (thisName: " + thisName +
", obj.children: " + JSON.stringify(obj.children, null, 4) +
")");
633 if (index === -1 && thisName == data.name) {
634 console.log(
"New Entry! Appending...");
636 for (var i in data) {
637 if (data.hasOwnProperty(i) && data[i] !==
"") {
638 dataObj[i] = data[i];
641 obj.children.push(dataObj);
644 obj.children[index] = ReplaceByPath(obj.children[index], pathArr, data);
658 function UpdateTable(configName, tablePath, data, dirs) {
659 console.log(
"UpdateTable: tablePath:" + tablePath +
", configName: " + configName +
", data: " + JSON.stringify(data, null, 4) +
", dirs: " + JSON.stringify(dirs, null, 4));
660 var tableArray = tablePath.split(
'/');
661 var collection = tableArray.shift();
662 var entity = tableArray.shift();
663 var fileName = GetFilePath(configName, collection, entity, dirs,
false);
664 if (!fs.existsSync(fileName)) {
throw { name:
"FileNotFoundException", message:
"The requested file: \"" + fileName +
"\" was not found" }; }
665 console.log(
"Reading from file " + fileName);
666 var jsonFile = JSON.parse(
"" + fs.readFileSync(fileName));
667 var oldFile = jsonFile.document.converted.guidata;
669 var curName = tableArray.shift();
670 var dataIdx = Utils.ContainsName(oldFile, curName,
"name");
672 console.log(
"Cannot find data in file!");
675 var oldData = oldFile[dataIdx];
676 console.log(
"oldData: " + JSON.stringify(oldData, null, 4));
677 ReplaceByPath(oldData, tableArray, data);
679 jsonFile.document.converted.guidata[dataIdx] = oldData;
680 console.log(
"After replacement, table data is " + JSON.stringify(oldData, null, 4));
682 var filePath = GetTempFilePath(configName, collection, entity, dirs);
683 console.log(
"Writing to file " + filePath);
684 fs.writeFileSync(filePath, JSON.stringify(jsonFile, null, 4));
694 function DiscardWorkingDir(dirs) {
695 console.log(
"Deleting existing trash dir (if any)");
696 Utils.ExecSync(
"rm -rf " + dirs.trash);
697 console.log(
"DiscardWorkingDir: Moving db temp to TRASH: mv " + dirs.db +
" " + dirs.trash);
698 Utils.ExecSync(
"mv " + dirs.db +
" " + dirs.trash);
699 console.log(
"DiscardWorkingDir: Moving temp files to TRASH: mv " + dirs.tmp +
"/* " + dirs.trash);
700 Utils.ExecSync(
"mv " + dirs.tmp +
"/* " + dirs.trash);
701 console.log(
"DiscardWorkingDir: Deleting temp directory: rmdir " + dirs.tmp);
702 Utils.ExecSync(
"rmdir " + dirs.tmp);
717 function SaveConfigurationChanges(oldConfig, newConfig, files, dirs, dbConfig) {
718 console.log(
"Saving Configuration Changes, oldConfig: " + oldConfig +
", newConfig: " + newConfig +
", files: " + JSON.stringify(files, null, 4) +
", dirs: " + JSON.stringify(dirs, null, 4));
719 var fileInfo = Connector.RunBuildFilterQuery(dbConfig, oldConfig).search;
723 if (files.hasOwnProperty(f)) {
724 console.log(
"Current file information: " + JSON.stringify(files[f], null, 4));
725 var collectionName = files[f].collection;
726 var entities = files[f].entities;
727 var entity = files[f].entity ? files[f].entity :
"notprovided";
728 var version = files[f].version;
729 var thisFileInfo = {};
730 for (var fi in fileInfo) {
731 if (fileInfo.hasOwnProperty(fi)) {
732 if (collectionName === fileInfo[fi].query.collection && Utils.ContainsString(entities, fileInfo[fi].query.filter[
"entities.name"]) != -1) {
733 console.log(
"Matched file information to Document: " + JSON.stringify(fileInfo[fi], null, 4));
734 thisFileInfo = fileInfo[fi];
735 entity = fileInfo[fi].query.filter[
"entities.name"];
736 fileInfo.splice(fi, 1);
741 console.log(
"Getting metadata from original and changed files");
742 var modified = GetFilePath(oldConfig, collectionName, entity, dirs,
false);
743 var newMetadata = ReadFileMetadata(dirs, thisFileInfo, dbConfig);
747 newMetadata.version = version;
748 console.log(
"newMetadata: " + JSON.stringify(newMetadata, null, 4));
750 console.log(
"Checking metadata version strings");
751 while (VersionExists(entity, collectionName, newMetadata.version, dbConfig)) {
752 console.log(
"Inferring new version string...");
753 version = Utils.Uniquify(newMetadata.version);
754 console.log(
"Changing version from " + newMetadata.version +
" to " + version);
755 newMetadata.version = version;
758 console.log(
"Prepending changelog");
759 newMetadata.changelog = files[f].changelog + newMetadata.changelog;
761 console.log(
"Writing new metadata to file");
762 if (WriteFileMetadata(newMetadata, modified)) {
764 console.log(
"Running store query");
765 var data =
"" + fs.readFileSync(modified);
766 console.log(
"Writing " + data +
" to database");
767 Connector.RunStoreQuery(dbConfig, data, collectionName, newMetadata.version, entity,
"gui", newConfig);
769 console.log(
"ERROR: Could not find file " + modified);
770 console.log(
"Please check if it exists.");
775 console.log(
"Running addconfig for unmodified files: " + JSON.stringify(fileInfo, null, 4));
776 for (f in fileInfo) {
777 if (fileInfo.hasOwnProperty(f)) {
778 var unmodifiedVersion = GetVersion(fileInfo[f], dirs, dbConfig);
779 Connector.RunAddConfigQuery(dbConfig, newConfig, unmodifiedVersion, fileInfo[f].query.collection, { name: fileInfo[f].query.filter[
"entities.name"] });
783 DiscardWorkingDir(dirs);
794 function CreateNewConfiguration(configName, configData, dbConfig) {
795 console.log(
"CreateNewConfigration: configName: " + configName +
", configData: " + JSON.stringify(configData, null, 4));
800 for (var d in configData.entities) {
801 if (configData.entities.hasOwnProperty(d)) {
802 var entityData = configData.entities[d];
803 console.log(
"Entity: " + JSON.stringify(entityData, null, 4));
804 query.operations.push({
805 filter: { version: entityData.version,
"entities.name": entityData.name },
806 configuration: configName,
807 collection: entityData.collection,
808 dbprovider: dbConfig.dbprovider,
809 operation:
"addconfig",
815 Connector.RunNewConfigQuery(dbConfig, query);
824 function ReadConfigurationMetadata(configName, dirs, dbConfig) {
825 console.log(
"ReadConfigurationMetadata: configname: " + configName +
", dirs: " + JSON.stringify(dirs, null, 4));
827 var data = Connector.RunBuildFilterQuery(dbConfig, configName ).search;
832 for (var i in data) {
833 if (data.hasOwnProperty(i)) {
834 console.log(
"Loading metadata: File " + i +
" of " + data.length);
835 var version = GetVersion(data[i], dirs, dbConfig);
836 metadata.entities.push({ name: data[i].query.filter[
"entities.name"], file: data[i].name +
"_" + data[i].query.collection, version: version, collection: data[i].query.collection });
840 console.log(
"Returning entity list: " + JSON.stringify(metadata.entities, null, 4));
852 function GetVersion(query, dirs, dbConfig) {
853 console.log(
"\"GetVersion\": {\"query\":" + JSON.stringify(query, null, 4) +
", \"dirs\":" + JSON.stringify(dirs, null, 4) +
"},");
854 var ver = LoadFile(query, dirs, dbConfig).data.version;
855 console.log(
"GetVersion Returning " + ver);
868 function ReadFileMetadata(dirs, query, dbConfig) {
869 console.log(
"ReadFileMetadata: query=" + JSON.stringify(query, null, 4) +
", dirs=" + JSON.stringify(dirs, null, 4) +
", dbConfig: " + JSON.stringify(dbConfig, null, 4));
871 var jsonFile = LoadFile(query, dirs, dbConfig).data;
872 if (jsonFile.changelog === undefined) {
873 jsonFile.changelog =
"";
876 entities: jsonFile.entities,
877 bookkeeping: jsonFile.bookkeeping,
878 aliases: jsonFile.aliases,
879 configurations: jsonFile.configurations,
880 version: jsonFile.version,
881 changelog: jsonFile.changelog,
882 collection: query.query.collection
885 console.log(
"ReadFileMetadata returning: " + JSON.stringify(metadata, null, 4));
895 function WriteFileMetadata(newMetadata, fileName) {
896 console.log(
"WriteFileMetadata: newMetadata=" + JSON.stringify(newMetadata, null, 4) +
", fileName=" + fileName);
898 console.log(
"Reading file: " + fileName);
899 if (!fs.existsSync(fileName))
return false;
900 var jsonFile = JSON.parse(
"" + fs.readFileSync(fileName));
903 console.log(
"Setting fields: " + JSON.stringify(newMetadata, null, 4));
904 jsonFile.configurable_entity = newMetadata.configurable_entity;
905 jsonFile.bookkeeping = newMetadata.bookkeeping;
906 jsonFile.aliases = newMetadata.aliases;
907 jsonFile.configurations = newMetadata.configurations;
908 jsonFile.version = newMetadata.version;
909 jsonFile.document.converted.changelog = newMetadata.changelog;
911 console.log(
"Writing data to file");
913 fs.writeFileSync(fileName, JSON.stringify(jsonFile, null, 4));
923 function GetDirectories(userId, dbConfig) {
924 console.log(
"GetDirectories: userid=" + userId);
925 if (dbConfig.baseDir ===
"" || !dbConfig.baseDir) {
926 dbConfig.baseDir = process.env[
"HOME"] +
"/databases";
927 console.log(
"WARNING: ARTDAQ_DATABASE_DATADIR not set. Using $HOME/databases instead!!!");
930 if (!fs.existsSync(dbConfig.baseDir)) {
931 console.log(
"ERROR: Base Directory " +dbConfig.baseDir +
" doesn't exist!!!");
932 throw { name:
"BaseDirectoryMissingException", message:
"ERROR: Base Directory doesn't exist!!!" };
934 if (!fs.existsSync(path_module.join(dbConfig.baseDir,
"db"))) {
935 fs.mkdirSync(path_module.join(dbConfig.baseDir,
"db"));
937 if (!fs.existsSync(path_module.join(dbConfig.baseDir,
"tmp"))) {
938 fs.mkdirSync(path_module.join(dbConfig.baseDir,
"tmp"));
940 if (!fs.existsSync(path_module.join(dbConfig.baseDir,
"TRASH"))) {
941 fs.mkdirSync(path_module.join(dbConfig.baseDir,
"TRASH"));
945 var db = path_module.join(dbConfig.baseDir,
"db", userId);
946 var tmp = path_module.join(dbConfig.baseDir,
"tmp", userId);
947 var trash = path_module.join(dbConfig.baseDir,
"TRASH", userId);
950 if (!fs.existsSync(db)) {
953 if (!fs.existsSync(tmp)) {
956 if (!fs.existsSync(trash)) {
960 return { db: db, tmp: tmp, trash: trash };
971 function VersionExists(entity, collection, version, dbConfig) {
972 console.log(
"\"VersionExists\": { version:\"" + version +
"\", entity:" + JSON.stringify(entity, null, 4) +
", collection: \"" + collection +
"\"}");
975 "entities.name": entity
977 collection: collection,
978 dbprovider: dbConfig.dbprovider,
979 operation:
"findversions",
982 var vers = Connector.RunGetVersionsQuery(dbConfig, query).search;
983 console.log(
"Search returned: " + JSON.stringify(vers, null, 4));
984 return Utils.ContainsName(vers, version,
"name") >= 0;
993 if (fs.existsSync(
"/tmp/node_db_lockfile")) {
994 if (Date.now() - fs.fstatSync(
"/tmp/node_db_lockfile").ctime.getTime() > 1000) {
995 console.log(
"Stale Lockfile detected, deleting...");
998 console.log(
"Lockfile detected and is not stale, aborting...");
1003 fs.writeFileSync(
"/tmp/node_db_lockfile",
"locked");
1012 console.log(
"Unlock");
1013 fs.unlinkSync(
"/tmp/node_db_lockfile");
1018 db.RO_GetData =
function (post, dbConfig) {
1019 console.log(
"RO_GetData: " + JSON.stringify(post, null, 4));
1020 var ret = { Success:
false, data: {} };
1022 ret.data = GetData(post.configName, post.collection, post.entity, GetDirectories(post.user, dbConfig));
1025 console.log(
"Exception occurred: " + e.name +
": " + e.message);
1028 console.log(
"GetData complete");
1032 db.RW_MakeNewConfig =
function (post, dbConfig) {
1034 console.log(
"RW_MakeNewConfig: Request to make new configuration received: " + JSON.stringify(post, null, 4));
1035 var res = { Success:
false };
1039 var configs = Connector.RunGetConfigsQuery(dbConfig).search;
1040 if (!Utils.ValidatePath(post.name)) {
1041 console.log(
"Invalid name detected!");
1044 while (Utils.ContainsName(configs, post.name,
"name") >= 0) {
1045 console.log(
"Inferring new configuration name");
1046 post.name = Utils.Uniquify(post.name);
1050 console.log(
"Exception occurred: " + e.name +
": " + e.message);
1053 console.log(
"Creating Configuration");
1055 CreateNewConfiguration(post.name, JSON.parse(post.config), dbConfig);
1058 console.log(
"Exception occurred: " + e.name +
": " + e.message);
1062 console.log(
"MakeNewConfig completed");
1068 db.RW_saveConfig =
function (post, dbConfig) {
1070 console.log(
"RW_saveConfig: Request to save configuration recieved. Configuration data: " + JSON.stringify(post, null, 4));
1071 var res = { Success:
false };
1075 console.log(
"Checking for unique Configuration name");
1076 var configs = Connector.RunGetConfigsQuery(dbConfig).search;
1077 if (!Utils.ValidatePath(post.newConfigName) || Utils.ContainsName(configs, post.oldConfigName,
"name") < 0) {
1078 console.log(
"Invalid name detected!");
1081 while (Utils.ContainsName(configs, post.newConfigName,
"name") >= 0) {
1082 console.log(
"Inferring new configuration name");
1083 post.newConfigName = Utils.Uniquify(post.newConfigName);
1087 console.log(
"Exception occurred: " + e.name +
": " + e.message);
1091 console.log(
"Updating Configuration Files");
1092 SaveConfigurationChanges(post.oldConfigName, post.newConfigName, post.files, GetDirectories(post.user, dbConfig), dbConfig);
1095 console.log(
"Exception occurred: " + e.name +
": " + e.message);
1099 console.log(
"SaveConfig completed");
1105 db.RO_LoadNamedConfig =
function (post, dbConfig) {
1106 console.log(
"RO_LoadNamedConfig: Request for configuration with name \"" + post.configName +
"\" and search query \"" + post.query +
"\" received.");
1107 if (post.query.length === 0 || post.configName ===
"No Configurations Found") {
1108 return { collections: [] };
1110 return LoadConfigFiles(post.configName, GetDirectories(post.user, dbConfig), JSON.parse(post.query), dbConfig);
1113 db.RW_discardConfig =
function (post, dbConfig) {
1114 console.log(
"RW_discardConfig: Discarding configuration with parameters: " + JSON.stringify(post, null, 4));
1115 DiscardWorkingDir(GetDirectories(post.user, dbConfig));
1116 return { Success:
true };
1119 db.RO_AddOrUpdate =
function (post, dbConfig) {
1120 console.log(
"RO_AddOrUpdate: Request to update table row recieved: " + JSON.stringify(post, null, 4));
1121 UpdateTable(post.configName, post.table, post.row, GetDirectories(post.user, dbConfig));
1122 return { Success:
true };
1125 db.RO_Update =
function (post, dbConfig) {
1126 console.log(
"RO_Update: Request to update table received: " + JSON.stringify(post, null, 4));
1127 UpdateTable(post.configName, post.table, {
id: post.id, name: post.name, column: post.column, value: post.value }, GetDirectories(post.user, dbConfig));
1128 console.log(
"Update Complete");
1129 return { Success:
true };
1132 db.RO_LoadConfigMetadata =
function (post, dbConfig) {
1133 console.log(
"RO_LoadConfigMetadata: Request to load configuration metadata received: " + JSON.stringify(post, null, 4) +
", dbConfig: " + JSON.stringify(dbConfig, null, 4));
1134 var ret = { Success:
false, data: {} };
1136 ret.data = ReadConfigurationMetadata(post.configName, GetDirectories(post.user, dbConfig), dbConfig);
1139 console.log(
"Exception caught: " + e.name +
": " + e.message);
1141 console.log(
"LoadConfigMetadata complete");
1145 db.RO_LoadFileMetadata =
function (post, dbConfig) {
1146 console.log(
"RO_LoadFileMetadata: Request to load file metadata received: " + JSON.stringify(post, null, 4));
1147 var ret = { Success:
false, data: {} };
1148 var dirs = GetDirectories(post.user, dbConfig);
1153 var search = Connector.RunBuildFilterQuery(dbConfig, post.configName).search;
1154 for (var s in search) {
1155 if (search.hasOwnProperty(s)) {
1156 if (search[s].query.collection === post.collection && search[s].query.filter[
"entities.name"] === post.entity) {
1163 console.log(
"Exception caught: " + e.name +
": " + e.message);
1167 ret.data = ReadFileMetadata(dirs, query, dbConfig);
1170 console.log(
"Exception caught: " + e.name +
": " + e.message);
1173 console.log(
"LoadFileMetadata complete");
1177 db.RW_UploadConfigurationFile =
function (post, dbConfig) {
1179 console.log(
"RW_UploadConfigurationFile: Recieved request to upload file: " + JSON.stringify(post, null, 4));
1182 var ret = { Success:
false };
1184 while (VersionExists(post.entity, post.collection, post.version, dbConfig)) {
1185 console.log(
"Version already exists. Running uniquifier...");
1186 post.version = Utils.Uniquify(post.version);
1190 console.log(
"Exception caught: " + e.name +
": " + e.message);
1194 console.log(
"Running store fhicl query");
1196 Connector.RunStoreQuery(dbConfig, post.file, post.collection, post.version, post.entity, post.type);
1199 console.log(
"Exception caught: " + e.name +
": " + e.message);
1203 console.log(
"UploadConfigurationFile complete");
1209 db.RO_DownloadConfigurationFile =
function (post, dbConfig) {
1210 console.log(
"RO_DownloadConfigurationFile: Request to download file(s) received: " + JSON.stringify(post, null, 4));
1211 var dirs = GetDirectories(post.user, dbConfig);
1212 var configObj = JSON.parse(post.config);
1214 if (configObj.entities.length === 1) {
1215 console.log(
"Single file mode: Fetching file...");
1216 var fileInfo = FetchFile(configObj.entities[0], post.type, dirs.db, dbConfig);
1219 'Content-Type':
'text/plain',
1220 'Content-Length': fileInfo.size,
1221 'Content-Disposition':
'attachment filename=' + fileInfo.fileName
1223 console.log(
"Headers: " + JSON.stringify(fclhdrs, null, 4) +
", fileInfo: " + JSON.stringify(fileInfo, null, 4));
1225 var fclStream = fs.createReadStream(fileInfo.filePath);
1226 db.emit(
"stream", fclStream, fclhdrs, 200);
1227 }
else if (configObj.entities.length > 1) {
1229 for (var e in configObj.entities) {
1230 if (configObj.entities.hasOwnProperty(e)) {
1231 args.push(FetchFile(configObj.entities[e], post.type, dirs.db, dbConfig).fileName);
1234 var fileName = post.tarFileName +
".tar.gz";
1236 'Content-Type':
"application/x-gzip",
1237 'Content-Disposition':
'attachment filename=' + fileName
1240 console.log(
"Spawning: tar " + args.join(
" "));
1241 var tar = child_process.spawn(
"tar", args, { cwd: dirs.db, stdio: [0,
'pipe', 0] });
1242 db.emit(
"stream", tar.stdout, tarhdrs, 200);
1246 console.log(
"Exception caught: " + err.name +
": " + err.message);
1248 var s =
new stream.Readable();
1249 s._read =
function noop() { };
1254 'Content-Type':
'text/plain'
1256 db.emit(
"stream", s, errhdrs, 500);
1261 db.RO_NamedConfigs =
function (post, dbConfig) {
1262 console.log(
"RO_NamedConfigs: Request for Named Configurations received");
1263 var filter =
"" + post.configFilter;
1264 return FindConfigs(filter, dbConfig);
1267 db.RW_updateDbConfig =
function (post, dbConfig) {
1268 console.log(
"RW_updateDbConfig: Request to update module configuration received: " + JSON.stringify(post, null, 4));
1269 var output = { Success:
false };
1270 if (fs.existsSync(post.baseDir) && (post.dbprovider ===
"filesystem" || post.dbprovider ===
"mongo")) {
1271 if (dbConfig.baseDir !== post.baseDir) {
1272 dbConfig.baseDir = post.baseDir;
1273 db.emit(
"message", { name:
"db", target:
"baseDir", data: post.baseDir });
1275 if (dbConfig.dbprovider !== post.dbprovider) {
1276 dbConfig.dbprovider = post.dbprovider;
1277 db.emit(
"message", { name:
"db", target:
"dbprovider", data: post.dbprovider });
1279 if (dbConfig.instanceName !== post.instanceName) {
1280 dbConfig.instanceName = post.instanceName;
1281 db.emit(
"message", { name:
"db", target:
"instanceName", data: post.instanceName });
1283 console.log(
"DB Config is now: " + JSON.stringify(dbConfig, null, 4));
1284 output.Success =
true;
1289 db.RO_SearchLoadedConfig =
function (post, dbConfig) {
1290 console.log(
"RO_SearchLoadedConfig: Searching for keys containing name \"" + post.searchKey +
"\" from configuration name \"" + post.configName +
"\" received.");
1291 if (post.configName ===
"No Configurations Found") {
1292 return { Success:
false, collections: [] };
1294 return SearchConfigFiles(post.searchKey, post.configName, GetDirectories(post.user, dbConfig), dbConfig);
1297 db.RW_makeNewDBInstance =
function (post, dbConfig) {
1298 console.log(
"RW_makeNewDBInstance: Creating new database instance \"" + post.name +
"\".");
1299 dbConfig.instanceName = post.name;
1300 return { Success:
true };
1303 db.RW_AddEntityToFile =
function (post, dbConfig) {
1304 console.log(
"RW_AddEntityToFile: request to add entity name \"" + post.name +
"\" to configuration file " + post.configName +
"/" + post.collection +
"/" + post.entity);
1305 return { Success:
true };
1309 db.GET_EntitiesAndVersions =
function (dbConfig) {
1310 console.log(
"GET_EntitiesAndVersions: Request for current Entities and Versions received");
1316 var entities = Connector.RunGetEntitiesQuery(dbConfig).search;
1317 console.log(
"Returned entities: " + JSON.stringify(entities, null, 4));
1318 for (var ent in entities) {
1319 if (entities.hasOwnProperty(ent)) {
1320 var entity = entities[ent];
1321 var versions = Connector.RunGetVersionsQuery(dbConfig, entity.query);
1322 if (Utils.ContainsName(output.collections, entity.query.collection,
"name") < 0) {
1323 output.collections.push({ name: entity.query.collection, entities: [] });
1326 var index = Utils.ContainsName(output.collections, entity.query.collection,
"name");
1329 collection: entity.query.collection,
1333 output.collections[index].entities.push(entityObj);
1336 output.Success =
true;
1338 console.log(
"Exception caught: " + e.name +
": " + e.message);
1340 console.log(
"EntitiesAndVersions complete");
1344 db.GET_getDbConfig =
function (dbConfig) {
1345 console.log(
"GET_getDbConfig Request for Database Module Configuration received");
1347 var instances = FindInstances(dbConfig);
1350 baseDir: dbConfig.baseDir,
1351 dbprovider: dbConfig.dbprovider,
1352 instanceName: dbConfig.instanceName,
1353 data: instances.data
1359 db.MasterInitFunction =
function (workerData, config) {
1360 var dbConfig = MakeDbConfig(config);
1361 workerData[
"db"] = dbConfig;
1362 GetDirectories(
"", dbConfig);
1365 module.exports =
function (moduleHolder) {
1366 moduleHolder[
"db"] = db;