otsdaq  v1_01_03
 All Classes Namespaces Functions
ConfigurationView.cc
1 #include "otsdaq-core/ConfigurationDataFormats/ConfigurationView.h"
2 
3 #include <iostream>
4 #include <sstream>
5 #include <cstdlib>
6 #include <regex>
7 
8 using namespace ots;
9 
10 #undef __MF_SUBJECT__
11 #define __MF_SUBJECT__ "ConfigurationView"
12 #undef __MF_HDR__
13 #define __MF_HDR__ tableName_ << ":v" << version_ << ":" << __COUT_HDR_FL__
14 
15 const unsigned int ConfigurationView::INVALID = -1;
16 
17 //==============================================================================
18 ConfigurationView::ConfigurationView(const std::string &name)
19 : uniqueStorageIdentifier_ (""),
20  tableName_ (name),
21  version_ (ConfigurationVersion::INVALID),
22  comment_ (""),
23  author_ (""),
24  creationTime_ (time(0)),
25  lastAccessTime_ (0),
26  colUID_ (INVALID),
27  fillWithLooseColumnMatching_(false),
28  sourceColumnMismatchCount_ (0),
29  sourceColumnMissingCount_ (0)
30 {}
31 
32 //==============================================================================
33 ConfigurationView::~ConfigurationView(void)
34 {}
35 
36 //==============================================================================
37 //operator=
38 // Do NOT allow!... use ConfigurationView::copy
39 // copy is used to maintain consistency with version, creationTime, lastAccessTime, etc)
40 ConfigurationView& ConfigurationView::operator=(const ConfigurationView src)
41 {
42  __SS__ << "Invalid use of operator=... Should not directly copy a ConfigurationView. Please use ConfigurationView::copy(sourceView,author,comment)";
43  throw std::runtime_error(ss.str());
44 }
45 
46 //==============================================================================
47 ConfigurationView& ConfigurationView::copy(const ConfigurationView &src,
48  ConfigurationVersion destinationVersion,
49  const std::string &author)
50 {
51  tableName_ = src.tableName_;
52  version_ = destinationVersion;
53  comment_ = src.comment_;
54  author_ = author; //take new author
55  //creationTime_ = time(0); //don't change creation time
56  lastAccessTime_ = time(0);
57  columnsInfo_ = src.columnsInfo_;
58  theDataView_ = src.theDataView_;
59  sourceColumnNames_ = src.sourceColumnNames_;
60  init(); // verify consistency
61  return *this;
62 }
63 
64 //==============================================================================
65 //init
66 // Should be called after being filled to setup special members
67 // and verify consistency.
68 // e.g. identifying the UID column
69 //
70 // Note: this function also sanitizes yes/no, on/off, and true/false types
71 void ConfigurationView::init(void)
72 {
73  try
74  {
75  //verify column names are unique
76  // make set of names,.. and CommentDescription == COMMENT
77  std::set<std::string> colNameSet;
78  std::string capsColName, colName;
79  for(auto &colInfo: columnsInfo_)
80  {
81  colName = colInfo.getStorageName();
82  if(colName == "COMMENT_DESCRIPTION")
83  colName = "COMMENT";
84  capsColName = "";
85  for(unsigned int i=0;i<colName.size(); ++i)
86  {
87  if(colName[i] == '_') continue;
88  capsColName += colName[i];
89  }
90 
91  colNameSet.emplace(capsColName);
92  }
93 
94  if(colNameSet.size() != columnsInfo_.size())
95  {
96  __SS__ << "Configuration Error:\t" <<
97  " Columns names must be unique! There are " << columnsInfo_.size() <<
98  " columns and the unique name count is " << colNameSet.size() << std::endl;
99  __MOUT_ERR__ << "\n" << ss.str();
100 
101  throw std::runtime_error(ss.str());
102  }
103 
104  getOrInitColUID(); //setup UID column
105 
106  //require one comment column
107  unsigned int colPos;
108  if((colPos = findColByType(ViewColumnInfo::TYPE_COMMENT)) != INVALID)
109  {
110  if(columnsInfo_[colPos].getName() != "CommentDescription")
111  {
112  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_COMMENT <<
113  " data type column must have name=" <<
114  "CommentDescription" << std::endl;
115  __MOUT_ERR__ << "\n" << ss.str();
116  throw std::runtime_error(ss.str());
117  }
118 
119  if(findColByType(ViewColumnInfo::TYPE_COMMENT,colPos+1) != INVALID) //found two!
120  {
121  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_COMMENT <<
122  " data type in column " <<
123  columnsInfo_[colPos].getName() <<
124  " is repeated. This is not allowed." << std::endl;
125  __MOUT_ERR__ << "\n" << ss.str();
126  throw std::runtime_error(ss.str());
127  }
128 
129  if(colPos != getNumberOfColumns()-3)
130  {
131  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_COMMENT <<
132  " data type column must be 3rd to last (in column " <<
133  getNumberOfColumns()-3 << ")." << std::endl;
134  __MOUT_ERR__ << "\n" << ss.str();
135  throw std::runtime_error(ss.str());
136  }
137  }
138  else
139  {
140  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_COMMENT <<
141  " data type column " <<" is missing. This is not allowed." << std::endl;
142  __MOUT_ERR__ << "\n" << ss.str();
143  throw std::runtime_error(ss.str());
144  }
145 
146  //require one author column
147  if((colPos = findColByType(ViewColumnInfo::TYPE_AUTHOR)) != INVALID)
148  {
149  if(findColByType(ViewColumnInfo::TYPE_AUTHOR,colPos+1) != INVALID) //found two!
150  {
151  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_AUTHOR <<
152  " data type in column " <<
153  columnsInfo_[colPos].getName() <<
154  " is repeated. This is not allowed." << std::endl;
155  __MOUT_ERR__ << "\n" << ss.str();
156  throw std::runtime_error(ss.str());
157  }
158 
159  if(colPos != getNumberOfColumns()-2)
160  {
161  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_AUTHOR <<
162  " data type column must be 2nd to last (in column " <<
163  getNumberOfColumns()-2 << ")." << std::endl;
164  __MOUT_ERR__ << "\n" << ss.str();
165  throw std::runtime_error(ss.str());
166  }
167  }
168  else
169  {
170  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_AUTHOR <<
171  " data type column " <<" is missing. This is not allowed." << std::endl;
172  __MOUT_ERR__ << "\n" << ss.str();
173  throw std::runtime_error(ss.str());
174  }
175 
176  //require one timestamp column
177  if((colPos = findColByType(ViewColumnInfo::TYPE_TIMESTAMP)) != INVALID)
178  {
179  if(findColByType(ViewColumnInfo::TYPE_TIMESTAMP,colPos+1) != INVALID) //found two!
180  {
181  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_TIMESTAMP <<
182  " data type in column " <<
183  columnsInfo_[colPos].getName() << " is repeated. This is not allowed." <<
184  std::endl;
185  __MOUT_ERR__ << "\n" << ss.str();
186  throw std::runtime_error(ss.str());
187  }
188 
189  if(colPos != getNumberOfColumns()-1)
190  {
191  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_TIMESTAMP <<
192  " data type column must be last (in column " <<
193  getNumberOfColumns()-1 << ")." << std::endl;
194  __MOUT_ERR__ << "\n" << ss.str();
195  throw std::runtime_error(ss.str());
196  }
197  }
198  else
199  {
200  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_TIMESTAMP <<
201  " data type column " <<" is missing. This is not allowed." << std::endl;
202  __MOUT_ERR__ << "\n" << ss.str();
203  throw std::runtime_error(ss.str());
204  }
205 
206  //check that UID is really unique ID (no repeats)
207  // and ... allow letters, numbers, dash, underscore
208  // and ... force size 1
209  std::set<std::string /*uid*/> uidSet;
210  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
211  {
212  if(uidSet.find(theDataView_[row][colUID_]) != uidSet.end())
213  {
214  __SS__ << ("Entries in UID are not unique. Specifically at row=" +
215  std::to_string(row) + " value=" + theDataView_[row][colUID_])<< std::endl;
216  __MOUT_ERR__ << "\n" << ss.str();
217  throw std::runtime_error(ss.str());
218  }
219 
220  if(theDataView_[row][colUID_].size() == 0)
221  {
222  __SS__ << "An invalid UID '" << theDataView_[row][colUID_] << "' " <<
223  " was identified. UIDs must contain at least 1 character." <<
224  std::endl;
225  throw std::runtime_error(ss.str());
226  }
227 
228  for(unsigned int i=0;i<theDataView_[row][colUID_].size();++i)
229  if(!(
230  (theDataView_[row][colUID_][i] >= 'A' && theDataView_[row][colUID_][i] <= 'Z') ||
231  (theDataView_[row][colUID_][i] >= 'a' && theDataView_[row][colUID_][i] <= 'z') ||
232  (theDataView_[row][colUID_][i] >= '0' && theDataView_[row][colUID_][i] <= '9') ||
233  (theDataView_[row][colUID_][i] == '-' || theDataView_[row][colUID_][i] <= '_')
234  ))
235  {
236  __SS__ << "An invalid UID '" << theDataView_[row][colUID_] << "' " <<
237  " was identified. UIDs must contain only letters, numbers," <<
238  "dashes, and underscores." << std::endl;
239  throw std::runtime_error(ss.str());
240  }
241 
242  uidSet.insert(theDataView_[row][colUID_]);
243  }
244  if(uidSet.size() != getNumberOfRows())
245  {
246  __SS__ << "Entries in UID are not unique!" <<
247  "There are " << getNumberOfRows() <<
248  " rows and the unique UID count is " << uidSet.size() << std::endl;
249  __MOUT_ERR__ << "\n" << ss.str();
250  throw std::runtime_error(ss.str());
251  }
252 
253  //check that any TYPE_UNIQUE_DATA columns are really unique (no repeats)
254  colPos = (unsigned int)-1;
255  while((colPos = findColByType(ViewColumnInfo::TYPE_UNIQUE_DATA,colPos+1)) != INVALID)
256  {
257  std::set<std::string /*unique data*/> uDataSet;
258  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
259  {
260  if(uDataSet.find(theDataView_[row][colPos]) != uDataSet.end())
261  {
262  __SS__ << "Entries in Unique Data column " <<
263  columnsInfo_[colPos].getName() <<
264  (" are not unique. Specifically at row=" +
265  std::to_string(row) + " value=" + theDataView_[row][colPos]) <<
266  std::endl;
267  __MOUT_ERR__ << "\n" << ss.str();
268  throw std::runtime_error(ss.str());
269  }
270  uDataSet.insert(theDataView_[row][colPos]);
271  }
272  if(uDataSet.size() != getNumberOfRows())
273  {
274  __SS__ << "Entries in Unique Data column " <<
275  columnsInfo_[colPos].getName() << " are not unique!" <<
276  "There are " << getNumberOfRows() <<
277  " rows and the unique data count is " << uDataSet.size() << std::endl;
278  __MOUT_ERR__ << "\n" << ss.str();
279  throw std::runtime_error(ss.str());
280  }
281  }
282 
283  auto rowDefaults = getDefaultRowValues();
284 
285  //check that column types are well behaved
286  // - check that fixed choice data is one of choices
287  // - sanitize booleans
288  // - check that child link I are unique
289  // note: childLinkId refers to childLinkGroupIDs AND childLinkUIDs
290  std::set<std::string> groupIdIndexes, childLinkIndexes, childLinkIdLabels;
291  unsigned int groupIdIndexesCount = 0, childLinkIndexesCount = 0, childLinkIdLabelsCount = 0;
292  for(unsigned int col = 0; col < getNumberOfColumns(); ++col)
293  {
294  if(columnsInfo_[col].getType() == ViewColumnInfo::TYPE_FIXED_CHOICE_DATA)
295  {
296  const std::vector<std::string>& theDataChoices =
297  columnsInfo_[col].getDataChoices();
298 
299  //check if arbitrary values allowed
300  if(theDataChoices.size() && theDataChoices[0] ==
301  "arbitraryBool=1")
302  continue; //arbitrary values allowed
303 
304  bool found;
305  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
306  {
307  found = false;
308  //check against default value first
309  if(theDataView_[row][col] == rowDefaults[col])
310  continue; //default is always ok
311 
312  for(const auto &choice:theDataChoices)
313  {
314  if(theDataView_[row][col] == choice)
315  {
316  found = true;
317  break;
318  }
319  }
320  if(!found)
321  {
322  __SS__ << "Configuration Error:\t'" << theDataView_[row][col] << "' in column " <<
323  columnsInfo_[col].getName() << " is not a valid Fixed Choice option. " <<
324  "Possible values are as follows: ";
325 
326  for(unsigned int i = 0; i < columnsInfo_[col].getDataChoices().size(); ++i)
327  {
328  if(i) ss << ", ";
329  ss << columnsInfo_[col].getDataChoices()[i];
330  }
331  ss << "." << std::endl;
332  __MOUT_ERR__ << "\n" << ss.str();
333  throw std::runtime_error(ss.str());
334  }
335  }
336  }
337  if(columnsInfo_[col].isChildLink())
338  {
339  //check if forcing fixed choices
340 
341  const std::vector<std::string>& theDataChoices =
342  columnsInfo_[col].getDataChoices();
343 
344  //check if arbitrary values allowed
345  if(!theDataChoices.size() ||
346  theDataChoices[0] == "arbitraryBool=1")
347  continue; //arbitrary values allowed
348 
349  bool found;
350  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
351  {
352  found = false;
353 
354  for(const auto &choice:theDataChoices)
355  {
356  if(theDataView_[row][col] == choice)
357  {
358  found = true;
359  break;
360  }
361  }
362  if(!found)
363  {
364  __SS__ << "Configuration Error:\t'" << theDataView_[row][col] << "' in column " <<
365  columnsInfo_[col].getName() << " is not a valid Fixed Choice option. " <<
366  "Possible values are as follows: ";
367 
368  for(unsigned int i = 0; i < columnsInfo_[col].getDataChoices().size(); ++i)
369  {
370  if(i) ss << ", ";
371  ss << columnsInfo_[col].getDataChoices()[i];
372  }
373  ss << "." << std::endl;
374  __MOUT_ERR__ << "\n" << ss.str();
375  throw std::runtime_error(ss.str());
376  }
377  }
378  }
379  else if(columnsInfo_[col].getType() == ViewColumnInfo::TYPE_ON_OFF)
380  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
381  {
382  if (theDataView_[row][col] == "1" || theDataView_[row][col] == "on" || theDataView_[row][col] == "On" || theDataView_[row][col] == "ON")
383  theDataView_[row][col] = ViewColumnInfo::TYPE_VALUE_ON;
384  else if (theDataView_[row][col] == "0" || theDataView_[row][col] == "off" || theDataView_[row][col] == "Off" || theDataView_[row][col] == "OFF")
385  theDataView_[row][col] = ViewColumnInfo::TYPE_VALUE_OFF;
386  else
387  {
388  __SS__ << "Configuration Error:\t" << theDataView_[row][col] << " in column " <<
389  columnsInfo_[col].getName() << " is not a valid Type (On/Off) std::string. Possible values are 1, on, On, ON, 0, off, Off, OFF." << std::endl;
390  __MOUT_ERR__ << "\n" << ss.str();
391  throw std::runtime_error(ss.str());
392  }
393  }
394  else if(columnsInfo_[col].getType() == ViewColumnInfo::TYPE_TRUE_FALSE)
395  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
396  {
397  if (theDataView_[row][col] == "1" || theDataView_[row][col] == "true" || theDataView_[row][col] == "True" || theDataView_[row][col] == "TRUE")
398  theDataView_[row][col] = ViewColumnInfo::TYPE_VALUE_TRUE;
399  else if (theDataView_[row][col] == "0" || theDataView_[row][col] == "false" || theDataView_[row][col] == "False" || theDataView_[row][col] == "FALSE")
400  theDataView_[row][col] = ViewColumnInfo::TYPE_VALUE_FALSE;
401  else
402  {
403  __SS__ << "Configuration Error:\t" << theDataView_[row][col] << " in column " <<
404  columnsInfo_[col].getName() << " is not a valid Type (True/False) std::string. Possible values are 1, true, True, TRUE, 0, false, False, FALSE." << std::endl;
405  __MOUT_ERR__ << "\n" << ss.str();
406  throw std::runtime_error("Invalid true/false usage");
407  }
408  }
409  else if(columnsInfo_[col].getType() == ViewColumnInfo::TYPE_YES_NO)
410  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
411  {
412  if (theDataView_[row][col] == "1" || theDataView_[row][col] == "yes" || theDataView_[row][col] == "Yes" || theDataView_[row][col] == "YES")
413  theDataView_[row][col] = ViewColumnInfo::TYPE_VALUE_YES;
414  else if (theDataView_[row][col] == "0" || theDataView_[row][col] == "no" || theDataView_[row][col] == "No" || theDataView_[row][col] == "NO")
415  theDataView_[row][col] = ViewColumnInfo::TYPE_VALUE_NO;
416  else
417  {
418  __SS__ << "Configuration Error:\t" << theDataView_[row][col] << " in column " <<
419  columnsInfo_[col].getName() << " is not a valid Type (Yes/No) std::string. Possible values are 1, yes, Yes, YES, 0, no, No, NO." << std::endl;
420  __MOUT_ERR__ << "\n" << ss.str();
421  throw std::runtime_error(ss.str());
422  }
423  }
424  else if(columnsInfo_[col].isGroupID()) //GroupID type
425  {
426  colLinkGroupIDs_[columnsInfo_[col].getChildLinkIndex()] = col; //add to groupid map
427  //check uniqueness
428  groupIdIndexes.emplace(columnsInfo_[col].getChildLinkIndex());
429  ++groupIdIndexesCount;
430  }
431  else if(columnsInfo_[col].isChildLink()) //Child Link type
432  {
433  //sanitize no link to default
434  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
435  if(theDataView_[row][col] == "NoLink" ||
436  theDataView_[row][col] == "No_Link" ||
437  theDataView_[row][col] == "NOLINK" ||
438  theDataView_[row][col] == "NO_LINK" ||
439  theDataView_[row][col] == "Nolink" ||
440  theDataView_[row][col] == "nolink" ||
441  theDataView_[row][col] == "noLink")
442  theDataView_[row][col] = ViewColumnInfo::DATATYPE_LINK_DEFAULT;
443 
444  //check uniqueness
445  childLinkIndexes.emplace(columnsInfo_[col].getChildLinkIndex());
446  ++childLinkIndexesCount;
447 
448  //force data type to ViewColumnInfo::DATATYPE_STRING
449  if(columnsInfo_[col].getDataType() != ViewColumnInfo::DATATYPE_STRING)
450  {
451  __SS__ << "Configuration Error:\t" << "Column " << col <<
452  " with name " << columnsInfo_[col].getName() <<
453  " is a Child Link column and has an illegal data type of '" <<
454  columnsInfo_[col].getDataType() <<
455  "'. The data type for Child Link columns must be " <<
456  ViewColumnInfo::DATATYPE_STRING << std::endl;
457  __MOUT_ERR__ << "\n" << ss.str();
458  throw std::runtime_error(ss.str());
459  }
460 
461  }
462  else if(columnsInfo_[col].isChildLinkUID() || //Child Link ID type
463  columnsInfo_[col].isChildLinkGroupID())
464  {
465  //check uniqueness
466  childLinkIdLabels.emplace(columnsInfo_[col].getChildLinkIndex());
467  ++childLinkIdLabelsCount;
468 
469  //check that the Link ID is not empty, and force to default
470  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
471  if(theDataView_[row][col] == "")
472  theDataView_[row][col] = rowDefaults[col];
473 
474  }
475  }
476 
477  //verify child link index uniqueness
478  if(groupIdIndexes.size() != groupIdIndexesCount)
479  {
480  __SS__ << ("GroupId Labels are not unique!") <<
481  "There are " << groupIdIndexesCount <<
482  " GroupId Labels and the unique count is " << groupIdIndexes.size() << std::endl;
483  __MOUT_ERR__ << "\n" << ss.str();
484  throw std::runtime_error(ss.str());
485  }
486  if(childLinkIndexes.size() != childLinkIndexesCount)
487  {
488  __SS__ << ("Child Link Labels are not unique!") <<
489  "There are " << childLinkIndexesCount <<
490  " Child Link Labels and the unique count is " << childLinkIndexes.size() << std::endl;
491  __MOUT_ERR__ << "\n" << ss.str();
492  throw std::runtime_error(ss.str());
493  }
494  if(childLinkIdLabels.size() != childLinkIdLabelsCount)
495  {
496  __SS__ << ("Child Link ID Labels are not unique!") <<
497  "There are " << childLinkIdLabelsCount <<
498  " Child Link ID Labels and the unique count is " << childLinkIdLabels.size() << std::endl;
499  __MOUT_ERR__ << "\n" << ss.str();
500  throw std::runtime_error(ss.str());
501  }
502 
503  }
504  catch(...)
505  {
506  __MOUT__ << "Error occured in ConfigurationView::init() for version=" << version_ << std::endl;
507  throw;
508  }
509 }
510 
511 //==============================================================================
512 //getValue
513 // string version
514 // Note: necessary because types of std::basic_string<char> cause compiler problems if no string specific function
515 void ConfigurationView::getValue(std::string& value, unsigned int row, unsigned int col, bool convertEnvironmentVariables) const
516 {
517  //getValue<std::string>(value,row,col,convertEnvironmentVariables);
518 
519  if(!(col < columnsInfo_.size() && row < getNumberOfRows()))
520  {
521  __SS__ << "Invalid row col requested" << std::endl;
522  __MOUT_ERR__ << "\n" << ss.str();
523  throw std::runtime_error(ss.str());
524  }
525 
526  value = validateValueForColumn(theDataView_[row][col],col,convertEnvironmentVariables);
527 
528 // if(!(col < columnsInfo_.size() && row < getNumberOfRows()))
529 // {
530 // __SS__ << "Invalid row col requested" << std::endl;
531 // __MOUT_ERR__ << "\n" << ss.str();
532 // throw std::runtime_error(ss.str());
533 // }
534 //
535 //
536 // if(columnsInfo_[col].getDataType() == ViewColumnInfo::DATATYPE_STRING)
537 // {
538 // value = theDataView_[row][col];
539 // if(convertEnvironmentVariables)
540 // value = convertEnvVariables(value);
541 // }
542 // else if(columnsInfo_[col].getDataType() == ViewColumnInfo::DATATYPE_TIME)
543 // {
544 // value.resize(30); //known fixed size: Thu Aug 23 14:55:02 2001 CST
545 // time_t timestamp(strtol(theDataView_[row][col].c_str(),0,10));
546 // struct tm tmstruct;
547 // ::localtime_r(&timestamp, &tmstruct);
548 // ::strftime(&value[0], 30, "%c %Z", &tmstruct);
549 // value.resize(strlen(value.c_str()));
550 // }
551 // else
552 // {
553 // __SS__ << "\tUnrecognized column data type: " << columnsInfo_[col].getDataType()
554 // << " in configuration " << tableName_
555 // << " at column=" << columnsInfo_[col].getName()
556 // << " for getValue with type '" << ots_demangle(typeid(value).name())
557 // << "'" << std::endl;
558 // throw std::runtime_error(ss.str());
559 // }
560 
561 
562 }
563 
564 
565 //==============================================================================
566 //validateValueForColumn
567 // string version
568 // Note: necessary because types of std::basic_string<char> cause compiler problems if no string specific function
569 std::string ConfigurationView::validateValueForColumn(const std::string& value, unsigned int col, bool convertEnvironmentVariables) const
570 {
571  //return validateValueForColumn<std::string>(value,col,convertEnvironmentVariables);
572  if(col >= columnsInfo_.size())
573  {
574  __SS__ << "Invalid col requested" << std::endl;
575  __MOUT_ERR__ << "\n" << ss.str();
576  throw std::runtime_error(ss.str());
577  }
578 
579  std::string retValue;
580 
581  if(columnsInfo_[col].getDataType() == ViewColumnInfo::DATATYPE_STRING)
582  retValue = convertEnvironmentVariables?convertEnvVariables(value):value;
583  else if(columnsInfo_[col].getDataType() == ViewColumnInfo::DATATYPE_TIME)
584  {
585  retValue.resize(30); //known fixed size: Thu Aug 23 14:55:02 2001 CST
586  time_t timestamp(
587  strtol((convertEnvironmentVariables?convertEnvVariables(value):value).c_str(),
588  0,10));
589  struct tm tmstruct;
590  ::localtime_r(&timestamp, &tmstruct);
591  ::strftime(&retValue[0], 30, "%c %Z", &tmstruct);
592  retValue.resize(strlen(retValue.c_str()));
593  }
594  else
595  {
596  __SS__ << "\tUnrecognized column data type: " << columnsInfo_[col].getDataType()
597  << " in configuration " << tableName_
598  << " at column=" << columnsInfo_[col].getName()
599  << " for getValue with type '" << ots_demangle(typeid(retValue).name())
600  << "'" << std::endl;
601  __MOUT_ERR__ << "\n" << ss.str();
602  throw std::runtime_error(ss.str());
603  }
604 
605  return retValue;
606  }
607 
608 
609 //==============================================================================
610 //getValueAsString
611 // gets the value with the proper data type and converts to string
612 // as though getValue was called.
613 std::string ConfigurationView::getValueAsString(unsigned int row, unsigned int col,
614  bool convertEnvironmentVariables) const
615 {
616  if(!(col < columnsInfo_.size() && row < getNumberOfRows()))
617  throw std::runtime_error("Invalid row col requested");
618 
619  //__MOUT__ << columnsInfo_[col].getType() << " " << col << std::endl;
620 
621  if(columnsInfo_[col].getType() == ViewColumnInfo::TYPE_ON_OFF)
622  {
623  if (theDataView_[row][col] == "1" || theDataView_[row][col] == "on" || theDataView_[row][col] == "On" || theDataView_[row][col] == "ON")
624  return ViewColumnInfo::TYPE_VALUE_ON;
625  else
626  return ViewColumnInfo::TYPE_VALUE_OFF;
627  }
628  else if(columnsInfo_[col].getType() == ViewColumnInfo::TYPE_TRUE_FALSE)
629  {
630  if (theDataView_[row][col] == "1" || theDataView_[row][col] == "true" || theDataView_[row][col] == "True" || theDataView_[row][col] == "TRUE")
631  return ViewColumnInfo::TYPE_VALUE_TRUE;
632  else
633  return ViewColumnInfo::TYPE_VALUE_FALSE;
634  }
635  else if(columnsInfo_[col].getType() == ViewColumnInfo::TYPE_YES_NO)
636  {
637  if (theDataView_[row][col] == "1" || theDataView_[row][col] == "yes" || theDataView_[row][col] == "Yes" || theDataView_[row][col] == "YES")
638  return ViewColumnInfo::TYPE_VALUE_YES;
639  else
640  return ViewColumnInfo::TYPE_VALUE_NO;
641  }
642 
643  //__MOUT__ << std::endl;
644  return convertEnvironmentVariables?convertEnvVariables(theDataView_[row][col]):
645  theDataView_[row][col];
646 }
647 
648 
649 //==============================================================================
650 //getEscapedValueAsString
651 // gets the value with the proper data type and converts to string
652 // as though getValue was called.
653 // then escapes all special characters with slash.
654 // Note: this should be useful for values placed in double quotes, i.e. JSON.
655 std::string ConfigurationView::getEscapedValueAsString(unsigned int row, unsigned int col,
656  bool convertEnvironmentVariables) const
657 {
658  std::string val = getValueAsString(row,col,convertEnvironmentVariables);
659  std::string retVal = "";
660  retVal.reserve(val.size()); //reserve roughly right size
661  for(unsigned int i=0;i<val.size();++i)
662  {
663  if(val[i] == '\n')
664  retVal += "\\n";
665  else if(val[i] == '\t')
666  retVal += "\\t";
667  else if(val[i] == '\r')
668  retVal += "\\r";
669  else
670  {
671  //escaped characters need a
672  if(val[i] == '"' ||
673  val[i] == '\\')
674  retVal += '\\';
675  retVal += val[i];
676  }
677  }
678  return retVal;
679 }
680 
681 //==============================================================================
682 std::string ConfigurationView::convertEnvVariables(const std::string& data) const
683 {
684  std::string converted = data;
685  if(data.find("${") != std::string::npos)
686  {
687  unsigned int begin = data.find("${");
688  unsigned int end = data.find("}");
689  std::string envVariable = data.substr(begin+2, end-begin-2);
690  //std::cout << "Converted Value: " << envVariable << std::endl;
691  if(getenv(envVariable.c_str()) != nullptr)
692  {
693  return convertEnvVariables(converted.replace(begin,end-begin+1,getenv(envVariable.c_str())));
694  }
695  else
696  throw std::runtime_error("In configuration " + tableName_ + " the environmental variable: " + envVariable + " is not set! Please make sure you set it before continuing!");
697  }
698  return converted;
699 }
700 
701 //==============================================================================
702 //setValue
703 // string version
704 void ConfigurationView::setValue(const std::string &value, unsigned int row, unsigned int col)
705 {
706  if(!(col < columnsInfo_.size() && row < getNumberOfRows()))
707  {
708  __SS__ << "Invalid row (" << row << ") col (" << col << ") requested!" << std::endl;
709  throw std::runtime_error(ss.str());
710  }
711 
712  if(columnsInfo_[col].getDataType() == ViewColumnInfo::DATATYPE_STRING)
713  theDataView_[row][col] = value;
714  else // dont allow ViewColumnInfo::DATATYPE_TIME to be set as string.. force use as time_t to standardize string result
715  {
716 
717  __SS__ << "\tUnrecognized column data type: " << columnsInfo_[col].getDataType()
718  << " in configuration " << tableName_
719  << " at column=" << columnsInfo_[col].getName()
720  << " for setValue with type '" << ots_demangle(typeid(value).name())
721  << "'" << std::endl;
722  throw std::runtime_error(ss.str());
723  }
724 }
725 void ConfigurationView::setValue(const char *value, unsigned int row, unsigned int col)
726 { setValue(std::string(value),row,col); }
727 
728 //==============================================================================
729 //setValue
730 // string version
731 void ConfigurationView::setValueAsString(const std::string &value, unsigned int row, unsigned int col)
732 {
733  if(!(col < columnsInfo_.size() && row < getNumberOfRows()))
734  {
735  __SS__ << "Invalid row (" << row << ") col (" << col << ") requested!" << std::endl;
736  throw std::runtime_error(ss.str());
737  }
738 
739  theDataView_[row][col] = value;
740 }
741 
742 //==============================================================================
743 //getColOfUID
744 // if column not found throw error
745 const unsigned int ConfigurationView::getOrInitColUID (void)
746 {
747  if(colUID_ != INVALID) return colUID_;
748 
749  //if doesn't exist throw error! each view must have a UID column
750  colUID_ = findColByType(ViewColumnInfo::TYPE_UID);
751  if(colUID_ == INVALID)
752  {
753  __MOUT__ << "Column Types: " << std::endl;
754  for(unsigned int col=0; col<columnsInfo_.size(); ++col)
755  std::cout << columnsInfo_[col].getType() << "() " << columnsInfo_[col].getName() << std::endl;
756  __SS__ << "\tMissing UID Column in table named '" << tableName_ << "'" << std::endl;
757  __MOUT_ERR__ << "\n" << ss.str() << std::endl;
758  throw std::runtime_error(ss.str());
759  }
760  return colUID_;
761 }
762 //==============================================================================
763 //getColOfUID
764 // const version, so don't attempt to lookup
765 // if column not found throw error
766 const unsigned int ConfigurationView::getColUID (void) const
767 {
768  if(colUID_ != INVALID) return colUID_;
769 
770  __MOUT__ << "Column Types: " << std::endl;
771  for(unsigned int col=0; col<columnsInfo_.size(); ++col)
772  std::cout << columnsInfo_[col].getType() << "() " << columnsInfo_[col].getName() << std::endl;
773 
774  __SS__ << ("Missing UID Column in config named " + tableName_ +
775  ". (Possibly ConfigurationView was just not initialized?" +
776  "This is the const call so can not alter class members)") << std::endl;
777  __MOUT_ERR__ << "\n" << ss.str() << std::endl;
778  throw std::runtime_error(ss.str());
779 }
780 
781 //==============================================================================
782 //addRowToGroup
783 // Group entry can include | to place a record in multiple groups
784 void ConfigurationView::addRowToGroup(const unsigned int &row,
785  const unsigned int &col,
786  const std::string &groupID,
787  const std::string &colDefault)
788 {
789  if(isEntryInGroupCol(row,col,groupID))
790  {
791  __SS__ << "GroupID (" << groupID <<
792  ") added to row (" << row
793  << " is already present!" << std::endl;
794  throw std::runtime_error(ss.str());
795  }
796 
797  //not in group, so
798  // if no groups
799  // set groupid
800  // if other groups
801  // prepend groupId |
802  if(getDataView()[row][col] == "" ||
803  getDataView()[row][col] == colDefault)
804  setValue(
805  groupID,
806  row,col);
807  else
808  setValue(
809  groupID + " | " + getDataView()[row][col],
810  row,col);
811 
812  //__MOUT__ << getDataView()[row][col] << std::endl;
813 }
814 
815 //==============================================================================
816 //removeRowFromGroup
817 // Group entry can include | to place a record in multiple groups
818 void ConfigurationView::removeRowFromGroup(const unsigned int &row,
819  const unsigned int &col,
820  const std::string &groupNeedle)
821 {
822  __MOUT__ << "groupNeedle " << groupNeedle << std::endl;
823  std::set<std::string> groupIDList;
824  if(!isEntryInGroupCol(row,col,groupNeedle,&groupIDList))
825  {
826  __SS__ << "GroupID (" << groupNeedle <<
827  ") removed from row (" << row
828  << ") was already removed!" << std::endl;
829  throw std::runtime_error(ss.str());
830  }
831 
832  //is in group, so
833  // create new string based on set of groupids
834  // but skip groupNeedle
835 
836  std::string newValue = "";
837  unsigned int cnt = 0;
838  for(const auto & groupID : groupIDList)
839  {
840  //__MOUT__ << groupID << " " << groupNeedle << " " << newValue << std::endl;
841  if(groupID == groupNeedle) continue; //skip group to be removed
842 
843  if(cnt) newValue += " | ";
844  newValue += groupID;
845  }
846  setValue(newValue,row,col);
847 
848  //__MOUT__ << getDataView()[row][col] << std::endl;
849 }
850 
851 //==============================================================================
852 //isEntryInGroup
853 // All group link checking should use this function
854 // so that handling is consistent
855 //
856 // Group entry can include | to place a record in multiple groups
857 bool ConfigurationView::isEntryInGroup(const unsigned int &r,
858  const std::string &childLinkIndex,
859  const std::string &groupNeedle) const
860 {
861  unsigned int c = getColLinkGroupID(childLinkIndex); //column in question
862 
863  return isEntryInGroupCol(r,c,groupNeedle);
864 }
865 
866 //==============================================================================
867 // isEntryInGroupCol
868 // private function to prevent users from treating any column like a GroupID column
869 //
870 // if *groupIDList != 0 return vector of groupIDs found
871 // useful for removing groupIDs.
872 //
873 // Group entry can include | to place a record in multiple groups
874 //
875 // Note: should mirror what happens in ConfigurationView::getSetOfGroupIDs
876 bool ConfigurationView::isEntryInGroupCol(const unsigned int &r,
877  const unsigned int &c, const std::string &groupNeedle,
878  std::set<std::string> *groupIDList) const
879 {
880  unsigned int i=0;
881  unsigned int j=0;
882  bool found = false;
883 
884  //__MOUT__ << "groupNeedle " << groupNeedle << std::endl;
885 
886  //go through the full groupString extracting groups and comparing to groupNeedle
887  for(;j<theDataView_[r][c].size();++j)
888  if((theDataView_[r][c][j] == ' ' || //ignore leading white space or |
889  theDataView_[r][c][j] == '|')
890  && i == j)
891  ++i;
892  else if((theDataView_[r][c][j] == ' ' || //trailing white space or | indicates group
893  theDataView_[r][c][j] == '|')
894  && i != j) // assume end of group name
895  {
896  if(groupIDList) groupIDList->emplace(theDataView_[r][c].substr(i,j-i));
897 
898  //__MOUT__ << "Group found to compare: " <<
899  // theDataView_[r][c].substr(i,j-i) << std::endl;
900  if(groupNeedle == theDataView_[r][c].substr(i,j-i))
901  {
902  if(!groupIDList) //dont return if caller is trying to get group list
903  return true;
904  found = true;
905  }
906  //if no match, setup i and j for next find
907  i = j+1;
908  }
909 
910  if(i != j) //last group check (for case when no ' ' or '|')
911  {
912  if(groupIDList) groupIDList->emplace(theDataView_[r][c].substr(i,j-i));
913 
914  //__MOUT__ << "Group found to compare: " <<
915  // theDataView_[r][c].substr(i,j-i) << std::endl;
916  if(groupNeedle == theDataView_[r][c].substr(i,j-i))
917  return true;
918  }
919 
920  return found;
921 }
922 
923 //==============================================================================
924 //getSetOfGroupIDs
925 // if row == -1, then considers all rows
926 // else just that row
927 // returns unique set of groupIds in GroupID column
928 // associate with childLinkIndex
929 //
930 // Note: should mirror what happens in ConfigurationView::isEntryInGroupCol
931 std::set<std::string> ConfigurationView::getSetOfGroupIDs(const std::string &childLinkIndex,
932  unsigned int r) const
933 {
934  unsigned int c = getColLinkGroupID(childLinkIndex);
935 
936  //__MOUT__ << "GroupID col=" << c << std::endl;
937 
938  std::set<std::string> retSet;
939 
940  unsigned int i=0;
941  unsigned int j=0;
942 
943  if(r != (unsigned int)-1)
944  {
945  if(r >= getNumberOfRows())
946  {
947  __SS__ << "Invalid row requested!" << std::endl;
948  throw std::runtime_error(ss.str());
949  }
950 
951  //go through the full groupString extracting groups
952  //add each found groupId to set
953  for(;j<theDataView_[r][c].size();++j)
954  if((theDataView_[r][c][j] == ' ' || //ignore leading white space or |
955  theDataView_[r][c][j] == '|')
956  && i == j)
957  ++i;
958  else if((theDataView_[r][c][j] == ' ' || //trailing white space or | indicates group
959  theDataView_[r][c][j] == '|')
960  && i != j) // assume end of group name
961  {
962  //__MOUT__ << "Group found: " <<
963  // theDataView_[r][c].substr(i,j-i) << std::endl;
964 
965 
966  retSet.emplace(theDataView_[r][c].substr(i,j-i));
967 
968  //setup i and j for next find
969  i = j+1;
970  }
971 
972  if(i != j) //last group check (for case when no ' ' or '|')
973  retSet.emplace(theDataView_[r][c].substr(i,j-i));
974  }
975  else
976  {
977  //do all rows
978  for(r=0;r<getNumberOfRows();++r)
979  {
980  i=0;
981  j=0;
982 
983  //__MOUT__ << r << ": " << theDataView_[r][c] << std::endl;
984 
985  //go through the full groupString extracting groups
986  //add each found groupId to set
987  for(;j<theDataView_[r][c].size();++j)
988  {
989  //__MOUT__ << "i:" << i << " j:" << j << std::endl;
990 
991  if((theDataView_[r][c][j] == ' ' || //ignore leading white space or |
992  theDataView_[r][c][j] == '|')
993  && i == j)
994  ++i;
995  else if((theDataView_[r][c][j] == ' ' || //trailing white space or | indicates group
996  theDataView_[r][c][j] == '|')
997  && i != j) // assume end of group name
998  {
999  //__MOUT__ << "Group found: " <<
1000  // theDataView_[r][c].substr(i,j-i) << std::endl;
1001 
1002  retSet.emplace(theDataView_[r][c].substr(i,j-i));
1003 
1004  //setup i and j for next find
1005  i = j+1;
1006  }
1007  }
1008 
1009  if(i != j) //last group (for case when no ' ' or '|')
1010  {
1011  //__MOUT__ << "Group found: " <<
1012  // theDataView_[r][c].substr(i,j-i) << std::endl;
1013  retSet.emplace(theDataView_[r][c].substr(i,j-i));
1014  }
1015  }
1016  }
1017 
1018  return retSet;
1019 }
1020 
1021 //==============================================================================
1022 //getColOfLinkGroupID
1023 // const version, if column not found throw error
1024 const unsigned int ConfigurationView::getColLinkGroupID(const std::string &childLinkIndex) const
1025 {
1026  std::map<std::string, unsigned int>::const_iterator it = colLinkGroupIDs_.find(childLinkIndex);
1027  if(it != //if already known, return it
1028  colLinkGroupIDs_.end())
1029  return it->second;
1030 
1031  __MOUT__ << "Existing Column Types: " << std::endl;
1032  for(unsigned int col=0; col<columnsInfo_.size(); ++col)
1033  std::cout << columnsInfo_[col].getType() << "() " << columnsInfo_[col].getName() << std::endl;
1034 
1035  __SS__ << ("Incompatible table for this group link. Table '" +
1036  tableName_ + "' is missing a GroupID column with data type '" +
1037  (ViewColumnInfo::TYPE_START_GROUP_ID + "-" + childLinkIndex) +
1038  "'." ) << std::endl;
1039  __MOUT_ERR__ << "\n" << ss.str();
1040  throw std::runtime_error(ss.str());
1041 }
1042 
1043 //==============================================================================
1044 unsigned int ConfigurationView::findRow(unsigned int col, const std::string& value, unsigned int offsetRow) const
1045 {
1046  for(unsigned int row=offsetRow; row<theDataView_.size(); ++row)
1047  {
1048  if(theDataView_[row][col] == value)
1049  return row;
1050  }
1051 
1052  __SS__ << "\tIn view: " << tableName_
1053  << ", Can't find value=" << value
1054  << " in column named " << columnsInfo_[col].getName()
1055  << " with type=" << columnsInfo_[col].getType()
1056  << std::endl;
1057  //Note: findRow gets purposely called by configuration GUI a lot looking for exceptions
1058  // so may not want to print out
1059  //__MOUT__ << "\n" << ss.str();
1060  throw std::runtime_error(ss.str());
1061 }
1062 
1063 //==============================================================================
1064 unsigned int ConfigurationView::findRowInGroup(unsigned int col, const std::string& value,
1065  const std::string &groupId, const std::string &childLinkIndex, unsigned int offsetRow) const
1066 {
1067  unsigned int groupIdCol = getColLinkGroupID(childLinkIndex);
1068  for(unsigned int row=offsetRow; row<theDataView_.size(); ++row)
1069  {
1070  if(theDataView_[row][col] == value &&
1071  isEntryInGroupCol(row,groupIdCol,groupId))
1072  return row;
1073  }
1074 
1075  __SS__ << "\tIn view: " << tableName_ << ", Can't find in group the value=" <<
1076  value << " in column named " <<
1077  columnsInfo_[col].getName() << " with type=" <<
1078  columnsInfo_[col].getType() << " and GroupID: " <<
1079  groupId << " in column " << groupIdCol <<
1080  " with GroupID child link index " << childLinkIndex << std::endl;
1081  __MOUT__ << "\n" << ss.str();
1082  throw std::runtime_error(ss.str());
1083 }
1084 
1085 //==============================================================================
1086 //findCol
1087 // throws exception if column not found by name
1088 unsigned int ConfigurationView::findCol(const std::string& name) const
1089 {
1090  for(unsigned int col=0; col<columnsInfo_.size(); ++col)
1091  if(columnsInfo_[col].getName() == name)
1092  return col;
1093 
1094  __SS__ << "\tIn view: " << tableName_ <<
1095  ", Can't find column named " << name << std::endl;
1096  throw std::runtime_error(ss.str());
1097 }
1098 
1099 //==============================================================================
1100 //findColByType
1101 // return invalid if type not found
1102 unsigned int ConfigurationView::findColByType(const std::string& type, int startingCol) const
1103 {
1104  for(unsigned int col=startingCol; col<columnsInfo_.size(); ++col)
1105  if(columnsInfo_[col].getType() == type)
1106  return col;
1107 
1108  return INVALID;
1109 }
1110 
1111 //Getters
1112 //==============================================================================
1113 const std::string& ConfigurationView::getUniqueStorageIdentifier(void) const
1114 {
1115  return uniqueStorageIdentifier_;
1116 }
1117 
1118 //==============================================================================
1119 const std::string& ConfigurationView::getTableName(void) const
1120 {
1121  return tableName_;
1122 }
1123 
1124 //==============================================================================
1125 const ConfigurationVersion& ConfigurationView::getVersion(void) const
1126 {
1127  return version_;
1128 }
1129 
1130 //==============================================================================
1131 const std::string& ConfigurationView::getComment(void) const
1132 {
1133  return comment_;
1134 }
1135 
1136 //==============================================================================
1137 const std::string& ConfigurationView::getAuthor(void) const
1138 {
1139  return author_;
1140 }
1141 
1142 //==============================================================================
1143 const time_t& ConfigurationView::getCreationTime(void) const
1144 {
1145  return creationTime_;
1146 }
1147 
1148 //==============================================================================
1149 const time_t& ConfigurationView::getLastAccessTime(void) const
1150 {
1151  return lastAccessTime_;
1152 }
1153 
1154 //==============================================================================
1155 const bool& ConfigurationView::getLooseColumnMatching(void) const
1156 {
1157  return fillWithLooseColumnMatching_;
1158 }
1159 
1160 //==============================================================================
1161 //getDataColumnSize
1162 const unsigned int ConfigurationView::getDataColumnSize(void) const
1163 {
1164  //if no data, give benefit of the doubt that phantom data has mockup column size
1165  if(!getNumberOfRows()) return getNumberOfColumns();
1166  return theDataView_[0].size(); //number of columns in first row of data
1167 }
1168 
1169 //==============================================================================
1170 //getSourceColumnMismatch
1171 // The source information is only valid after modifying the table with ::fillFromJSON
1172 const unsigned int& ConfigurationView::getSourceColumnMismatch(void) const
1173 {
1174  return sourceColumnMismatchCount_;
1175 }
1176 
1177 //==============================================================================
1178 //getSourceColumnMissing
1179 // The source information is only valid after modifying the table with ::fillFromJSON
1180 const unsigned int& ConfigurationView::getSourceColumnMissing(void) const
1181 {
1182  return sourceColumnMissingCount_;
1183 }
1184 
1185 //==============================================================================
1186 //getSourceColumnNames
1187 // The source information is only valid after modifying the table with ::fillFromJSON
1188 const std::set<std::string>& ConfigurationView::getSourceColumnNames(void) const
1189 {
1190  return sourceColumnNames_;
1191 }
1192 
1193 //==============================================================================
1194 std::set<std::string> ConfigurationView::getColumnNames(void) const
1195 {
1196  std::set<std::string> retSet;
1197  for(auto &colInfo: columnsInfo_)
1198  retSet.emplace(colInfo.getName());
1199  return retSet;
1200 }
1201 
1202 //==============================================================================
1203 std::set<std::string> ConfigurationView::getColumnStorageNames(void) const
1204 {
1205  std::set<std::string> retSet;
1206  for(auto &colInfo: columnsInfo_)
1207  retSet.emplace(colInfo.getStorageName());
1208  return retSet;
1209 }
1210 
1211 //==============================================================================
1212 std::vector<std::string> ConfigurationView::getDefaultRowValues(void) const
1213 {
1214  std::vector<std::string> retVec;
1215 
1216  //fill each col of new row with default values
1217  for(unsigned int col=0;col<getNumberOfColumns();++col)
1218  retVec.push_back(columnsInfo_[col].getDefaultValue());
1219 
1220  return retVec;
1221 }
1222 
1223 //==============================================================================
1224 unsigned int ConfigurationView::getNumberOfRows(void) const
1225 {
1226  return theDataView_.size();
1227 }
1228 
1229 //==============================================================================
1230 unsigned int ConfigurationView::getNumberOfColumns(void) const
1231 {
1232  return columnsInfo_.size();
1233 }
1234 
1235 //==============================================================================
1236 const ConfigurationView::DataView& ConfigurationView::getDataView(void) const
1237 {
1238  return theDataView_;
1239 }
1240 
1241 //==============================================================================
1242 //ConfigurationView::DataView* ConfigurationView::getDataViewP(void)
1243 //{
1244 // return &theDataView_;
1245 //}
1246 
1247 //==============================================================================
1248 const std::vector<ViewColumnInfo>& ConfigurationView::getColumnsInfo (void) const
1249 {
1250  return columnsInfo_;
1251 }
1252 
1253 //==============================================================================
1254 std::vector<ViewColumnInfo>* ConfigurationView::getColumnsInfoP(void)
1255 {
1256  return &columnsInfo_;
1257 }
1258 //==============================================================================
1259 const ViewColumnInfo& ConfigurationView::getColumnInfo(unsigned int column) const
1260 {
1261  if(column >= columnsInfo_.size())
1262  {
1263  std::stringstream errMsg;
1264  errMsg << __COUT_HDR_FL__ << "\nCan't find column " << column <<
1265  "\n\n\n\nThe column info is likely missing due to incomplete Configuration View filling.\n\n"
1266  << std::endl;
1267  throw std::runtime_error(errMsg.str().c_str());
1268  }
1269  return columnsInfo_[column];
1270 }
1271 
1272 //Setters
1273 //==============================================================================
1274 void ConfigurationView::setUniqueStorageIdentifier(const std::string &storageUID)
1275 {
1276  uniqueStorageIdentifier_ = storageUID;
1277 }
1278 
1279 //==============================================================================
1280 void ConfigurationView::setTableName(const std::string &name)
1281 {
1282  tableName_ = name;
1283 }
1284 
1285 //==============================================================================
1286 void ConfigurationView::setComment(const std::string &comment)
1287 {
1288  comment_ = comment;
1289 }
1290 
1291 //==============================================================================
1292 void ConfigurationView::setURIEncodedComment(const std::string &uriComment)
1293 {
1294  comment_ = decodeURIComponent(uriComment);
1295 }
1296 
1297 //==============================================================================
1298 void ConfigurationView::setAuthor(const std::string &author)
1299 {
1300  author_ = author;
1301 }
1302 
1303 //==============================================================================
1304 void ConfigurationView::setCreationTime(time_t t)
1305 {
1306  creationTime_ = t;
1307 }
1308 
1309 //==============================================================================
1310 void ConfigurationView::setLastAccessTime(time_t t)
1311 {
1312  lastAccessTime_ = t;
1313 }
1314 
1315 //==============================================================================
1316 void ConfigurationView::setLooseColumnMatching(bool setValue)
1317 {
1318  fillWithLooseColumnMatching_ = setValue;
1319 }
1320 
1321 //==============================================================================
1322 void ConfigurationView::reset (void)
1323 {
1324  version_ = -1;
1325  comment_ = "";
1326  author_ + "";
1327  columnsInfo_.clear();
1328  theDataView_.clear();
1329 }
1330 
1331 //==============================================================================
1332 void ConfigurationView::print (std::ostream &out) const
1333 {
1334  out << "==============================================================================" << std::endl;
1335  out << "Print: " << tableName_ << " Version: " << version_ << " Comment: " << comment_ <<
1336  " Author: " << author_ << " Creation Time: " << ctime(&creationTime_) << std::endl;
1337  out << "\t\tNumber of Cols " << getNumberOfColumns() << std::endl;
1338  out << "\t\tNumber of Rows " << getNumberOfRows() << std::endl;
1339 
1340  out << "Columns:\t";
1341  for(int i=0;i<(int)columnsInfo_.size();++i)
1342  out << i << ":" << columnsInfo_[i].getName() << ":" << columnsInfo_[i].getStorageName() << ":" << columnsInfo_[i].getDataType() << "\t ";
1343  out << std::endl;
1344 
1345  out << "Rows:" << std::endl;
1346  int num;
1347  std::string val;
1348  for(int r=0;r<(int)getNumberOfRows();++r)
1349  {
1350  out << r << ":\t";
1351  for(int c=0;c<(int)getNumberOfColumns();++c)
1352  {
1353  out << c << ":";
1354 
1355  //if fixed choice type, print index in choice
1356  if(columnsInfo_[c].getType() == ViewColumnInfo::TYPE_FIXED_CHOICE_DATA)
1357  {
1358  int choiceIndex = -1;
1359  std::vector<std::string> choices = columnsInfo_[c].getDataChoices();
1360  val = convertEnvVariables(theDataView_[r][c]);
1361 
1362  if(val == columnsInfo_[c].getDefaultValue())
1363  choiceIndex = 0;
1364  else
1365  {
1366  for(int i=0;i<(int)choices.size();++i)
1367  if(val == choices[i])
1368  choiceIndex = i+1;
1369  }
1370 
1371  out << "ChoiceIndex=" << choiceIndex << ":";
1372  }
1373 
1374 
1375  out << theDataView_[r][c];
1376  //stopped using below, because it is called sometimes during debugging when
1377  // numbers are set to environment variables:
1378  // if(columnsInfo_[c].getDataType() == "NUMBER")
1379  // {
1380  // getValue(num,r,c,false);
1381  // out << num;
1382  // }
1383  // else
1384  // {
1385  // getValue(val,r,c,false);
1386  // out << val;
1387  // }
1388  out << "\t\t";
1389  }
1390  out << std::endl;
1391  }
1392 }
1393 
1394 
1395 //==============================================================================
1396 void ConfigurationView::printJSON (std::ostream &out) const
1397 {
1398  out << "{\n";
1399  out << "\"NAME\" : \"" << tableName_ << "\",\n";
1400 
1401  //out << "\"VERSION\" : \"" << version_ << "\"\n";
1402 
1403  out << "\"COMMENT\" : ";
1404 
1405  //output escaped comment
1406  std::string val;
1407  val = comment_;
1408  out << "\"";
1409  for(unsigned int i=0;i<val.size();++i)
1410  {
1411 
1412  if(val[i] == '\n')
1413  out << "\\n";
1414  else if(val[i] == '\t')
1415  out << "\\t";
1416  else if(val[i] == '\r')
1417  out << "\\r";
1418  else
1419  {
1420  //escaped characters need a
1421  if(val[i] == '"' ||
1422  val[i] == '\\')
1423  out << '\\';
1424  out << val[i];
1425  }
1426  }
1427  out << "\",\n";
1428 
1429  out << "\"AUTHOR\" : \"" << author_ << "\",\n";
1430  out << "\"CREATION_TIME\" : " << creationTime_ << ",\n";
1431 
1432  //USELESS... out << "\"NUM_OF_COLS\" : " << getNumberOfColumns() << ",\n";
1433  //USELESS... out << "\"NUM_OF_ROWS\" : " << getNumberOfRows() << ",\n";
1434 
1435 
1436 
1437  out << "\"COL_TYPES\" : {\n";
1438  for(int c=0;c<(int)getNumberOfColumns();++c)
1439  {
1440  out << "\t\t\"" << columnsInfo_[c].getStorageName() << "\" : ";
1441  out << "\"" << columnsInfo_[c].getDataType() << "\"";
1442  if(c+1 < (int)getNumberOfColumns())
1443  out << ",";
1444  out << "\n";
1445  }
1446  out << "},\n"; //close COL_TYPES
1447 
1448  out << "\"DATA_SET\" : [\n";
1449  int num;
1450  for(int r=0;r<(int)getNumberOfRows();++r)
1451  {
1452  out << "\t{\n";
1453  for(int c=0;c<(int)getNumberOfColumns();++c)
1454  {
1455  out << "\t\t\"" << columnsInfo_[c].getStorageName() << "\" : ";
1456 
1457  out << "\"" << getEscapedValueAsString(r,c,false) << "\""; //do not convert env variables
1458 
1459  if(c+1 < (int)getNumberOfColumns())
1460  out << ",";
1461  out << "\n";
1462  }
1463  out << "\t}";
1464  if(r+1 < (int)getNumberOfRows())
1465  out << ",";
1466  out << "\n";
1467  }
1468  out << "]\n"; //close DATA_SET
1469 
1470  out << "}";
1471 }
1472 
1473 //==============================================================================
1474 //restoreJSONStringEntities
1475 // returns string with literals \n \t \" \r \\ replaced with char
1476 std::string restoreJSONStringEntities(const std::string &str)
1477 {
1478  unsigned int sz = str.size();
1479  if(!sz) return ""; //empty string, returns empty string
1480 
1481  std::stringstream retStr;
1482  unsigned int i=0;
1483  for(;i<sz-1;++i)
1484  {
1485  if(str[i] == '\\') //if 2 char escape sequence, replace with char
1486  switch(str[i+1])
1487  {
1488  case 'n':
1489  retStr << '\n'; ++i; break;
1490  case '"':
1491  retStr << '"'; ++i; break;
1492  case 't':
1493  retStr << '\t'; ++i; break;
1494  case 'r':
1495  retStr << '\r'; ++i; break;
1496  case '\\':
1497  retStr << '\\'; ++i; break;
1498  default:
1499  retStr << str[i];
1500  }
1501  else
1502  retStr << str[i];
1503  }
1504  if(i == sz-1)
1505  retStr << str[sz-1]; //output last character (which can't escape anything)
1506 
1507  return retStr.str();
1508 }
1509 
1510 //==============================================================================
1511 //fillFromJSON
1512 // Clears and fills the view from the JSON string.
1513 // Returns -1 on failure
1514 //
1515 // first level keys:
1516 // NAME
1517 // DATA_SET
1518 
1519 int ConfigurationView::fillFromJSON(const std::string &json)
1520 {
1521  std::vector<std::string> keys;
1522  keys.push_back ("NAME");
1523  keys.push_back ("COMMENT");
1524  keys.push_back ("AUTHOR");
1525  keys.push_back ("CREATION_TIME");
1526  //keys.push_back ("COL_TYPES");
1527  keys.push_back ("DATA_SET");
1528  enum {
1529  CV_JSON_FILL_NAME,
1530  CV_JSON_FILL_COMMENT,
1531  CV_JSON_FILL_AUTHOR,
1532  CV_JSON_FILL_CREATION_TIME,
1533  // CV_JSON_FILL_COL_TYPES,
1534  CV_JSON_FILL_DATA_SET
1535  };
1536 
1537  //__MOUT__ << json << std::endl;
1538 
1539  sourceColumnMismatchCount_ = 0;
1540  sourceColumnMissingCount_ = 0;
1541  sourceColumnNames_.clear(); //reset
1542  unsigned int colFoundCount = 0;
1543  unsigned int i = 0;
1544  unsigned int row = -1;
1545  unsigned int colSpeedup = 0;
1546  unsigned int startString, startNumber, endNumber = -1;
1547  unsigned int bracketCount = 0;
1548  unsigned int sqBracketCount = 0;
1549  bool inQuotes = 0;
1550  bool newString = 0;
1551  bool newValue = 0;
1552  bool isDataArray = 0;
1553  bool keyIsMatch, keyIsComment;
1554  unsigned int keyIsMatchIndex, keyIsMatchStorageIndex, keyIsMatchCommentIndex;
1555  const std::string COMMENT_ALT_KEY = "COMMENT";
1556 
1557  std::string extractedString = "", currKey = "", currVal = "";
1558  unsigned int currDepth;
1559 
1560  std::vector<std::string> jsonPath;
1561  std::vector<char> jsonPathType; // indicator of type in jsonPath: { [ K
1562  char lastPopType = '_'; // either: _ { [ K
1563  // _ indicates reset pop (this happens when a new {obj} starts)
1564  unsigned int matchedKey = -1;
1565  unsigned int lastCol = -1;
1566 
1567 
1568  //find all depth 1 matching keys
1569  for(;i<json.size();++i)
1570  {
1571  switch(json[i])
1572  {
1573  case '"':
1574  if(i-1 < json.size() && //ignore if escaped
1575  json[i-1] == '\\') break;
1576 
1577  inQuotes = !inQuotes; //toggle in quotes if not escaped
1578  if(inQuotes)
1579  startString = i;
1580  else
1581  {
1582  extractedString = restoreJSONStringEntities(
1583  json.substr(startString+1,i-startString-1));
1584  newString = 1; //have new string!
1585  }
1586  break;
1587  case ':':
1588  if(inQuotes) break; //skip if in quote
1589 
1590  //must be a json object level to have a key
1591  if(jsonPathType[jsonPathType.size()-1] != '{'
1592  || !newString) //and must have a string for key
1593  {
1594  __MOUT__ << "Invalid ':' position" << std::endl;
1595  return -1;
1596  }
1597 
1598  //valid, so take key
1599  jsonPathType.push_back('K');
1600  jsonPath.push_back(extractedString);
1601  startNumber = i;
1602  newString = 0; //clear flag
1603  endNumber = -1; //reset end number index
1604  break;
1605 
1606  // if(isKey ||
1607  // isDataArray)
1608  // {
1609  // std::cout << "Invalid ':' position" << std::endl;
1610  // return -1;
1611  // }
1612  // isKey = 1; //new value is a key
1613  // newValue = 1;
1614  // startNumber = i;
1615  // break;
1616  case ',':
1617  if(inQuotes) break; //skip if in quote
1618  if(lastPopType == '{') //don't need value again of nested object
1619  {
1620  //check if the nested object was the value to a key, if so, pop key
1621  if(jsonPathType[jsonPathType.size()-1] == 'K')
1622  {
1623  lastPopType = 'K';
1624  jsonPath.pop_back();
1625  jsonPathType.pop_back();
1626  }
1627  break; //skip , handling if {obj} just ended
1628  }
1629 
1630  if(newString)
1631  currVal = extractedString;
1632  else //number value
1633  {
1634  if(endNumber == (unsigned int)-1 || //take i as end number if needed
1635  endNumber <= startNumber)
1636  endNumber = i;
1637  //extract number value
1638  if(endNumber <= startNumber) //empty data, could be {}
1639  currVal = "";
1640  else
1641  currVal = json.substr(startNumber+1,endNumber-startNumber-1);
1642  }
1643 
1644  currDepth = bracketCount;
1645 
1646  if(jsonPathType[jsonPathType.size()-1] == 'K') //this is the value to key
1647  {
1648  currKey = jsonPath[jsonPathType.size()-1];
1649  newValue = 1; //new value to consider!
1650 
1651  //pop key
1652  lastPopType = 'K';
1653  jsonPath.pop_back();
1654  jsonPathType.pop_back();
1655  }
1656  else if(jsonPathType[jsonPathType.size()-1] == '[') //this is a value in array
1657  {
1658  //key is last key
1659  for(unsigned int k=jsonPathType.size()-2; k<jsonPathType.size(); --k)
1660  if(jsonPathType[k] == 'K')
1661  {
1662  currKey = jsonPath[k];
1663  break;
1664  }
1665  else if(k == 0)
1666  {
1667  __MOUT__ << "Invalid array position" << std::endl;
1668  return -1;
1669  }
1670 
1671  newValue = 1; //new value to consider!
1672  isDataArray = 1;
1673  }
1674  else // { is an error
1675  {
1676  __MOUT__ << "Invalid ',' position" << std::endl;
1677  return -1;
1678  }
1679 
1680  startNumber = i;
1681  break;
1682 
1683  case '{':
1684  if(inQuotes) break; //skip if in quote
1685  lastPopType = '_'; //reset because of new object
1686  jsonPathType.push_back('{');
1687  jsonPath.push_back("{");
1688  ++bracketCount;
1689  break;
1690 
1691  // ++bracketCount;
1692  // isDataArray = 0;
1693  // isKey = 0;
1694  // endingObject = 0;
1695  // break;
1696  case '}':
1697  if(inQuotes) break; //skip if in quote
1698 
1699  if(lastPopType != '{' && //don't need value again of nested object
1700  jsonPathType[jsonPathType.size()-1] == 'K') //this is the value to key
1701  {
1702  currDepth = bracketCount;
1703  currKey = jsonPath[jsonPathType.size()-1];
1704  if(newString)
1705  currVal = extractedString;
1706  else //number value
1707  {
1708  if(endNumber == (unsigned int)-1 || //take i as end number if needed
1709  endNumber <= startNumber)
1710  endNumber = i;
1711  //extract val
1712  if(endNumber <= startNumber) //empty data, could be {}
1713  currVal = "";
1714  else
1715  currVal = json.substr(startNumber+1,endNumber-startNumber-1);
1716  }
1717  newValue = 1; //new value to consider!
1718  //pop key
1719  jsonPath.pop_back();
1720  jsonPathType.pop_back();
1721  }
1722  //pop {
1723  if(jsonPathType[jsonPathType.size()-1] != '{')
1724  {
1725  __MOUT__ << "Invalid '}' position" << std::endl;
1726  return -1;
1727  }
1728  lastPopType = '{';
1729  jsonPath.pop_back();
1730  jsonPathType.pop_back();
1731  --bracketCount;
1732  break;
1733  case '[':
1734  if(inQuotes) break; //skip if in quote
1735  jsonPathType.push_back('[');
1736  jsonPath.push_back("[");
1737  ++sqBracketCount;
1738  startNumber = i;
1739  break;
1740  case ']':
1741  if(inQuotes) break; //skip if in quote
1742 
1743  //must be an array at this level (in order to close it)
1744  if(jsonPathType[jsonPathType.size()-1] != '[')
1745  {
1746  __MOUT__ << "Invalid ']' position" << std::endl;
1747  return -1;
1748  }
1749 
1750  currDepth = bracketCount;
1751 
1752  //This is an array value
1753  if(newString)
1754  currVal = extractedString;
1755  else //number value
1756  {
1757  if(endNumber == (unsigned int)-1 || //take i as end number if needed
1758  endNumber <= startNumber)
1759  endNumber = i;
1760  //extract val
1761  if(endNumber <= startNumber) //empty data, could be {}
1762  currVal = "";
1763  else
1764  currVal = json.substr(startNumber+1,endNumber-startNumber-1);
1765  }
1766  isDataArray = 1;
1767 
1768  //key is last key
1769  for(unsigned int k=jsonPathType.size()-2; k<jsonPathType.size(); --k)
1770  if(jsonPathType[k] == 'K')
1771  {
1772  currKey = jsonPath[k];
1773  break;
1774  }
1775  else if(k == 0)
1776  {
1777  __MOUT__ << "Invalid array position" << std::endl;
1778  return -1;
1779  }
1780 
1781  //pop [
1782  if(jsonPathType[jsonPathType.size()-1] != '[')
1783  {
1784  __MOUT__ << "Invalid ']' position" << std::endl;
1785  return -1;
1786  }
1787  lastPopType = '[';
1788  jsonPath.pop_back();
1789  jsonPathType.pop_back();
1790  --sqBracketCount;
1791  break;
1792  case ' ': //white space handling for numbers
1793  case '\t':
1794  case '\n':
1795  case '\r':
1796  if(inQuotes) break; //skip if in quote
1797  if(startNumber != (unsigned int)-1 &&
1798  endNumber == (unsigned int)-1)
1799  endNumber = i;
1800  startNumber = i;
1801  break;
1802  default:;
1803  }
1804 
1805  if(0) //for debugging
1806  {
1807  std::cout << i << ":\t" << json[i] << " - ";
1808 
1809  std::cout << "ExtKey=";
1810  for(unsigned int k=0;k<jsonPath.size();++k)
1811  std::cout << jsonPath[k] << "/";
1812  std::cout << " - ";
1813  std::cout << lastPopType << " ";
1814  std::cout << bracketCount << " ";
1815  std::cout << sqBracketCount << " ";
1816  std::cout << inQuotes << " ";
1817  std::cout << newValue << "-";
1818  std::cout << currKey << "-{" << currDepth << "}:" ;
1819  std::cout << currVal << " ";
1820  std::cout << startNumber << "-";
1821  std::cout << endNumber << " ";
1822  std::cout << "\n";
1823  }
1824 
1825  //continue;
1826 
1827  //handle a new completed value
1828  if(newValue)
1829  {
1830 
1831  // std::cout << "ExtKey=";
1832  // for(unsigned int k=0;k<jsonPath.size();++k)
1833  // std::cout << jsonPath[k] << "/";
1834  //
1835  // std::cout << currKey << "-{" << currDepth << "}: - ";
1836  //
1837  // if(isDataArray)
1838  // std::cout << "Array:: ";
1839  // if(newString)
1840  // std::cout << "New String:: ";
1841  // else
1842  // std::cout << "New Number:: ";
1843  //
1844  // std::cout << currVal << "\n";
1845 
1846 
1847  //extract only what we care about
1848  // for ConfigurationView only care about matching depth 1
1849 
1850  //handle matching depth 1 keys
1851 
1852  matchedKey = -1; //init to unfound
1853  for(unsigned int k=0;k<keys.size();++k)
1854  if((currDepth == 1 && keys[k] == currKey) ||
1855  (currDepth > 1 && keys[k] == jsonPath[1]))
1856  matchedKey = k;
1857 
1858  if(matchedKey != (unsigned int)-1)
1859  {
1860  // std::cout << "New Data for:: key[" << matchedKey << "]-" <<
1861  // keys[matchedKey] << "\n";
1862 
1863  switch(matchedKey)
1864  {
1865  case CV_JSON_FILL_NAME:
1866  if(currDepth == 1) setTableName(currVal);
1867  break;
1868  case CV_JSON_FILL_COMMENT:
1869  if(currDepth == 1) setComment(currVal);
1870  break;
1871  case CV_JSON_FILL_AUTHOR:
1872  if(currDepth == 1) setAuthor(currVal);
1873  break;
1874  case CV_JSON_FILL_CREATION_TIME:
1875  if(currDepth == 1) setCreationTime(strtol(currVal.c_str(),0,10));
1876  break;
1877  //case CV_JSON_FILL_COL_TYPES:
1878  //
1879  //break;
1880  case CV_JSON_FILL_DATA_SET:
1881  // std::cout << "CV_JSON_FILL_DATA_SET New Data for:: " << matchedKey << "]-" <<
1882  // keys[matchedKey] << "/" << currDepth << ".../" <<
1883  // currKey << "\n";
1884 
1885  if(currDepth == 2) //second level depth
1886  {
1887  //if matches first column name.. then add new row
1888  //else add to current row
1889  unsigned int col, ccnt = 0;
1890  unsigned int noc = getNumberOfColumns();
1891  for(;ccnt<noc;++ccnt)
1892  {
1893  //use colSpeedup to change the first column we search
1894  //for each iteration.. since we expect the data to
1895  //be arranged in column order
1896 
1897  if(fillWithLooseColumnMatching_)
1898  {
1899  //loose column matching makes no attempt to
1900  // match the column names
1901  //just assumes the data is in the correct order
1902 
1903  col = colSpeedup;
1904 
1905  //auto matched
1906  if(col <= lastCol) //add row (use lastCol in case new column-0 was added
1907  row = addRow();
1908  lastCol = col;
1909  if(getNumberOfRows() == 1) //only for first row
1910  sourceColumnNames_.emplace(currKey);
1911 
1912  //add value to row and column
1913 
1914  if(row >= getNumberOfRows())
1915  {
1916  __SS__ << "Invalid row" << std::endl; //should be impossible?
1917  std::cout << ss.str();
1918  throw std::runtime_error(ss.str());
1919  return -1;
1920  }
1921 
1922  theDataView_[row][col] = currVal;//THERE IS NO CHECK FOR WHAT IS READ FROM THE DATABASE. IT SHOULD BE ALREADY CONSISTENT
1923  break;
1924  }
1925  else
1926  {
1927  col = (ccnt + colSpeedup) % noc;
1928 
1929  //match key by ignoring '_'
1930  // also accept COMMENT == COMMENT_DESCRIPTION
1931  // (this is for backwards compatibility..)
1932  keyIsMatch = true;
1933  keyIsComment = true;
1934  for(keyIsMatchIndex=0, keyIsMatchStorageIndex=0, keyIsMatchCommentIndex=0;
1935  keyIsMatchIndex<currKey.size();++keyIsMatchIndex)
1936  {
1937  if(columnsInfo_[col].getStorageName()[keyIsMatchStorageIndex] == '_')
1938  ++keyIsMatchStorageIndex; //skip to next storage character
1939  if(currKey[keyIsMatchIndex] == '_')
1940  continue; //skip to next character
1941 
1942  //match to storage name
1943  if(keyIsMatchStorageIndex >= columnsInfo_[col].getStorageName().size() ||
1944  currKey[keyIsMatchIndex] !=
1945  columnsInfo_[col].getStorageName()[keyIsMatchStorageIndex])
1946  {
1947  //size mismatch or character mismatch
1948  keyIsMatch = false;
1949  if(!keyIsComment)
1950  break;
1951  }
1952 
1953  //check also if alternate comment is matched
1954  if(keyIsComment && keyIsMatchCommentIndex < COMMENT_ALT_KEY.size())
1955  {
1956  if(currKey[keyIsMatchIndex] != COMMENT_ALT_KEY[keyIsMatchCommentIndex])
1957  {
1958  //character mismatch with COMMENT
1959  keyIsComment = false;
1960  }
1961  }
1962 
1963  ++keyIsMatchStorageIndex; //go to next character
1964  }
1965 
1966  if(keyIsMatch || keyIsComment) //currKey == columnsInfo_[c].getStorageName())
1967  {
1968  //matched
1969  if(col <= lastCol) //add row (use lastCol in case new column-0 was added
1970  {
1971  if(getNumberOfRows()) //skip first time
1972  sourceColumnMissingCount_ += getNumberOfColumns() - colFoundCount;
1973 
1974  colFoundCount = 0; //reset column found count
1975  row = addRow();
1976  }
1977  lastCol = col;
1978  ++colFoundCount;
1979 
1980  if(getNumberOfRows() == 1) //only for first row
1981  sourceColumnNames_.emplace(currKey);
1982 
1983  //add value to row and column
1984 
1985  if(row >= getNumberOfRows())
1986  {
1987  __SS__ << "Invalid row" << std::endl; //should be impossible?!
1988  __MOUT__ << "\n" << ss.str();
1989  throw std::runtime_error(ss.str());
1990  return -1; //never gets here
1991  }
1992 
1993  theDataView_[row][col] = currVal;
1994  break;
1995  }
1996  }
1997  }
1998 
1999  colSpeedup = (colSpeedup + 1) % noc; //short cut to proper column hopefully in next search
2000 
2001  if(ccnt >= getNumberOfColumns())
2002  {
2003  __SS__ << "\n\nInvalid column in JSON source data: " <<
2004  currKey << " not found in column names of table named " <<
2005  getTableName() << "." <<
2006  std::endl; //input data doesn't match config description
2007  __MOUT__ << "\n" << ss.str();
2008  //CHANGED on 11/10/2016
2009  // to.. try just not populating data instead of error
2010  ++sourceColumnMismatchCount_; //but count errors
2011  if(getNumberOfRows() == 1) //only for first row
2012  sourceColumnNames_.emplace(currKey);
2013 
2014  //throw std::runtime_error("\n\n" + ss.str());
2015  //return -1; //never gets here
2016  }
2017 
2018  }
2019  break;
2020  default:; //unknown match?
2021  }
2022 
2023  }
2024 
2025  //clean up handling of new value
2026 
2027  newString = 0; //toggle flag
2028  newValue = 0; //toggle flag
2029  isDataArray = 0;
2030  endNumber = -1; //reset end number index
2031  }
2032 
2033  //if(i>200) break; //185
2034  }
2035 
2036  //__MOUT__ << "Done!" << std::endl;
2037 
2038  //print();
2039 
2040  return 0; //success
2041 }
2042 
2043 
2044 
2045 //==============================================================================
2046 bool ConfigurationView::isURIEncodedCommentTheSame(const std::string &comment) const
2047 {
2048  std::string compareStr = decodeURIComponent(comment);
2049  return comment_ == compareStr;
2050 }
2051 //
2053 //bool ConfigurationView::isValueTheSame(const std::string &valueStr,
2054 // unsigned int r, unsigned int c) const
2055 //{
2056 // __MOUT__ << "valueStr " << valueStr << std::endl;
2057 //
2058 // if(!(c < columnsInfo_.size() && r < getNumberOfRows()))
2059 // {
2060 // __SS__ << "Invalid row (" << r << ") col (" << c << ") requested!" << std::endl;
2061 // throw std::runtime_error(ss.str());
2062 // }
2063 //
2064 // __MOUT__ << "originalValueStr " << theDataView_[r][c] << std::endl;
2065 //
2066 // if(columnsInfo_[c].getDataType() == ViewColumnInfo::DATATYPE_TIME)
2067 // {
2068 // time_t valueTime(strtol(valueStr.c_str(),0,10));
2069 // time_t originalValueTime;
2070 // getValue(originalValueTime,r,c);
2071 // __MOUT__ << "time_t valueStr " << valueTime << std::endl;
2072 // __MOUT__ << "time_t originalValueStr " << originalValueTime << std::endl;
2073 // return valueTime == originalValueTime;
2074 // }
2075 // else
2076 // {
2077 // return valueStr == theDataView_[r][c];
2078 // }
2079 //}
2080 
2081 //==============================================================================
2082 //fillFromCSV
2083 // Fills the view from the CSV string.
2084 //
2085 // Note: converts all %## to the ascii character, # is hex nibble
2086 // (e.g. '%' must be represented as "%25")
2087 //
2088 // dataOffset := starting destination row
2089 //
2090 // while there are row entries in the data.. replace
2091 // data range from [dataOffset, dataOffset+chunkSize-1]
2092 // ... note if less rows, this means rows were deleted
2093 // ... if more, then rows were added.
2094 //
2095 // ',' next cell delimiter
2096 // ';' next row delimiter
2097 //
2098 //
2099 // if author == "", do nothing special for author and timestamp column
2100 // if author != "", assign author for any row that has been modified, and assign now as timestamp
2101 //
2102 // Returns -1 if data was same and pre-existing content
2103 // otherwise 0
2104 int ConfigurationView::fillFromCSV(const std::string &data, const int &dataOffset,
2105  const std::string &author)
2106 throw(std::runtime_error)
2107 {
2108  int r = dataOffset;
2109  int c = 0;
2110 
2111  int i = 0; //use to parse data std::string
2112  int j = data.find(',',i); //find next cell delimiter
2113  int k = data.find(';',i); //find next row delimiter
2114 
2115  bool rowWasModified;
2116  unsigned int countRowsModified = 0;
2117  int authorCol = findColByType(ViewColumnInfo::TYPE_AUTHOR);
2118  int timestampCol = findColByType(ViewColumnInfo::TYPE_TIMESTAMP);
2119  //std::string valueStr, tmpTimeStr, originalValueStr;
2120 
2121  while(k != (int)(std::string::npos))
2122  {
2123  rowWasModified = false;
2124  if(r >= (int)getNumberOfRows())
2125  {
2126  addRow();
2127  //__MOUT__ << "Row added" << std::endl;
2128  rowWasModified = true;
2129  }
2130 
2131  while(j < k && j != (int)(std::string::npos))
2132  {
2133  //__MOUT__ << "Col " << c << std::endl;
2134 
2135  //skip last 2 columns
2136  if(c >= (int)getNumberOfColumns()-2)
2137  {
2138  i=j+1;
2139  j = data.find(',',i); //find next cell delimiter
2140  ++c;
2141  continue;
2142  }
2143 
2144  if(setURIEncodedValue(data.substr(i,j-i),r,c))
2145  rowWasModified = true;
2146 
2147  i=j+1;
2148  j = data.find(',',i); //find next cell delimiter
2149  ++c;
2150  }
2151 
2152  //if row was modified, assign author and timestamp
2153  if(author != "" && rowWasModified)
2154  {
2155  __MOUT__ << "Row=" << r << " was modified!" << std::endl;
2156  setValue(author,r,authorCol);
2157  setValue(time(0),r,timestampCol);
2158  }
2159 
2160  if(rowWasModified) ++countRowsModified;
2161 
2162  ++r;
2163  c = 0;
2164 
2165 
2166  i = k+1;
2167  j = data.find(',',i); //find next cell delimiter
2168  k = data.find(';',i); //find new row delimiter
2169  }
2170 
2171  //delete excess rows
2172  while(r < (int)getNumberOfRows())
2173  {
2174  deleteRow(r);
2175  __MOUT__ << "Row deleted: " << r << std::endl;
2176  ++countRowsModified;
2177  }
2178 
2179  __MOUT_INFO__ << "countRowsModified=" <<
2180  countRowsModified << std::endl;
2181 
2182  if(!countRowsModified)
2183  {
2184  __SS__ << "No rows were modified! No reason to fill a view with same content." << std::endl;
2185  __MOUT__ << "\n" << ss.str();
2186  return -1;
2187  }
2188 
2189  //print(); //for debugging
2190 
2191  //setup sourceColumnNames_ to be correct
2192  sourceColumnNames_.clear();
2193  for(unsigned int i=0;i<getNumberOfColumns();++i)
2194  sourceColumnNames_.emplace(getColumnsInfo()[i].getStorageName());
2195 
2196  init(); //verify new table (throws runtime_errors)
2197 
2198  //printout for debugging
2199  // __SS__ << "\n";
2200  // print(ss);
2201  // __MOUT__ << "\n" << ss.str() << std::endl;
2202 
2203  return 0;
2204 }
2205 
2206 //==============================================================================
2207 //setURIEncodedValue
2208 // converts all %## to the ascii character
2209 // returns true if value was different than original value
2210 //
2211 //
2212 // if author == "", do nothing special for author and timestamp column
2213 // if author != "", assign author for any row that has been modified, and assign now as timestamp
2214 bool ConfigurationView::setURIEncodedValue(const std::string &value, const unsigned int &r,
2215  const unsigned int &c, const std::string &author)
2216 {
2217  if(!(c < columnsInfo_.size() && r < getNumberOfRows()))
2218  {
2219  __SS__ << "Invalid row (" << r << ") col (" << c << ") requested!" << std::endl;
2220  throw std::runtime_error(ss.str());
2221  }
2222 
2223  std::string valueStr = decodeURIComponent(value);
2224  std::string originalValueStr = getValueAsString(r,c,false); //do not convert env variables
2225 
2226 
2227  //__MOUT__ << "valueStr " << valueStr << std::endl;
2228  //__MOUT__ << "originalValueStr " << originalValueStr << std::endl;
2229 
2230  if(columnsInfo_[c].getDataType() == ViewColumnInfo::DATATYPE_NUMBER)
2231  {
2232  //check if valid number
2233  std::string convertedString = convertEnvVariables(valueStr);
2234  if(!isNumber(convertedString))
2235  {
2236  __SS__ << "\tIn configuration " << tableName_
2237  << " at column=" << columnsInfo_[c].getName()
2238  << " the value set (" << convertedString << ")"
2239  << " is not a number! Please fix it or change the column type..." << std::endl;
2240  throw std::runtime_error(ss.str());
2241  }
2242  theDataView_[r][c] = valueStr;
2243  }
2244  else if(columnsInfo_[c].getDataType() == ViewColumnInfo::DATATYPE_TIME)
2245  {
2246  // valueStr = decodeURIComponent(data.substr(i,j-i));
2247  //
2248  // getValue(tmpTimeStr,r,c);
2249  // if(valueStr != tmpTimeStr)//theDataView_[r][c])
2250  // {
2251  // __MOUT__ << "valueStr=" << valueStr <<
2252  // " theDataView_[r][c]=" << tmpTimeStr << std::endl;
2253  // rowWasModified = true;
2254  // }
2255 
2256  setValue(time_t(strtol(valueStr.c_str(),0,10)),
2257  r,c);
2258  }
2259  else
2260  theDataView_[r][c] = valueStr;
2261 
2262  bool rowWasModified = (originalValueStr != getValueAsString(r,c,false)); //do not convert env variables
2263 
2264  //if row was modified, assign author and timestamp
2265  if(author != "" && rowWasModified)
2266  {
2267  __MOUT__ << "Row=" << r << " was modified!" << std::endl;
2268  int authorCol = findColByType(ViewColumnInfo::TYPE_AUTHOR);
2269  int timestampCol = findColByType(ViewColumnInfo::TYPE_TIMESTAMP);
2270  setValue(author,r,authorCol);
2271  setValue(time(0),r,timestampCol);
2272  }
2273 
2274  return rowWasModified;
2275 }
2276 
2277 //==============================================================================
2278 //decodeURIComponent
2279 // converts all %## to the ascii character
2280 std::string ConfigurationView::decodeURIComponent(const std::string &data)
2281 {
2282  std::string decodeURIString(data.size(),0); //init to same size
2283  unsigned int j=0;
2284  for(unsigned int i=0;i<data.size();++i,++j)
2285  {
2286  if(data[i] == '%')
2287  {
2288  //high order hex nibble digit
2289  if(data[i+1] > '9') //then ABCDEF
2290  decodeURIString[j] += (data[i+1]-55)*16;
2291  else
2292  decodeURIString[j] += (data[i+1]-48)*16;
2293 
2294  //low order hex nibble digit
2295  if(data[i+2] > '9') //then ABCDEF
2296  decodeURIString[j] += (data[i+2]-55);
2297  else
2298  decodeURIString[j] += (data[i+2]-48);
2299 
2300  i+=2; //skip to next char
2301  }
2302  else
2303  decodeURIString[j] = data[i];
2304  }
2305  decodeURIString.resize(j);
2306  return decodeURIString;
2307 }
2308 
2309 //==============================================================================
2310 void ConfigurationView::resizeDataView(unsigned int nRows, unsigned int nCols)
2311 {
2312  //FIXME This maybe should disappear but I am using it in ConfigurationHandler still...
2313  theDataView_.resize(nRows, std::vector<std::string>(nCols));
2314 }
2315 
2316 //==============================================================================
2317 //addRow
2318 // returns index of added row, always is last row
2319 // return -1 on failure (throw error)
2320 int ConfigurationView::addRow (const std::string &author)
2321 {
2322  int row = getNumberOfRows();
2323  theDataView_.resize(getNumberOfRows()+1,std::vector<std::string>(getNumberOfColumns()));
2324 
2325  std::vector<std::string> defaultRowValues =
2326  getDefaultRowValues();
2327 
2328  //fill each col of new row with default values
2329  for(unsigned int col=0;col<getNumberOfColumns();++col)
2330  theDataView_[row][col] = defaultRowValues[col];
2331 
2332  if(author != "")
2333  {
2334  __MOUT__ << "Row=" << row << " was created!" << std::endl;
2335  int authorCol = findColByType(ViewColumnInfo::TYPE_AUTHOR);
2336  int timestampCol = findColByType(ViewColumnInfo::TYPE_TIMESTAMP);
2337  setValue(author,row,authorCol);
2338  setValue(time(0),row,timestampCol);
2339  }
2340 
2341  return row;
2342 }
2343 
2344 //==============================================================================
2345 //deleteRow
2346 // throws exception on failure
2347 void ConfigurationView::deleteRow(int r)
2348 {
2349  if(r >= (int)getNumberOfRows())
2350  {
2351  //out of bounds
2352  __SS__ << "Row " << r << " is out of bounds (Row Count = " <<
2353  getNumberOfRows() << ") and can not be deleted." <<
2354  std::endl;
2355  throw std::runtime_error(ss.str());
2356  }
2357 
2358  theDataView_.erase(theDataView_.begin()+r);
2359 }
2360 
2361 //==============================================================================
2362 //isColChildLinkUnique ~
2363 // find the pair of columns associated with a child link.
2364 //
2365 // c := a member column of the pair
2366 //
2367 // returns:
2368 // isGroup := indicates pair found is a group link
2369 // linkPair := pair of columns that are part of the link
2370 //
2371 // a unique link is defined by two columns: TYPE_START_CHILD_LINK, TYPE_START_CHILD_LINK_UID
2372 // a group link is defined by two columns: TYPE_START_CHILD_LINK, TYPE_START_CHILD_LINK_GROUP_ID
2373 //
2374 // returns true if column is member of a group or unique link.
2375 const bool ConfigurationView::getChildLink(const unsigned int& c, bool& isGroup,
2376  std::pair<unsigned int /*link col*/, unsigned int /*link id col*/>& linkPair) const
2377 {
2378 
2379  if(!(c < columnsInfo_.size()))
2380  {
2381  __SS__ << "Invalid col (" << c << ") requested!" << std::endl;
2382  throw std::runtime_error(ss.str());
2383  }
2384 
2385  //__MOUT__ << "getChildLink for col: " << c << "-" <<
2386  // columnsInfo_[c].getType() << "-" << columnsInfo_[c].getName() << std::endl;
2387 
2388  //check if column is a child link UID
2389  if((isGroup = columnsInfo_[c].isChildLinkGroupID()) ||
2390  columnsInfo_[c].isChildLinkUID())
2391  {
2392  //must be part of unique link, (or invalid table?)
2393  //__MOUT__ << "col: " << c << std::endl;
2394  linkPair.second = c;
2395  std::string index = columnsInfo_[c].getChildLinkIndex();
2396 
2397  //__MOUT__ << "index: " << index << std::endl;
2398 
2399  //find pair link
2400  for(unsigned int col=0; col<columnsInfo_.size(); ++col)
2401  {
2402  //__MOUT__ << "try: " << col << "-" << columnsInfo_[col].getType() << "-" << columnsInfo_[col].getName() << std::endl;
2403  if(col == c) continue; //skip column c that we know
2404  else if(columnsInfo_[col].isChildLink() &&
2405  index == columnsInfo_[col].getChildLinkIndex())
2406  {
2407  //found match!
2408  __MOUT__ << "getChildLink Found match for col: " << c << " at " << col << std::endl;
2409  linkPair.first = col;
2410  return true;
2411  }
2412  }
2413 
2414  //if here then invalid table!
2415  __SS__ << "\tIn view: " << tableName_ <<
2416  ", Can't find complete child link for column name " << columnsInfo_[c].getName() << std::endl;
2417  throw std::runtime_error(ss.str());
2418  }
2419 
2420  if(!columnsInfo_[c].isChildLink())
2421  return false; //cant be unique link
2422 
2423  //this is child link, so find pair link uid or gid column
2424  linkPair.first = c;
2425  std::string index = columnsInfo_[c].getChildLinkIndex();
2426 
2427  //__MOUT__ << "index: " << index << std::endl;
2428 
2429  //find pair link
2430  for(unsigned int col=0; col<columnsInfo_.size(); ++col)
2431  {
2432  //__MOUT__ << "try: " << col << "-" << columnsInfo_[col].getType() << "-" << columnsInfo_[col].getName() << std::endl;
2433  if(col == c) continue; //skip column c that we know
2434  // __MOUT__ << "try: " << col << "-" << columnsInfo_[col].getType() <<
2435  // "-" << columnsInfo_[col].getName() <<
2436  // "-u" << columnsInfo_[col].isChildLinkUID() <<
2437  // "-g" << columnsInfo_[col].isChildLinkGroupID() << std::endl;
2438  //
2439  // if(columnsInfo_[col].isChildLinkUID())
2440  // __MOUT__ << "-L" << columnsInfo_[col].getChildLinkIndex() << std::endl;
2441  //
2442  // if(columnsInfo_[col].isChildLinkGroupID())
2443  // __MOUT__ << "-L" << columnsInfo_[col].getChildLinkIndex() << std::endl;
2444 
2445  if(((columnsInfo_[col].isChildLinkUID() && !(isGroup = false)) ||
2446  (columnsInfo_[col].isChildLinkGroupID() && (isGroup = true)))
2447  && index == columnsInfo_[col].getChildLinkIndex())
2448  {
2449  //found match!
2450  //__MOUT__ << "getChildLink Found match for col: " << c << " at " << col << std::endl;
2451  linkPair.second = col;
2452  return true;
2453  }
2454  }
2455 
2456  //if here then invalid table!
2457  __SS__ << "\tIn view: " << tableName_ <<
2458  ", Can't find complete child link id for column name " << columnsInfo_[c].getName() << std::endl;
2459  throw std::runtime_error(ss.str());
2460 }
2461 
2462 //==============================================================================
2463 //isNumber ~~
2464 // returns true if hex ("0x.."), binary("b..."), or base10 number
2465 bool ConfigurationView::isNumber(const std::string& s) const
2466 {
2467  //__MOUT__ << "string " << s << std::endl;
2468  if(s.find("0x") == 0) //indicates hex
2469  {
2470  //__MOUT__ << "0x found" << std::endl;
2471  for(unsigned int i=2;i<s.size();++i)
2472  {
2473  if(!((s[i] >= '0' && s[i] <= '9') ||
2474  (s[i] >= 'A' && s[i] <= 'F') ||
2475  (s[i] >= 'a' && s[i] <= 'f')
2476  ))
2477  {
2478  //__MOUT__ << "prob " << s[i] << std::endl;
2479  return false;
2480  }
2481  }
2482  //return std::regex_match(s.substr(2), std::regex("^[0-90-9a-fA-F]+"));
2483  }
2484  else if(s[0] == 'b') //indicates binary
2485  {
2486  //__MOUT__ << "b found" << std::endl;
2487 
2488  for(unsigned int i=1;i<s.size();++i)
2489  {
2490  if(!((s[i] >= '0' && s[i] <= '1')
2491  ))
2492  {
2493  //__MOUT__ << "prob " << s[i] << std::endl;
2494  return false;
2495  }
2496  }
2497  }
2498  else
2499  {
2500  //__MOUT__ << "base 10 " << std::endl;
2501  for(unsigned int i=0;i<s.size();++i)
2502  if(!((s[i] >= '0' && s[i] <= '9') ||
2503  s[i] == '.' ||
2504  s[i] == '+' ||
2505  s[i] == '-'))
2506  return false;
2507  //Note: std::regex crashes in unresolvable ways (says Ryan.. also, stop using libraries)
2508  //return std::regex_match(s, std::regex("^(\\-|\\+)?[0-9]*(\\.[0-9]+)?"));
2509  }
2510 
2511  return true;
2512 }
2513 
2514 
2515 
2516 
2517 #ifdef __GNUG__
2518 #include <cstdlib>
2519 #include <memory>
2520 #include <cxxabi.h>
2521 
2522 std::string ots_demangle(const char* name) {
2523 
2524  int status = -4; // some arbitrary value to eliminate the compiler warning
2525 
2526  // enable c++11 by passing the flag -std=c++11 to g++
2527  std::unique_ptr<char, void(*)(void*)> res {
2528  abi::__cxa_demangle(name, NULL, NULL, &status),
2529  std::free
2530  };
2531 
2532  return (status==0) ? res.get() : name ;
2533 }
2534 
2535 #else
2536 
2537 // does nothing if not g++
2538 std::string ots_demangle(const char* name) {
2539  return name;
2540 }
2541 
2542 
2543 #endif