otsdaq  v1_01_02
 All Classes Namespaces Functions
ConfigurationView.cc
1 #include "otsdaq-core/ConfigurationDataFormats/ConfigurationView.h"
2 
3 #include <iostream>
4 #include <sstream>
5 #include <time.h> /* time_t, time, ctime */
6 #include <cstdlib>
7 #include <regex>
8 
9 using namespace ots;
10 
11 #undef __MF_SUBJECT__
12 #define __MF_SUBJECT__ "ConfigurationView"
13 #undef __MF_HDR__
14 #define __MF_HDR__ tableName_ << ":v" << version_ << ":" << __COUT_HDR_FL__
15 
16 const unsigned int ConfigurationView::INVALID = -1;
17 
18 //==============================================================================
19 ConfigurationView::ConfigurationView(const std::string &name)
20 : uniqueStorageIdentifier_ (""),
21  tableName_ (name),
22  version_ (ConfigurationVersion::INVALID),
23  comment_ (""),
24  author_ (""),
25  creationTime_ (time(0)),
26  lastAccessTime_ (0),
27  colUID_ (INVALID),
28  fillWithLooseColumnMatching_(false),
29  sourceColumnMismatchCount_ (0),
30  sourceColumnMissingCount_ (0)
31 {}
32 
33 //==============================================================================
34 ConfigurationView::~ConfigurationView(void)
35 {}
36 
37 //==============================================================================
38 //operator=
39 // Do NOT allow!... use ConfigurationView::copy
40 // copy is used to maintain consistency with version, creationTime, lastAccessTime, etc)
41 ConfigurationView& ConfigurationView::operator=(const ConfigurationView src)
42 {
43  __SS__ << "Invalid use of operator=... Should not directly copy a ConfigurationView. Please use ConfigurationView::copy(sourceView,author,comment)";
44  throw std::runtime_error(ss.str());
45 }
46 
47 //==============================================================================
48 ConfigurationView& ConfigurationView::copy(const ConfigurationView &src,
49  ConfigurationVersion destinationVersion,
50  const std::string &author)
51 {
52  tableName_ = src.tableName_;
53  version_ = destinationVersion;
54  comment_ = src.comment_;
55  author_ = author; //take new author
56  //creationTime_ = time(0); //don't change creation time
57  lastAccessTime_ = time(0);
58  columnsInfo_ = src.columnsInfo_;
59  theDataView_ = src.theDataView_;
60  sourceColumnNames_ = src.sourceColumnNames_;
61  init(); // verify consistency
62  return *this;
63 }
64 
65 //==============================================================================
66 //init
67 // Should be called after being filled to setup special members
68 // and verify consistency.
69 // e.g. identifying the UID column
70 //
71 // Note: this function also sanitizes yes/no, on/off, and true/false types
72 void ConfigurationView::init(void)
73 {
74  try
75  {
76  //verify column names are unique
77  // make set of names,.. and CommentDescription == COMMENT
78  std::set<std::string> colNameSet;
79  std::string capsColName, colName;
80  for(auto &colInfo: columnsInfo_)
81  {
82  colName = colInfo.getStorageName();
83  if(colName == "COMMENT_DESCRIPTION")
84  colName = "COMMENT";
85  capsColName = "";
86  for(unsigned int i=0;i<colName.size(); ++i)
87  {
88  if(colName[i] == '_') continue;
89  capsColName += colName[i];
90  }
91 
92  colNameSet.emplace(capsColName);
93  }
94 
95  if(colNameSet.size() != columnsInfo_.size())
96  {
97  __SS__ << "Configuration Error:\t" <<
98  " Columns names must be unique! There are " << columnsInfo_.size() <<
99  " columns and the unique name count is " << colNameSet.size() << std::endl;
100  __MOUT_ERR__ << ss.str();
101 
102  throw std::runtime_error(ss.str());
103  }
104 
105  getOrInitColUID(); //setup UID column
106 
107  //require one comment column
108  unsigned int colPos;
109  if((colPos = findColByType(ViewColumnInfo::TYPE_COMMENT)) != INVALID)
110  {
111  if(columnsInfo_[colPos].getName() != "CommentDescription")
112  {
113  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_COMMENT <<
114  " data type column must have name=" <<
115  "CommentDescription" << std::endl;
116  __MOUT_ERR__ << ss.str();
117  throw std::runtime_error(ss.str());
118  }
119 
120  if(findColByType(ViewColumnInfo::TYPE_COMMENT,colPos+1) != INVALID) //found two!
121  {
122  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_COMMENT <<
123  " data type in column " <<
124  columnsInfo_[colPos].getName() <<
125  " is repeated. This is not allowed." << std::endl;
126  __MOUT_ERR__ << ss.str();
127  throw std::runtime_error(ss.str());
128  }
129 
130  if(colPos != getNumberOfColumns()-3)
131  {
132  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_COMMENT <<
133  " data type column must be 3rd to last (in column " <<
134  getNumberOfColumns()-3 << ")." << std::endl;
135  __MOUT_ERR__ << ss.str();
136  throw std::runtime_error(ss.str());
137  }
138  }
139  else
140  {
141  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_COMMENT <<
142  " data type column " <<" is missing. This is not allowed." << std::endl;
143  __MOUT_ERR__ << ss.str();
144  throw std::runtime_error(ss.str());
145  }
146 
147  //require one author column
148  if((colPos = findColByType(ViewColumnInfo::TYPE_AUTHOR)) != INVALID)
149  {
150  if(findColByType(ViewColumnInfo::TYPE_AUTHOR,colPos+1) != INVALID) //found two!
151  {
152  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_AUTHOR <<
153  " data type in column " <<
154  columnsInfo_[colPos].getName() <<
155  " is repeated. This is not allowed." << std::endl;
156  __MOUT_ERR__ << ss.str();
157  throw std::runtime_error(ss.str());
158  }
159 
160  if(colPos != getNumberOfColumns()-2)
161  {
162  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_AUTHOR <<
163  " data type column must be 2nd to last (in column " <<
164  getNumberOfColumns()-2 << ")." << std::endl;
165  __MOUT_ERR__ << ss.str();
166  throw std::runtime_error(ss.str());
167  }
168  }
169  else
170  {
171  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_AUTHOR <<
172  " data type column " <<" is missing. This is not allowed." << std::endl;
173  __MOUT_ERR__ << ss.str();
174  throw std::runtime_error(ss.str());
175  }
176 
177  //require one timestamp column
178  if((colPos = findColByType(ViewColumnInfo::TYPE_TIMESTAMP)) != INVALID)
179  {
180  if(findColByType(ViewColumnInfo::TYPE_TIMESTAMP,colPos+1) != INVALID) //found two!
181  {
182  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_TIMESTAMP <<
183  " data type in column " <<
184  columnsInfo_[colPos].getName() << " is repeated. This is not allowed." <<
185  std::endl;
186  __MOUT_ERR__ << ss.str();
187  throw std::runtime_error(ss.str());
188  }
189 
190  if(colPos != getNumberOfColumns()-1)
191  {
192  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_TIMESTAMP <<
193  " data type column must be last (in column " <<
194  getNumberOfColumns()-1 << ")." << std::endl;
195  __MOUT_ERR__ << ss.str();
196  throw std::runtime_error(ss.str());
197  }
198  }
199  else
200  {
201  __SS__ << "Configuration Error:\t" << ViewColumnInfo::TYPE_TIMESTAMP <<
202  " data type column " <<" is missing. This is not allowed." << std::endl;
203  __MOUT_ERR__ << ss.str();
204  throw std::runtime_error(ss.str());
205  }
206 
207  //check that UID is really unique ID (no repeats)
208  // and ... allow letters, numbers, dash, underscore
209  // and ... force size 1
210  std::set<std::string /*uid*/> uidSet;
211  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
212  {
213  if(uidSet.find(theDataView_[row][colUID_]) != uidSet.end())
214  {
215  __SS__ << ("Entries in UID are not unique. Specifically at row=" +
216  std::to_string(row) + " value=" + theDataView_[row][colUID_])<< std::endl;
217  __MOUT_ERR__ << ss.str();
218  throw std::runtime_error(ss.str());
219  }
220 
221  if(theDataView_[row][colUID_].size() == 0)
222  {
223  __SS__ << "An invalid UID '" << theDataView_[row][colUID_] << "' " <<
224  " was identified. UIDs must contain at least 1 character." <<
225  std::endl;
226  throw std::runtime_error(ss.str());
227  }
228 
229  for(unsigned int i=0;i<theDataView_[row][colUID_].size();++i)
230  if(!(
231  (theDataView_[row][colUID_][i] >= 'A' && theDataView_[row][colUID_][i] <= 'Z') ||
232  (theDataView_[row][colUID_][i] >= 'a' && theDataView_[row][colUID_][i] <= 'z') ||
233  (theDataView_[row][colUID_][i] >= '0' && theDataView_[row][colUID_][i] <= '9') ||
234  (theDataView_[row][colUID_][i] == '-' || theDataView_[row][colUID_][i] <= '_')
235  ))
236  {
237  __SS__ << "An invalid UID '" << theDataView_[row][colUID_] << "' " <<
238  " was identified. UIDs must contain only letters, numbers," <<
239  "dashes, and underscores." << std::endl;
240  throw std::runtime_error(ss.str());
241  }
242 
243  uidSet.insert(theDataView_[row][colUID_]);
244  }
245  if(uidSet.size() != getNumberOfRows())
246  {
247  __SS__ << "Entries in UID are not unique!" <<
248  "There are " << getNumberOfRows() <<
249  " rows and the unique UID count is " << uidSet.size() << std::endl;
250  __MOUT_ERR__ << ss.str();
251  throw std::runtime_error(ss.str());
252  }
253 
254  //check that any TYPE_UNIQUE_DATA columns are really unique (no repeats)
255  colPos = (unsigned int)-1;
256  while((colPos = findColByType(ViewColumnInfo::TYPE_UNIQUE_DATA,colPos+1)) != INVALID)
257  {
258  std::set<std::string /*unique data*/> uDataSet;
259  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
260  {
261  if(uDataSet.find(theDataView_[row][colPos]) != uDataSet.end())
262  {
263  __SS__ << "Entries in Unique Data column " <<
264  columnsInfo_[colPos].getName() <<
265  (" are not unique. Specifically at row=" +
266  std::to_string(row) + " value=" + theDataView_[row][colPos]) <<
267  std::endl;
268  __MOUT_ERR__ << ss.str();
269  throw std::runtime_error(ss.str());
270  }
271  uDataSet.insert(theDataView_[row][colPos]);
272  }
273  if(uDataSet.size() != getNumberOfRows())
274  {
275  __SS__ << "Entries in Unique Data column " <<
276  columnsInfo_[colPos].getName() << " are not unique!" <<
277  "There are " << getNumberOfRows() <<
278  " rows and the unique data count is " << uDataSet.size() << std::endl;
279  __MOUT_ERR__ << ss.str();
280  throw std::runtime_error(ss.str());
281  }
282  }
283 
284  auto rowDefaults = getDefaultRowValues();
285 
286  //check that column types are well behaved
287  // - check that fixed choice data is one of choices
288  // - sanitize booleans
289  // - check that child link I are unique
290  // note: childLinkId refers to childLinkGroupIDs AND childLinkUIDs
291  std::set<std::string> groupIdIndexes, childLinkIndexes, childLinkIdLabels;
292  unsigned int groupIdIndexesCount = 0, childLinkIndexesCount = 0, childLinkIdLabelsCount = 0;
293  for(unsigned int col = 0; col < getNumberOfColumns(); ++col)
294  {
295  if(columnsInfo_[col].getType() == ViewColumnInfo::TYPE_FIXED_CHOICE_DATA)
296  {
297  bool found;
298  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
299  {
300  found = false;
301  //check against default value first
302  if(theDataView_[row][col] == rowDefaults[col])
303  continue; //default is always ok
304 
305  const std::vector<std::string>& theDataChoices =
306  columnsInfo_[col].getDataChoices();
307 
308  //check if arbitrary values allowed
309  if(theDataChoices.size() && theDataChoices[0] ==
310  "arbitraryBool=1")
311  continue; //arbitrary values allowed
312 
313  for(const auto &choice:theDataChoices)
314  {
315  if(theDataView_[row][col] == choice)
316  {
317  found = true;
318  break;
319  }
320  }
321  if(!found)
322  {
323  __SS__ << "Configuration Error:\t'" << theDataView_[row][col] << "' in column " <<
324  columnsInfo_[col].getName() << " is not a valid Fixed Choice option. " <<
325  "Possible values are as follows: ";
326 
327  for(unsigned int i = 0; i < columnsInfo_[col].getDataChoices().size(); ++i)
328  {
329  if(i) ss << ", ";
330  ss << columnsInfo_[col].getDataChoices()[i];
331  }
332  ss << "." << std::endl;
333  __MOUT_ERR__ << ss.str();
334  throw std::runtime_error(ss.str());
335  }
336  }
337  }
338  else if(columnsInfo_[col].getType() == ViewColumnInfo::TYPE_ON_OFF)
339  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
340  {
341  if (theDataView_[row][col] == "1" || theDataView_[row][col] == "on" || theDataView_[row][col] == "On" || theDataView_[row][col] == "ON")
342  theDataView_[row][col] = ViewColumnInfo::TYPE_VALUE_ON;
343  else if (theDataView_[row][col] == "0" || theDataView_[row][col] == "off" || theDataView_[row][col] == "Off" || theDataView_[row][col] == "OFF")
344  theDataView_[row][col] = ViewColumnInfo::TYPE_VALUE_OFF;
345  else
346  {
347  __SS__ << "Configuration Error:\t" << theDataView_[row][col] << " in column " <<
348  columnsInfo_[col].getName() << " is not a valid Type (On/Off) std::string. Possible values are 1, on, On, ON, 0, off, Off, OFF." << std::endl;
349  __MOUT_ERR__ << ss.str();
350  throw std::runtime_error(ss.str());
351  }
352  }
353  else if(columnsInfo_[col].getType() == ViewColumnInfo::TYPE_TRUE_FALSE)
354  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
355  {
356  if (theDataView_[row][col] == "1" || theDataView_[row][col] == "true" || theDataView_[row][col] == "True" || theDataView_[row][col] == "TRUE")
357  theDataView_[row][col] = ViewColumnInfo::TYPE_VALUE_TRUE;
358  else if (theDataView_[row][col] == "0" || theDataView_[row][col] == "false" || theDataView_[row][col] == "False" || theDataView_[row][col] == "FALSE")
359  theDataView_[row][col] = ViewColumnInfo::TYPE_VALUE_FALSE;
360  else
361  {
362  __SS__ << "Configuration Error:\t" << theDataView_[row][col] << " in column " <<
363  columnsInfo_[col].getName() << " is not a valid Type (True/False) std::string. Possible values are 1, true, True, TRUE, 0, false, False, FALSE." << std::endl;
364  __MOUT_ERR__ << ss.str();
365  throw std::runtime_error("Invalid true/false usage");
366  }
367  }
368  else if(columnsInfo_[col].getType() == ViewColumnInfo::TYPE_YES_NO)
369  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
370  {
371  if (theDataView_[row][col] == "1" || theDataView_[row][col] == "yes" || theDataView_[row][col] == "Yes" || theDataView_[row][col] == "YES")
372  theDataView_[row][col] = ViewColumnInfo::TYPE_VALUE_YES;
373  else if (theDataView_[row][col] == "0" || theDataView_[row][col] == "no" || theDataView_[row][col] == "No" || theDataView_[row][col] == "NO")
374  theDataView_[row][col] = ViewColumnInfo::TYPE_VALUE_NO;
375  else
376  {
377  __SS__ << "Configuration Error:\t" << theDataView_[row][col] << " in column " <<
378  columnsInfo_[col].getName() << " is not a valid Type (Yes/No) std::string. Possible values are 1, yes, Yes, YES, 0, no, No, NO." << std::endl;
379  __MOUT_ERR__ << ss.str();
380  throw std::runtime_error(ss.str());
381  }
382  }
383  else if(columnsInfo_[col].isGroupID()) //GroupID type
384  {
385  colLinkGroupIDs_[columnsInfo_[col].getChildLinkIndex()] = col; //add to groupid map
386  //check uniqueness
387  groupIdIndexes.emplace(columnsInfo_[col].getChildLinkIndex());
388  ++groupIdIndexesCount;
389  }
390  else if(columnsInfo_[col].isChildLink()) //Child Link type
391  {
392  //sanitize no link to default
393  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
394  if(theDataView_[row][col] == "NoLink" ||
395  theDataView_[row][col] == "No_Link" ||
396  theDataView_[row][col] == "NOLINK" ||
397  theDataView_[row][col] == "NO_LINK" ||
398  theDataView_[row][col] == "Nolink" ||
399  theDataView_[row][col] == "nolink" ||
400  theDataView_[row][col] == "noLink")
401  theDataView_[row][col] = ViewColumnInfo::DATATYPE_LINK_DEFAULT;
402 
403  //check uniqueness
404  childLinkIndexes.emplace(columnsInfo_[col].getChildLinkIndex());
405  ++childLinkIndexesCount;
406 
407  //force data type to ViewColumnInfo::DATATYPE_STRING
408  if(columnsInfo_[col].getDataType() != ViewColumnInfo::DATATYPE_STRING)
409  {
410  __SS__ << "Configuration Error:\t" << "Column " << col <<
411  " with name " << columnsInfo_[col].getName() <<
412  " is a Child Link column and has an illegal data type of '" <<
413  columnsInfo_[col].getDataType() <<
414  "'. The data type for Child Link columns must be " <<
415  ViewColumnInfo::DATATYPE_STRING << std::endl;
416  __MOUT_ERR__ << ss.str();
417  throw std::runtime_error(ss.str());
418  }
419 
420  }
421  else if(columnsInfo_[col].isChildLinkUID() || //Child Link ID type
422  columnsInfo_[col].isChildLinkGroupID())
423  {
424  //check uniqueness
425  childLinkIdLabels.emplace(columnsInfo_[col].getChildLinkIndex());
426  ++childLinkIdLabelsCount;
427 
428  //check that the Link ID is not empty, and force to default
429  for(unsigned int row = 0; row < getNumberOfRows(); ++row)
430  if(theDataView_[row][col] == "")
431  theDataView_[row][col] = rowDefaults[col];
432 
433  }
434  }
435 
436  //verify child link index uniqueness
437  if(groupIdIndexes.size() != groupIdIndexesCount)
438  {
439  __SS__ << ("GroupId Labels are not unique!") <<
440  "There are " << groupIdIndexesCount <<
441  " GroupId Labels and the unique count is " << groupIdIndexes.size() << std::endl;
442  __MOUT_ERR__ << ss.str();
443  throw std::runtime_error(ss.str());
444  }
445  if(childLinkIndexes.size() != childLinkIndexesCount)
446  {
447  __SS__ << ("Child Link Labels are not unique!") <<
448  "There are " << childLinkIndexesCount <<
449  " Child Link Labels and the unique count is " << childLinkIndexes.size() << std::endl;
450  __MOUT_ERR__ << ss.str();
451  throw std::runtime_error(ss.str());
452  }
453  if(childLinkIdLabels.size() != childLinkIdLabelsCount)
454  {
455  __SS__ << ("Child Link ID Labels are not unique!") <<
456  "There are " << childLinkIdLabelsCount <<
457  " Child Link ID Labels and the unique count is " << childLinkIdLabels.size() << std::endl;
458  __MOUT_ERR__ << ss.str();
459  throw std::runtime_error(ss.str());
460  }
461 
462  }
463  catch(...)
464  {
465  __MOUT__ << "Error occured in ConfigurationView::init() for version=" << version_ << std::endl;
466  throw;
467  }
468 }
469 
470 //==============================================================================
471 //getValue
472 // string version
473 void ConfigurationView::getValue(std::string& value, unsigned int row, unsigned int col, bool convertEnvironmentVariables) const
474 {
475  if(!(col < columnsInfo_.size() && row < getNumberOfRows()))
476  throw std::runtime_error("Invalid row col requested");
477 
478  if(columnsInfo_[col].getDataType() == ViewColumnInfo::DATATYPE_STRING)
479  {
480  value = theDataView_[row][col];
481  if(convertEnvironmentVariables)
482  value = convertEnvVariables(value);
483  }
484  else if(columnsInfo_[col].getDataType() == ViewColumnInfo::DATATYPE_TIME)
485  {
486  value.resize(30); //known fixed size: Thu Aug 23 14:55:02 2001 CST
487  time_t timestamp(strtol(theDataView_[row][col].c_str(),0,10));
488  struct tm tmstruct;
489  ::localtime_r(&timestamp, &tmstruct);
490  ::strftime(&value[0], 30, "%c %Z", &tmstruct);
491  value.resize(strlen(value.c_str()));
492  }
493  else
494  {
495  __SS__ << "\tUnrecognized column data type: " << columnsInfo_[col].getDataType()
496  << " in configuration " << tableName_
497  << " at column=" << columnsInfo_[col].getName()
498  << " for getValue with type '" << ots_demangle(typeid(value).name())
499  << "'" << std::endl;
500  throw std::runtime_error(ss.str());
501  }
502 
503 
504 }
505 
506 
507 //==============================================================================
508 //getValueAsString
509 // gets the value with the proper data type and converts to string
510 // as though getValue was called.
511 std::string ConfigurationView::getValueAsString(unsigned int row, unsigned int col,
512  bool convertEnvironmentVariables) const
513 {
514  if(!(col < columnsInfo_.size() && row < getNumberOfRows()))
515  throw std::runtime_error("Invalid row col requested");
516 
517 
518  if(columnsInfo_[col].getType() == ViewColumnInfo::TYPE_ON_OFF)
519  {
520  if (theDataView_[row][col] == "1" || theDataView_[row][col] == "on" || theDataView_[row][col] == "On" || theDataView_[row][col] == "ON")
521  return ViewColumnInfo::TYPE_VALUE_ON;
522  else
523  return ViewColumnInfo::TYPE_VALUE_OFF;
524  }
525  else if(columnsInfo_[col].getType() == ViewColumnInfo::TYPE_TRUE_FALSE)
526  {
527  if (theDataView_[row][col] == "1" || theDataView_[row][col] == "true" || theDataView_[row][col] == "True" || theDataView_[row][col] == "TRUE")
528  return ViewColumnInfo::TYPE_VALUE_TRUE;
529  else
530  return ViewColumnInfo::TYPE_VALUE_FALSE;
531  }
532  else if(columnsInfo_[col].getType() == ViewColumnInfo::TYPE_YES_NO)
533  {
534  if (theDataView_[row][col] == "1" || theDataView_[row][col] == "yes" || theDataView_[row][col] == "Yes" || theDataView_[row][col] == "YES")
535  return ViewColumnInfo::TYPE_VALUE_YES;
536  else
537  return ViewColumnInfo::TYPE_VALUE_NO;
538  }
539 
540  return convertEnvironmentVariables?convertEnvVariables(theDataView_[row][col]):
541  theDataView_[row][col];
542 }
543 
544 
545 //==============================================================================
546 //getEscapedValueAsString
547 // gets the value with the proper data type and converts to string
548 // as though getValue was called.
549 // then escapes all special characters with slash.
550 // Note: this should be useful for values placed in double quotes, i.e. JSON.
551 std::string ConfigurationView::getEscapedValueAsString(unsigned int row, unsigned int col,
552  bool convertEnvironmentVariables) const
553 {
554  std::string val = getValueAsString(row,col,convertEnvironmentVariables);
555  std::string retVal = "";
556  retVal.reserve(val.size()); //reserve roughly right size
557  for(unsigned int i=0;i<val.size();++i)
558  {
559  if(val[i] == '\n')
560  retVal += "\\n";
561  else if(val[i] == '\t')
562  retVal += "\\t";
563  else if(val[i] == '\r')
564  retVal += "\\r";
565  else
566  {
567  //escaped characters need a
568  if(val[i] == '"' ||
569  val[i] == '\\')
570  retVal += '\\';
571  retVal += val[i];
572  }
573  }
574  return retVal;
575 }
576 
577 //==============================================================================
578 std::string ConfigurationView::convertEnvVariables(const std::string& data) const
579 {
580  std::string converted = data;
581  if(data.find("${") != std::string::npos)
582  {
583  unsigned int begin = data.find("${");
584  unsigned int end = data.find("}");
585  std::string envVariable = data.substr(begin+2, end-begin-2);
586  //std::cout << "Converted Value: " << envVariable << std::endl;
587  if(getenv(envVariable.c_str()) != nullptr)
588  {
589  return convertEnvVariables(converted.replace(begin,end-begin+1,getenv(envVariable.c_str())));
590  }
591  else
592  throw std::runtime_error("In configuration " + tableName_ + " the environmental variable: " + envVariable + " is not set! Please make sure you set it before continuing!");
593  }
594  return converted;
595 }
596 
597 //==============================================================================
598 //setValue
599 // string version
600 void ConfigurationView::setValue(const std::string &value, unsigned int row, unsigned int col)
601 {
602  if(!(col < columnsInfo_.size() && row < getNumberOfRows()))
603  {
604  __SS__ << "Invalid row (" << row << ") col (" << col << ") requested!" << std::endl;
605  throw std::runtime_error(ss.str());
606  }
607 
608  if(columnsInfo_[col].getDataType() == ViewColumnInfo::DATATYPE_STRING)
609  theDataView_[row][col] = value;
610  else // dont allow ViewColumnInfo::DATATYPE_TIME to be set as string.. force use as time_t to standardize string result
611  {
612 
613  __SS__ << "\tUnrecognized column data type: " << columnsInfo_[col].getDataType()
614  << " in configuration " << tableName_
615  << " at column=" << columnsInfo_[col].getName()
616  << " for setValue with type '" << ots_demangle(typeid(value).name())
617  << "'" << std::endl;
618  throw std::runtime_error(ss.str());
619  }
620 }
621 void ConfigurationView::setValue(const char *value, unsigned int row, unsigned int col)
622 { setValue(std::string(value),row,col); }
623 
624 //==============================================================================
625 //setValue
626 // string version
627 void ConfigurationView::setValueAsString(const std::string &value, unsigned int row, unsigned int col)
628 {
629  if(!(col < columnsInfo_.size() && row < getNumberOfRows()))
630  {
631  __SS__ << "Invalid row (" << row << ") col (" << col << ") requested!" << std::endl;
632  throw std::runtime_error(ss.str());
633  }
634 
635  theDataView_[row][col] = value;
636 }
637 
638 //==============================================================================
639 //getColOfUID
640 // if column not found throw error
641 const unsigned int ConfigurationView::getOrInitColUID (void)
642 {
643  if(colUID_ != INVALID) return colUID_;
644 
645  //if doesn't exist throw error! each view must have a UID column
646  colUID_ = findColByType(ViewColumnInfo::TYPE_UID);
647  if(colUID_ == INVALID)
648  {
649  __MOUT__ << "Column Types: " << std::endl;
650  for(unsigned int col=0; col<columnsInfo_.size(); ++col)
651  std::cout << columnsInfo_[col].getType() << "() " << columnsInfo_[col].getName() << std::endl;
652  __SS__ << "\tMissing UID Column in table named '" << tableName_ << "'" << std::endl;
653  __MOUT_ERR__ << "\n" << ss.str() << std::endl;
654  throw std::runtime_error(ss.str());
655  }
656  return colUID_;
657 }
658 //==============================================================================
659 //getColOfUID
660 // const version, so don't attempt to lookup
661 // if column not found throw error
662 const unsigned int ConfigurationView::getColUID (void) const
663 {
664  if(colUID_ != INVALID) return colUID_;
665 
666  __MOUT__ << "Column Types: " << std::endl;
667  for(unsigned int col=0; col<columnsInfo_.size(); ++col)
668  std::cout << columnsInfo_[col].getType() << "() " << columnsInfo_[col].getName() << std::endl;
669 
670  __SS__ << ("Missing UID Column in config named " + tableName_ +
671  ". (Possibly ConfigurationView was just not initialized?" +
672  "This is the const call so can not alter class members)") << std::endl;
673  __MOUT_ERR__ << "\n" << ss.str() << std::endl;
674  throw std::runtime_error(ss.str());
675 }
676 
677 //==============================================================================
678 //addRowToGroup
679 // Group entry can include | to place a record in multiple groups
680 void ConfigurationView::addRowToGroup(const unsigned int &row,
681  const unsigned int &col,
682  const std::string &groupID,
683  const std::string &colDefault)
684 {
685  if(isEntryInGroupCol(row,col,groupID))
686  {
687  __SS__ << "GroupID (" << groupID <<
688  ") added to row (" << row
689  << " is already present!" << std::endl;
690  throw std::runtime_error(ss.str());
691  }
692 
693  //not in group, so
694  // if no groups
695  // set groupid
696  // if other groups
697  // prepend groupId |
698  if(getDataView()[row][col] == "" ||
699  getDataView()[row][col] == colDefault)
700  setValue(
701  groupID,
702  row,col);
703  else
704  setValue(
705  groupID + " | " + getDataView()[row][col],
706  row,col);
707 
708  //__MOUT__ << getDataView()[row][col] << std::endl;
709 }
710 
711 //==============================================================================
712 //removeRowFromGroup
713 // Group entry can include | to place a record in multiple groups
714 void ConfigurationView::removeRowFromGroup(const unsigned int &row,
715  const unsigned int &col,
716  const std::string &groupNeedle)
717 {
718  __MOUT__ << "groupNeedle " << groupNeedle << std::endl;
719  std::set<std::string> groupIDList;
720  if(!isEntryInGroupCol(row,col,groupNeedle,&groupIDList))
721  {
722  __SS__ << "GroupID (" << groupNeedle <<
723  ") removed from row (" << row
724  << ") was already removed!" << std::endl;
725  throw std::runtime_error(ss.str());
726  }
727 
728  //is in group, so
729  // create new string based on set of groupids
730  // but skip groupNeedle
731 
732  std::string newValue = "";
733  unsigned int cnt = 0;
734  for(const auto & groupID : groupIDList)
735  {
736  //__MOUT__ << groupID << " " << groupNeedle << " " << newValue << std::endl;
737  if(groupID == groupNeedle) continue; //skip group to be removed
738 
739  if(cnt) newValue += " | ";
740  newValue += groupID;
741  }
742  setValue(newValue,row,col);
743 
744  //__MOUT__ << getDataView()[row][col] << std::endl;
745 }
746 
747 //==============================================================================
748 //isEntryInGroup
749 // All group link checking should use this function
750 // so that handling is consistent
751 //
752 // Group entry can include | to place a record in multiple groups
753 bool ConfigurationView::isEntryInGroup(const unsigned int &r,
754  const std::string &childLinkIndex,
755  const std::string &groupNeedle) const
756 {
757  unsigned int c = getColLinkGroupID(childLinkIndex); //column in question
758 
759  return isEntryInGroupCol(r,c,groupNeedle);
760 }
761 
762 //==============================================================================
763 // isEntryInGroupCol
764 // private function to prevent users from treating any column like a GroupID column
765 //
766 // if *groupIDList != 0 return vector of groupIDs found
767 // useful for removing groupIDs.
768 //
769 // Group entry can include | to place a record in multiple groups
770 //
771 // Note: should mirror what happens in ConfigurationView::getSetOfGroupIDs
772 bool ConfigurationView::isEntryInGroupCol(const unsigned int &r,
773  const unsigned int &c, const std::string &groupNeedle,
774  std::set<std::string> *groupIDList) const
775 {
776  unsigned int i=0;
777  unsigned int j=0;
778  bool found = false;
779 
780  //__MOUT__ << "groupNeedle " << groupNeedle << std::endl;
781 
782  //go through the full groupString extracting groups and comparing to groupNeedle
783  for(;j<theDataView_[r][c].size();++j)
784  if((theDataView_[r][c][j] == ' ' || //ignore leading white space or |
785  theDataView_[r][c][j] == '|')
786  && i == j)
787  ++i;
788  else if((theDataView_[r][c][j] == ' ' || //trailing white space or | indicates group
789  theDataView_[r][c][j] == '|')
790  && i != j) // assume end of group name
791  {
792  if(groupIDList) groupIDList->emplace(theDataView_[r][c].substr(i,j-i));
793 
794  //__MOUT__ << "Group found to compare: " <<
795  // theDataView_[r][c].substr(i,j-i) << std::endl;
796  if(groupNeedle == theDataView_[r][c].substr(i,j-i))
797  {
798  if(!groupIDList) //dont return if caller is trying to get group list
799  return true;
800  found = true;
801  }
802  //if no match, setup i and j for next find
803  i = j+1;
804  }
805 
806  if(i != j) //last group check (for case when no ' ' or '|')
807  {
808  if(groupIDList) groupIDList->emplace(theDataView_[r][c].substr(i,j-i));
809 
810  //__MOUT__ << "Group found to compare: " <<
811  // theDataView_[r][c].substr(i,j-i) << std::endl;
812  if(groupNeedle == theDataView_[r][c].substr(i,j-i))
813  return true;
814  }
815 
816  return found;
817 }
818 
819 //==============================================================================
820 //getSetOfGroupIDs
821 // if row == -1, then considers all rows
822 // else just that row
823 // returns unique set of groupIds in GroupID column
824 // associate with childLinkIndex
825 //
826 // Note: should mirror what happens in ConfigurationView::isEntryInGroupCol
827 std::set<std::string> ConfigurationView::getSetOfGroupIDs(const std::string &childLinkIndex,
828  unsigned int r) const
829 {
830  unsigned int c = getColLinkGroupID(childLinkIndex);
831 
832  //__MOUT__ << "GroupID col=" << c << std::endl;
833 
834  std::set<std::string> retSet;
835 
836  unsigned int i=0;
837  unsigned int j=0;
838 
839  if(r != (unsigned int)-1)
840  {
841  if(r >= getNumberOfRows())
842  {
843  __SS__ << "Invalid row requested!" << std::endl;
844  throw std::runtime_error(ss.str());
845  }
846 
847  //go through the full groupString extracting groups
848  //add each found groupId to set
849  for(;j<theDataView_[r][c].size();++j)
850  if((theDataView_[r][c][j] == ' ' || //ignore leading white space or |
851  theDataView_[r][c][j] == '|')
852  && i == j)
853  ++i;
854  else if((theDataView_[r][c][j] == ' ' || //trailing white space or | indicates group
855  theDataView_[r][c][j] == '|')
856  && i != j) // assume end of group name
857  {
858  //__MOUT__ << "Group found: " <<
859  // theDataView_[r][c].substr(i,j-i) << std::endl;
860 
861 
862  retSet.emplace(theDataView_[r][c].substr(i,j-i));
863 
864  //setup i and j for next find
865  i = j+1;
866  }
867 
868  if(i != j) //last group check (for case when no ' ' or '|')
869  retSet.emplace(theDataView_[r][c].substr(i,j-i));
870  }
871  else
872  {
873  //do all rows
874  for(r=0;r<getNumberOfRows();++r)
875  {
876  i=0;
877  j=0;
878 
879  //__MOUT__ << r << ": " << theDataView_[r][c] << std::endl;
880 
881  //go through the full groupString extracting groups
882  //add each found groupId to set
883  for(;j<theDataView_[r][c].size();++j)
884  {
885  //__MOUT__ << "i:" << i << " j:" << j << std::endl;
886 
887  if((theDataView_[r][c][j] == ' ' || //ignore leading white space or |
888  theDataView_[r][c][j] == '|')
889  && i == j)
890  ++i;
891  else if((theDataView_[r][c][j] == ' ' || //trailing white space or | indicates group
892  theDataView_[r][c][j] == '|')
893  && i != j) // assume end of group name
894  {
895  //__MOUT__ << "Group found: " <<
896  // theDataView_[r][c].substr(i,j-i) << std::endl;
897 
898  retSet.emplace(theDataView_[r][c].substr(i,j-i));
899 
900  //setup i and j for next find
901  i = j+1;
902  }
903  }
904 
905  if(i != j) //last group (for case when no ' ' or '|')
906  {
907  //__MOUT__ << "Group found: " <<
908  // theDataView_[r][c].substr(i,j-i) << std::endl;
909  retSet.emplace(theDataView_[r][c].substr(i,j-i));
910  }
911  }
912  }
913 
914  return retSet;
915 }
916 
917 //==============================================================================
918 //getColOfLinkGroupID
919 // const version, if column not found throw error
920 const unsigned int ConfigurationView::getColLinkGroupID(const std::string &childLinkIndex) const
921 {
922  std::map<std::string, unsigned int>::const_iterator it = colLinkGroupIDs_.find(childLinkIndex);
923  if(it != //if already known, return it
924  colLinkGroupIDs_.end())
925  return it->second;
926 
927  __MOUT__ << "Existing Column Types: " << std::endl;
928  for(unsigned int col=0; col<columnsInfo_.size(); ++col)
929  std::cout << columnsInfo_[col].getType() << "() " << columnsInfo_[col].getName() << std::endl;
930 
931  __SS__ << ("Missing column with data type " +
932  (ViewColumnInfo::TYPE_START_GROUP_ID + "-" + childLinkIndex) +
933  " in config named " + tableName_) << std::endl;
934  __MOUT_ERR__ << "\n" << ss.str();
935  throw std::runtime_error(ss.str());
936 }
937 
938 //==============================================================================
939 unsigned int ConfigurationView::findRow(unsigned int col, const std::string& value, unsigned int offsetRow) const
940 {
941  for(unsigned int row=offsetRow; row<theDataView_.size(); ++row)
942  {
943  if(theDataView_[row][col] == value)
944  return row;
945  }
946 
947  __SS__ << "\tIn view: " << tableName_
948  << ", Can't find value=" << value
949  << " in column named " << columnsInfo_[col].getName()
950  << " with type=" << columnsInfo_[col].getType()
951  << std::endl;
952  //Note: findRow gets purposely called by configuration GUI a lot looking for exceptions
953  // so may not want to print out
954  //__MOUT__ << ss.str();
955  throw std::runtime_error(ss.str());
956 }
957 
958 //==============================================================================
959 unsigned int ConfigurationView::findRowInGroup(unsigned int col, const std::string& value,
960  const std::string &groupId, const std::string &childLinkIndex, unsigned int offsetRow) const
961 {
962  unsigned int groupIdCol = getColLinkGroupID(childLinkIndex);
963  for(unsigned int row=offsetRow; row<theDataView_.size(); ++row)
964  {
965  if(theDataView_[row][col] == value &&
966  isEntryInGroupCol(row,groupIdCol,groupId))
967  return row;
968  }
969 
970  __SS__ << "\tIn view: " << tableName_ << ", Can't find in group the value=" <<
971  value << " in column named " <<
972  columnsInfo_[col].getName() << " with type=" <<
973  columnsInfo_[col].getType() << " and GroupID: " <<
974  groupId << " in column " << groupIdCol <<
975  " with GroupID child link index " << childLinkIndex << std::endl;
976  __MOUT__ << ss.str();
977  throw std::runtime_error(ss.str());
978 }
979 
980 //==============================================================================
981 //findCol
982 // throws exception if column not found by name
983 unsigned int ConfigurationView::findCol(const std::string& name) const
984 {
985  for(unsigned int col=0; col<columnsInfo_.size(); ++col)
986  if(columnsInfo_[col].getName() == name)
987  return col;
988 
989  __SS__ << "\tIn view: " << tableName_ <<
990  ", Can't find column named " << name << std::endl;
991  throw std::runtime_error(ss.str());
992 }
993 
994 //==============================================================================
995 //findColByType
996 // return invalid if type not found
997 unsigned int ConfigurationView::findColByType(const std::string& type, int startingCol) const
998 {
999  for(unsigned int col=startingCol; col<columnsInfo_.size(); ++col)
1000  if(columnsInfo_[col].getType() == type)
1001  return col;
1002 
1003  return INVALID;
1004 }
1005 
1006 //Getters
1007 //==============================================================================
1008 const std::string& ConfigurationView::getUniqueStorageIdentifier(void) const
1009 {
1010  return uniqueStorageIdentifier_;
1011 }
1012 
1013 //==============================================================================
1014 const std::string& ConfigurationView::getTableName(void) const
1015 {
1016  return tableName_;
1017 }
1018 
1019 //==============================================================================
1020 const ConfigurationVersion& ConfigurationView::getVersion(void) const
1021 {
1022  return version_;
1023 }
1024 
1025 //==============================================================================
1026 const std::string& ConfigurationView::getComment(void) const
1027 {
1028  return comment_;
1029 }
1030 
1031 //==============================================================================
1032 const std::string& ConfigurationView::getAuthor(void) const
1033 {
1034  return author_;
1035 }
1036 
1037 //==============================================================================
1038 const time_t& ConfigurationView::getCreationTime(void) const
1039 {
1040  return creationTime_;
1041 }
1042 
1043 //==============================================================================
1044 const time_t& ConfigurationView::getLastAccessTime(void) const
1045 {
1046  return lastAccessTime_;
1047 }
1048 
1049 //==============================================================================
1050 const bool& ConfigurationView::getLooseColumnMatching(void) const
1051 {
1052  return fillWithLooseColumnMatching_;
1053 }
1054 
1055 //==============================================================================
1056 const unsigned int ConfigurationView::getSourceColumnSize(void) const
1057 {
1058  //if no rows.. then assume source had no data
1059  // (and give benefit of the doubt that columns were ok for the phantom data)
1060  if(!getNumberOfRows()) return getNumberOfColumns();
1061  return sourceColumnNames_.size();
1062 }
1063 
1064 //==============================================================================
1065 const unsigned int& ConfigurationView::getSourceColumnMismatch(void) const
1066 {
1067  return sourceColumnMismatchCount_;
1068 }
1069 
1070 //==============================================================================
1071 const unsigned int& ConfigurationView::getSourceColumnMissing(void) const
1072 {
1073  return sourceColumnMissingCount_;
1074 }
1075 
1076 //==============================================================================
1077 const std::set<std::string>& ConfigurationView::getSourceColumnNames(void) const
1078 {
1079 
1080  return sourceColumnNames_;
1081 }
1082 
1083 //==============================================================================
1084 std::set<std::string> ConfigurationView::getColumnNames(void) const
1085 {
1086  std::set<std::string> retSet;
1087  for(auto &colInfo: columnsInfo_)
1088  retSet.emplace(colInfo.getName());
1089  return retSet;
1090 }
1091 
1092 //==============================================================================
1093 std::set<std::string> ConfigurationView::getColumnStorageNames(void) const
1094 {
1095  std::set<std::string> retSet;
1096  for(auto &colInfo: columnsInfo_)
1097  retSet.emplace(colInfo.getStorageName());
1098  return retSet;
1099 }
1100 
1101 //==============================================================================
1102 std::vector<std::string> ConfigurationView::getDefaultRowValues(void) const
1103 {
1104  std::vector<std::string> retVec;
1105 
1106  //fill each col of new row with default values
1107  for(unsigned int col=0;col<getNumberOfColumns();++col)
1108  retVec.push_back(columnsInfo_[col].getDefaultValue());
1109 
1110  return retVec;
1111 }
1112 
1113 //==============================================================================
1114 unsigned int ConfigurationView::getNumberOfRows(void) const
1115 {
1116  return theDataView_.size();
1117 }
1118 
1119 //==============================================================================
1120 unsigned int ConfigurationView::getNumberOfColumns(void) const
1121 {
1122  return columnsInfo_.size();
1123 }
1124 
1125 //==============================================================================
1126 const ConfigurationView::DataView& ConfigurationView::getDataView(void) const
1127 {
1128  return theDataView_;
1129 }
1130 
1131 //==============================================================================
1132 //ConfigurationView::DataView* ConfigurationView::getDataViewP(void)
1133 //{
1134 // return &theDataView_;
1135 //}
1136 
1137 //==============================================================================
1138 const std::vector<ViewColumnInfo>& ConfigurationView::getColumnsInfo (void) const
1139 {
1140  return columnsInfo_;
1141 }
1142 
1143 //==============================================================================
1144 std::vector<ViewColumnInfo>* ConfigurationView::getColumnsInfoP(void)
1145 {
1146  return &columnsInfo_;
1147 }
1148 //==============================================================================
1149 const ViewColumnInfo& ConfigurationView::getColumnInfo(unsigned int column) const
1150 {
1151  if(column >= columnsInfo_.size())
1152  {
1153  std::stringstream errMsg;
1154  errMsg << __COUT_HDR_FL__ << "\nCan't find column " << column <<
1155  "\n\n\n\nThe column info is likely missing due to incomplete Configuration View filling.\n\n"
1156  << std::endl;
1157  throw std::runtime_error(errMsg.str().c_str());
1158  }
1159  return columnsInfo_[column];
1160 }
1161 
1162 //Setters
1163 //==============================================================================
1164 void ConfigurationView::setUniqueStorageIdentifier(const std::string &storageUID)
1165 {
1166  uniqueStorageIdentifier_ = storageUID;
1167 }
1168 
1169 //==============================================================================
1170 void ConfigurationView::setTableName(const std::string &name)
1171 {
1172  tableName_ = name;
1173 }
1174 
1175 //==============================================================================
1176 void ConfigurationView::setComment(const std::string &comment)
1177 {
1178  comment_ = comment;
1179 }
1180 
1181 //==============================================================================
1182 void ConfigurationView::setURIEncodedComment(const std::string &uriComment)
1183 {
1184  comment_ = decodeURIComponent(uriComment);
1185 }
1186 
1187 //==============================================================================
1188 void ConfigurationView::setAuthor(const std::string &author)
1189 {
1190  author_ = author;
1191 }
1192 
1193 //==============================================================================
1194 void ConfigurationView::setCreationTime(time_t t)
1195 {
1196  creationTime_ = t;
1197 }
1198 
1199 //==============================================================================
1200 void ConfigurationView::setLastAccessTime(time_t t)
1201 {
1202  lastAccessTime_ = t;
1203 }
1204 
1205 //==============================================================================
1206 void ConfigurationView::setLooseColumnMatching(bool setValue)
1207 {
1208  fillWithLooseColumnMatching_ = setValue;
1209 }
1210 
1211 //==============================================================================
1212 void ConfigurationView::reset (void)
1213 {
1214  version_ = -1;
1215  comment_ = "";
1216  author_ + "";
1217  columnsInfo_.clear();
1218  theDataView_.clear();
1219 }
1220 
1221 //==============================================================================
1222 void ConfigurationView::print (std::ostream &out) const
1223 {
1224  out << "==============================================================================" << std::endl;
1225  out << "Print: " << tableName_ << " Version: " << version_ << " Comment: " << comment_ <<
1226  " Author: " << author_ << " Creation Time: " << ctime(&creationTime_) << std::endl;
1227  out << "\t\tNumber of Cols " << getNumberOfColumns() << std::endl;
1228  out << "\t\tNumber of Rows " << getNumberOfRows() << std::endl;
1229 
1230  out << "Columns:\t";
1231  for(int i=0;i<(int)columnsInfo_.size();++i)
1232  out << i << ":" << columnsInfo_[i].getName() << ":" << columnsInfo_[i].getStorageName() << ":" << columnsInfo_[i].getDataType() << "\t ";
1233  out << std::endl;
1234 
1235  out << "Rows:" << std::endl;
1236  int num;
1237  std::string val;
1238  for(int r=0;r<(int)getNumberOfRows();++r)
1239  {
1240  out << r << ":\t";
1241  for(int c=0;c<(int)getNumberOfColumns();++c)
1242  {
1243  out << c << ":";
1244  out << theDataView_[r][c];
1245  //stopped using below, because it is called sometimes during debugging when
1246  // numbers are set to environment variables:
1247  // if(columnsInfo_[c].getDataType() == "NUMBER")
1248  // {
1249  // getValue(num,r,c,false);
1250  // out << num;
1251  // }
1252  // else
1253  // {
1254  // getValue(val,r,c,false);
1255  // out << val;
1256  // }
1257  out << "\t\t";
1258  }
1259  out << std::endl;
1260  }
1261 }
1262 
1263 
1264 //==============================================================================
1265 void ConfigurationView::printJSON (std::ostream &out) const
1266 {
1267  out << "{\n";
1268  out << "\"NAME\" : \"" << tableName_ << "\",\n";
1269 
1270  //out << "\"VERSION\" : \"" << version_ << "\"\n";
1271 
1272  out << "\"COMMENT\" : ";
1273 
1274  //output escaped comment
1275  std::string val;
1276  val = comment_;
1277  out << "\"";
1278  for(unsigned int i=0;i<val.size();++i)
1279  {
1280 
1281  if(val[i] == '\n')
1282  out << "\\n";
1283  else if(val[i] == '\t')
1284  out << "\\t";
1285  else if(val[i] == '\r')
1286  out << "\\r";
1287  else
1288  {
1289  //escaped characters need a
1290  if(val[i] == '"' ||
1291  val[i] == '\\')
1292  out << '\\';
1293  out << val[i];
1294  }
1295  }
1296  out << "\",\n";
1297 
1298  out << "\"AUTHOR\" : \"" << author_ << "\",\n";
1299  out << "\"CREATION_TIME\" : " << creationTime_ << ",\n";
1300 
1301  //USELESS... out << "\"NUM_OF_COLS\" : " << getNumberOfColumns() << ",\n";
1302  //USELESS... out << "\"NUM_OF_ROWS\" : " << getNumberOfRows() << ",\n";
1303 
1304 
1305 
1306  out << "\"COL_TYPES\" : {\n";
1307  for(int c=0;c<(int)getNumberOfColumns();++c)
1308  {
1309  out << "\t\t\"" << columnsInfo_[c].getStorageName() << "\" : ";
1310  out << "\"" << columnsInfo_[c].getDataType() << "\"";
1311  if(c+1 < (int)getNumberOfColumns())
1312  out << ",";
1313  out << "\n";
1314  }
1315  out << "},\n"; //close COL_TYPES
1316 
1317  out << "\"DATA_SET\" : [\n";
1318  int num;
1319  for(int r=0;r<(int)getNumberOfRows();++r)
1320  {
1321  out << "\t{\n";
1322  for(int c=0;c<(int)getNumberOfColumns();++c)
1323  {
1324  out << "\t\t\"" << columnsInfo_[c].getStorageName() << "\" : ";
1325 
1326  out << "\"" << getEscapedValueAsString(r,c,false) << "\""; //do not convert env variables
1327 
1328  if(c+1 < (int)getNumberOfColumns())
1329  out << ",";
1330  out << "\n";
1331  }
1332  out << "\t}";
1333  if(r+1 < (int)getNumberOfRows())
1334  out << ",";
1335  out << "\n";
1336  }
1337  out << "]\n"; //close DATA_SET
1338 
1339  out << "}";
1340 }
1341 
1342 //==============================================================================
1343 //restoreJSONStringEntities
1344 // returns string with literals \n \t \" \r \\ replaced with char
1345 std::string restoreJSONStringEntities(const std::string &str)
1346 {
1347  unsigned int sz = str.size();
1348  if(!sz) return ""; //empty string, returns empty string
1349 
1350  std::stringstream retStr;
1351  unsigned int i=0;
1352  for(;i<sz-1;++i)
1353  {
1354  if(str[i] == '\\') //if 2 char escape sequence, replace with char
1355  switch(str[i+1])
1356  {
1357  case 'n':
1358  retStr << '\n'; ++i; break;
1359  case '"':
1360  retStr << '"'; ++i; break;
1361  case 't':
1362  retStr << '\t'; ++i; break;
1363  case 'r':
1364  retStr << '\r'; ++i; break;
1365  case '\\':
1366  retStr << '\\'; ++i; break;
1367  default:
1368  retStr << str[i];
1369  }
1370  else
1371  retStr << str[i];
1372  }
1373  if(i == sz-1)
1374  retStr << str[sz-1]; //output last character (which can't escape anything)
1375 
1376  return retStr.str();
1377 }
1378 
1379 //==============================================================================
1380 //fillFromJSON
1381 // Clears and fills the view from the JSON string.
1382 // Returns -1 on failure
1383 //
1384 // first level keys:
1385 // NAME
1386 // DATA_SET
1387 
1388 int ConfigurationView::fillFromJSON(const std::string &json)
1389 {
1390  std::vector<std::string> keys;
1391  keys.push_back ("NAME");
1392  keys.push_back ("COMMENT");
1393  keys.push_back ("AUTHOR");
1394  keys.push_back ("CREATION_TIME");
1395  //keys.push_back ("COL_TYPES");
1396  keys.push_back ("DATA_SET");
1397  enum {
1398  CV_JSON_FILL_NAME,
1399  CV_JSON_FILL_COMMENT,
1400  CV_JSON_FILL_AUTHOR,
1401  CV_JSON_FILL_CREATION_TIME,
1402  // CV_JSON_FILL_COL_TYPES,
1403  CV_JSON_FILL_DATA_SET
1404  };
1405 
1406  //__MOUT__ << json << std::endl;
1407 
1408  sourceColumnMismatchCount_ = 0;
1409  sourceColumnMissingCount_ = 0;
1410  sourceColumnNames_.clear(); //reset
1411  unsigned int colFoundCount = 0;
1412  unsigned int i = 0;
1413  unsigned int row = -1;
1414  unsigned int colSpeedup = 0;
1415  unsigned int startString, startNumber, endNumber = -1;
1416  unsigned int bracketCount = 0;
1417  unsigned int sqBracketCount = 0;
1418  bool inQuotes = 0;
1419  bool newString = 0;
1420  bool newValue = 0;
1421  bool isDataArray = 0;
1422  bool keyIsMatch, keyIsComment;
1423  unsigned int keyIsMatchIndex, keyIsMatchStorageIndex, keyIsMatchCommentIndex;
1424  const std::string COMMENT_ALT_KEY = "COMMENT";
1425 
1426  std::string extractedString = "", currKey = "", currVal = "";
1427  unsigned int currDepth;
1428 
1429  std::vector<std::string> jsonPath;
1430  std::vector<char> jsonPathType; // indicator of type in jsonPath: { [ K
1431  char lastPopType = '_'; // either: _ { [ K
1432  // _ indicates reset pop (this happens when a new {obj} starts)
1433  unsigned int matchedKey = -1;
1434  unsigned int lastCol = -1;
1435 
1436 
1437  //find all depth 1 matching keys
1438  for(;i<json.size();++i)
1439  {
1440  switch(json[i])
1441  {
1442  case '"':
1443  if(i-1 < json.size() && //ignore if escaped
1444  json[i-1] == '\\') break;
1445 
1446  inQuotes = !inQuotes; //toggle in quotes if not escaped
1447  if(inQuotes)
1448  startString = i;
1449  else
1450  {
1451  extractedString = restoreJSONStringEntities(
1452  json.substr(startString+1,i-startString-1));
1453  newString = 1; //have new string!
1454  }
1455  break;
1456  case ':':
1457  if(inQuotes) break; //skip if in quote
1458 
1459  //must be a json object level to have a key
1460  if(jsonPathType[jsonPathType.size()-1] != '{'
1461  || !newString) //and must have a string for key
1462  {
1463  __MOUT__ << "Invalid ':' position" << std::endl;
1464  return -1;
1465  }
1466 
1467  //valid, so take key
1468  jsonPathType.push_back('K');
1469  jsonPath.push_back(extractedString);
1470  startNumber = i;
1471  newString = 0; //clear flag
1472  endNumber = -1; //reset end number index
1473  break;
1474 
1475  // if(isKey ||
1476  // isDataArray)
1477  // {
1478  // std::cout << "Invalid ':' position" << std::endl;
1479  // return -1;
1480  // }
1481  // isKey = 1; //new value is a key
1482  // newValue = 1;
1483  // startNumber = i;
1484  // break;
1485  case ',':
1486  if(inQuotes) break; //skip if in quote
1487  if(lastPopType == '{') //don't need value again of nested object
1488  {
1489  //check if the nested object was the value to a key, if so, pop key
1490  if(jsonPathType[jsonPathType.size()-1] == 'K')
1491  {
1492  lastPopType = 'K';
1493  jsonPath.pop_back();
1494  jsonPathType.pop_back();
1495  }
1496  break; //skip , handling if {obj} just ended
1497  }
1498 
1499  if(newString)
1500  currVal = extractedString;
1501  else //number value
1502  {
1503  if(endNumber == (unsigned int)-1 || //take i as end number if needed
1504  endNumber <= startNumber)
1505  endNumber = i;
1506  //extract number value
1507  if(endNumber <= startNumber) //empty data, could be {}
1508  currVal = "";
1509  else
1510  currVal = json.substr(startNumber+1,endNumber-startNumber-1);
1511  }
1512 
1513  currDepth = bracketCount;
1514 
1515  if(jsonPathType[jsonPathType.size()-1] == 'K') //this is the value to key
1516  {
1517  currKey = jsonPath[jsonPathType.size()-1];
1518  newValue = 1; //new value to consider!
1519 
1520  //pop key
1521  lastPopType = 'K';
1522  jsonPath.pop_back();
1523  jsonPathType.pop_back();
1524  }
1525  else if(jsonPathType[jsonPathType.size()-1] == '[') //this is a value in array
1526  {
1527  //key is last key
1528  for(unsigned int k=jsonPathType.size()-2; k<jsonPathType.size(); --k)
1529  if(jsonPathType[k] == 'K')
1530  {
1531  currKey = jsonPath[k];
1532  break;
1533  }
1534  else if(k == 0)
1535  {
1536  __MOUT__ << "Invalid array position" << std::endl;
1537  return -1;
1538  }
1539 
1540  newValue = 1; //new value to consider!
1541  isDataArray = 1;
1542  }
1543  else // { is an error
1544  {
1545  __MOUT__ << "Invalid ',' position" << std::endl;
1546  return -1;
1547  }
1548 
1549  startNumber = i;
1550  break;
1551 
1552  case '{':
1553  if(inQuotes) break; //skip if in quote
1554  lastPopType = '_'; //reset because of new object
1555  jsonPathType.push_back('{');
1556  jsonPath.push_back("{");
1557  ++bracketCount;
1558  break;
1559 
1560  // ++bracketCount;
1561  // isDataArray = 0;
1562  // isKey = 0;
1563  // endingObject = 0;
1564  // break;
1565  case '}':
1566  if(inQuotes) break; //skip if in quote
1567 
1568  if(lastPopType != '{' && //don't need value again of nested object
1569  jsonPathType[jsonPathType.size()-1] == 'K') //this is the value to key
1570  {
1571  currDepth = bracketCount;
1572  currKey = jsonPath[jsonPathType.size()-1];
1573  if(newString)
1574  currVal = extractedString;
1575  else //number value
1576  {
1577  if(endNumber == (unsigned int)-1 || //take i as end number if needed
1578  endNumber <= startNumber)
1579  endNumber = i;
1580  //extract val
1581  if(endNumber <= startNumber) //empty data, could be {}
1582  currVal = "";
1583  else
1584  currVal = json.substr(startNumber+1,endNumber-startNumber-1);
1585  }
1586  newValue = 1; //new value to consider!
1587  //pop key
1588  jsonPath.pop_back();
1589  jsonPathType.pop_back();
1590  }
1591  //pop {
1592  if(jsonPathType[jsonPathType.size()-1] != '{')
1593  {
1594  __MOUT__ << "Invalid '}' position" << std::endl;
1595  return -1;
1596  }
1597  lastPopType = '{';
1598  jsonPath.pop_back();
1599  jsonPathType.pop_back();
1600  --bracketCount;
1601  break;
1602  case '[':
1603  if(inQuotes) break; //skip if in quote
1604  jsonPathType.push_back('[');
1605  jsonPath.push_back("[");
1606  ++sqBracketCount;
1607  startNumber = i;
1608  break;
1609  case ']':
1610  if(inQuotes) break; //skip if in quote
1611 
1612  //must be an array at this level (in order to close it)
1613  if(jsonPathType[jsonPathType.size()-1] != '[')
1614  {
1615  __MOUT__ << "Invalid ']' position" << std::endl;
1616  return -1;
1617  }
1618 
1619  currDepth = bracketCount;
1620 
1621  //This is an array value
1622  if(newString)
1623  currVal = extractedString;
1624  else //number value
1625  {
1626  if(endNumber == (unsigned int)-1 || //take i as end number if needed
1627  endNumber <= startNumber)
1628  endNumber = i;
1629  //extract val
1630  if(endNumber <= startNumber) //empty data, could be {}
1631  currVal = "";
1632  else
1633  currVal = json.substr(startNumber+1,endNumber-startNumber-1);
1634  }
1635  isDataArray = 1;
1636 
1637  //key is last key
1638  for(unsigned int k=jsonPathType.size()-2; k<jsonPathType.size(); --k)
1639  if(jsonPathType[k] == 'K')
1640  {
1641  currKey = jsonPath[k];
1642  break;
1643  }
1644  else if(k == 0)
1645  {
1646  __MOUT__ << "Invalid array position" << std::endl;
1647  return -1;
1648  }
1649 
1650  //pop [
1651  if(jsonPathType[jsonPathType.size()-1] != '[')
1652  {
1653  __MOUT__ << "Invalid ']' position" << std::endl;
1654  return -1;
1655  }
1656  lastPopType = '[';
1657  jsonPath.pop_back();
1658  jsonPathType.pop_back();
1659  --sqBracketCount;
1660  break;
1661  case ' ': //white space handling for numbers
1662  case '\t':
1663  case '\n':
1664  case '\r':
1665  if(inQuotes) break; //skip if in quote
1666  if(startNumber != (unsigned int)-1 &&
1667  endNumber == (unsigned int)-1)
1668  endNumber = i;
1669  startNumber = i;
1670  break;
1671  default:;
1672  }
1673 
1674  if(0) //for debugging
1675  {
1676  std::cout << i << ":\t" << json[i] << " - ";
1677 
1678  std::cout << "ExtKey=";
1679  for(unsigned int k=0;k<jsonPath.size();++k)
1680  std::cout << jsonPath[k] << "/";
1681  std::cout << " - ";
1682  std::cout << lastPopType << " ";
1683  std::cout << bracketCount << " ";
1684  std::cout << sqBracketCount << " ";
1685  std::cout << inQuotes << " ";
1686  std::cout << newValue << "-";
1687  std::cout << currKey << "-{" << currDepth << "}:" ;
1688  std::cout << currVal << " ";
1689  std::cout << startNumber << "-";
1690  std::cout << endNumber << " ";
1691  std::cout << "\n";
1692  }
1693 
1694  //continue;
1695 
1696  //handle a new completed value
1697  if(newValue)
1698  {
1699 
1700  // std::cout << "ExtKey=";
1701  // for(unsigned int k=0;k<jsonPath.size();++k)
1702  // std::cout << jsonPath[k] << "/";
1703  //
1704  // std::cout << currKey << "-{" << currDepth << "}: - ";
1705  //
1706  // if(isDataArray)
1707  // std::cout << "Array:: ";
1708  // if(newString)
1709  // std::cout << "New String:: ";
1710  // else
1711  // std::cout << "New Number:: ";
1712  //
1713  // std::cout << currVal << "\n";
1714 
1715 
1716  //extract only what we care about
1717  // for ConfigurationView only care about matching depth 1
1718 
1719  //handle matching depth 1 keys
1720 
1721  matchedKey = -1; //init to unfound
1722  for(unsigned int k=0;k<keys.size();++k)
1723  if((currDepth == 1 && keys[k] == currKey) ||
1724  (currDepth > 1 && keys[k] == jsonPath[1]))
1725  matchedKey = k;
1726 
1727  if(matchedKey != (unsigned int)-1)
1728  {
1729  // std::cout << "New Data for:: key[" << matchedKey << "]-" <<
1730  // keys[matchedKey] << "\n";
1731 
1732  switch(matchedKey)
1733  {
1734  case CV_JSON_FILL_NAME:
1735  if(currDepth == 1) setTableName(currVal);
1736  break;
1737  case CV_JSON_FILL_COMMENT:
1738  if(currDepth == 1) setComment(currVal);
1739  break;
1740  case CV_JSON_FILL_AUTHOR:
1741  if(currDepth == 1) setAuthor(currVal);
1742  break;
1743  case CV_JSON_FILL_CREATION_TIME:
1744  if(currDepth == 1) setCreationTime(strtol(currVal.c_str(),0,10));
1745  break;
1746  //case CV_JSON_FILL_COL_TYPES:
1747  //
1748  //break;
1749  case CV_JSON_FILL_DATA_SET:
1750  // std::cout << "CV_JSON_FILL_DATA_SET New Data for:: " << matchedKey << "]-" <<
1751  // keys[matchedKey] << "/" << currDepth << ".../" <<
1752  // currKey << "\n";
1753 
1754  if(currDepth == 2) //second level depth
1755  {
1756  //if matches first column name.. then add new row
1757  //else add to current row
1758  unsigned int col, ccnt = 0;
1759  unsigned int noc = getNumberOfColumns();
1760  for(;ccnt<noc;++ccnt)
1761  {
1762  //use colSpeedup to change the first column we search
1763  //for each iteration.. since we expect the data to
1764  //be arranged in column order
1765 
1766  if(fillWithLooseColumnMatching_)
1767  {
1768  //loose column matching makes no attempt to
1769  // match the column names
1770  //just assumes the data is in the correct order
1771 
1772  col = colSpeedup;
1773 
1774  //auto matched
1775  if(col <= lastCol) //add row (use lastCol in case new column-0 was added
1776  row = addRow();
1777  lastCol = col;
1778  if(getNumberOfRows() == 1) //only for first row
1779  sourceColumnNames_.emplace(currKey);
1780 
1781  //add value to row and column
1782 
1783  if(row >= getNumberOfRows())
1784  {
1785  __SS__ << "Invalid row" << std::endl; //should be impossible?
1786  std::cout << ss.str();
1787  throw std::runtime_error(ss.str());
1788  return -1;
1789  }
1790 
1791  theDataView_[row][col] = currVal;//THERE IS NO CHECK FOR WHAT IS READ FROM THE DATABASE. IT SHOULD BE ALREADY CONSISTENT
1792  break;
1793  }
1794  else
1795  {
1796  col = (ccnt + colSpeedup) % noc;
1797 
1798  //match key by ignoring '_'
1799  // also accept COMMENT == COMMENT_DESCRIPTION
1800  // (this is for backwards compatibility..)
1801  keyIsMatch = true;
1802  keyIsComment = true;
1803  for(keyIsMatchIndex=0, keyIsMatchStorageIndex=0, keyIsMatchCommentIndex=0;
1804  keyIsMatchIndex<currKey.size();++keyIsMatchIndex)
1805  {
1806  if(columnsInfo_[col].getStorageName()[keyIsMatchStorageIndex] == '_')
1807  ++keyIsMatchStorageIndex; //skip to next storage character
1808  if(currKey[keyIsMatchIndex] == '_')
1809  continue; //skip to next character
1810 
1811  //match to storage name
1812  if(keyIsMatchStorageIndex >= columnsInfo_[col].getStorageName().size() ||
1813  currKey[keyIsMatchIndex] !=
1814  columnsInfo_[col].getStorageName()[keyIsMatchStorageIndex])
1815  {
1816  //size mismatch or character mismatch
1817  keyIsMatch = false;
1818  if(!keyIsComment)
1819  break;
1820  }
1821 
1822  //check also if alternate comment is matched
1823  if(keyIsComment && keyIsMatchCommentIndex < COMMENT_ALT_KEY.size())
1824  {
1825  if(currKey[keyIsMatchIndex] != COMMENT_ALT_KEY[keyIsMatchCommentIndex])
1826  {
1827  //character mismatch with COMMENT
1828  keyIsComment = false;
1829  }
1830  }
1831 
1832  ++keyIsMatchStorageIndex; //go to next character
1833  }
1834 
1835  if(keyIsMatch || keyIsComment) //currKey == columnsInfo_[c].getStorageName())
1836  {
1837  //matched
1838  if(col <= lastCol) //add row (use lastCol in case new column-0 was added
1839  {
1840  if(getNumberOfRows()) //skip first time
1841  sourceColumnMissingCount_ += getNumberOfColumns() - colFoundCount;
1842 
1843  colFoundCount = 0; //reset column found count
1844  row = addRow();
1845  }
1846  lastCol = col;
1847  ++colFoundCount;
1848 
1849  if(getNumberOfRows() == 1) //only for first row
1850  sourceColumnNames_.emplace(currKey);
1851 
1852  //add value to row and column
1853 
1854  if(row >= getNumberOfRows())
1855  {
1856  __SS__ << "Invalid row" << std::endl; //should be impossible?!
1857  __MOUT__ << "\n" << ss.str();
1858  throw std::runtime_error(ss.str());
1859  return -1; //never gets here
1860  }
1861 
1862  theDataView_[row][col] = currVal;
1863  break;
1864  }
1865  }
1866  }
1867 
1868  colSpeedup = (colSpeedup + 1) % noc; //short cut to proper column hopefully in next search
1869 
1870  if(ccnt >= getNumberOfColumns())
1871  {
1872  __SS__ << "\n\nInvalid column in JSON source data: " <<
1873  currKey << " not found in column names of table named " <<
1874  getTableName() << "." <<
1875  std::endl; //input data doesn't match config description
1876  __MOUT__ << "\n" << ss.str();
1877  //CHANGED on 11/10/2016
1878  // to.. try just not populating data instead of error
1879  ++sourceColumnMismatchCount_; //but count errors
1880  if(getNumberOfRows() == 1) //only for first row
1881  sourceColumnNames_.emplace(currKey);
1882 
1883  //throw std::runtime_error("\n\n" + ss.str());
1884  //return -1; //never gets here
1885  }
1886 
1887  }
1888  break;
1889  default:; //unknown match?
1890  }
1891 
1892  }
1893 
1894  //clean up handling of new value
1895 
1896  newString = 0; //toggle flag
1897  newValue = 0; //toggle flag
1898  isDataArray = 0;
1899  endNumber = -1; //reset end number index
1900  }
1901 
1902  //if(i>200) break; //185
1903  }
1904 
1905  //__MOUT__ << "Done!" << std::endl;
1906 
1907  //print();
1908 
1909  return 0; //success
1910 }
1911 
1912 
1913 
1914 //==============================================================================
1915 bool ConfigurationView::isURIEncodedCommentTheSame(const std::string &comment) const
1916 {
1917  std::string compareStr = decodeURIComponent(comment);
1918  return comment_ == compareStr;
1919 }
1920 //
1922 //bool ConfigurationView::isValueTheSame(const std::string &valueStr,
1923 // unsigned int r, unsigned int c) const
1924 //{
1925 // __MOUT__ << "valueStr " << valueStr << std::endl;
1926 //
1927 // if(!(c < columnsInfo_.size() && r < getNumberOfRows()))
1928 // {
1929 // __SS__ << "Invalid row (" << r << ") col (" << c << ") requested!" << std::endl;
1930 // throw std::runtime_error(ss.str());
1931 // }
1932 //
1933 // __MOUT__ << "originalValueStr " << theDataView_[r][c] << std::endl;
1934 //
1935 // if(columnsInfo_[c].getDataType() == ViewColumnInfo::DATATYPE_TIME)
1936 // {
1937 // time_t valueTime(strtol(valueStr.c_str(),0,10));
1938 // time_t originalValueTime;
1939 // getValue(originalValueTime,r,c);
1940 // __MOUT__ << "time_t valueStr " << valueTime << std::endl;
1941 // __MOUT__ << "time_t originalValueStr " << originalValueTime << std::endl;
1942 // return valueTime == originalValueTime;
1943 // }
1944 // else
1945 // {
1946 // return valueStr == theDataView_[r][c];
1947 // }
1948 //}
1949 
1950 //==============================================================================
1951 //fillFromCSV
1952 // Fills the view from the CSV string.
1953 //
1954 // Note: converts all %## to the ascii character, # is hex nibble
1955 // (e.g. '%' must be represented as "%25")
1956 //
1957 // dataOffset := starting destination row
1958 //
1959 // while there are row entries in the data.. replace
1960 // data range from [dataOffset, dataOffset+chunkSize-1]
1961 // ... note if less rows, this means rows were deleted
1962 // ... if more, then rows were added.
1963 //
1964 // ',' next cell delimiter
1965 // ';' next row delimiter
1966 //
1967 //
1968 // if author == "", do nothing special for author and timestamp column
1969 // if author != "", assign author for any row that has been modified, and assign now as timestamp
1970 //
1971 // Returns -1 if data was same and pre-existing content
1972 // otherwise 0
1973 int ConfigurationView::fillFromCSV(const std::string &data, const int &dataOffset,
1974  const std::string &author)
1975 throw(std::runtime_error)
1976 {
1977  int r = dataOffset;
1978  int c = 0;
1979 
1980  int i = 0; //use to parse data std::string
1981  int j = data.find(',',i); //find next cell delimiter
1982  int k = data.find(';',i); //find next row delimiter
1983 
1984  bool rowWasModified;
1985  unsigned int countRowsModified = 0;
1986  int authorCol = findColByType(ViewColumnInfo::TYPE_AUTHOR);
1987  int timestampCol = findColByType(ViewColumnInfo::TYPE_TIMESTAMP);
1988  //std::string valueStr, tmpTimeStr, originalValueStr;
1989 
1990  while(k != (int)(std::string::npos))
1991  {
1992  rowWasModified = false;
1993  if(r >= (int)getNumberOfRows())
1994  {
1995  addRow();
1996  //__MOUT__ << "Row added" << std::endl;
1997  rowWasModified = true;
1998  }
1999 
2000  while(j < k && j != (int)(std::string::npos))
2001  {
2002  //__MOUT__ << "Col " << c << std::endl;
2003 
2004  //skip last 2 columns
2005  if(c >= (int)getNumberOfColumns()-2)
2006  {
2007  i=j+1;
2008  j = data.find(',',i); //find next cell delimiter
2009  ++c;
2010  continue;
2011  }
2012 
2013  if(setURIEncodedValue(data.substr(i,j-i),r,c))
2014  rowWasModified = true;
2015 
2016  i=j+1;
2017  j = data.find(',',i); //find next cell delimiter
2018  ++c;
2019  }
2020 
2021  //if row was modified, assign author and timestamp
2022  if(author != "" && rowWasModified)
2023  {
2024  __MOUT__ << "Row=" << r << " was modified!" << std::endl;
2025  setValue(author,r,authorCol);
2026  setValue(time(0),r,timestampCol);
2027  }
2028 
2029  if(rowWasModified) ++countRowsModified;
2030 
2031  ++r;
2032  c = 0;
2033 
2034 
2035  i = k+1;
2036  j = data.find(',',i); //find next cell delimiter
2037  k = data.find(';',i); //find new row delimiter
2038  }
2039 
2040  //delete excess rows
2041  while(r < (int)getNumberOfRows())
2042  {
2043  deleteRow(r);
2044  __MOUT__ << "Row deleted: " << r << std::endl;
2045  ++countRowsModified;
2046  }
2047 
2048  __MOUT_INFO__ << "countRowsModified=" <<
2049  countRowsModified << std::endl;
2050 
2051  if(!countRowsModified)
2052  {
2053  __SS__ << "No rows were modified! No reason to fill a view with same content." << std::endl;
2054  __MOUT__ << ss.str();
2055  return -1;
2056  }
2057 
2058  //print(); //for debugging
2059 
2060  //setup sourceColumnNames_ to be correct
2061  sourceColumnNames_.clear();
2062  for(unsigned int i=0;i<getNumberOfColumns();++i)
2063  sourceColumnNames_.emplace(getColumnsInfo()[i].getStorageName());
2064 
2065  init(); //verify new table (throws runtime_errors)
2066 
2067  //printout for debugging
2068  // __SS__ << "\n";
2069  // print(ss);
2070  // __MOUT__ << "\n" << ss.str() << std::endl;
2071 
2072  return 0;
2073 }
2074 
2075 //==============================================================================
2076 //setURIEncodedValue
2077 // converts all %## to the ascii character
2078 // returns true if value was different than original value
2079 bool ConfigurationView::setURIEncodedValue(const std::string &value, const unsigned int &r, const unsigned int &c)
2080 {
2081  if(!(c < columnsInfo_.size() && r < getNumberOfRows()))
2082  {
2083  __SS__ << "Invalid row (" << r << ") col (" << c << ") requested!" << std::endl;
2084  throw std::runtime_error(ss.str());
2085  }
2086 
2087  std::string valueStr = decodeURIComponent(value);
2088  std::string originalValueStr = getValueAsString(r,c,false); //do not convert env variables
2089  //__MOUT__ << "valueStr " << valueStr << std::endl;
2090  //__MOUT__ << "originalValueStr " << originalValueStr << std::endl;
2091 
2092  if(columnsInfo_[c].getDataType() == ViewColumnInfo::DATATYPE_NUMBER)
2093  {
2094  //check if valid number
2095  std::string convertedString = convertEnvVariables(valueStr);
2096  if(!isNumber(convertedString))
2097  {
2098  __SS__ << "\tIn configuration " << tableName_
2099  << " at column=" << columnsInfo_[c].getName()
2100  << " the value set (" << convertedString << ")"
2101  << " is not a number! Please fix it or change the column type..." << std::endl;
2102  throw std::runtime_error(ss.str());
2103  }
2104  theDataView_[r][c] = valueStr;
2105  }
2106  else if(columnsInfo_[c].getDataType() == ViewColumnInfo::DATATYPE_TIME)
2107  {
2108  // valueStr = decodeURIComponent(data.substr(i,j-i));
2109  //
2110  // getValue(tmpTimeStr,r,c);
2111  // if(valueStr != tmpTimeStr)//theDataView_[r][c])
2112  // {
2113  // __MOUT__ << "valueStr=" << valueStr <<
2114  // " theDataView_[r][c]=" << tmpTimeStr << std::endl;
2115  // rowWasModified = true;
2116  // }
2117 
2118  setValue(time_t(strtol(valueStr.c_str(),0,10)),
2119  r,c);
2120  }
2121  else
2122  theDataView_[r][c] = valueStr;
2123 
2124 
2125  return(originalValueStr != getValueAsString(r,c,false)); //do not convert env variables
2126 }
2127 
2128 //==============================================================================
2129 //decodeURIComponent
2130 // converts all %## to the ascii character
2131 std::string ConfigurationView::decodeURIComponent(const std::string &data)
2132 {
2133  std::string decodeURIString(data.size(),0); //init to same size
2134  unsigned int j=0;
2135  for(unsigned int i=0;i<data.size();++i,++j)
2136  {
2137  if(data[i] == '%')
2138  {
2139  //high order hex nibble digit
2140  if(data[i+1] > '9') //then ABCDEF
2141  decodeURIString[j] += (data[i+1]-55)*16;
2142  else
2143  decodeURIString[j] += (data[i+1]-48)*16;
2144 
2145  //low order hex nibble digit
2146  if(data[i+2] > '9') //then ABCDEF
2147  decodeURIString[j] += (data[i+2]-55);
2148  else
2149  decodeURIString[j] += (data[i+2]-48);
2150 
2151  i+=2; //skip to next char
2152  }
2153  else
2154  decodeURIString[j] = data[i];
2155  }
2156  decodeURIString.resize(j);
2157  return decodeURIString;
2158 }
2159 
2160 //==============================================================================
2161 void ConfigurationView::resizeDataView(unsigned int nRows, unsigned int nCols)
2162 {
2163  //FIXME This maybe should disappear but I am using it in ConfigurationHandler still...
2164  theDataView_.resize(nRows, std::vector<std::string>(nCols));
2165 }
2166 
2167 //==============================================================================
2168 //addRow
2169 // returns index of added row, always is last row
2170 // return -1 on failure (throw error)
2171 int ConfigurationView::addRow (void)
2172 {
2173  int row = getNumberOfRows();
2174  theDataView_.resize(getNumberOfRows()+1,std::vector<std::string>(getNumberOfColumns()));
2175 
2176  std::vector<std::string> defaultRowValues =
2177  getDefaultRowValues();
2178 
2179  //fill each col of new row with default values
2180  for(unsigned int col=0;col<getNumberOfColumns();++col)
2181  theDataView_[row][col] = defaultRowValues[col];
2182 
2183  return row;
2184 }
2185 
2186 //==============================================================================
2187 //deleteRow
2188 // throws exception on failure
2189 void ConfigurationView::deleteRow(int r)
2190 {
2191  if(r >= (int)getNumberOfRows())
2192  {
2193  //out of bounds
2194  __SS__ << "Row " << r << " is out of bounds (Row Count = " <<
2195  getNumberOfRows() << ") and can not be deleted." <<
2196  std::endl;
2197  throw std::runtime_error(ss.str());
2198  }
2199 
2200  theDataView_.erase(theDataView_.begin()+r);
2201 }
2202 
2203 //==============================================================================
2204 //isColChildLinkUnique ~
2205 // find the pair of columns associated with a child link.
2206 //
2207 // c := a member column of the pair
2208 //
2209 // returns:
2210 // isGroup := indicates pair found is a group link
2211 // linkPair := pair of columns that are part of the link
2212 //
2213 // a unique link is defined by two columns: TYPE_START_CHILD_LINK, TYPE_START_CHILD_LINK_UID
2214 // a group link is defined by two columns: TYPE_START_CHILD_LINK, TYPE_START_CHILD_LINK_GROUP_ID
2215 //
2216 // returns true if column is member of a group or unique link.
2217 const bool ConfigurationView::getChildLink(const unsigned int &c, bool *isGroup,
2218  std::pair<unsigned int /*link col*/, unsigned int /*link id col*/> *linkPair) const
2219 {
2220  //__MOUT__ << "getChildLink for col: " << c << "-" <<
2221  // columnsInfo_[c].getType() << "-" << columnsInfo_[c].getName() << std::endl;
2222 
2223  //check if column is a child link UID
2224  if((*isGroup = columnsInfo_[c].isChildLinkGroupID()) ||
2225  columnsInfo_[c].isChildLinkUID())
2226  {
2227  //must be part of unique link, (or invalid table?)
2228  //__MOUT__ << "col: " << c << std::endl;
2229  linkPair->second = c;
2230  std::string index = columnsInfo_[c].getChildLinkIndex();
2231 
2232  //__MOUT__ << "index: " << index << std::endl;
2233 
2234  //find pair link
2235  for(unsigned int col=0; col<columnsInfo_.size(); ++col)
2236  {
2237  //__MOUT__ << "try: " << col << "-" << columnsInfo_[col].getType() << "-" << columnsInfo_[col].getName() << std::endl;
2238  if(col == c) continue; //skip column c that we know
2239  else if(columnsInfo_[col].isChildLink() &&
2240  index == columnsInfo_[col].getChildLinkIndex())
2241  {
2242  //found match!
2243  __MOUT__ << "getChildLink Found match for col: " << c << " at " << col << std::endl;
2244  linkPair->first = col;
2245  return true;
2246  }
2247  }
2248 
2249  //if here then invalid table!
2250  __SS__ << "\tIn view: " << tableName_ <<
2251  ", Can't find complete child link for column name " << columnsInfo_[c].getName() << std::endl;
2252  throw std::runtime_error(ss.str());
2253  }
2254 
2255  if(!columnsInfo_[c].isChildLink())
2256  return false; //cant be unique link
2257 
2258  //this is child link, so find pair link uid or gid column
2259  linkPair->first = c;
2260  std::string index = columnsInfo_[c].getChildLinkIndex();
2261 
2262  //__MOUT__ << "index: " << index << std::endl;
2263 
2264  //find pair link
2265  for(unsigned int col=0; col<columnsInfo_.size(); ++col)
2266  {
2267  //__MOUT__ << "try: " << col << "-" << columnsInfo_[col].getType() << "-" << columnsInfo_[col].getName() << std::endl;
2268  if(col == c) continue; //skip column c that we know
2269  // __MOUT__ << "try: " << col << "-" << columnsInfo_[col].getType() <<
2270  // "-" << columnsInfo_[col].getName() <<
2271  // "-u" << columnsInfo_[col].isChildLinkUID() <<
2272  // "-g" << columnsInfo_[col].isChildLinkGroupID() << std::endl;
2273  //
2274  // if(columnsInfo_[col].isChildLinkUID())
2275  // __MOUT__ << "-L" << columnsInfo_[col].getChildLinkIndex() << std::endl;
2276  //
2277  // if(columnsInfo_[col].isChildLinkGroupID())
2278  // __MOUT__ << "-L" << columnsInfo_[col].getChildLinkIndex() << std::endl;
2279 
2280  if(((columnsInfo_[col].isChildLinkUID() && !(*isGroup = false)) ||
2281  (columnsInfo_[col].isChildLinkGroupID() && (*isGroup = true)))
2282  && index == columnsInfo_[col].getChildLinkIndex())
2283  {
2284  //found match!
2285  //__MOUT__ << "getChildLink Found match for col: " << c << " at " << col << std::endl;
2286  linkPair->second = col;
2287  return true;
2288  }
2289  }
2290 
2291  //if here then invalid table!
2292  __SS__ << "\tIn view: " << tableName_ <<
2293  ", Can't find complete child link id for column name " << columnsInfo_[c].getName() << std::endl;
2294  throw std::runtime_error(ss.str());
2295 }
2296 
2297 //==============================================================================
2298 //isNumber ~~
2299 // returns true if hex ("0x.."), binary("b..."), or base10 number
2300 bool ConfigurationView::isNumber(const std::string& s) const
2301 {
2302  //__MOUT__ << "string " << s << std::endl;
2303  if(s.find("0x") == 0) //indicates hex
2304  {
2305  //__MOUT__ << "0x found" << std::endl;
2306  for(unsigned int i=2;i<s.size();++i)
2307  {
2308  if(!((s[i] >= '0' && s[i] <= '9') ||
2309  (s[i] >= 'A' && s[i] <= 'F') ||
2310  (s[i] >= 'a' && s[i] <= 'f')
2311  ))
2312  {
2313  //__MOUT__ << "prob " << s[i] << std::endl;
2314  return false;
2315  }
2316  }
2317  //return std::regex_match(s.substr(2), std::regex("^[0-90-9a-fA-F]+"));
2318  }
2319  else if(s[0] == 'b') //indicates binary
2320  {
2321  //__MOUT__ << "b found" << std::endl;
2322 
2323  for(unsigned int i=1;i<s.size();++i)
2324  {
2325  if(!((s[i] >= '0' && s[i] <= '1')
2326  ))
2327  {
2328  //__MOUT__ << "prob " << s[i] << std::endl;
2329  return false;
2330  }
2331  }
2332  }
2333  else
2334  {
2335  //__MOUT__ << "base 10 " << std::endl;
2336  for(unsigned int i=0;i<s.size();++i)
2337  if(!((s[i] >= '0' && s[i] <= '9') ||
2338  s[i] == '.' ||
2339  s[i] == '+' ||
2340  s[i] == '-'))
2341  return false;
2342  //Note: std::regex crashes in unresolvable ways (says Ryan.. also, stop using libraries)
2343  //return std::regex_match(s, std::regex("^(\\-|\\+)?[0-9]*(\\.[0-9]+)?"));
2344  }
2345 
2346  return true;
2347 }
2348 
2349 
2350 
2351 
2352 #ifdef __GNUG__
2353 #include <cstdlib>
2354 #include <memory>
2355 #include <cxxabi.h>
2356 
2357 std::string ots_demangle(const char* name) {
2358 
2359  int status = -4; // some arbitrary value to eliminate the compiler warning
2360 
2361  // enable c++11 by passing the flag -std=c++11 to g++
2362  std::unique_ptr<char, void(*)(void*)> res {
2363  abi::__cxa_demangle(name, NULL, NULL, &status),
2364  std::free
2365  };
2366 
2367  return (status==0) ? res.get() : name ;
2368 }
2369 
2370 #else
2371 
2372 // does nothing if not g++
2373 std::string ots_demangle(const char* name) {
2374  return name;
2375 }
2376 
2377 
2378 #endif