otsdaq  v2_00_00
DataManager.cc
1 #include "otsdaq-core/DataManager/DataManager.h"
2 #include "otsdaq-core/DataManager/CircularBuffer.h"
3 #include "otsdaq-core/DataManager/DataProducer.h"
4 #include "otsdaq-core/DataManager/DataConsumer.h"
5 #include "otsdaq-core/MessageFacility/MessageFacility.h"
6 #include "otsdaq-core/Macros/CoutHeaderMacros.h"
7 #include "otsdaq-core/PluginMakers/MakeDataProcessor.h"
8 #include "otsdaq-core/ConfigurationInterface/ConfigurationManager.h"
9 
10 /*
11 #include "otsdaq-core/DataProcessorPlugins/RawDataSaverConsumer.h"
12 #include "otsdaq-core/DataProcessorPlugins/ARTDAQConsumer.h"
13 #include "otsdaq-core/EventBuilder/EventDataSaver.h"
14 #include "otsdaq-core/DataProcessorPlugins/DataListenerProducer.h"
15 #include "otsdaq-core/DataProcessorPlugins/DataStreamer.h"
16 #include "otsdaq-core/DataProcessorPlugins/DQMHistosConsumer.h"
17 #include "otsdaq-core/EventBuilder/AssociativeMemoryEventBuilder.h"
18 #include "otsdaq-core/EventBuilder/Event.h"
19 #include "otsdaq-core/DataProcessorPlugins/DataListenerProducer.h"
20 #include "otsdaq-core/ConfigurationPluginDataFormats/UDPDataListenerProducerConfiguration.h"
21 #include "otsdaq-core/ConfigurationPluginDataFormats/ARTDAQConsumerConfiguration.h"
22 #include "otsdaq-core/ConfigurationPluginDataFormats/DataManagerConfiguration.h"
23 #include "otsdaq-core/ConfigurationPluginDataFormats/DataBufferConfiguration.h"
24 */
25 
26 #include <iostream>
27 #include <vector>
28 #include <unistd.h> //usleep
29 
30 using namespace ots;
31 
32 //========================================================================================================================
33 DataManager::DataManager(const ConfigurationTree& theXDAQContextConfigTree, const std::string& supervisorConfigurationPath)
34  : Configurable(theXDAQContextConfigTree, supervisorConfigurationPath)
35 {}
36 
37 //========================================================================================================================
38 DataManager::~DataManager(void)
39 {
40  eraseAllBuffers();
41 }
42 
43 //========================================================================================================================
44 void DataManager::configure(void)
45 {
46  eraseAllBuffers(); //Deletes all pointers created and given to the DataManager!
47 
48  for (const auto& buffer : theXDAQContextConfigTree_.getNode(theConfigurationPath_ + "/LinkToDataManagerConfiguration").getChildren())
49  {
50  __COUT__ << "Data Buffer Name: " << buffer.first << std::endl;
51  if (buffer.second.getNode(ViewColumnInfo::COL_NAME_STATUS).getValue<bool>())
52  {
53  std::vector<unsigned int> producersVectorLocation;
54  std::vector<unsigned int> consumersVectorLocation;
55  auto bufferConfigurationList = buffer.second.getNode("LinkToDataBufferConfiguration").getChildren();
56  unsigned int location = 0;
57  for (const auto& bufferConfiguration : bufferConfigurationList)
58  {
59  __COUT__ << "Processor id: " << bufferConfiguration.first << std::endl;
60  if (bufferConfiguration.second.getNode(ViewColumnInfo::COL_NAME_STATUS).getValue<bool>())
61  {
62  if (bufferConfiguration.second.getNode("ProcessorType").getValue<std::string>() == "Producer")
63  {
64  producersVectorLocation.push_back(location);
65  }
66  else if (bufferConfiguration.second.getNode("ProcessorType").getValue<std::string>() == "Consumer")
67  {
68  consumersVectorLocation.push_back(location);
69  }
70  else
71  {
72  __SS__ << "Node ProcessorType in "
73  << bufferConfiguration.first
74  << " of type "
75  << bufferConfiguration.second.getNode("ProcessorType").getValue<std::string>()
76  << " is invalid. The only accepted types are Producer and Consumer" << std::endl;
77  throw std::runtime_error(ss.str());
78  }
79  }
80  ++location;
81 
82  }
83 
84  if (producersVectorLocation.size() == 0)// || consumersVectorLocation.size() == 0)
85  {
86  __SS__ << "Node Data Buffer "
87  << buffer.first
88  << " has " << producersVectorLocation.size() << " Producers"
89  << " and " << consumersVectorLocation.size() << " Consumers"
90  << " there must be at least 1 Producer " << // of both configured
91  "for the buffer!" << std::endl;
92  throw std::runtime_error(ss.str());
93 
94  }
95 
96  configureBuffer<std::string, std::map<std::string, std::string>>(buffer.first);
97  for (auto& producerLocation : producersVectorLocation)
98  {
99  // __COUT__ << theConfigurationPath_ << std::endl;
100  // __COUT__ << buffer.first << std::endl;
101  __COUT__ << bufferConfigurationList[producerLocation].first << std::endl;
102  // __COUT__ << bufferConfigurationMap[producer].getNode("ProcessorPluginName").getValue<std::string>() << std::endl;
103  // __COUT__ << bufferConfigurationMap[producer].getNode("LinkToProcessorConfiguration") << std::endl;
104  // __COUT__ << "THIS DATA MANAGER POINTER: " << this << std::endl;
105  // __COUT__ << "PASSED" << std::endl;
106  buffers_[buffer.first].producers_.push_back(std::shared_ptr<DataProducer>(dynamic_cast<DataProducer*>(
107  makeDataProcessor
108  (
109  bufferConfigurationList[producerLocation].second.getNode("ProcessorPluginName").getValue<std::string>()
110  , theXDAQContextConfigTree_.getBackNode(theConfigurationPath_).getNode("ApplicationUID").getValue<std::string>()
111  , buffer.first
112  , bufferConfigurationList[producerLocation].first
113  , theXDAQContextConfigTree_
114  , theConfigurationPath_ + "/LinkToDataManagerConfiguration/" + buffer.first + "/LinkToDataBufferConfiguration/" + bufferConfigurationList[producerLocation].first + "/LinkToProcessorConfiguration"
115  ))));
116  __COUT__ << bufferConfigurationList[producerLocation].first << " has been created!" << std::endl;
117  }
118  for (auto& consumerLocation : consumersVectorLocation)
119  {
120  // __COUT__ << theConfigurationPath_ << std::endl;
121  // __COUT__ << buffer.first << std::endl;
122  __COUT__ << bufferConfigurationList[consumerLocation].first << std::endl;
123  // __COUT__ << bufferConfigurationMap[consumer].getNode("ProcessorPluginName").getValue<std::string>() << std::endl;
124  // __COUT__ << bufferConfigurationMap[consumer].getNode("LinkToProcessorConfiguration") << std::endl;
125  // __COUT__ << theXDAQContextConfigTree_.getBackNode(theConfigurationPath_) << std::endl;
126  // __COUT__ << "THIS DATA MANAGER POINTER: " << this << std::endl;
127  // __COUT__ << "PASSED" << std::endl;
128  buffers_[buffer.first].consumers_.push_back(std::shared_ptr<DataConsumer>(dynamic_cast<DataConsumer*>(
129  makeDataProcessor
130  (
131  bufferConfigurationList[consumerLocation].second.getNode("ProcessorPluginName").getValue<std::string>()
132  , theXDAQContextConfigTree_.getBackNode(theConfigurationPath_).getNode("ApplicationUID").getValue<std::string>()
133  , buffer.first
134  , bufferConfigurationList[consumerLocation].first
135  , theXDAQContextConfigTree_
136  , theConfigurationPath_ + "/LinkToDataManagerConfiguration/" + buffer.first + "/LinkToDataBufferConfiguration/" + bufferConfigurationList[consumerLocation].first + "/LinkToProcessorConfiguration"
137  ))));
138  __COUT__ << bufferConfigurationList[consumerLocation].first << " has been created!" << std::endl;
139  }
140  }
141  //__COUT__ << "Interface Name: "<< interface.first << std::endl;
142  //__COUT__ << "XDAQContext Node: "<< theConfigurationManager_->getNode("/XDAQContextConfiguration") << std::endl;
143  //__COUT__ << "Path to configuration: "<< (theSupervisorConfigurationPath_ + "/LinkToFEInterfaceConfiguration/" + interface.first + "/LinkToFETypeConfiguration") << std::endl;
144  // theFEInterfaces_[interface.first] = makeInterface(
145  // interface.second.getNode("FEInterfacePluginName").getValue<std::string>(),
146  // interface.first,
147  // theXDAQContextConfigTree_,
148  // (supervisorConfigurationPath_ + "/LinkToFEInterfaceConfiguration/" + interface.first + "/LinkToFETypeConfiguration")
149  // );
150  }
151  /*
152  __COUT__ << "supervisor instance: " << supervisorInstance_ << std::endl;
153  const DataManagerConfiguration* dataManagerConfiguration = theConfigurationManager_->__GET_CONFIG__(DataManagerConfiguration);
154  const DataBufferConfiguration* dataBufferConfiguration = theConfigurationManager_->__GET_CONFIG__(DataBufferConfiguration);
155  std::cout << __PRETTY_FUNCTION__ << dataManagerConfiguration->getConfigurationName() << std::endl;
156  // const FEConfiguration* frontEndConfiguration = theConfigurationManager_->__GET_CONFIG__(FEConfiguration);
157  // std::vector<unsigned int> fedList = frontEndConfiguration->getListOfFERs(supervisorInstance_);
158  // std::stringstream fedName;
159 
160  auto bufferList = dataManagerConfiguration->getListOfDataBuffers(supervisorType_, supervisorInstance_);
161  for(const auto& itbufferID: bufferList)
162  {
163  if(dataManagerConfiguration->getDataBufferStatus(supervisorType_, supervisorInstance_,itbufferID))
164  {
165  configureBuffer<std::string,std::map<std::string,std::string>>(itbufferID);
166  DataProcessor* aDataProcessor = nullptr;
167  std::cout << __PRETTY_FUNCTION__ << "Buffer ID: " << itbufferID << std::endl;
168  auto producerIDList = dataBufferConfiguration->getProducerIDList(itbufferID);
169  for(const auto& itProducerID: producerIDList)
170  {
171  std::cout << __PRETTY_FUNCTION__ << "Processor ID: " << itProducerID << std::endl;
172  const ConfigurationBase* dataProcessorConfiguration = theConfigurationManager_->getConfigurationByName(dataBufferConfiguration->getProducerClass(itbufferID, itProducerID) + "Configuration");
173  if(dataBufferConfiguration->getProducerStatus(itbufferID, itProducerID))
174  {
175  buffers_[itbufferID].producers_.push_back( std::shared_ptr<DataProducer>(dynamic_cast<DataProducer*>(makeDataProcessor(
176  dataBufferConfiguration->getProducerClass(itbufferID, itProducerID)
177  , supervisorType_
178  , supervisorInstance_
179  , itbufferID
180  , itProducerID
181  , dataProcessorConfiguration
182  ))));
183  //aDataProcessor->registerToBuffer();
184  }
185  }
186  auto consumerIDList = dataBufferConfiguration->getConsumerIDList(itbufferID);
187  for(const auto& itConsumerID : consumerIDList)
188  {
189  std::cout << __PRETTY_FUNCTION__ << "Consumer ID: " << itConsumerID << std::endl;
190  std::cout << __PRETTY_FUNCTION__ << "Consumer Type: " <<
191  dataBufferConfiguration->getConsumerClass(itbufferID, itConsumerID) << std::endl;
192 
193  if(dataBufferConfiguration->getConsumerStatus(itbufferID, itConsumerID))
194  {
195  std::cout << __PRETTY_FUNCTION__ << "\tStatus on: " << itConsumerID << std::endl;
196  const ConfigurationBase* dataProcessorConfiguration =
197  theConfigurationManager_->getConfigurationByName(
198  dataBufferConfiguration->getConsumerClass(itbufferID, itConsumerID)+ "Configuration"
199  );
200  //std::cout << "\tGot configuration for: " << itConsumerID << " pointer: " << dataProcessorConfiguration << std::endl;
201  std::cout << __PRETTY_FUNCTION__ << "\tGot configuration for: " << itConsumerID << " pointer: " << dataProcessorConfiguration << std::endl;
202  buffers_[itbufferID].consumers_.push_back( std::shared_ptr<DataConsumer>(dynamic_cast<DataConsumer*>(makeDataProcessor(
203  dataBufferConfiguration->getConsumerClass(itbufferID, itConsumerID)
204  , supervisorType_
205  , supervisorInstance_
206  , itbufferID
207  , itConsumerID
208  , dataProcessorConfiguration
209  ))));
210  }
211  }
212  }
213  }
214  */
215 }
216 
217 //========================================================================================================================
218 void DataManager::halt(void)
219 {
220  stop();
221  DataManager::eraseAllBuffers(); //Stop all Buffers and deletes all pointers created and given to the DataManager!
222 }
223 
224 //========================================================================================================================
225 void DataManager::pause(void)
226 {
227  __COUT__ << "Pausing..." << std::endl;
228  DataManager::pauseAllBuffers();
229 }
230 
231 //========================================================================================================================
232 void DataManager::resume(void)
233 {
234  DataManager::resumeAllBuffers();
235 }
236 //========================================================================================================================
237 void DataManager::start(std::string runNumber)
238 {
239  DataManager::startAllBuffers(runNumber);
240 }
241 
242 //========================================================================================================================
243 void DataManager::stop()
244 {
245  DataManager::stopAllBuffers();
246 }
247 
248 //========================================================================================================================
249 void DataManager::eraseAllBuffers(void)
250 {
251  for (auto& it : buffers_)
252  deleteBuffer(it.first);
253 
254  buffers_.clear();
255 }
256 
257 //========================================================================================================================
258 void DataManager::eraseBuffer(std::string bufferUID)
259 {
260  if (deleteBuffer(bufferUID))
261  buffers_.erase(bufferUID);
262 }
263 
264 //========================================================================================================================
265 bool DataManager::unregisterConsumer(std::string consumerID)
266 {
267  for (auto it = buffers_.begin(); it != buffers_.end(); it++)
268  for (auto& itc : it->second.consumers_)
269  {
270  if (itc->getProcessorID() == consumerID)
271  {
272  it->second.buffer_->unregisterConsumer(itc.get());
273  return true;
274  }
275  }
276 
277  return false;
278 }
279 
280 //========================================================================================================================
281 bool DataManager::deleteBuffer(std::string bufferUID)
282 {
283  auto it = buffers_.find(bufferUID);
284  if (it != buffers_.end())
285  {
286  auto aBuffer = it->second;
287  if (aBuffer.status_ == Running)
288  stopBuffer(bufferUID);
289 
290  for (auto& itc : aBuffer.consumers_)
291  {
292  aBuffer.buffer_->unregisterConsumer(itc.get());
293  // delete itc;
294  }
295  aBuffer.consumers_.clear();
296  // for(auto& itp: aBuffer.producers_)
297  // delete itp;
298  aBuffer.producers_.clear();
299 
300  delete aBuffer.buffer_;
301  return true;
302  }
303  return false;
304 }
305 
306 //========================================================================================================================
307 void DataManager::registerProducer(std::string bufferUID, DataProducer* producer)
308 {
309  buffers_[bufferUID].buffer_->registerProducer(producer, producer->getBufferSize());
310 }
311 
312 //========================================================================================================================
313 void DataManager::registerConsumer(std::string bufferUID, DataConsumer* consumer, bool registerToBuffer)
314 {
315  if (registerToBuffer)
316  {
317  if (buffers_.find(bufferUID) == buffers_.end())
318  {
319  __SS__ << ("Can't find buffer UID: " + bufferUID + ". Make sure that your configuration is correct!") << std::endl;
320  __COUT_ERR__ << ss.str();
321  throw std::runtime_error(ss.str());
322  }
323  buffers_[bufferUID].buffer_->registerConsumer(consumer);
324  }
325 }
326 
327 //========================================================================================================================
328 //void DataManager::addConsumers(std::string bufferUID, std::vector<DataConsumer&> consumers)
329 //{
330 //
331 //}
332 
333 //========================================================================================================================
334 void DataManager::startAllBuffers(std::string runNumber)
335 {
336  for (auto it = buffers_.begin(); it != buffers_.end(); it++)
337  startBuffer(it->first, runNumber);
338 }
339 
340 //========================================================================================================================
341 void DataManager::stopAllBuffers(void)
342 {
343  for (auto it = buffers_.begin(); it != buffers_.end(); it++)
344  stopBuffer(it->first);
345 }
346 
347 //========================================================================================================================
348 void DataManager::resumeAllBuffers(void)
349 {
350  for (auto it = buffers_.begin(); it != buffers_.end(); it++)
351  resumeBuffer(it->first);
352 }
353 
354 //========================================================================================================================
355 void DataManager::pauseAllBuffers(void)
356 {
357  for (auto it = buffers_.begin(); it != buffers_.end(); it++)
358  pauseBuffer(it->first);
359 }
360 
361 //========================================================================================================================
362 void DataManager::startBuffer(std::string bufferUID, std::string runNumber)
363 {
364  buffers_[bufferUID].buffer_->reset();
365  for (auto& it : buffers_[bufferUID].consumers_)
366  it->startProcessingData(runNumber);
367  for (auto& it : buffers_[bufferUID].producers_)
368  it->startProcessingData(runNumber);
369  buffers_[bufferUID].status_ = Running;
370 }
371 
372 //========================================================================================================================
373 void DataManager::stopBuffer(std::string bufferUID)
374 {
375  for (auto& it : buffers_[bufferUID].producers_)
376  it->stopProcessingData();
377 
378  //Wait until all buffers are flushed
379  unsigned int timeOut = 0;
380  const unsigned int ratio = 100;
381  const unsigned int sleepTime = 1000 * ratio;
382  unsigned int totalSleepTime = sleepTime / ratio * buffers_[bufferUID].buffer_->getNumberOfBuffers();//1 milliseconds for each buffer!!!!
383  if (totalSleepTime < 5000000)
384  totalSleepTime = 5000000;//At least 5 seconds
385  while (!buffers_[bufferUID].buffer_->isEmpty())
386  {
387  usleep(sleepTime);
388  timeOut += sleepTime;
389  if (timeOut > totalSleepTime)
390  {
391  std::cout << "Couldn't flush all buffers! Timing out after " << totalSleepTime / 1000000. << " seconds!" << std::endl;
392  buffers_[bufferUID].buffer_->isEmpty();
393  break;
394  }
395  }
396  std::cout << "Stopping consumers, buffer MUST BE EMPTY. Is buffer empty? " << buffers_[bufferUID].buffer_->isEmpty() << std::endl;
397  for (auto& it : buffers_[bufferUID].consumers_)
398  it->stopProcessingData();
399  buffers_[bufferUID].buffer_->reset();
400  buffers_[bufferUID].status_ = Initialized;
401 }
402 
403 //========================================================================================================================
404 void DataManager::resumeBuffer(std::string bufferUID)
405 {
406  for (auto& it : buffers_[bufferUID].consumers_)
407  it->resumeProcessingData();
408  for (auto& it : buffers_[bufferUID].producers_)
409  it->resumeProcessingData();
410  buffers_[bufferUID].status_ = Running;
411 }
412 
413 //========================================================================================================================
414 void DataManager::pauseBuffer(std::string bufferUID)
415 {
416  __COUT__ << "Pausing..." << std::endl;
417 
418  for (auto& it : buffers_[bufferUID].producers_)
419  it->pauseProcessingData();
420  //Wait until all buffers are flushed
421  unsigned int timeOut = 0;
422  const unsigned int sleepTime = 1000;
423  while (!buffers_[bufferUID].buffer_->isEmpty())
424  {
425  usleep(sleepTime);
426  timeOut += sleepTime;
427  if (timeOut > sleepTime*buffers_[bufferUID].buffer_->getNumberOfBuffers())//1 milliseconds for each buffer!!!!
428  {
429  std::cout << "Couldn't flush all buffers! Timing out after " << buffers_[bufferUID].buffer_->getNumberOfBuffers()*sleepTime / 1000000. << " seconds!" << std::endl;
430  break;
431  }
432  }
433  for (auto& it : buffers_[bufferUID].consumers_)
434  it->pauseProcessingData();
435  buffers_[bufferUID].status_ = Initialized;
436 }
void eraseBuffer(std::string bufferUID)
!!!!Delete all Buffers and all the pointers of the producers and consumers
Definition: DataManager.cc:258
void startAllBuffers(std::string runNumber)
!!!!Delete all the pointers of the producers and consumers
Definition: DataManager.cc:334