otsdaq  v1_01_03
 All Classes Namespaces Functions
DataManager.cc
1 #include "otsdaq-core/DataManager/DataManager.h"
2 #include "otsdaq-core/DataManager/CircularBuffer.h"
3 #include "otsdaq-core/DataManager/DataProducer.h"
4 #include "otsdaq-core/DataManager/DataConsumer.h"
5 #include "otsdaq-core/MessageFacility/MessageFacility.h"
6 #include "otsdaq-core/Macros/CoutHeaderMacros.h"
7 #include "otsdaq-core/PluginMakers/MakeDataProcessor.h"
8 #include "otsdaq-core/ConfigurationInterface/ConfigurationManager.h"
9 
10 /*
11 #include "otsdaq-core/DataProcessorPlugins/RawDataSaverConsumer.h"
12 #include "otsdaq-core/DataProcessorPlugins/ARTDAQConsumer.h"
13 #include "otsdaq-core/EventBuilder/EventDataSaver.h"
14 #include "otsdaq-core/DataProcessorPlugins/DataListenerProducer.h"
15 #include "otsdaq-core/DataProcessorPlugins/DataStreamer.h"
16 #include "otsdaq-core/DataProcessorPlugins/DQMHistosConsumer.h"
17 #include "otsdaq-core/EventBuilder/AssociativeMemoryEventBuilder.h"
18 #include "otsdaq-core/EventBuilder/Event.h"
19 #include "otsdaq-core/DataProcessorPlugins/DataListenerProducer.h"
20 #include "otsdaq-core/ConfigurationPluginDataFormats/UDPDataListenerProducerConfiguration.h"
21 #include "otsdaq-core/ConfigurationPluginDataFormats/ARTDAQConsumerConfiguration.h"
22 #include "otsdaq-core/ConfigurationPluginDataFormats/DataManagerConfiguration.h"
23 #include "otsdaq-core/ConfigurationPluginDataFormats/DataBufferConfiguration.h"
24 */
25 
26 #include <iostream>
27 #include <vector>
28 #include <unistd.h> //usleep
29 
30 using namespace ots;
31 
32 //========================================================================================================================
33 DataManager::DataManager(const ConfigurationTree& theXDAQContextConfigTree, const std::string& supervisorConfigurationPath)
34 : Configurable(theXDAQContextConfigTree, supervisorConfigurationPath)
35 {}
36 
37 //========================================================================================================================
38 DataManager::~DataManager(void)
39 {
40  eraseAllBuffers();
41 }
42 
43 //========================================================================================================================
44 void DataManager::configure(void)
45 {
46  eraseAllBuffers(); //Deletes all pointers created and given to the DataManager!
47 
48  for(const auto& buffer: theXDAQContextConfigTree_.getNode(theConfigurationPath_+"/LinkToDataManagerConfiguration").getChildren())
49  {
50  __MOUT__ << "Data Buffer Name: "<< buffer.first << std::endl;
51  if(buffer.second.getNode("Status").getValue<bool>())
52  {
53  std::vector<unsigned int> producersVectorLocation;
54  std::vector<unsigned int> consumersVectorLocation;
55  auto bufferConfigurationList = buffer.second.getNode("LinkToDataBufferConfiguration").getChildren();
56  unsigned int location = 0;
57  for(const auto& bufferConfiguration: bufferConfigurationList)
58  {
59  __MOUT__ << "Processor id: " << bufferConfiguration.first << std::endl;
60  if(bufferConfiguration.second.getNode("Status").getValue<bool>())
61  {
62  if(bufferConfiguration.second.getNode("ProcessorType").getValue<std::string>() == "Producer")
63  {
64  producersVectorLocation.push_back(location);
65  }
66  else if(bufferConfiguration.second.getNode("ProcessorType").getValue<std::string>() == "Consumer")
67  {
68  consumersVectorLocation.push_back(location);
69  }
70  else
71  {
72  __SS__ << "Node ProcessorType in "
73  << bufferConfiguration.first
74  << " of type "
75  << bufferConfiguration.second.getNode("ProcessorType").getValue<std::string>()
76  << " is invalid. The only accepted types are Producer and Consumer" << std::endl;
77  throw std::runtime_error(ss.str());
78  }
79  }
80  ++location;
81 
82  }
83 
84  if(producersVectorLocation.size() == 0)// || consumersVectorLocation.size() == 0)
85  {
86  __SS__ << "Node Data Buffer "
87  << buffer.first
88  << " has " << producersVectorLocation.size() << " Producers"
89  << " and " << consumersVectorLocation.size() << " Consumers"
90  << " there must be at least 1 Producer " << // of both configured
91  "for the buffer!" << std::endl;
92  throw std::runtime_error(ss.str());
93 
94  }
95 
96  configureBuffer<std::string,std::map<std::string,std::string>>(buffer.first);
97  for(auto& producerLocation: producersVectorLocation)
98  {
99 // __MOUT__ << theConfigurationPath_ << std::endl;
100 // __MOUT__ << buffer.first << std::endl;
101  __MOUT__ << bufferConfigurationList[producerLocation].first << std::endl;
102 // __MOUT__ << bufferConfigurationMap[producer].getNode("ProcessorPluginName").getValue<std::string>() << std::endl;
103 // __MOUT__ << bufferConfigurationMap[producer].getNode("LinkToProcessorConfiguration") << std::endl;
104 // __MOUT__ << "THIS DATA MANAGER POINTER: " << this << std::endl;
105 // __MOUT__ << "PASSED" << std::endl;
106  buffers_[buffer.first].producers_.push_back( std::shared_ptr<DataProducer>(dynamic_cast<DataProducer*>(
107  makeDataProcessor
108  (
109  bufferConfigurationList[producerLocation].second.getNode("ProcessorPluginName").getValue<std::string>()
110  , theXDAQContextConfigTree_.getBackNode(theConfigurationPath_).getNode("ApplicationUID").getValue<std::string>()
111  , buffer.first
112  , bufferConfigurationList[producerLocation].first
113  , theXDAQContextConfigTree_
114  , theConfigurationPath_ + "/LinkToDataManagerConfiguration/" + buffer.first + "/LinkToDataBufferConfiguration/" + bufferConfigurationList[producerLocation].first + "/LinkToProcessorConfiguration"
115  ))));
116  __MOUT__ << bufferConfigurationList[producerLocation].first << " has been created!" << std::endl;
117  }
118  for(auto& consumerLocation: consumersVectorLocation)
119  {
120 // __MOUT__ << theConfigurationPath_ << std::endl;
121 // __MOUT__ << buffer.first << std::endl;
122  __MOUT__ << bufferConfigurationList[consumerLocation].first << std::endl;
123 // __MOUT__ << bufferConfigurationMap[consumer].getNode("ProcessorPluginName").getValue<std::string>() << std::endl;
124 // __MOUT__ << bufferConfigurationMap[consumer].getNode("LinkToProcessorConfiguration") << std::endl;
125 // __MOUT__ << theXDAQContextConfigTree_.getBackNode(theConfigurationPath_) << std::endl;
126 // __MOUT__ << "THIS DATA MANAGER POINTER: " << this << std::endl;
127 // __MOUT__ << "PASSED" << std::endl;
128  buffers_[buffer.first].consumers_.push_back( std::shared_ptr<DataConsumer>(dynamic_cast<DataConsumer*>(
129  makeDataProcessor
130  (
131  bufferConfigurationList[consumerLocation].second.getNode("ProcessorPluginName").getValue<std::string>()
132  , theXDAQContextConfigTree_.getBackNode(theConfigurationPath_).getNode("ApplicationUID").getValue<std::string>()
133  , buffer.first
134  , bufferConfigurationList[consumerLocation].first
135  , theXDAQContextConfigTree_
136  , theConfigurationPath_ + "/LinkToDataManagerConfiguration/" + buffer.first + "/LinkToDataBufferConfiguration/" + bufferConfigurationList[consumerLocation].first + "/LinkToProcessorConfiguration"
137  ))));
138  __MOUT__ << bufferConfigurationList[consumerLocation].first << " has been created!" << std::endl;
139  }
140  }
141  //__MOUT__ << "Interface Name: "<< interface.first << std::endl;
142  //__MOUT__ << "XDAQContext Node: "<< theConfigurationManager_->getNode("/XDAQContextConfiguration") << std::endl;
143  //__MOUT__ << "Path to configuration: "<< (theSupervisorConfigurationPath_ + "/LinkToFEInterfaceConfiguration/" + interface.first + "/LinkToFETypeConfiguration") << std::endl;
144  // theFEInterfaces_[interface.first] = makeInterface(
145  // interface.second.getNode("FEInterfacePluginName").getValue<std::string>(),
146  // interface.first,
147  // theXDAQContextConfigTree_,
148  // (supervisorConfigurationPath_ + "/LinkToFEInterfaceConfiguration/" + interface.first + "/LinkToFETypeConfiguration")
149  // );
150  }
151  /*
152  __MOUT__ << "supervisor instance: " << supervisorInstance_ << std::endl;
153  const DataManagerConfiguration* dataManagerConfiguration = theConfigurationManager_->__GET_CONFIG__(DataManagerConfiguration);
154  const DataBufferConfiguration* dataBufferConfiguration = theConfigurationManager_->__GET_CONFIG__(DataBufferConfiguration);
155  std::cout << __PRETTY_FUNCTION__ << dataManagerConfiguration->getConfigurationName() << std::endl;
156  // const FEConfiguration* frontEndConfiguration = theConfigurationManager_->__GET_CONFIG__(FEConfiguration);
157  // std::vector<unsigned int> fedList = frontEndConfiguration->getListOfFERs(supervisorInstance_);
158  // std::stringstream fedName;
159 
160  auto bufferList = dataManagerConfiguration->getListOfDataBuffers(supervisorType_, supervisorInstance_);
161  for(const auto& itbufferID: bufferList)
162  {
163  if(dataManagerConfiguration->getDataBufferStatus(supervisorType_, supervisorInstance_,itbufferID))
164  {
165  configureBuffer<std::string,std::map<std::string,std::string>>(itbufferID);
166  DataProcessor* aDataProcessor = nullptr;
167  std::cout << __PRETTY_FUNCTION__ << "Buffer ID: " << itbufferID << std::endl;
168  auto producerIDList = dataBufferConfiguration->getProducerIDList(itbufferID);
169  for(const auto& itProducerID: producerIDList)
170  {
171  std::cout << __PRETTY_FUNCTION__ << "Processor ID: " << itProducerID << std::endl;
172  const ConfigurationBase* dataProcessorConfiguration = theConfigurationManager_->getConfigurationByName(dataBufferConfiguration->getProducerClass(itbufferID, itProducerID) + "Configuration");
173  if(dataBufferConfiguration->getProducerStatus(itbufferID, itProducerID))
174  {
175  buffers_[itbufferID].producers_.push_back( std::shared_ptr<DataProducer>(dynamic_cast<DataProducer*>(makeDataProcessor(
176  dataBufferConfiguration->getProducerClass(itbufferID, itProducerID)
177  , supervisorType_
178  , supervisorInstance_
179  , itbufferID
180  , itProducerID
181  , dataProcessorConfiguration
182  ))));
183  //aDataProcessor->registerToBuffer();
184  }
185  }
186  auto consumerIDList = dataBufferConfiguration->getConsumerIDList(itbufferID);
187  for(const auto& itConsumerID : consumerIDList)
188  {
189  std::cout << __PRETTY_FUNCTION__ << "Consumer ID: " << itConsumerID << std::endl;
190  std::cout << __PRETTY_FUNCTION__ << "Consumer Type: " <<
191  dataBufferConfiguration->getConsumerClass(itbufferID, itConsumerID) << std::endl;
192 
193  if(dataBufferConfiguration->getConsumerStatus(itbufferID, itConsumerID))
194  {
195  std::cout << __PRETTY_FUNCTION__ << "\tStatus on: " << itConsumerID << std::endl;
196  const ConfigurationBase* dataProcessorConfiguration =
197  theConfigurationManager_->getConfigurationByName(
198  dataBufferConfiguration->getConsumerClass(itbufferID, itConsumerID)+ "Configuration"
199  );
200  //std::cout << "\tGot configuration for: " << itConsumerID << " pointer: " << dataProcessorConfiguration << std::endl;
201  std::cout << __PRETTY_FUNCTION__ << "\tGot configuration for: " << itConsumerID << " pointer: " << dataProcessorConfiguration << std::endl;
202  buffers_[itbufferID].consumers_.push_back( std::shared_ptr<DataConsumer>(dynamic_cast<DataConsumer*>(makeDataProcessor(
203  dataBufferConfiguration->getConsumerClass(itbufferID, itConsumerID)
204  , supervisorType_
205  , supervisorInstance_
206  , itbufferID
207  , itConsumerID
208  , dataProcessorConfiguration
209  ))));
210  }
211  }
212  }
213  }
214  */
215 }
216 
217 //========================================================================================================================
218 void DataManager::halt(void)
219 {
220  stop();
221  DataManager::eraseAllBuffers(); //Stop all Buffers and deletes all pointers created and given to the DataManager!
222 }
223 
224 //========================================================================================================================
225 void DataManager::pause(void)
226 {
227  DataManager::pauseAllBuffers();
228 }
229 
230 //========================================================================================================================
231 void DataManager::resume(void)
232 {
233  DataManager::resumeAllBuffers();
234 }
235 //========================================================================================================================
236 void DataManager::start(std::string runNumber)
237 {
238  DataManager::startAllBuffers(runNumber);
239 }
240 
241 //========================================================================================================================
242 void DataManager::stop()
243 {
244  DataManager::stopAllBuffers();
245 }
246 
247 //========================================================================================================================
248 void DataManager::eraseAllBuffers(void)
249 {
250  for(auto& it: buffers_)
251  deleteBuffer(it.first);
252 
253  buffers_.clear();
254 }
255 
256 //========================================================================================================================
257 void DataManager::eraseBuffer(std::string bufferUID)
258 {
259  if(deleteBuffer(bufferUID))
260  buffers_.erase(bufferUID);
261 }
262 
263 //========================================================================================================================
264 bool DataManager::unregisterConsumer(std::string consumerID)
265 {
266  for(auto it=buffers_.begin(); it!=buffers_.end(); it++)
267  for(auto& itc: it->second.consumers_)
268  {
269  if(itc->getProcessorID() == consumerID)
270  {
271  it->second.buffer_->unregisterConsumer(itc.get());
272  return true;
273  }
274  }
275 
276  return false;
277 }
278 
279 //========================================================================================================================
280 bool DataManager::deleteBuffer(std::string bufferUID)
281 {
282  auto it = buffers_.find(bufferUID);
283  if(it != buffers_.end())
284  {
285  auto aBuffer = it->second;
286  if(aBuffer.status_ == Running)
287  stopBuffer(bufferUID);
288 
289  for(auto& itc: aBuffer.consumers_)
290  {
291  aBuffer.buffer_->unregisterConsumer(itc.get());
292  // delete itc;
293  }
294  aBuffer.consumers_.clear();
295  // for(auto& itp: aBuffer.producers_)
296  // delete itp;
297  aBuffer.producers_.clear();
298 
299  delete aBuffer.buffer_;
300  return true;
301  }
302  return false;
303 }
304 
305 //========================================================================================================================
306 void DataManager::registerProducer(std::string bufferUID, DataProducer* producer)
307 {
308  buffers_[bufferUID].buffer_->registerProducer(producer, producer->getBufferSize());
309 }
310 
311 //========================================================================================================================
312 void DataManager::registerConsumer(std::string bufferUID, DataConsumer* consumer, bool registerToBuffer)
313 {
314  if(registerToBuffer)
315  if(buffers_.find(bufferUID) == buffers_.end())
316  throw std::runtime_error("Can't find buffer UID: " + bufferUID + ". Make sure that your configuration is correct!");
317  buffers_[bufferUID].buffer_->registerConsumer(consumer);
318 }
319 
320 //========================================================================================================================
321 //void DataManager::addConsumers(std::string bufferUID, std::vector<DataConsumer&> consumers)
322 //{
323 //
324 //}
325 
326 //========================================================================================================================
327 void DataManager::startAllBuffers(std::string runNumber)
328 {
329  for(auto it=buffers_.begin(); it!=buffers_.end(); it++)
330  startBuffer(it->first, runNumber);
331 }
332 
333 //========================================================================================================================
334 void DataManager::stopAllBuffers (void)
335 {
336  for(auto it=buffers_.begin(); it!=buffers_.end(); it++)
337  stopBuffer(it->first);
338 }
339 
340 //========================================================================================================================
341 void DataManager::resumeAllBuffers(void)
342 {
343  for(auto it=buffers_.begin(); it!=buffers_.end(); it++)
344  resumeBuffer(it->first);
345 }
346 
347 //========================================================================================================================
348 void DataManager::pauseAllBuffers (void)
349 {
350  for(auto it=buffers_.begin(); it!=buffers_.end(); it++)
351  pauseBuffer(it->first);
352 }
353 
354 //========================================================================================================================
355 void DataManager::startBuffer(std::string bufferUID, std::string runNumber)
356 {
357  buffers_[bufferUID].buffer_->reset();
358  for(auto& it: buffers_[bufferUID].consumers_)
359  it->startProcessingData(runNumber);
360  for(auto& it: buffers_[bufferUID].producers_)
361  it->startProcessingData(runNumber);
362  buffers_[bufferUID].status_ = Running;
363 }
364 
365 //========================================================================================================================
366 void DataManager::stopBuffer(std::string bufferUID)
367 {
368  for(auto& it: buffers_[bufferUID].producers_)
369  it->stopProcessingData();
370 
371  //Wait until all buffers are flushed
372  unsigned int timeOut = 0;
373  const unsigned int ratio = 100;
374  const unsigned int sleepTime = 1000*ratio;
375  unsigned int totalSleepTime = sleepTime/ratio*buffers_[bufferUID].buffer_->getNumberOfBuffers();//1 milliseconds for each buffer!!!!
376  if(totalSleepTime < 5000000)
377  totalSleepTime = 5000000;//At least 5 seconds
378  while(!buffers_[bufferUID].buffer_->isEmpty())
379  {
380  usleep(sleepTime);
381  timeOut += sleepTime;
382  if(timeOut > totalSleepTime)
383  {
384  std::cout << "Couldn't flush all buffers! Timing out after " << totalSleepTime/1000000. << " seconds!" << std::endl;
385  buffers_[bufferUID].buffer_->isEmpty();
386  break;
387  }
388  }
389  std::cout << "Stopping consumers, buffer MUST BE EMPTY. Is buffer empty? "<< buffers_[bufferUID].buffer_->isEmpty() << std::endl;
390  for(auto& it: buffers_[bufferUID].consumers_)
391  it->stopProcessingData();
392  buffers_[bufferUID].buffer_->reset();
393  buffers_[bufferUID].status_ = Initialized;
394 }
395 
396 //========================================================================================================================
397 void DataManager::resumeBuffer(std::string bufferUID)
398 {
399  for(auto& it: buffers_[bufferUID].consumers_)
400  it->resumeProcessingData();
401  for(auto& it: buffers_[bufferUID].producers_)
402  it->resumeProcessingData();
403  buffers_[bufferUID].status_ = Running;
404 }
405 
406 //========================================================================================================================
407 void DataManager::pauseBuffer(std::string bufferUID)
408 {
409  for(auto& it: buffers_[bufferUID].producers_)
410  it->pauseProcessingData();
411  //Wait until all buffers are flushed
412  unsigned int timeOut = 0;
413  const unsigned int sleepTime = 1000;
414  while(!buffers_[bufferUID].buffer_->isEmpty())
415  {
416  usleep(sleepTime);
417  timeOut += sleepTime;
418  if(timeOut > sleepTime*buffers_[bufferUID].buffer_->getNumberOfBuffers())//1 milliseconds for each buffer!!!!
419  {
420  std::cout << "Couldn't flush all buffers! Timing out after " << buffers_[bufferUID].buffer_->getNumberOfBuffers()*sleepTime/1000000. << " seconds!" << std::endl;
421  break;
422  }
423  }
424  for(auto& it: buffers_[bufferUID].consumers_)
425  it->pauseProcessingData();
426  buffers_[bufferUID].status_ = Initialized;
427 }
void eraseBuffer(std::string bufferUID)
!!!!Delete all Buffers and all the pointers of the producers and consumers
Definition: DataManager.cc:257
void startAllBuffers(std::string runNumber)
!!!!Delete all the pointers of the producers and consumers
Definition: DataManager.cc:327