otsdaq  v1_01_02
 All Classes Namespaces Functions
DataManager.cc
1 #include "otsdaq-core/DataManager/DataManager.h"
2 #include "otsdaq-core/DataManager/CircularBuffer.h"
3 #include "otsdaq-core/DataManager/DataProducer.h"
4 #include "otsdaq-core/DataManager/DataConsumer.h"
5 #include "otsdaq-core/MessageFacility/MessageFacility.h"
6 #include "otsdaq-core/Macros/CoutHeaderMacros.h"
7 #include "otsdaq-core/PluginMakers/MakeDataProcessor.h"
8 #include "otsdaq-core/ConfigurationInterface/ConfigurationManager.h"
9 
10 /*
11 #include "otsdaq-core/DataProcessorPlugins/RawDataSaverConsumer.h"
12 #include "otsdaq-core/DataProcessorPlugins/ARTDAQConsumer.h"
13 #include "otsdaq-core/EventBuilder/EventDataSaver.h"
14 #include "otsdaq-core/DataProcessorPlugins/DataListenerProducer.h"
15 #include "otsdaq-core/DataProcessorPlugins/DataStreamer.h"
16 #include "otsdaq-core/DataProcessorPlugins/DQMHistosConsumer.h"
17 #include "otsdaq-core/EventBuilder/AssociativeMemoryEventBuilder.h"
18 #include "otsdaq-core/EventBuilder/Event.h"
19 #include "otsdaq-core/DataProcessorPlugins/DataListenerProducer.h"
20 #include "otsdaq-core/ConfigurationPluginDataFormats/UDPDataListenerProducerConfiguration.h"
21 #include "otsdaq-core/ConfigurationPluginDataFormats/ARTDAQConsumerConfiguration.h"
22 #include "otsdaq-core/ConfigurationPluginDataFormats/DataManagerConfiguration.h"
23 #include "otsdaq-core/ConfigurationPluginDataFormats/DataBufferConfiguration.h"
24 */
25 
26 #include <iostream>
27 #include <vector>
28 #include <unistd.h> //usleep
29 
30 using namespace ots;
31 
32 //========================================================================================================================
33 DataManager::DataManager(const ConfigurationTree& theXDAQContextConfigTree, const std::string& supervisorConfigurationPath)
34 : Configurable(theXDAQContextConfigTree, supervisorConfigurationPath)
35 {}
36 
37 //========================================================================================================================
38 DataManager::~DataManager(void)
39 {
40  eraseAllBuffers();
41 }
42 
43 //========================================================================================================================
44 void DataManager::configure(void)
45 {
46  eraseAllBuffers(); //Deletes all pointers created and given to the DataManager!
47 
48  for(const auto& buffer: theXDAQContextConfigTree_.getNode(theConfigurationPath_+"/LinkToDataManagerConfiguration").getChildren())
49  {
50  __MOUT__ << "Data Buffer Name: "<< buffer.first << std::endl;
51  if(buffer.second.getNode("Status").getValue<bool>())
52  {
53  std::vector<unsigned int> producersVectorLocation;
54  std::vector<unsigned int> consumersVectorLocation;
55  auto bufferConfigurationList = buffer.second.getNode("LinkToDataBufferConfiguration").getChildren();
56  unsigned int location = 0;
57  for(const auto& bufferConfiguration: bufferConfigurationList)
58  {
59  __MOUT__ << "Processor id: " << bufferConfiguration.first << std::endl;
60  if(bufferConfiguration.second.getNode("Status").getValue<bool>())
61  {
62  if(bufferConfiguration.second.getNode("ProcessorType").getValue<std::string>() == "Producer")
63  {
64  producersVectorLocation.push_back(location);
65  }
66  else if(bufferConfiguration.second.getNode("ProcessorType").getValue<std::string>() == "Consumer")
67  {
68  consumersVectorLocation.push_back(location);
69  }
70  else
71  {
72  __SS__ << "Node ProcessorType in "
73  << bufferConfiguration.first
74  << " of type "
75  << bufferConfiguration.second.getNode("ProcessorType").getValue<std::string>()
76  << " is invalid. The only accepted types are Producer and Consumer" << std::endl;
77  throw std::runtime_error(ss.str());
78  }
79  }
80  ++location;
81 
82  }
83  if(producersVectorLocation.size() == 0 || consumersVectorLocation.size() == 0)
84  {
85  __SS__ << "Node Data Buffer "
86  << buffer.first
87  << " has " << producersVectorLocation.size() << " Producers"
88  << " and " << consumersVectorLocation.size() << " Consumers"
89  << " there must be at least 1 of both configured for the buffer!" << std::endl;
90  throw std::runtime_error(ss.str());
91 
92  }
93 
94  configureBuffer<std::string,std::map<std::string,std::string>>(buffer.first);
95  for(auto& producerLocation: producersVectorLocation)
96  {
97 // __MOUT__ << theConfigurationPath_ << std::endl;
98 // __MOUT__ << buffer.first << std::endl;
99  __MOUT__ << bufferConfigurationList[producerLocation].first << std::endl;
100 // __MOUT__ << bufferConfigurationMap[producer].getNode("ProcessorPluginName").getValue<std::string>() << std::endl;
101 // __MOUT__ << bufferConfigurationMap[producer].getNode("LinkToProcessorConfiguration") << std::endl;
102 // __MOUT__ << "THIS DATA MANAGER POINTER: " << this << std::endl;
103 // __MOUT__ << "PASSED" << std::endl;
104  buffers_[buffer.first].producers_.push_back( std::shared_ptr<DataProducer>(dynamic_cast<DataProducer*>(
105  makeDataProcessor
106  (
107  bufferConfigurationList[producerLocation].second.getNode("ProcessorPluginName").getValue<std::string>()
108  , theXDAQContextConfigTree_.getBackNode(theConfigurationPath_).getNode("ApplicationUID").getValue<std::string>()
109  , buffer.first
110  , bufferConfigurationList[producerLocation].first
111  , theXDAQContextConfigTree_
112  , theConfigurationPath_ + "/LinkToDataManagerConfiguration/" + buffer.first + "/LinkToDataBufferConfiguration/" + bufferConfigurationList[producerLocation].first + "/LinkToProcessorConfiguration"
113  ))));
114  __MOUT__ << bufferConfigurationList[producerLocation].first << " has been created!" << std::endl;
115  }
116  for(auto& consumerLocation: consumersVectorLocation)
117  {
118 // __MOUT__ << theConfigurationPath_ << std::endl;
119 // __MOUT__ << buffer.first << std::endl;
120  __MOUT__ << bufferConfigurationList[consumerLocation].first << std::endl;
121 // __MOUT__ << bufferConfigurationMap[consumer].getNode("ProcessorPluginName").getValue<std::string>() << std::endl;
122 // __MOUT__ << bufferConfigurationMap[consumer].getNode("LinkToProcessorConfiguration") << std::endl;
123 // __MOUT__ << theXDAQContextConfigTree_.getBackNode(theConfigurationPath_) << std::endl;
124 // __MOUT__ << "THIS DATA MANAGER POINTER: " << this << std::endl;
125 // __MOUT__ << "PASSED" << std::endl;
126  buffers_[buffer.first].consumers_.push_back( std::shared_ptr<DataConsumer>(dynamic_cast<DataConsumer*>(
127  makeDataProcessor
128  (
129  bufferConfigurationList[consumerLocation].second.getNode("ProcessorPluginName").getValue<std::string>()
130  , theXDAQContextConfigTree_.getBackNode(theConfigurationPath_).getNode("ApplicationUID").getValue<std::string>()
131  , buffer.first
132  , bufferConfigurationList[consumerLocation].first
133  , theXDAQContextConfigTree_
134  , theConfigurationPath_ + "/LinkToDataManagerConfiguration/" + buffer.first + "/LinkToDataBufferConfiguration/" + bufferConfigurationList[consumerLocation].first + "/LinkToProcessorConfiguration"
135  ))));
136  __MOUT__ << bufferConfigurationList[consumerLocation].first << " has been created!" << std::endl;
137  }
138  }
139  //__MOUT__ << "Interface Name: "<< interface.first << std::endl;
140  //__MOUT__ << "XDAQContext Node: "<< theConfigurationManager_->getNode("/XDAQContextConfiguration") << std::endl;
141  //__MOUT__ << "Path to configuration: "<< (theSupervisorConfigurationPath_ + "/LinkToFEInterfaceConfiguration/" + interface.first + "/LinkToFETypeConfiguration") << std::endl;
142  // theFEInterfaces_[interface.first] = makeInterface(
143  // interface.second.getNode("FEInterfacePluginName").getValue<std::string>(),
144  // interface.first,
145  // theXDAQContextConfigTree_,
146  // (supervisorConfigurationPath_ + "/LinkToFEInterfaceConfiguration/" + interface.first + "/LinkToFETypeConfiguration")
147  // );
148  }
149  /*
150  __MOUT__ << "supervisor instance: " << supervisorInstance_ << std::endl;
151  const DataManagerConfiguration* dataManagerConfiguration = theConfigurationManager_->__GET_CONFIG__(DataManagerConfiguration);
152  const DataBufferConfiguration* dataBufferConfiguration = theConfigurationManager_->__GET_CONFIG__(DataBufferConfiguration);
153  std::cout << __PRETTY_FUNCTION__ << dataManagerConfiguration->getConfigurationName() << std::endl;
154  // const FEConfiguration* frontEndConfiguration = theConfigurationManager_->__GET_CONFIG__(FEConfiguration);
155  // std::vector<unsigned int> fedList = frontEndConfiguration->getListOfFERs(supervisorInstance_);
156  // std::stringstream fedName;
157 
158  auto bufferList = dataManagerConfiguration->getListOfDataBuffers(supervisorType_, supervisorInstance_);
159  for(const auto& itbufferID: bufferList)
160  {
161  if(dataManagerConfiguration->getDataBufferStatus(supervisorType_, supervisorInstance_,itbufferID))
162  {
163  configureBuffer<std::string,std::map<std::string,std::string>>(itbufferID);
164  DataProcessor* aDataProcessor = nullptr;
165  std::cout << __PRETTY_FUNCTION__ << "Buffer ID: " << itbufferID << std::endl;
166  auto producerIDList = dataBufferConfiguration->getProducerIDList(itbufferID);
167  for(const auto& itProducerID: producerIDList)
168  {
169  std::cout << __PRETTY_FUNCTION__ << "Processor ID: " << itProducerID << std::endl;
170  const ConfigurationBase* dataProcessorConfiguration = theConfigurationManager_->getConfigurationByName(dataBufferConfiguration->getProducerClass(itbufferID, itProducerID) + "Configuration");
171  if(dataBufferConfiguration->getProducerStatus(itbufferID, itProducerID))
172  {
173  buffers_[itbufferID].producers_.push_back( std::shared_ptr<DataProducer>(dynamic_cast<DataProducer*>(makeDataProcessor(
174  dataBufferConfiguration->getProducerClass(itbufferID, itProducerID)
175  , supervisorType_
176  , supervisorInstance_
177  , itbufferID
178  , itProducerID
179  , dataProcessorConfiguration
180  ))));
181  //aDataProcessor->registerToBuffer();
182  }
183  }
184  auto consumerIDList = dataBufferConfiguration->getConsumerIDList(itbufferID);
185  for(const auto& itConsumerID : consumerIDList)
186  {
187  std::cout << __PRETTY_FUNCTION__ << "Consumer ID: " << itConsumerID << std::endl;
188  std::cout << __PRETTY_FUNCTION__ << "Consumer Type: " <<
189  dataBufferConfiguration->getConsumerClass(itbufferID, itConsumerID) << std::endl;
190 
191  if(dataBufferConfiguration->getConsumerStatus(itbufferID, itConsumerID))
192  {
193  std::cout << __PRETTY_FUNCTION__ << "\tStatus on: " << itConsumerID << std::endl;
194  const ConfigurationBase* dataProcessorConfiguration =
195  theConfigurationManager_->getConfigurationByName(
196  dataBufferConfiguration->getConsumerClass(itbufferID, itConsumerID)+ "Configuration"
197  );
198  //std::cout << "\tGot configuration for: " << itConsumerID << " pointer: " << dataProcessorConfiguration << std::endl;
199  std::cout << __PRETTY_FUNCTION__ << "\tGot configuration for: " << itConsumerID << " pointer: " << dataProcessorConfiguration << std::endl;
200  buffers_[itbufferID].consumers_.push_back( std::shared_ptr<DataConsumer>(dynamic_cast<DataConsumer*>(makeDataProcessor(
201  dataBufferConfiguration->getConsumerClass(itbufferID, itConsumerID)
202  , supervisorType_
203  , supervisorInstance_
204  , itbufferID
205  , itConsumerID
206  , dataProcessorConfiguration
207  ))));
208  }
209  }
210  }
211  }
212  */
213 }
214 
215 //========================================================================================================================
216 void DataManager::halt(void)
217 {
218  stop();
219  DataManager::eraseAllBuffers(); //Stop all Buffers and deletes all pointers created and given to the DataManager!
220 }
221 
222 //========================================================================================================================
223 void DataManager::pause(void)
224 {
225  DataManager::pauseAllBuffers();
226 }
227 
228 //========================================================================================================================
229 void DataManager::resume(void)
230 {
231  DataManager::resumeAllBuffers();
232 }
233 //========================================================================================================================
234 void DataManager::start(std::string runNumber)
235 {
236  DataManager::startAllBuffers(runNumber);
237 }
238 
239 //========================================================================================================================
240 void DataManager::stop()
241 {
242  DataManager::stopAllBuffers();
243 }
244 
245 //========================================================================================================================
246 void DataManager::eraseAllBuffers(void)
247 {
248  for(auto& it: buffers_)
249  deleteBuffer(it.first);
250 
251  buffers_.clear();
252 }
253 
254 //========================================================================================================================
255 void DataManager::eraseBuffer(std::string bufferUID)
256 {
257  if(deleteBuffer(bufferUID))
258  buffers_.erase(bufferUID);
259 }
260 
261 //========================================================================================================================
262 bool DataManager::unregisterConsumer(std::string consumerID)
263 {
264  for(auto it=buffers_.begin(); it!=buffers_.end(); it++)
265  for(auto& itc: it->second.consumers_)
266  {
267  if(itc->getProcessorID() == consumerID)
268  {
269  it->second.buffer_->unregisterConsumer(itc.get());
270  return true;
271  }
272  }
273 
274  return false;
275 }
276 
277 //========================================================================================================================
278 bool DataManager::deleteBuffer(std::string bufferUID)
279 {
280  auto it = buffers_.find(bufferUID);
281  if(it != buffers_.end())
282  {
283  auto aBuffer = it->second;
284  if(aBuffer.status_ == Running)
285  stopBuffer(bufferUID);
286 
287  for(auto& itc: aBuffer.consumers_)
288  {
289  aBuffer.buffer_->unregisterConsumer(itc.get());
290  // delete itc;
291  }
292  aBuffer.consumers_.clear();
293  // for(auto& itp: aBuffer.producers_)
294  // delete itp;
295  aBuffer.producers_.clear();
296 
297  delete aBuffer.buffer_;
298  return true;
299  }
300  return false;
301 }
302 
303 //========================================================================================================================
304 void DataManager::registerProducer(std::string bufferUID, DataProducer* producer)
305 {
306  buffers_[bufferUID].buffer_->registerProducer(producer, producer->getBufferSize());
307 }
308 
309 //========================================================================================================================
310 void DataManager::registerConsumer(std::string bufferUID, DataConsumer* consumer, bool registerToBuffer)
311 {
312  if(registerToBuffer)
313  if(buffers_.find(bufferUID) == buffers_.end())
314  throw std::runtime_error("Can't find buffer UID: " + bufferUID + ". Make sure that your configuration is correct!");
315  buffers_[bufferUID].buffer_->registerConsumer(consumer);
316 }
317 
318 //========================================================================================================================
319 //void DataManager::addConsumers(std::string bufferUID, std::vector<DataConsumer&> consumers)
320 //{
321 //
322 //}
323 
324 //========================================================================================================================
325 void DataManager::startAllBuffers(std::string runNumber)
326 {
327  for(auto it=buffers_.begin(); it!=buffers_.end(); it++)
328  startBuffer(it->first, runNumber);
329 }
330 
331 //========================================================================================================================
332 void DataManager::stopAllBuffers (void)
333 {
334  for(auto it=buffers_.begin(); it!=buffers_.end(); it++)
335  stopBuffer(it->first);
336 }
337 
338 //========================================================================================================================
339 void DataManager::resumeAllBuffers(void)
340 {
341  for(auto it=buffers_.begin(); it!=buffers_.end(); it++)
342  resumeBuffer(it->first);
343 }
344 
345 //========================================================================================================================
346 void DataManager::pauseAllBuffers (void)
347 {
348  for(auto it=buffers_.begin(); it!=buffers_.end(); it++)
349  pauseBuffer(it->first);
350 }
351 
352 //========================================================================================================================
353 void DataManager::startBuffer(std::string bufferUID, std::string runNumber)
354 {
355  buffers_[bufferUID].buffer_->reset();
356  for(auto& it: buffers_[bufferUID].consumers_)
357  it->startProcessingData(runNumber);
358  for(auto& it: buffers_[bufferUID].producers_)
359  it->startProcessingData(runNumber);
360  buffers_[bufferUID].status_ = Running;
361 }
362 
363 //========================================================================================================================
364 void DataManager::stopBuffer(std::string bufferUID)
365 {
366  for(auto& it: buffers_[bufferUID].producers_)
367  it->stopProcessingData();
368 
369  //Wait until all buffers are flushed
370  unsigned int timeOut = 0;
371  const unsigned int ratio = 100;
372  const unsigned int sleepTime = 1000*ratio;
373  unsigned int totalSleepTime = sleepTime/ratio*buffers_[bufferUID].buffer_->getNumberOfBuffers();//1 milliseconds for each buffer!!!!
374  if(totalSleepTime < 5000000)
375  totalSleepTime = 5000000;//At least 5 seconds
376  while(!buffers_[bufferUID].buffer_->isEmpty())
377  {
378  usleep(sleepTime);
379  timeOut += sleepTime;
380  if(timeOut > totalSleepTime)
381  {
382  std::cout << "Couldn't flush all buffers! Timing out after " << totalSleepTime/1000000. << " seconds!" << std::endl;
383  buffers_[bufferUID].buffer_->isEmpty();
384  break;
385  }
386  }
387  std::cout << "Stopping consumers, buffer MUST BE EMPTY. Is buffer empty? "<< buffers_[bufferUID].buffer_->isEmpty() << std::endl;
388  for(auto& it: buffers_[bufferUID].consumers_)
389  it->stopProcessingData();
390  buffers_[bufferUID].buffer_->reset();
391  buffers_[bufferUID].status_ = Initialized;
392 }
393 
394 //========================================================================================================================
395 void DataManager::resumeBuffer(std::string bufferUID)
396 {
397  for(auto& it: buffers_[bufferUID].consumers_)
398  it->resumeProcessingData();
399  for(auto& it: buffers_[bufferUID].producers_)
400  it->resumeProcessingData();
401  buffers_[bufferUID].status_ = Running;
402 }
403 
404 //========================================================================================================================
405 void DataManager::pauseBuffer(std::string bufferUID)
406 {
407  for(auto& it: buffers_[bufferUID].producers_)
408  it->pauseProcessingData();
409  //Wait until all buffers are flushed
410  unsigned int timeOut = 0;
411  const unsigned int sleepTime = 1000;
412  while(!buffers_[bufferUID].buffer_->isEmpty())
413  {
414  usleep(sleepTime);
415  timeOut += sleepTime;
416  if(timeOut > sleepTime*buffers_[bufferUID].buffer_->getNumberOfBuffers())//1 milliseconds for each buffer!!!!
417  {
418  std::cout << "Couldn't flush all buffers! Timing out after " << buffers_[bufferUID].buffer_->getNumberOfBuffers()*sleepTime/1000000. << " seconds!" << std::endl;
419  break;
420  }
421  }
422  for(auto& it: buffers_[bufferUID].consumers_)
423  it->pauseProcessingData();
424  buffers_[bufferUID].status_ = Initialized;
425 }
void eraseBuffer(std::string bufferUID)
!!!!Delete all Buffers and all the pointers of the producers and consumers
Definition: DataManager.cc:255
void startAllBuffers(std::string runNumber)
!!!!Delete all the pointers of the producers and consumers
Definition: DataManager.cc:325