artdaq  v3_09_00
SharedMemoryEventManager.hh
1 #ifndef ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
2 #define ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
3 
4 #include "artdaq/DAQdata/Globals.hh" // Before trace.h gets included in ConcurrentQueue (from GlobalQueue)
5 
6 #include <sys/stat.h>
7 #include <deque>
8 #include <fstream>
9 #include <iomanip>
10 #include <set>
11 #include "artdaq-core/Core/SharedMemoryManager.hh"
12 #include "artdaq-core/Data/RawEvent.hh"
13 #include "artdaq/DAQrate/RequestSender.hh"
14 #include "artdaq/DAQrate/StatisticsHelper.hh"
15 #include "fhiclcpp/fwd.h"
16 #define ART_SUPPORTS_DUPLICATE_EVENTS 0
17 
18 namespace artdaq {
19 
24 {
25 public:
30  explicit art_config_file(fhicl::ParameterSet const& ps /*, uint32_t shm_key, uint32_t broadcast_key*/)
31  : dir_name_("/tmp/partition_" + std::to_string(GetPartitionNumber()))
32  , file_name_(dir_name_ + "/artConfig_" + std::to_string(my_rank) + "_" + std::to_string(artdaq::TimeUtils::gettimeofday_us()) + ".fcl")
33  {
34  mkdir(dir_name_.c_str(), 0777); // Allowed to fail if directory already exists
35 
36  std::ofstream of(file_name_, std::ofstream::trunc);
37  if (of.fail())
38  {
39  // Probably a permissions error...
40  dir_name_ = "/tmp/partition_" + std::to_string(GetPartitionNumber()) + "_" + std::to_string(getuid());
41  mkdir(dir_name_.c_str(), 0777); // Allowed to fail if directory already exists
42  file_name_ = dir_name_ + "/artConfig_" + std::to_string(my_rank) + "_" + std::to_string(artdaq::TimeUtils::gettimeofday_us()) + ".fcl";
43 
44  of.open(file_name_, std::ofstream::trunc);
45  if (of.fail())
46  {
47  TLOG(TLVL_ERROR) << "Failed to open configuration file after two attemps! ABORTING!";
48  exit(46);
49  }
50  }
51  of << ps.to_string();
52 
53  //if (ps.has_key("services.NetMonTransportServiceInterface"))
54  //{
55  // of << " services.NetMonTransportServiceInterface.shared_memory_key: 0x" << std::hex << shm_key;
56  // of << " services.NetMonTransportServiceInterface.broadcast_shared_memory_key: 0x" << std::hex << broadcast_key;
57  // of << " services.NetMonTransportServiceInterface.rank: " << std::dec << my_rank;
58  //}
59  if (!ps.has_key("services.message"))
60  {
61  of << " services.message: { " << generateMessageFacilityConfiguration("art") << "} ";
62  }
63  //of << " source.shared_memory_key: 0x" << std::hex << shm_key;
64  //of << " source.broadcast_shared_memory_key: 0x" << std::hex << broadcast_key;
65  //of << " source.rank: " << std::dec << my_rank;
66  of.close();
67  }
69  {
70  remove(file_name_.c_str());
71  rmdir(dir_name_.c_str()); // Will only delete directory if no config files are left over
72  }
77  std::string getFileName() const { return file_name_; }
78 
79 private:
80  art_config_file(art_config_file const&) = delete;
81  art_config_file(art_config_file&&) = delete;
82  art_config_file& operator=(art_config_file const&) = delete;
83  art_config_file& operator=(art_config_file&&) = delete;
84 
85  std::string dir_name_;
86  std::string file_name_;
87 };
88 
92 class SharedMemoryEventManager : public SharedMemoryManager
93 {
94 public:
95  static const std::string FRAGMENTS_RECEIVED_STAT_KEY;
96  static const std::string EVENTS_RELEASED_STAT_KEY;
97 
98  typedef RawEvent::run_id_t run_id_t;
99  typedef RawEvent::subrun_id_t subrun_id_t;
100  typedef Fragment::sequence_id_t sequence_id_t;
101  typedef std::map<sequence_id_t, RawEvent_ptr> EventMap;
102 
106  struct Config
107  {
110  fhicl::Atom<size_t> max_event_size_bytes{fhicl::Name{"max_event_size_bytes"}, fhicl::Comment{"Maximum event size (all Fragments), in bytes"}};
112  fhicl::Atom<size_t> stale_buffer_timeout_usec{fhicl::Name{"stale_buffer_timeout_usec"}, fhicl::Comment{"Maximum amount of time elapsed before a buffer is marked as abandoned. Time is reset each time an operation is performed on the buffer."}, 5000000};
114  fhicl::Atom<bool> overwrite_mode{fhicl::Name{"overwrite_mode"}, fhicl::Comment{"Whether buffers are allowed to be overwritten when safe (state == Full or Reading)"}, false};
116  fhicl::Atom<bool> restart_crashed_art_processes{fhicl::Name{"restart_crashed_art_processes"}, fhicl::Comment{"Whether to automatically restart art processes that fail for any reason"}, true};
118  fhicl::Atom<uint32_t> shared_memory_key{fhicl::Name{"shared_memory_key"}, fhicl::Comment{"Key to use for shared memory access"}, 0xBEE70000 + getpid()};
120  fhicl::Atom<size_t> buffer_count{fhicl::Name{"buffer_count"}, fhicl::Comment{"Number of events in the Shared Memory (incomplete + pending art)"}};
123  fhicl::Atom<size_t> max_fragment_size_bytes{fhicl::Name{"max_fragment_size_bytes"}, fhicl::Comment{" Maximum Fragment size, in bytes"}};
125  fhicl::Atom<size_t> event_queue_wait_time{fhicl::Name{"event_queue_wait_time"}, fhicl::Comment{"Amount of time (in seconds) an event can exist in shared memory before being released to art. Used as input to default parameter of \"stale_buffer_timeout_usec\"."}, 5};
127  fhicl::Atom<bool> broadcast_mode{fhicl::Name{"broadcast_mode"}, fhicl::Comment{"When true, buffers are not marked Empty when read, but return to Full state. Buffers are overwritten in order received."}, false};
129  fhicl::Atom<size_t> art_analyzer_count{fhicl::Name{"art_analyzer_count"}, fhicl::Comment{"Number of art procceses to start"}, 1};
131  fhicl::Atom<size_t> expected_fragments_per_event{fhicl::Name{"expected_fragments_per_event"}, fhicl::Comment{"Number of Fragments to expect per event"}};
133  fhicl::Atom<int> maximum_oversize_fragment_count{fhicl::Name{"maximum_oversize_fragment_count"}, fhicl::Comment{"Maximum number of over-size Fragments to drop before throwing an exception. Default is 1, which means to throw an exception if any over-size Fragments are dropped. Set to 0 to disable."}, 1};
135  fhicl::Atom<bool> update_run_ids_on_new_fragment{fhicl::Name{"update_run_ids_on_new_fragment"}, fhicl::Comment{"Whether the run and subrun ID of an event should be updated whenever a Fragment is added."}, true};
137  fhicl::Atom<bool> use_sequence_id_for_event_number{fhicl::Name{"use_sequence_id_for_event_number"}, fhicl::Comment{"Whether to use the artdaq Sequence ID (true) or the Timestamp (false) for art Event numbers"}, true};
139  fhicl::Atom<size_t> max_subrun_lookup_table_size{fhicl::Name{"max_subrun_lookup_table_size"}, fhicl::Comment{"The maximum number of entries to store in the sequence ID-SubRun ID lookup table"}, 100};
141  fhicl::Atom<size_t> max_event_list_length{fhicl::Name{"max_event_list_length"}, fhicl::Comment{" The maximum number of entries to store in the released events list"}, 100};
143  fhicl::Atom<bool> send_init_fragments{fhicl::Name{"send_init_fragments"}, fhicl::Comment{"Whether Init Fragments are expected to be sent to art. If true, a Warning message is printed when an Init Fragment is requested but none are available."}, true};
145  fhicl::Atom<int> incomplete_event_report_interval_ms{fhicl::Name{"incomplete_event_report_interval_ms"}, fhicl::Comment{"Interval at which an incomplete event report should be written"}, -1};
148  fhicl::Atom<int> fragment_broadcast_timeout_ms{fhicl::Name{"fragment_broadcast_timeout_ms"}, fhicl::Comment{"Amount of time broadcast fragments should live in the broadcast shared memory segment"}, 3000};
150  fhicl::Atom<double> minimum_art_lifetime_s{fhicl::Name{"minimum_art_lifetime_s"}, fhicl::Comment{"Amount of time that an art process should run to not be considered \"DOA\""}, 2.0};
153  fhicl::Atom<size_t> expected_art_event_processing_time_us{fhicl::Name{"expected_art_event_processing_time_us"}, fhicl::Comment{"During shutdown, SMEM will wait for this amount of time while it is checking that the art threads are done reading buffers."}, 100000};
155  fhicl::Atom<uint32_t> broadcast_shared_memory_key{fhicl::Name{"broadcast_shared_memory_key"}, fhicl::Comment{""}, 0xCEE70000 + getpid()};
157  fhicl::Atom<size_t> broadcast_buffer_count{fhicl::Name{"broadcast_buffer_count"}, fhicl::Comment{"Buffers in the broadcast shared memory segment"}, 10};
159  fhicl::Atom<size_t> broadcast_buffer_size{fhicl::Name{"broadcast_buffer_size"}, fhicl::Comment{"Size of the buffers in the broadcast shared memory segment"}, 0x100000};
161  fhicl::Atom<bool> use_art{fhicl::Name{"use_art"}, fhicl::Comment{"Whether to start and manage art threads (Sets art_analyzer count to 0 and overwrite_mode to true when false)"}, true};
163  fhicl::Atom<bool> manual_art{fhicl::Name{"manual_art"}, fhicl::Comment{"Prints the startup command line for the art process so that the user may (for example) run it in GDB or valgrind"}, false};
164 
165  fhicl::TableFragment<artdaq::RequestSender::Config> requestSenderConfig;
166  };
168  using Parameters = fhicl::WrappedTable<Config>;
169 
175  SharedMemoryEventManager(const fhicl::ParameterSet& pset, fhicl::ParameterSet art_pset);
179  virtual ~SharedMemoryEventManager();
180 
181 private:
188  bool AddFragment(detail::RawFragmentHeader frag, void* dataPtr);
189 
190 public:
198  bool AddFragment(FragmentPtr frag, size_t timeout_usec, FragmentPtr& outfrag);
199 
206  RawDataType* WriteFragmentHeader(detail::RawFragmentHeader frag, bool dropIfNoBuffersAvailable = false);
207 
212  void DoneWritingFragment(detail::RawFragmentHeader frag);
213 
218  size_t GetIncompleteEventCount() { return active_buffers_.size(); }
219 
224  size_t GetPendingEventCount() { return pending_buffers_.size(); }
225 
230  size_t GetLockedBufferCount() { return GetBuffersOwnedByManager().size(); }
231 
236  size_t GetArtEventCount() { return run_event_count_; }
237 
244  size_t GetFragmentCount(Fragment::sequence_id_t seqID, Fragment::type_t type = Fragment::InvalidFragmentType);
245 
252  size_t GetFragmentCountInBuffer(int buffer, Fragment::type_t type = Fragment::InvalidFragmentType);
253 
257  void RunArt(const std::shared_ptr<art_config_file>& config_file, const std::shared_ptr<std::atomic<pid_t>>& pid_out);
261  void StartArt();
262 
268  pid_t StartArtProcess(fhicl::ParameterSet pset);
269 
274  void ShutdownArtProcesses(std::set<pid_t>& pids);
275 
282  void ReconfigureArt(fhicl::ParameterSet art_pset, run_id_t newRun = 0, int n_art_processes = -1);
283 
293  bool endOfData();
294 
299  void startRun(run_id_t runID);
300 
305  run_id_t runID() const { return run_id_; }
306 
311  bool endRun();
312 
318  void rolloverSubrun(sequence_id_t boundary, subrun_id_t subrun);
319 
323  void rolloverSubrun();
324 
328  void sendMetrics();
329 
335  {
336  if (requests_) requests_->SetRequestMode(mode);
337  }
338 
343  void setOverwrite(bool overwrite) { overwrite_mode_ = overwrite; }
344 
348  void AddInitFragment(FragmentPtr& frag);
349 
354  uint32_t GetBroadcastKey() { return broadcasts_.GetKey(); }
355 
361  RawDataType* GetDroppedDataAddress(Fragment::fragment_id_t frag)
362  {
363  if (dropped_data_.count(frag) && dropped_data_[frag] != nullptr)
364  {
365  return dropped_data_[frag]->dataBegin();
366  }
367  return nullptr;
368  }
369 
379  void UpdateArtConfiguration(fhicl::ParameterSet art_pset);
380 
384  void CheckPendingBuffers();
385 
391  subrun_id_t GetSubrunForSequenceID(Fragment::sequence_id_t seqID);
392 
397  subrun_id_t GetCurrentSubrun() { return GetSubrunForSequenceID(Fragment::InvalidSequenceID); }
398 
399 private:
402  SharedMemoryEventManager& operator=(SharedMemoryEventManager const&) = delete;
403  SharedMemoryEventManager& operator=(SharedMemoryEventManager&&) = delete;
404 
405  size_t get_art_process_count_()
406  {
407  std::unique_lock<std::mutex> lk(art_process_mutex_);
408  return art_processes_.size();
409  }
410 
411  std::string buildStatisticsString_() const;
412 
413 private:
414  size_t num_art_processes_;
415  size_t const num_fragments_per_event_;
416  size_t const queue_size_;
417  run_id_t run_id_;
418 
419  std::map<sequence_id_t, subrun_id_t> subrun_event_map_;
420  size_t max_subrun_event_map_length_;
421  static std::mutex subrun_event_map_mutex_;
422 
423  std::set<int> active_buffers_;
424  std::set<int> pending_buffers_;
425  std::unordered_map<Fragment::sequence_id_t, size_t> released_incomplete_events_;
426  std::set<Fragment::sequence_id_t> released_events_;
427  size_t max_event_list_length_;
428 
429  bool update_run_ids_;
430  bool use_sequence_id_for_event_number_;
431  bool overwrite_mode_;
432  size_t init_fragment_count_;
433  bool running_;
434 
435  std::unordered_map<int, std::atomic<int>> buffer_writes_pending_;
436  std::unordered_map<int, std::mutex> buffer_mutexes_;
437  static std::mutex sequence_id_mutex_;
438 
439  int incomplete_event_report_interval_ms_;
440  std::chrono::steady_clock::time_point last_incomplete_event_report_time_;
441  std::chrono::steady_clock::time_point last_backpressure_report_time_;
442  std::chrono::steady_clock::time_point last_fragment_header_write_time_;
443 
444  StatisticsHelper statsHelper_;
445 
446  int broadcast_timeout_ms_;
447 
448  std::atomic<int> run_event_count_;
449  std::atomic<int> run_incomplete_event_count_;
450  std::atomic<int> subrun_event_count_;
451  std::atomic<int> subrun_incomplete_event_count_;
452  std::atomic<int> oversize_fragment_count_;
453  int maximum_oversize_fragment_count_;
454 
455  mutable std::mutex art_process_mutex_;
456  std::set<pid_t> art_processes_;
457  std::atomic<bool> restart_art_;
458  bool always_restart_art_;
459  std::atomic<bool> manual_art_;
460  fhicl::ParameterSet current_art_pset_;
461  std::shared_ptr<art_config_file> current_art_config_file_;
462  double minimum_art_lifetime_s_;
463  size_t art_event_processing_time_us_;
464 
465  std::unique_ptr<RequestSender> requests_;
466  fhicl::ParameterSet data_pset_;
467 
468  FragmentPtrs init_fragments_;
469  std::unordered_map<Fragment::fragment_id_t, FragmentPtr> dropped_data_;
470 
471  bool broadcastFragments_(FragmentPtrs& frags);
472 
473  detail::RawEventHeader* getEventHeader_(int buffer);
474 
475  int getBufferForSequenceID_(Fragment::sequence_id_t seqID, bool create_new, Fragment::timestamp_t timestamp = Fragment::InvalidTimestamp);
476  bool hasFragments_(int buffer);
477  void complete_buffer_(int buffer);
478  bool bufferComparator(int bufA, int bufB);
479  void check_pending_buffers_(std::unique_lock<std::mutex> const& lock);
480 
481  void send_init_frags_();
482  SharedMemoryManager broadcasts_;
483 };
484 } // namespace artdaq
485 
486 #endif //ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
fhicl::Atom< size_t > event_queue_wait_time
&quot;event_queue_wait_time&quot; (Default: 5) : Amount of time(in seconds) an event can exist in shared memory...
fhicl::Atom< bool > overwrite_mode
&quot;overwite_mode&quot; (Default: false): Whether new data is allowed to overwrite buffers in the &quot;Full&quot; stat...
art_config_file wraps a temporary file used to configure art
void AddInitFragment(FragmentPtr &frag)
Set the stored Init fragment, if one has not yet been set already.
void ShutdownArtProcesses(std::set< pid_t > &pids)
Shutdown a set of art processes.
size_t GetLockedBufferCount()
Returns the number of buffers currently owned by this manager.
virtual ~SharedMemoryEventManager()
SharedMemoryEventManager Destructor.
The SharedMemoryEventManager is a SharedMemoryManger which tracks events as they are built...
Fragment::sequence_id_t sequence_id_t
Copy Fragment::sequence_id_t into local scope.
fhicl::Atom< size_t > broadcast_buffer_size
&quot;broadcast_buffer_size&quot; (Default: 0x100000): Size of the buffers in the broadcast shared memory segme...
subrun_id_t GetCurrentSubrun()
Get the current subrun number (Gets the last defined subrun)
fhicl::Atom< bool > restart_crashed_art_processes
&quot;restart_crashed_art_processes&quot; (Default: true) : Whether to automatically restart art processes that...
void ReconfigureArt(fhicl::ParameterSet art_pset, run_id_t newRun=0, int n_art_processes=-1)
Restart all art processes, using the given fhicl code to configure the new art processes.
void RunArt(const std::shared_ptr< art_config_file > &config_file, const std::shared_ptr< std::atomic< pid_t >> &pid_out)
Run an art instance, recording the return codes and restarting it until the end flag is raised...
pid_t StartArtProcess(fhicl::ParameterSet pset)
Start one art process.
fhicl::Atom< double > minimum_art_lifetime_s
&quot;minimum_art_lifetime_s&quot; (Default: 2 seconds): Amount of time that an art process should run to not b...
fhicl::Atom< int > maximum_oversize_fragment_count
&quot;maximum_oversize_fragment_count&quot; (Default: 1): Maximum number of over-size Fragments to drop before ...
fhicl::Atom< bool > update_run_ids_on_new_fragment
&quot;update_run_ids_on_new_fragment&quot; (Default: true) : Whether the run and subrun ID of an event should b...
RawDataType * WriteFragmentHeader(detail::RawFragmentHeader frag, bool dropIfNoBuffersAvailable=false)
Get a pointer to a reserved memory area for the given Fragment header.
void setRequestMode(detail::RequestMessageMode mode)
Set the RequestMessageMode for all outgoing data requests.
size_t GetArtEventCount()
Returns the number of events sent to art this run.
fhicl::Atom< size_t > art_analyzer_count
&quot;art_analyzer_count&quot; (Default: 1) : Number of art procceses to start
Configuration of the SharedMemoryEventManager. May be used for parameter validation ...
fhicl::Atom< bool > use_sequence_id_for_event_number
&quot;use_sequence_id_for_event_number&quot; (Default: true): Whether to use the artdaq Sequence ID (true) or t...
fhicl::Atom< uint32_t > broadcast_shared_memory_key
&quot;broadcast_shared_memory_key&quot; (Default: 0xCEE7000 + PID): Key to use for broadcast shared memory acce...
RawEvent::run_id_t run_id_t
Copy RawEvent::run_id_t into local scope.
fhicl::WrappedTable< Config > Parameters
Used for ParameterSet validation (if desired)
size_t GetFragmentCount(Fragment::sequence_id_t seqID, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in an event.
void UpdateArtConfiguration(fhicl::ParameterSet art_pset)
Updates the internally-stored copy of the art configuration.
RawDataType * GetDroppedDataAddress(Fragment::fragment_id_t frag)
Gets the address of the &quot;dropped data&quot; fragment. Used for testing.
fhicl::Atom< size_t > max_subrun_lookup_table_size
&quot;max_subrun_lookup_table_size&quot; (Default: 100): The maximum number of entries to store in the sequence...
size_t GetPendingEventCount()
Returns the number of events which are complete but waiting on lower sequenced events to finish...
void StartArt()
Start all the art processes.
run_id_t runID() const
Get the current Run number.
fhicl::Atom< size_t > buffer_count
&quot;buffer_count&quot; REQUIRED: Number of events in the Shared Memory(incomplete + pending art) ...
subrun_id_t GetSubrunForSequenceID(Fragment::sequence_id_t seqID)
Get the subrun number that the given Sequence ID would be assigned to.
fhicl::TableFragment< artdaq::RequestSender::Config > requestSenderConfig
Configuration of the RequestSender. See artdaq::RequestSender::Config.
SharedMemoryEventManager(const fhicl::ParameterSet &pset, fhicl::ParameterSet art_pset)
SharedMemoryEventManager Constructor.
fhicl::Atom< uint32_t > shared_memory_key
&quot;shared_memory_key&quot; (Default 0xBEE70000 + PID) : Key to use for shared memory access ...
std::map< sequence_id_t, RawEvent_ptr > EventMap
An EventMap is a map of RawEvent_ptr objects, keyed by sequence ID.
RequestMessageMode
Mode used to indicate current run conditions to the request receiver.
fhicl::Atom< size_t > max_event_list_length
&quot;max_event_list_length&quot; (Default: 100): The maximum number of entries to store in the released events...
void rolloverSubrun()
Add a subrun transition immediately after the highest currently define sequence ID.
void sendMetrics()
Send metrics to the MetricManager, if one has been instantiated in the application.
fhicl::Atom< size_t > stale_buffer_timeout_usec
&quot;stale_buffer_timeout_usec&quot; (Default: event_queue_wait_time * 1, 000, 000) : Maximum amount of time e...
fhicl::Atom< size_t > expected_fragments_per_event
&quot;expected_fragments_per_event&quot; (REQUIRED) : Number of Fragments to expect per event ...
size_t GetIncompleteEventCount()
Returns the number of buffers which contain data but are not yet complete.
fhicl::Atom< bool > send_init_fragments
&quot;send_init_fragments&quot; (Default: true): Whether Init Fragments are expected to be sent to art...
static const std::string FRAGMENTS_RECEIVED_STAT_KEY
Key for Fragments Received MonitoredQuantity.
art_config_file(fhicl::ParameterSet const &ps)
art_config_file Constructor
bool endRun()
Send an EndOfRunFragment to the art thread.
void setOverwrite(bool overwrite)
Set the overwrite flag (non-reliable data transfer) for the Shared Memory.
fhicl::Atom< int > incomplete_event_report_interval_ms
&quot;incomplete_event_report_interval_ms&quot; (Default: -1): Interval at which an incomplete event report sho...
std::string getFileName() const
Get the path of the temporary file.
fhicl::Atom< bool > manual_art
&quot;manual_art&quot; (Default: false): Prints the startup command line for the art process so that the user m...
fhicl::Atom< bool > broadcast_mode
&quot;broadcast_mode&quot; (Default: false) : When true, buffers are not marked Empty when read, but return to Full state.Buffers are overwritten in order received.
fhicl::Atom< size_t > broadcast_buffer_count
&quot;broadcast_buffer_count&quot; (Default: 10): Buffers in the broadcast shared memory segment ...
void DoneWritingFragment(detail::RawFragmentHeader frag)
Used to indicate that the given Fragment is now completely in the buffer. Will check for buffer compl...
static const std::string EVENTS_RELEASED_STAT_KEY
Key for the Events Released MonitoredQuantity.
uint32_t GetBroadcastKey()
Gets the shared memory key of the broadcast SharedMemoryManager.
bool endOfData()
Indicate that the end of input has been reached to the art processes.
fhicl::Atom< bool > use_art
&quot;use_art&quot; (Default: true): Whether to start and manage art threads (Sets art_analyzer count to 0 and ...
RawEvent::subrun_id_t subrun_id_t
Copy RawEvent::subrun_id_t into local scope.
void startRun(run_id_t runID)
Start a Run.
size_t GetFragmentCountInBuffer(int buffer, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in a buffer.
void CheckPendingBuffers()
Check for buffers which are ready to be marked incomplete and released to art and issue tokens for an...