Line data Source code
1 : #ifndef TOHOST_BUFFER_H_ 2 : #define TOHOST_BUFFER_H_ 3 : 4 : #include <cstddef> 5 : #include <cstdint> 6 : #include <thread> 7 : #include <span> 8 : #include <mutex> 9 : #include <shared_mutex> 10 : #include <cstdio> 11 : #include <condition_variable> 12 : 13 : #include "device.hpp" 14 : #include "cmem_buffer.hpp" 15 : #include "block.hpp" 16 : 17 : 18 : #if REGMAP_VERSION < 0x500 19 : #define IRQ_WRAP_AROUND_FROM_HOST 0 20 : #define IRQ_WRAP_AROUND_TO_HOST 1 21 : #define IRQ_DATA_AVAILABLE 2 22 : #define IRQ_FIFO_FULL_FROM_HOST 3 23 : #define IRQ_BUSY_CHANGE_TO_HOST 6 24 : #define IRQ_FIFO_FULL_TO_HOST 7 25 : #else 26 : #define IRQ_DATA_AVAILABLE 0 27 : #define IRQ_XOFF_TO_HOST 5 28 : #define IRQ_BUSY_CHANGE_TO_HOST 6 29 : #define IRQ_FIFO_FULL_TO_HOST 7 30 : #endif 31 : 32 : #define MIN(X, Y) (((X) < (Y)) ? (X) : (Y)) 33 : 34 : 35 : /** 36 : * Abstract class representing a ToHost DMA buffer. 37 : * The elinks enabled for the corresponding DMA buffers are read from the 38 : * device in the constructors. 39 : */ 40 : class ToHostBuffer 41 : { 42 : public: 43 : ToHostBuffer(int dmaid, std::shared_ptr<Device> d); 44 : 45 : //Interrupts 46 : /** 47 : * \defgroup Interrupt handling 48 : * @{ 49 : * Functions to enable/disable for MSI-X interrupts. 50 : */ 51 : /** 52 : * @brief enable on_data interrupt for this DMA buffer. 53 : */ 54 0 : void irq_data_enable() {m_device->irq_enable(m_irq_on_data);} 55 : 56 : /** 57 : * @brief disable on_data interrupt for this DMA buffer. 58 : */ 59 : void irq_data_disable(){m_device->irq_disable(m_irq_on_data);} 60 : 61 : /** 62 : * @brief enable busy interrupt for this DMA buffer. 63 : */ 64 : void irq_busy_enable() {m_device->irq_enable(IRQ_BUSY_CHANGE_TO_HOST);} 65 : 66 : /** 67 : * @brief disable busy interrupt for this DMA buffer. 68 : */ 69 : void irq_busy_disable(){m_device->irq_disable(IRQ_BUSY_CHANGE_TO_HOST);} 70 : /** 71 : * @} 72 : */ 73 : 74 : /** 75 : * @return a shared pointer to the device serving this DMA buffer. 76 : */ 77 120 : std::shared_ptr<Device> get_device(){return m_device;} 78 : 79 : /** 80 : * @return identifying number of this DMA buffer. 81 : */ 82 : int get_dmaid() const {return m_dmaid;} 83 : 84 : /** 85 : * @return read e-links enabled for this DMA buffer. 86 : */ 87 : std::vector<Elink> get_elinks(){return m_device->read_enabled_elinks(m_dmaid);} 88 : 89 : /** 90 : * @return read e-links enabled for this DMA buffer of a given type. 91 : * @param t the e-link type, determined from the encoding. 92 : */ 93 66 : std::vector<Elink> get_elinks_of_type(elink_type_t t){return m_device->get_enabled_elinks_of_type(m_dmaid, t);}; 94 : 95 : /** 96 : * @return fraction of e-links enabled for this DMA buffer of a given type. 97 : * @param t the e-link type, determined from the encoding. 98 : */ 99 : std::vector<std::vector<Elink>> split_elinks_of_type(elink_type_t t, size_t n); 100 : 101 : //DMA buffer operations 102 : /** 103 : * \defgroup DMA operations 104 : * @{ 105 : * Functions to manage DMA read/write operations. 106 : */ 107 : /** 108 : * @brief fraction of e-links enabled for this DMA buffer of a given type. 109 : * @param name of the buffers passed to the CMEM driver. 110 : * @param vmem if true, do not use CMEM but malloc a buffer in memory. 111 : * @param free_previous_cmem if true, re-claim a buffer with the same name if present and not locked. 112 : */ 113 : virtual void allocate_buffer(size_t size, 114 : const std::string& name, 115 : bool vmem, bool free_previous_cmem=true) = 0; 116 : 117 : /** 118 : * @brief inform the FELIX card of the allocated buffer, to be used as a circular DMA buffer. 119 : */ 120 : virtual void dma_start_continuous() = 0; //start_circular_dma 121 : 122 : /** 123 : * @brief wait for an on-data MSI-X interrupt. 124 : */ 125 : virtual void dma_wait_for_data_irq() = 0; //wait_data_available 126 : 127 : /** 128 : * @return number of bytes written by firmware, to be read by software. 129 : */ 130 : virtual size_t dma_bytes_available() = 0; //bytes_available 131 : 132 : /** 133 : * @return number of bytes written by firmware up to the end of the buffer, 134 : * to be read by software. 135 : */ 136 : virtual size_t dma_bytes_available_nowrap() = 0; //bytes_available_to_read 137 : 138 : /** 139 : * @return true if the DMA buffer is full. 140 : */ 141 : virtual bool dma_is_full() = 0; 142 : 143 : /** 144 : * @return address in virtual memory of write pointer (moved by firmware). 145 : */ 146 : virtual uint64_t dma_get_write_ptr() = 0; 147 : 148 : /** 149 : * @return address in virtual memory of read pointer (moved by software). 150 : */ 151 : virtual uint64_t dma_get_read_ptr() = 0; 152 : 153 : /** 154 : * @brief set a new read pointer. 155 : * @details software uses the virtual addresses, firmware uses the physical ones. 156 : * @param v_addr virtual address of new read pointer. 157 : */ 158 : virtual void dma_set_read_ptr_vaddr(uint64_t v_addr) = 0; 159 : 160 : /** 161 : * @brief set a new read pointer. 162 : * @details software uses the virtual addresses, firmware uses the physical ones. 163 : * @param p_addr physical address of new read pointer. 164 : */ 165 : virtual void dma_set_read_ptr_paddr(uint64_t p_addr) = 0; 166 : 167 : /** 168 : * @return read pointer offset with respect to the start of buffer. 169 : */ 170 : virtual uint64_t dma_get_read_offset() = 0; 171 : 172 : /** 173 : * @return write pointer offset with respect to the start of buffer. 174 : */ 175 : virtual uint64_t dma_get_write_offset() = 0; 176 : 177 : /** 178 : * @brief advance read pointer. 179 : * @param bytes number of bytes processed by software. 180 : */ 181 : virtual void dma_advance_read_ptr(size_t bytes) = 0; 182 : 183 : /** 184 : * @return virtual address of start of buffer. 185 : */ 186 36 : uint64_t dma_get_vaddr(){return m_buffer->vaddr;} 187 : 188 : /** 189 : * @return buffer size. 190 : */ 191 37025 : size_t dma_get_size(){return m_buffer->size;} 192 : /** 193 : * @} 194 : */ 195 : 196 : /** 197 : * \defgroup Multi-reader functions 198 : * @{ 199 : * Functions to manage readers' opearation 200 : */ 201 : /** 202 : * @brief register a new reader to this DMA buffer. 203 : * @param zc_flag true if the reader uses zero-copy. 204 : * @return reader identifier. 205 : */ 206 : uint32_t reader_register(bool zc_flag); 207 : 208 : /** 209 : * @param reader_id reader identifier. 210 : * @return whether there are data in the DMA buffer for this reader to read. 211 : */ 212 : bool reader_is_data_available(uint32_t reader_id); 213 : 214 : /** 215 : * @param reader_id reader identifier. 216 : * @return bytes available for reading for the given reader. 217 : */ 218 : size_t reader_get_available_bytes(uint32_t reader_id); 219 : 220 : /** 221 : * @param reader_id reader identifier. 222 : * @return blocks to be read by the given reader. 223 : */ 224 : std::span<Block> reader_get_available_blocks(uint32_t reader_id); 225 : 226 : /** 227 : * @brief advance the DMA buffer read pointer according to the readers' progress 228 : * @details single synchronisation point of the different ToHostReaders. 229 : * @param reader_id reader identifier. 230 : * @param read_bytes bytes read by the reader. 231 : * @param reader_id bytes sent by the reader (sent != read only in zero-copy mode). 232 : */ 233 : void reader_advance_read_ptr(uint32_t reader_id, uint32_t read_bytes, uint32_t sent_bytes); 234 : /** 235 : * @} 236 : */ 237 : 238 : /** 239 : * \defgroup Support for zero-copy module 240 : * @{ 241 : * Functions to enable zero-copy flags. 242 : */ 243 : /** 244 : * @return whether a zero-copy reader is present among those registered to this buffer. 245 : */ 246 : bool has_zero_copy_reader() const {return m_has_zero_copy_reader;} 247 : 248 : /** 249 : * @brief set flag for the presence of a zero-copy reader. 250 : */ 251 10 : void set_zero_copy_reader(){m_has_zero_copy_reader = true;} 252 : /** 253 : * @} 254 : */ 255 : 256 : /** 257 : * @brief stop reading operations. 258 : */ 259 22 : void stop(){m_stop_flag = true;} 260 : 261 : /** 262 : * @return whether read operations have been stopped. 263 : */ 264 6940698 : bool is_stopped() const {return m_stop_flag;} 265 : 266 : /** 267 : * \defgroup Monitoring information 268 : * @{ 269 : * Functions to manage readers' opearation 270 : */ 271 : /** 272 : * @return free space in the buffer in MB. 273 : */ 274 : uint32_t dma_get_free_MB(); 275 : 276 : /** 277 : * @brief increase the on-data interrupt counter. 278 : */ 279 0 : void dma_increase_irq_counter(){++m_irq_counter;} 280 : 281 : /** 282 : * @return the number of on-data interrupts. 283 : */ 284 177 : uint64_t dma_get_irq_counter() const {return m_irq_counter;} 285 : /** 286 : * @} 287 : */ 288 : 289 : protected: 290 : int m_dmaid; 291 : std::shared_ptr<Device> m_device; 292 : unsigned int m_block_size; 293 : int m_irq_on_data; //IRQ_DATA_AVAILABLE + m_dmaid; 294 : uint64_t m_irq_counter; 295 : size_t m_size; 296 : bool m_stop_flag; //stop flag for readers 297 : bool m_has_zero_copy_reader; 298 : size_t m_min_bytes_sent; 299 : 300 : std::mutex m_mutex; 301 : std::unique_ptr<DmaBuffer> m_buffer; 302 : 303 : //support for multiple ToHostBufferReader 304 : std::vector<uint64_t> m_bytes_read; //read from DMA buffer 305 : std::vector<uint64_t> m_bytes_sent; //different from read only for zc mode 306 : 307 : size_t dma_compute_bytes_to_read(uint64_t rd_ptr, uint64_t wr_ptr, bool even); 308 : }; 309 : 310 : 311 177 : inline size_t ToHostBuffer::dma_compute_bytes_to_read(uint64_t rd_ptr, uint64_t wr_ptr, bool even) 312 : { 313 177 : if( wr_ptr > rd_ptr ) { 314 77 : return wr_ptr - rd_ptr; 315 : } 316 100 : else if( wr_ptr < rd_ptr ) { 317 40 : return m_buffer->size + wr_ptr -rd_ptr; 318 : } 319 : else { // dma_ptr == buf->pc_ptr 320 60 : if( even ) { 321 : return 0; // Buffer empty 322 : } 323 : else { 324 49 : return m_buffer->size; // Buffer full 325 : } 326 : } 327 : } 328 : 329 : 330 : /** 331 : * Reader of a Felix card ToHost DMA buffer. 332 : * */ 333 : class FlxToHostBuffer : public ToHostBuffer { 334 : 335 : public: 336 : FlxToHostBuffer(int m_dmaid, std::shared_ptr<Device> d); 337 : ~FlxToHostBuffer(); 338 : 339 : 340 : void allocate_buffer(size_t size, 341 : const std::string& name, 342 : bool vmem, bool free_previous_cmem=false) override; 343 : 344 : void dma_start_continuous() override; 345 : void dma_wait_for_data_irq() override; 346 : size_t dma_bytes_available() override; 347 : //uint32_t dma_get_free_MB() override; 348 : size_t dma_bytes_available_nowrap() override; 349 : bool dma_is_full() override; 350 : uint64_t dma_get_write_ptr() override; 351 : uint64_t dma_get_read_ptr() override; 352 : void dma_set_read_ptr_vaddr(uint64_t v_addr) override; 353 : void dma_set_read_ptr_paddr(uint64_t p_addr) override; 354 : uint64_t dma_get_read_offset() override; 355 : uint64_t dma_get_write_offset() override; 356 : void dma_advance_read_ptr(size_t bytes) override; 357 : }; 358 : 359 : 360 : /** 361 : * Reader for the ToHost DMA buffer with hardware emulated in software. 362 : * A dedicated thread fills writes in the DMA buffer in stead of the FELIX card. 363 : * */ 364 : class FileToHostBuffer : public ToHostBuffer { 365 : 366 : public: 367 : FileToHostBuffer(int m_dmaid, std::shared_ptr<Device> d, 368 : std::string& filename, unsigned int block_rate, bool repeat); 369 : ~FileToHostBuffer(); 370 : void allocate_buffer(size_t size, 371 : const std::string& name, 372 : bool vmem, bool free_previous_cmem=true) override; 373 : 374 : void dma_start_continuous() override; 375 : void dma_wait_for_data_irq() override; 376 : size_t dma_bytes_available() override; 377 : //uint32_t dma_get_free_MB() override; 378 : size_t dma_bytes_available_nowrap() override; 379 : bool dma_is_full() override; 380 : uint64_t dma_get_write_ptr() override; 381 : uint64_t dma_get_read_ptr() override; 382 : void dma_set_read_ptr_vaddr(uint64_t v_addr) override; 383 : void dma_set_read_ptr_paddr(uint64_t p_addr) override; 384 : uint64_t dma_get_read_offset() override; 385 : uint64_t dma_get_write_offset() override; 386 : void dma_advance_read_ptr(size_t bytes) override; 387 : 388 : private: 389 : void set_read_ptr_paddr(uint64_t p_addr); 390 : void write_in_dma_buffer(); 391 : size_t limit_block_rate(size_t max_blocks); 392 : size_t blocks_to_write(uint64_t rd_ptr, uint64_t wr_ptr, bool even); 393 : void throttle_writer(const std::chrono::nanoseconds &elapsed, size_t count); 394 : void writer_updates_fw_ptr(size_t written_blocks); 395 : bool reset_file(); 396 : 397 : private: 398 : //regulate the concurrent access to read and write pointers 399 : mutable std::shared_mutex m_driver_mutex; 400 : 401 : //emulate MSI-X on_data interrupts 402 : std::condition_variable m_irq_cond; 403 : mutable std::mutex m_irq_mutex; 404 : 405 : //suspend the card emulator thread when the DMA is full 406 : std::condition_variable m_writer_cond; 407 : mutable std::mutex m_stop_writer_mutex; 408 : 409 : unsigned int m_block_rate; 410 : bool m_repeat; 411 : bool m_rd_odd; 412 : bool m_wr_odd; 413 : FILE* m_fp; 414 : std::thread m_writer_thread; 415 : }; 416 : 417 : #endif /* TOHOST_BUFFER_H_ */
