Program Listing for File trickle.cpp

↰ Return to documentation for file (trickle/trickle.cpp)

#include "trickle.hpp"
#include "config.hpp"

namespace trickle {

template <class Config, class Device, class Buffer>
void Trickle<Config, Device, Buffer>::start()
{
    ERS_INFO(std::format("Number of PCIe endpoints to serve {}", cfg->get_number_devices()));

    int tot_elinks{0};

    for (const auto & dev : devices){

        //Get DMA ID
        const int dev_no = dev->get_device_number();
        constexpr static auto no_lock_mask = 0u;

        if(dev->open_device(no_lock_mask) != 0 ){
            ers::error(felix_log::trickle_issue(ERS_HERE, std::format("Trickle - cannot open device {} ", dev_no)));
            continue;
        };

        const int dmaid = dev->get_trickle_dmaid();
        dev->close_device();

        ERS_INFO(std::format("Opening PCIe endpoint {}", dev_no));
        const unsigned int lock_mask = dev->make_dma_lock_mask({dmaid});

        if(dev->open_device(lock_mask) != 0 ){
            ers::error(felix_log::trickle_issue(ERS_HERE, std::format("Trickle - cannot open device {} ", dev_no)));
            continue;
        };

        //Initialize bus and  DMA buffers
        auto dma_buf = dma_buffers.at(dev_no);
        dma_buf->set_dmaid(dmaid);

        const auto data_format = dev->get_fromhost_data_format();
        dma_buf->set_encoder_data_format(data_format);

        m_bus_map.try_emplace(dev_no, cfg->network.bus_dir, cfg->network.bus_groupname, dmaid, cfg->network.verbose_bus);

        const std::string dma_name = std::format("trickle-d{}-{}", dev_no, dmaid);
        ERS_INFO(std::format("Allocating DMA buffer {} of size {} MB (PCIe endpoint {}, DMAID {})", dma_name, cfg->resource.cmem_buffersize/1024/1024, dev_no, dmaid));
        dma_buf->allocate_buffer(cfg->resource.cmem_buffersize, dma_name, cfg->vmem, cfg->resource.free_previous_cmem);

        const std::vector<Elink> elinks = dma_buf->get_elinks();

        const auto base_port = cfg->network.ports.at(dev_no);
        const auto computed_port = base_port == 0 ? base_port : base_port + 1000u * dev_no;

        if (computed_port > std::numeric_limits<uint16_t>::max()) {
            ers::warning(felix_log::trickle_issue(std::format("Port {} number overflow! Starting port is too big", computed_port)));
        }
        const uint16_t port = static_cast<uint16_t>(computed_port);

        const std::string trickle_info = std::format("d{}Trkl", dev_no);
        m_trickle_managers[dev_no].emplace_back(dma_buf, network::utility::create_unbuffered_receiver(NetworkMode::tcp,
                                                                                                      cfg->network.ip,
                                                                                                      port,
                                                                                                      cfg->network.netio_pages,
                                                                                                      cfg->network.netio_pagesize,
                                                                                                      trickle_info));

        const uint16_t actual_port = m_trickle_managers.at(dev_no).back().get_receiver_port();

        std::vector<felixbus::FelixBusElinkInfo> bus_info{};
        std::ranges::transform(elinks, std::back_inserter(bus_info), [this, &actual_port](const Elink& e) {
            return felixbus::FelixBusElinkInfo{
                .fid = e.fid,
                .ip = cfg->network.ip,
                .port = actual_port,
                .unbuffered = true,
                .pubsub = false,
                .tcp = true,
                .stream = e.has_streams,
                .netio_pages = cfg->network.netio_pages,
                .netio_pagesize = cfg->network.netio_pagesize,
            };
        });
        m_bus_map.at(dev_no).publish(bus_info);

        ERS_INFO(std::format("PCIe endpoint {}, DMA {} receiving data on port {}", dev_no, dmaid, actual_port));
        for(const auto & e : elinks) {
            ERS_INFO(std::format("E-link {}: {:#5x} {:#18x} (channel {} egroup {} epath {})",
                     tot_elinks, e.lid, e.fid,
                     dev->get_channel(e.lid),
                     dev->get_egroup(e.lid),
                     dev->get_epath(e.lid)));
            ++tot_elinks;
        }
    }
}


template <class Config, class Device, class Buffer>
void Trickle<Config, Device, Buffer>::stop()
{
    for (auto& dma : dma_buffers) {
        dma.second->stop();
    }
}


} // namespace trickle

// Explicit instantiations for the specific template combinations used
namespace trickle {
    template class Trickle<ConfigTrickle, FlxDevice, FlxFromHostBuffer>;
    template class Trickle<ConfigTrickleToFile, FileDevice, FileFromHostBuffer>;
}