Program Listing for File device_flx.cpp

↰ Return to documentation for file (device_flx.cpp)

#include <cstdint>
#include <ers/ers.h>

#include "device.hpp"

#include "flxcard/FlxException.h"
#include "DFDebug/DFDebug.h"

#include "felix/felix_fid.h"
#include "regmap/regmap.h"
#include "block.hpp"


FlxDevice::FlxDevice(Config& cfg, unsigned dev_no)
    : Device(dev_no, cfg.resource.vid, cfg.resource.did, cfg.get_device_cid(dev_no))
{ }


FlxDevice::FlxDevice(unsigned dev_no)
    : Device(dev_no, 1, 0, dev_no)
{ }


int FlxDevice::open_device(u_int lock_mask)
{
    u_int current_lock{0};
    ERS_DEBUG(1, std::format("Initializing device {}", m_device_number));
    try {
        current_lock = m_dev.get_lock_mask(m_device_number);
        ERS_DEBUG(1, std::format("Device {} locks {}", m_device_number, current_lock));
    }
    catch(FlxException &ex) {
        ers::error(felix_log::device_flx_issue(ERS_HERE, std::format("Cannot get lock mask for device {}", m_device_number)));
        ers::error(felix_log::device_flx_issue(ERS_HERE, ex.errorString(ex.getErrorId())));
        return ex.getErrorId();
    }

    try {
        const bool enable_cached_elinks = true;
        m_dev.card_open(m_device_number, lock_mask, enable_cached_elinks);
    }
    catch(FlxException &ex) {
        ers::error(felix_log::device_flx_issue(ERS_HERE, std::format("Could not open device {}", m_device_number)));
        ers::error(felix_log::device_flx_issue(ERS_HERE, ex.errorString(ex.getErrorId())));
        return ex.getErrorId();
    }

    // Clear various latched signals
    if (current_lock == 0) {
        m_dev.m_bar2->TTC_BUSY_CLEAR = 1;
        uint64_t c;
        c = m_dev.m_bar2->XOFF_FM_HIGH_THRESH.CROSS_LATCHED;
        m_dev.m_bar2->XOFF_FM_HIGH_THRESH.CROSS_LATCHED = c;
        c = m_dev.m_bar2->DMA_BUSY_STATUS.ENABLE;
        m_dev.m_bar2->DMA_BUSY_STATUS.ENABLE = c;
        c = m_dev.m_bar2->FM_BUSY_CHANNEL_STATUS.BUSY;
        m_dev.m_bar2->FM_BUSY_CHANNEL_STATUS.BUSY = c;
        c = m_dev.m_bar2->BUSY_MAIN_OUTPUT_FIFO_STATUS.HIGH_THRESH_CROSSED_LATCHED;
        m_dev.m_bar2->BUSY_MAIN_OUTPUT_FIFO_STATUS.HIGH_THRESH_CROSSED_LATCHED = c;
    }

    m_firmware_type = m_dev.firmware_type();
    return 0;
}


void FlxDevice::close_device()
{
    //stop all DMAs ?
    //flxcard->dma_stop(dmaid);
    ERS_DEBUG(1, std::format("Closing device {}", m_device_number));
    m_dev.card_close();
}


bool FlxDevice::is_primary()
{
    return not static_cast<bool>(m_dev.m_bar2->PCIE_ENDPOINT);
}


unsigned int FlxDevice::get_card_endpoints()
{
    return m_dev.m_bar2->NUMBER_OF_PCIE_ENDPOINTS;
}


unsigned int FlxDevice::get_card_model()
{
    return m_dev.m_bar2->CARD_TYPE;
}


u_int FlxDevice::get_regmap_version()
{
    return m_dev.m_bar2->REG_MAP_VERSION;
}


u_int FlxDevice::get_number_of_channels()
{
    return m_dev.m_bar2->NUM_OF_CHANNELS;
}


bool FlxDevice::get_wide_mode()
{
    return (m_dev.m_bar2->WIDE_MODE == 1 ? true : false);
}


u_int FlxDevice::get_block_size()
{
    return m_dev.m_bar2->BLOCKSIZE;
}


u_int FlxDevice::get_trailer_size()
{
    u_int tsize = m_dev.m_bar2->CHUNK_TRAILER_32B == 1 ? 4 : 2;
    if( tsize != sizeof(subchunk_trailer_t) ){
        throw std::runtime_error("SW/FW trailer size mismatch");
    }
    return tsize;
}


flx_fromhost_format FlxDevice::get_fromhost_data_format()
{
#if REGMAP_VERSION < 0x0500
    m_fromHostDataFormat = static_cast<flx_fromhost_format>(FROMHOST_FORMAT_REGMAP4);
#else
    m_fromHostDataFormat = static_cast<flx_fromhost_format>(m_dev.m_bar2->FROMHOST_DATA_FORMAT);
#endif
    return m_fromHostDataFormat;
}


flx_tohost_format FlxDevice::get_tohost_data_format()
{
#if REGMAP_VERSION < 0x0500
    return TOHOST_SUBCHUNK_TRAILER;
#else
    return static_cast<flx_tohost_format>(m_dev.m_bar2->TOHOST_DATA_FORMAT);
#endif
}


int FlxDevice::get_fromhost_dmaid()
{
    if (m_dev.m_bar2 == nullptr) {
        ers::error(felix_log::device_flx_issue(ERS_HERE, "m_dev.m_bar2 is null"));
        return -1;
    }

    int dmaid = m_dev.m_bar2->GENERIC_CONSTANTS.DESCRIPTORS - 1;
    if (dmaid < 0) {
        ers::error(felix_log::device_flx_issue(ERS_HERE, std::format("Invalid ToFLx DMA ID {}", dmaid)));
    } else {
        ERS_DEBUG(1, std::format("ToFlx dmaid {}", dmaid));
    }
    return dmaid;
}


int FlxDevice::get_trickle_dmaid()
{
#if REGMAP_VERSION >= 0x0500
    int dmaid = m_dev.m_bar2->GENERIC_CONSTANTS.TRICKLE_DESCRIPTOR_INDEX;
    //int trickle_offset = m_dev.m_bar2->GENERIC_CONSTANTS.TRICKLE_DESCRIPTOR_INDEX;
    //dmaid = dmaid + trickle_offset;

    if (dmaid < 0) {
        ers::error(felix_log::device_flx_issue(ERS_HERE, std::format("Invalid ToFLx DMA ID {}", dmaid)));
    }
    ERS_DEBUG(1, std::format( "Trickle dmaid {}", dmaid));
    return dmaid;
#endif

    return -1;
}


int FlxDevice::dma_max_tlp_bytes()
{
    return m_dev.dma_max_tlp_bytes();
}


void FlxDevice::set_register(const char *key, uint64_t value)
{
    try {
        m_dev.cfg_set_option(key, value);
    }
    catch(FlxException &ex) {
        ERS_INFO(std::format("Register {} not available with current firmware version.", key));
    }
}


uint64_t FlxDevice::get_register(const char *key)
{
    try {
        return m_dev.cfg_get_option(key);
    }
    catch(FlxException &ex) {
        ERS_INFO(std::format("Register {} not available with current firmware version.", key));
        return 0;
    }
}


bool FlxDevice::check_register(const char *key)
{
  try {
        m_dev.cfg_get_option(key);
        return true;
    }
    catch(FlxException &ex) {
        ERS_INFO(std::format("Register {} not available with current firmware version.", key));
        return false;
    }
}


void FlxDevice::cancel_irq(int i)
{
    try{
        m_dev.irq_cancel(i);
    }
    catch(FlxException &ex) {
        ers::error(felix_log::device_flx_issue(ERS_HERE, ex.errorString(ex.getErrorId())));
    }
}


void FlxDevice::irq_wait(int i)
{
    m_dev.irq_wait(i);
}


void FlxDevice::irq_enable(int i)
{
    try{
        m_dev.irq_enable(i);
    }
    catch(FlxException &ex) {
        ers::error(felix_log::device_flx_issue(ERS_HERE, ex.errorString(ex.getErrorId())));
    }
}


void FlxDevice::irq_disable(int i)
{
    try{
        m_dev.irq_disable(i);
    }
    catch(FlxException &ex) {
        ers::error(felix_log::device_flx_issue(ERS_HERE, ex.errorString(ex.getErrorId())));
    }
}


#if REGMAP_VERSION < 0x0500
encoding_t FlxDevice::elink_get_encoding(u_int channel, u_int egroup, u_int epath, bool is_to_flx)
{
    if (!is_to_flx && (m_firmware_type == FIRMW_FULL || m_firmware_type == FIRMW_MROD)) {
        return ENC_8b10b;
    }
    else {
        int enc = 0;
        if (is_to_flx) {
            uint64_t modebits = m_dev.m_bar2->CR_GBT_CTRL[channel].EGROUP_FROMHOST[egroup].FROMHOST.PATH_ENCODING;
            uint64_t mask = 0xF << 4*epath; // 4 bits per E-path
            enc = ((modebits & mask) >> 4*epath);
        }
        else {
            uint64_t modebits = m_dev.m_bar2->CR_GBT_CTRL[channel].EGROUP_TOHOST[egroup].TOHOST.PATH_ENCODING;
            uint64_t mask = 0x3 << 2*epath; // 2 bits per E-path
            enc = ((modebits & mask) >> 2*epath);
        }
        if(enc < 0 || enc > 100){
            ers::error(felix_log::device_flx_issue(ERS_HERE, std::format("Invalid encoding type {} for elink channel {} group {}, epath {} using DIRECT",
                                                                        enc, channel, egroup, epath)));
            return ENC_DIRECT;
        }
        else {
            return static_cast<encoding_t>(enc);
        }
    }
}
#else
encoding_t FlxDevice::elink_get_encoding(u_int channel, u_int egroup, u_int epath, bool is_to_flx)
{
    if (!is_to_flx && (m_firmware_type == FIRMW_FULL || m_firmware_type == FIRMW_MROD)) {
        return ENC_8b10b;
    }
    else if (!is_to_flx && (m_firmware_type == FIRMW_INTERLAKEN)){
        return ENC_INTERLAKEN;
    }
    else {
        int enc = 0;
        if (is_to_flx) {
            uint64_t modebits = m_dev.m_bar2->ENCODING_EGROUP_CTRL_GEN[channel].ENCODING_EGROUP[egroup].ENCODING_EGROUP_CTRL.PATH_ENCODING;
            uint64_t mask = 0xF << 4*epath; // 4 bits per E-path
            enc = ((modebits & mask) >> 4*epath);
        }
        else {
            uint64_t modebits = m_dev.m_bar2->DECODING_EGROUP_CTRL_GEN[channel].DECODING_EGROUP[egroup].EGROUP.PATH_ENCODING;
            uint64_t mask = 0xF << 4*epath; // 4 bits per E-path
            enc = ((modebits & mask) >> 4*epath);
        }
        if(enc < 0 || enc > 100){
            ers::error(felix_log::device_flx_issue(ERS_HERE, std::format("Invalid encoding type {} for elink channel {} group {}, epath {} using DIRECT",
                                                                        enc, channel, egroup, epath)));
            return ENC_DIRECT;
        }
        else {
            return static_cast<encoding_t>(enc);
        }
    }
}
#endif


bool FlxDevice::elink_is_in_dma(local_elink_t elinknr, int dmaid)
{
#if REGMAP_VERSION < 0x0500
    return true;
#else
    // Check for special FROMHOST descriptor
    if (dmaid == m_dev.m_bar2->GENERIC_CONSTANTS.FROMHOST_DESCRIPTOR_INDEX) {
        return true;
    }

    const uint enabled_dma = m_dev.tohost_elink_dmaid(elinknr);
    const uint max_dma_id = m_dev.number_of_dma_tohost() - 1U;
    const uint dmaid_u = static_cast<uint>(dmaid);

    // For backwards compatibility, no value means DMA 0, even if CRTOHOST_DMA_DESCRIPTOR_MASK is enabled
    if (enabled_dma == 0 && dmaid == 0) {
        return true;
    }

    if (m_dev.uses_dma_index_mask()) {
        // Mask mode: check if bit at position [dmaid] is set
        const uint mask = 1U << dmaid;
        const uint masked_enabled_dma = enabled_dma & mask;

        if(masked_enabled_dma > (1U << max_dma_id)){
            ers::error(felix_log::device_flx_issue(
                std::format("Invalid DMA ID mask {} for E-link {} (DMA ID [0, {}]). Discarded elink.", masked_enabled_dma, elinknr, max_dma_id)));
            return false;
        }
        if (masked_enabled_dma == 0) {
            return false;
        }
        return true;

    } else {
        // Direct value mode: enabled_dma contains the descriptor ID
        if (enabled_dma > max_dma_id) {
            ers::error(felix_log::device_flx_issue(
                std::format("Invalid DMA ID {} for E-link {} (DMA ID [0, {}]). Discarded elink.", enabled_dma, elinknr, max_dma_id)));
            return false;
        }
        return (enabled_dma == dmaid_u);
    }
#endif
}


#if REGMAP_VERSION < 0x0500
bool FlxDevice::elink_is_enabled(u_int channel, u_int egroup, u_int epath, bool is_to_flx)
{
    if (m_firmware_type == FIRMW_FULL) {
        return !is_to_flx && ((m_dev.m_bar2->CR_FM_PATH_ENA >> channel) & 1);
    }
    // MROD
    else if (m_firmware_type == FIRMW_MROD) {
        u_long eproc_ena = m_dev.m_bar2->MROD_EP0_CSMENABLE;
        return !is_to_flx && ((eproc_ena >> channel) & 1);
    }
    // GBT Mode
    else {
        u_long eproc_ena = is_to_flx ?
        m_dev.m_bar2->CR_GBT_CTRL[channel].EGROUP_FROMHOST[egroup].FROMHOST.EPROC_ENA :
        m_dev.m_bar2->CR_GBT_CTRL[channel].EGROUP_TOHOST[egroup].TOHOST.EPROC_ENA;
        // Central Router documentation p. 13
        switch (epath) {
            case 0: return ((eproc_ena >> 3) & 1) || ((eproc_ena >> 7) & 1);
            case 1: return ((eproc_ena >> 1) & 1) || ((eproc_ena >> 8) & 1);
            case 2: return ((eproc_ena >> 4) & 1) || ((eproc_ena >> 9) & 1);
            case 3: return ((eproc_ena >> 0) & 1) || ((eproc_ena >> 10) & 1);
            case 4: return ((eproc_ena >> 5) & 1) || ((eproc_ena >> 11) & 1);
            case 5: return ((eproc_ena >> 2) & 1) || ((eproc_ena >> 12) & 1);
            case 6: return ((eproc_ena >> 6) & 1) || ((eproc_ena >> 13) & 1);
            case 7: return ((eproc_ena >> 14) & 1);
            default: return false;
        }
    }
    return false;
}
#else
bool FlxDevice::elink_is_enabled(u_int channel, u_int egroup, u_int epath, bool is_to_flx)
{
    uint32_t enables_phase2;
    if( is_to_flx ) {
        enables_phase2 = static_cast<uint32_t>(m_dev.m_bar2->ENCODING_EGROUP_CTRL_GEN[channel].ENCODING_EGROUP[egroup].ENCODING_EGROUP_CTRL.EPATH_ENA);
    } else {
        enables_phase2 = static_cast<uint32_t>(m_dev.m_bar2->DECODING_EGROUP_CTRL_GEN[channel].DECODING_EGROUP[egroup].EGROUP.EPATH_ENA);
    }
    return (enables_phase2 >> epath) & 0x01;
}
#endif

uint64_t FlxDevice::get_broadcast_enable_gen(u_int channel) {
#if REGMAP_VERSION >= 0x0500
  return m_dev.m_bar2->BROADCAST_ENABLE_GEN[channel].BROADCAST_ENABLE;
#else
  return 0;
#endif
}


local_elink_t FlxDevice::get_ec_elink(bool is_to_flx, u_int channel)
{
    std::pair<int,int> ec_indices = m_dev.ec_elink_indices();
    if( is_to_flx ) {
        return ((channel<<6) + ec_indices.second);
    } else {
        return ((channel<<6) + ec_indices.first);
    }
}


local_elink_t FlxDevice::get_ic_elink(bool is_to_flx, u_int channel)
{
    std::pair<int,int> ic_indices = m_dev.ic_elink_indices();
    if( is_to_flx ) {
        return ((channel<<6) + ic_indices.second);
    } else {
        return ((channel<<6) + ic_indices.first);
    }
}


local_elink_t FlxDevice::get_aux_elink(bool is_to_flx, u_int channel)
{
    local_elink_t ic_elink = get_ic_elink(is_to_flx, channel);
#if REGMAP_VERSION < 0x0500
    return (ic_elink - 1);
#else
    return (ic_elink + 1); // Not (yet) defined
#endif
}


bool FlxDevice::elink_is_ec_enabled(bool is_to_flx, u_int channel)
{
#if REGMAP_VERSION < 0x0500
    if( is_to_flx ) {
        return(m_dev.m_bar2->MINI_EGROUP_CTRL[channel].EC_FROMHOST.ENABLE);
    } else {
        return(m_dev.m_bar2->MINI_EGROUP_CTRL[channel].EC_TOHOST.ENABLE);
    }
#else
    if( is_to_flx ) {
        return(m_dev.m_bar2->MINI_EGROUP_FROMHOST_GEN[channel].MINI_EGROUP_FROMHOST.EC_ENABLE);
    } else {
        return(m_dev.m_bar2->MINI_EGROUP_TOHOST_GEN[channel].MINI_EGROUP_TOHOST.EC_ENABLE);
    }
#endif
}


bool FlxDevice::elink_is_ic_enabled(bool is_to_flx, u_int channel)
{
#if REGMAP_VERSION < 0x0500
    if( is_to_flx ) {
        return(m_dev.m_bar2->MINI_EGROUP_CTRL[channel].EC_FROMHOST.IC_ENABLE);
    } else {
        return(m_dev.m_bar2->MINI_EGROUP_CTRL[channel].EC_TOHOST.IC_ENABLE);
    }
#else
    if( is_to_flx ) {
        return(m_dev.m_bar2->MINI_EGROUP_FROMHOST_GEN[channel].MINI_EGROUP_FROMHOST.IC_ENABLE);
    } else {
        return(m_dev.m_bar2->MINI_EGROUP_TOHOST_GEN[channel].MINI_EGROUP_TOHOST.IC_ENABLE);
    }
#endif
}


bool FlxDevice::elink_is_aux_enabled(bool is_to_flx, u_int channel)
{
#if REGMAP_VERSION < 0x0500
    if( is_to_flx ) {
        return(m_dev.m_bar2->MINI_EGROUP_CTRL[channel].EC_FROMHOST.SCA_AUX_ENABLE);
    } else {
        return(m_dev.m_bar2->MINI_EGROUP_CTRL[channel].EC_TOHOST.SCA_AUX_ENABLE);
    }
#else
    if( is_to_flx ) {
        return(m_dev.m_bar2->MINI_EGROUP_FROMHOST_GEN[channel].MINI_EGROUP_FROMHOST.AUX_ENABLE);
    } else {
        return(m_dev.m_bar2->MINI_EGROUP_TOHOST_GEN[channel].MINI_EGROUP_TOHOST.AUX_ENABLE);
    }
#endif
}


bool FlxDevice::elink_has_stream_id(u_int channel, u_int egroup, u_int epath)
{
    if(channel > get_number_of_channels()){
        ers::error(felix_log::device_flx_issue(ERS_HERE, std::format("Checking stream status on invalid channel {}", channel)));
        return false;
    }
    u_long enable = 0;
#if REGMAP_VERSION < 0x0500
    switch (egroup)
    {
    case 0:
        enable = m_dev.m_bar2->PATH_HAS_STREAM_ID[channel].TOHOST.EGROUP0;
        break;
    case 1:
        enable = m_dev.m_bar2->PATH_HAS_STREAM_ID[channel].TOHOST.EGROUP1;
        break;
    case 2:
        enable = m_dev.m_bar2->PATH_HAS_STREAM_ID[channel].TOHOST.EGROUP2;
        break;
    case 3:
        enable = m_dev.m_bar2->PATH_HAS_STREAM_ID[channel].TOHOST.EGROUP3;
        break;
    case 4:
        enable = m_dev.m_bar2->PATH_HAS_STREAM_ID[channel].TOHOST.EGROUP4;
        break;
    case 5:
        enable = m_dev.m_bar2->PATH_HAS_STREAM_ID[channel].TOHOST.EGROUP5;
        break;
    case 6:
        enable = m_dev.m_bar2->PATH_HAS_STREAM_ID[channel].TOHOST.EGROUP6;
        break;
    default:
        ers::error(felix_log::device_flx_issue(ERS_HERE, std::format("Checking stream status on invalid egroup {}.", egroup)));
        break;
    }
#else
    switch (egroup)
    {
    case 0:
        enable = m_dev.m_bar2->PATH_HAS_STREAM_ID[channel].HAS_STREAM_ID.EGROUP0;
        break;
    case 1:
        enable = m_dev.m_bar2->PATH_HAS_STREAM_ID[channel].HAS_STREAM_ID.EGROUP1;
        break;
    case 2:
        enable = m_dev.m_bar2->PATH_HAS_STREAM_ID[channel].HAS_STREAM_ID.EGROUP2;
        break;
    case 3:
        enable = m_dev.m_bar2->PATH_HAS_STREAM_ID[channel].HAS_STREAM_ID.EGROUP3;
        break;
    case 4:
        enable = m_dev.m_bar2->PATH_HAS_STREAM_ID[channel].HAS_STREAM_ID.EGROUP4;
        break;
    case 5:
        enable = m_dev.m_bar2->PATH_HAS_STREAM_ID[channel].HAS_STREAM_ID.EGROUP5;
        break;
    case 6:
        enable = m_dev.m_bar2->PATH_HAS_STREAM_ID[channel].HAS_STREAM_ID.EGROUP6;
        break;
    default:
        ers::error(felix_log::device_flx_issue(ERS_HERE, std::format("Checking stream status on invalid egroup {}.", egroup)));
        break;
    }
#endif
    return static_cast<bool>( (enable >> epath) & 1);
}


std::vector<Elink> FlxDevice::read_enabled_elinks(int dmaid)
{
    std::vector<Elink> enabled_elinks;

    // If Trickle mode is active, only the Trickle E-link is enabled and returned
    bool is_trickle = (dmaid == get_trickle_dmaid()) ? true : false;
    if (is_trickle){
        const local_elink_t lid = 0xFFFF;
        uint64_t fid = get_fid_from_ids(m_did, m_cid, lid, 0, m_vid, true, true);
        enabled_elinks.emplace_back(Elink{fid, lid, 0, elink_type_t::DAQ, encoding_t::ENC_8b10b});
        return enabled_elinks;
    }

    bool is_to_flx = (dmaid == get_fromhost_dmaid()) ? true : false;
    bool fullmode              = is_full_mode();
    u_int no_of_channels       = get_number_of_channels();
    u_int egroups_per_chan     = get_egroups_per_channel(is_to_flx);
    u_int epaths_per_egroup    = get_epaths_per_egroup(is_to_flx);

  // TTCtoHost e-link = #Nchannels group #7 path #3 (RM4) or 0x600 (RM5)
#if REGMAP_VERSION < 0x0500
    if( !is_to_flx && m_dev.m_bar2->CR_TTC_TOHOST.ENABLE) {
#else
    if( !is_to_flx && m_dev.m_bar2->TTC_TOHOST_ENABLE && elink_is_in_dma(0x600, dmaid)) {
#endif
        local_elink_t lid = TTC2HOST_LINK(no_of_channels);
        uint64_t fid = get_fid_from_ids(m_did, m_cid, lid, 0, m_vid, false, true);
        bool streams = 0;
        enabled_elinks.emplace_back(Elink{fid, lid, streams, elink_type_t::TTC, encoding_t::ENC_TTC});
        ERS_DEBUG(1, std::format("E-Link {}: \t{:#4x}\t{:#18x}\t(TTC2H)", enabled_elinks.size(), lid, fid));
    }

    // IC, EC, AUX
    if( !fullmode ) {
        for( unsigned channel = 0; channel < no_of_channels; ++channel ) {

            local_elink_t ec_elink = get_ec_elink(is_to_flx, channel);
            bool ec_enabled = elink_is_ec_enabled(is_to_flx, channel) && elink_is_in_dma(ec_elink, dmaid);

            local_elink_t ic_elink = get_ic_elink(is_to_flx, channel);
            bool ic_enabled = elink_is_ic_enabled(is_to_flx, channel) && elink_is_in_dma(ic_elink, dmaid);

            local_elink_t aux_elink = get_aux_elink(is_to_flx, channel);
            bool aux_enabled = elink_is_aux_enabled(is_to_flx, channel) && elink_is_in_dma(aux_elink, dmaid);
            if( ec_enabled ){
                uint64_t fid = get_fid_from_ids(m_did, m_cid, ec_elink, 0, m_vid, is_to_flx, false);
                enabled_elinks.emplace_back(Elink{fid, ec_elink, false, elink_type_t::DCS, encoding_t::ENC_HDLC});
                ERS_DEBUG(1, std::format("E-Link {}:\t{:#4x}\t{:#18x}\t(EC channel {})", enabled_elinks.size(), ec_elink, fid, channel));
            }
            if( ic_enabled ){
                uint64_t fid = get_fid_from_ids(m_did, m_cid, ic_elink, 0, m_vid, is_to_flx, false);
                enabled_elinks.emplace_back(Elink{fid, ic_elink, false, elink_type_t::IC, encoding_t::ENC_HDLC});
                ERS_DEBUG(1, std::format("E-Link {}:\t{:#4x}\t{:#18x}\t(IC channel {})", enabled_elinks.size(), ic_elink, fid, channel));
            }
            if( m_firmware_type == FIRMW_LTDB && aux_enabled ) {
                uint64_t fid = get_fid_from_ids(m_did, m_cid, aux_elink, 0, m_vid, is_to_flx, false);
                enabled_elinks.emplace_back(Elink{fid, aux_elink, false, elink_type_t::DCS, encoding_t::ENC_HDLC});
                ERS_DEBUG(1, std::format("E-Link {}:\t{:#4x}\t{:#18x}\t(AUX channel {})", enabled_elinks.size(), aux_elink, fid, channel));
            }
        }
    }

    for( unsigned channel = 0; channel < no_of_channels; ++channel ) {
        for( unsigned egroup = 0; egroup < egroups_per_chan; ++egroup ) {
            for( unsigned epath = 0; epath < epaths_per_egroup; ++epath ) {

                local_elink_t elinknr = get_elink(channel, egroup, epath);
                bool enabled = elink_is_enabled(channel, egroup, epath, is_to_flx);
                bool in_enabled_dma = is_to_flx ? 1 : elink_is_in_dma(elinknr, dmaid);

                if( enabled && in_enabled_dma ) {
                    bool streams = !is_to_flx && elink_has_stream_id(channel, egroup, epath);
                    encoding_t enc = elink_get_encoding(channel, egroup, epath, is_to_flx);
                    elink_type_t type = elink_type_t::NONE_ELINK_TYPE;
                    if (enc == ENC_HDLC){
                        type = elink_type_t::DCS;
                    } else {
                        type = elink_type_t::DAQ;
                    }

                    uint64_t fid = get_fid_from_ids(m_did, m_cid, elinknr, 0, m_vid, is_to_flx, false);
                    enabled_elinks.emplace_back(Elink{fid, elinknr, streams, type, enc});
                    ERS_DEBUG(1, std::format("E-Link {}:\t{:#4x}\t{:#18x}\t(channel {} egroup {} epath {}) type {}, streams {}",
                        enabled_elinks.size(), elinknr, fid, channel, egroup, epath,
                        Device::get_elink_type_str(type),
                        streams ? 'Y' : 'N'));
                } //if enabled
            } //epath
        } //egroup
    } //channel

#if REGMAP_VERSION >= 0x0500
    // Add Broadcast Elinks
    if (is_to_flx) {
        // Calculate Broadcast Elinks
        uint64_t broadcast_elinks[MAX_BROADCAST_ELINKS] = {0};
        unsigned no_of_broadcast_elinks = get_broadcast_elinks(broadcast_elinks, no_of_channels);
        ERS_INFO(std::format("Adding {} broadcast elinks", no_of_broadcast_elinks));
        for( unsigned i=0; i < no_of_broadcast_elinks; ++i ) {
            bool streams = 0;
            //broadcast elinks use the same encoding as
            //elinks they broadcast into
            encoding_t enc = ENC_DIRECT;
            elink_type_t type = elink_type_t::DAQ;
            local_elink_t elinknr = static_cast<local_elink_t>(broadcast_elinks[i]);
            uint64_t fid = get_fid(0, elinknr, streams, m_vid, is_to_flx, 0);
            enabled_elinks.emplace_back(Elink{fid, elinknr, streams, type, enc});

            ERS_INFO(std::format("Broadcast for {:8}\t{:4} using \tE-Link {}:\t{:#4x}\t{:#18x}\t(channel {} egroup {} epath {})",
                                  Device::broadcast_type_str(elinknr),
                                  Device::broadcast_for(elinknr),
                                  elinknr,
                                  elinknr,
                                  fid,
                                  Device::get_channel(elinknr),
                                  Device::get_egroup(elinknr),
                                  Device::get_epath(elinknr)));
        }
    }
#endif
    return enabled_elinks;
}

bool FlxDevice::dma_enabled(int dmaid)
{
    return m_dev.dma_enabled(dmaid);
}

void FlxDevice::dma_to_host(DmaBuffer* buf, int dmaid)
{
    m_dev.dma_stop(dmaid);
    m_dev.dma_to_host(dmaid, buf->paddr, buf->size, FLX_DMA_WRAPAROUND);
    buf->pend = buf->paddr + buf->size;
    buf->pc_ptr = m_dev.dma_get_ptr(dmaid);
}


void FlxDevice::dma_from_host(DmaBuffer* buf, int dmaid)
{
    m_dev.dma_from_host(dmaid, buf->paddr, buf->size, FLX_DMA_WRAPAROUND);
    buf->pend = buf->paddr + buf->size;
    buf->pc_ptr = m_dev.dma_get_ptr(dmaid);
}


void FlxDevice::dma_from_host_trickle(DmaBuffer* buf, int dmaid, size_t config_size)
{
    if(config_size > buf->size){
        ers::error(felix_log::device_flx_issue(ERS_HERE, std::format("Trickle config size {} larger than buffer size {}", config_size, buf->size)));
        return;
    }
    // The firmware will read the buffer up until the config_size
    m_dev.dma_from_host(dmaid, buf->paddr, config_size, FLX_DMA_WRAPAROUND);
    buf->pend = buf->paddr + config_size;
}


void FlxDevice::dma_set_oneshot(DmaBuffer* buf, int dmaid, size_t trickle_config_size){
    // The firmware will read the buffer up until the config_size
    m_dev.dma_from_host(dmaid, buf->paddr, trickle_config_size, 0 /*FLX_DMA_ONESHOT*/);
}

uint64_t FlxDevice::dma_get_fw_ptr(int dmaid)
{
    return m_dev.dma_get_fw_ptr(dmaid);
}


uint64_t FlxDevice::dma_get_sw_ptr(int dmaid)
{
    return m_dev.dma_get_ptr(dmaid);
}


void FlxDevice::dma_set_sw_ptr(int dmaid, uint64_t p_phys)
{
    return m_dev.dma_set_ptr(dmaid, p_phys);
}


bool FlxDevice::dma_cmp_even_bits(int dmaid)
{
    return m_dev.dma_cmp_even_bits(dmaid);
}


void FlxDevice::dma_stop(unsigned int dmaid)
{
    m_dev.dma_stop(dmaid);
}

monitoring_data_t FlxDevice::hw_get_monitoring_data(unsigned int mon_mask)
{
    monitoring_data_t data;
    //memset(&data, 0, sizeof(data));
    try {
        data = m_dev.get_monitoring_data(mon_mask);
    } catch(const FlxException & e) {
        ers::warning(felix_log::device_flx_issue(ERS_HERE, e.what()));
    }
    return data;
}