Program Listing for File DomainManager.cpp

↰ Return to documentation for file (BackendLibfabric/DomainManager.cpp)

#include "DomainManager.hpp"

#include <tracy/Tracy.hpp>

#include <rdma/fabric.h>
#include <rdma/fi_domain.h>

#include "Issues.hpp"
#include "Helpers.hpp"

netio3::libfabric::DomainManager::DomainManager(NetworkMode mode,
                                                const EndPointAddress& address,
                                                std::uint64_t flags) :
  m_info(create_info(mode, address, flags)),
  m_fabric(create_fabric()),
  m_send_domain(create_domain()),
  m_listen_domain(create_domain())
{}

netio3::libfabric::DomainManager::~DomainManager()
{
  ZoneScoped;
  if (m_send_domain.get_domain() != nullptr) {
    fi_close(&m_send_domain.get_domain()->fid);
  }
  if (m_listen_domain.get_domain() != nullptr) {
    fi_close(&m_listen_domain.get_domain()->fid);
  }
  if (m_fabric != nullptr) {
    fi_close(&m_fabric->fid);
  }
}

netio3::libfabric::FiInfoUniquePtr netio3::libfabric::DomainManager::create_info(
  NetworkMode mode,
  const EndPointAddress& address,
  std::uint64_t flags)
{
  ZoneScoped;
  auto hints = FiInfoWrapper{};
  hints.get()->addr_format = FI_FORMAT_UNSPEC;
  hints.get()->ep_attr->type = FI_EP_MSG;
  hints.get()->caps = FI_MSG;
  hints.get()->mode = FI_LOCAL_MR;
  try {
    hints.get()->fabric_attr->prov_name = strdup(get_provider(mode).c_str());
  } catch (const LibFabricError& e) {
    throw LibfabricDomainError(ERS_HERE, e.message());
  }

  const auto port_str = std::to_string(address.port());
  ERS_DEBUG(1,
            std::format("Registering Fabric and domains for {}:{}", address.address(), port_str));
  fi_info* info = nullptr;

  if (auto ret = fi_getinfo(
        FI_VERSION(1, 1), address.address().c_str(), port_str.c_str(), flags, hints.get(), &info)) {
    throw LibfabricDomainError(
      ERS_HERE,
      std::format("Failed to get info on local interface {} - {}", ret, fi_strerror(-ret)));
  }
  ERS_DEBUG(1,
            std::format(
              "addr format: {:#x}, provider: {}", info->addr_format, info->fabric_attr->prov_name));
  return FiInfoUniquePtr{info};
}

fid_fabric* netio3::libfabric::DomainManager::create_fabric()
{
  ZoneScoped;
  fid_fabric* fabric{nullptr};
  if (auto ret = fi_fabric(m_info->fabric_attr, &fabric, nullptr)) {
    throw LibfabricDomainError(ERS_HERE,
                               std::format("Failed to open fabric - {}", fi_strerror(-ret)));
  }
  return fabric;
}

netio3::libfabric::DomainContext netio3::libfabric::DomainManager::create_domain() const
{
  ZoneScoped;
  fid_domain* domain = nullptr;
  if (auto ret = fi_domain(m_fabric, m_info.get(), &domain, nullptr)) {
    throw LibfabricDomainError(
      ERS_HERE, std::format("Cannot open send domain {} - {}", ret, fi_strerror(-ret)));
  }
  return DomainContext{domain};
}