#1 - quicr module

This commit is contained in:
Martin Slachta
2026-07-22 17:34:44 +02:00
parent a04f0dc262
commit e6dd954ded
149 changed files with 3997 additions and 2126 deletions
+33
View File
@@ -0,0 +1,33 @@
project(tw_message_protocol)
file(GLOB FILES
src/*.cpp
)
file(GLOB HEADERS
include/message_protocol/*.hpp
)
add_library(${PROJECT_NAME} OBJECT ${FILES})
add_library(tw::message_protocol ALIAS ${PROJECT_NAME})
target_sources(${PROJECT_NAME}
PUBLIC FILE_SET HEADERS
BASE_DIRS include
FILES ${HEADERS})
set_target_properties(${PROJECT_NAME} PROPERTIES POSITION_INDEPENDENT_CODE 1)
target_include_directories(${PROJECT_NAME}
PUBLIC
${PROJECT_SOURCE_DIR}/include/
)
target_link_libraries(${PROJECT_NAME}
PUBLIC
tw::io
tw::quicr
tl::expected
spdlog::spdlog
)
add_subdirectory(tests)
@@ -0,0 +1,127 @@
#pragma once
#include "message_protocol/MessageError.hpp"
#include "message_protocol/MessageHeader.hpp"
#include "message_protocol/MessageType.hpp"
#include "message_protocol/PeerId.hpp"
#include <tl/expected.hpp>
#include <chrono>
#include <cstddef>
#include <functional>
#include <span>
#include <unordered_map>
#include <vector>
namespace tw::net::quicr {
class QuicrConnection;
}
namespace tw::msg {
class MessageDispatcher;
/**
* Sends messages to one peer and routes the ones it sends back.
*
* Owned by the endpoint that created it and valid until the peer disconnects.
* Handlers for message addresses are registered on the endpoint and shared by
* every peer; a connection only holds the reply handlers for requests it made
* itself.
*/
class MessageConnection {
public:
using ReplyHandler = std::function<void(std::span<const std::byte>)>;
private:
struct PendingRequest {
ReplyHandler on_reply;
std::function<void()> on_timeout;
std::chrono::steady_clock::time_point expires_at;
};
net::quicr::QuicrConnection* m_connection;
MessageDispatcher* m_dispatcher;
PeerId m_peer_id;
std::vector<std::byte> m_send_buffer;
std::unordered_map<uint32_t, PendingRequest> m_pending;
uint32_t m_next_seq = 1;
uint64_t m_bytes_sent = 0;
uint64_t m_messages_sent = 0;
uint64_t m_messages_received = 0;
uint32_t next_seq();
tl::expected<void, MessageError> send_impl(MessageType type,
std::span<const std::byte> body,
uint32_t seq,
bool reliable);
public:
MessageConnection(PeerId peer_id,
net::quicr::QuicrConnection* connection,
MessageDispatcher* dispatcher);
MessageConnection(const MessageConnection&) = delete;
MessageConnection& operator=(const MessageConnection&) = delete;
PeerId peer_id() const {
return m_peer_id;
}
uint64_t bytes_sent() const {
return m_bytes_sent;
}
/** Messages handed over for sending since the connection was created. */
uint64_t messages_sent() const {
return m_messages_sent;
}
/** Messages routed from the peer since the connection was created. */
uint64_t messages_received() const {
return m_messages_received;
}
bool is_established() const;
tl::expected<void, MessageError> send(MessageType type,
std::span<const std::byte> body,
bool reliable = false);
/**
* Sends a message the caller has already written a header into, for
* callers that build the whole message in a buffer of their own.
*/
tl::expected<void, MessageError> send_framed(std::span<const std::byte> message,
bool reliable = false);
/**
* Sends `body` and calls `on_reply` with the reply carrying the same
* sequence number, or `on_timeout` if no reply arrives in time.
*/
tl::expected<void, MessageError> request(
MessageType type,
std::span<const std::byte> body,
ReplyHandler on_reply,
std::chrono::milliseconds timeout = std::chrono::seconds(5),
std::function<void()> on_timeout = nullptr,
bool reliable = true);
/**
* Reads everything the peer has sent, routing each message, and returns
* how many bytes were read. `scratch` is used to hold one message at a
* time and may be reused between peers.
*/
size_t receive(std::span<std::byte> scratch);
/** Routes one received message to its reply handler, or to the dispatcher. */
void on_message(std::span<const std::byte> message);
/** Fails every request whose reply did not arrive before `now`. */
void expire_requests(std::chrono::steady_clock::time_point now);
};
}
@@ -0,0 +1,47 @@
#pragma once
#include "message_protocol/MessageType.hpp"
#include "message_protocol/PeerId.hpp"
#include <functional>
#include <span>
#include <unordered_map>
namespace tw::msg {
/**
* Routes a message body to the handler registered for its address.
*
* Never inspects the body, so how it is encoded is entirely the caller's
* concern. At most one handler may be registered per address.
*/
class MessageDispatcher {
public:
using Handler = std::function<void(PeerId, std::span<const std::byte>)>;
private:
std::unordered_map<MessageType, Handler> m_handlers;
public:
/** Registers `handler` for `type`, replacing any handler already there. */
void set_handler(MessageType type, Handler handler) {
m_handlers[type] = std::move(handler);
}
bool has_handler(MessageType type) const {
return m_handlers.contains(type);
}
/** Invokes the handler for `type`. Returns false if there is none. */
bool dispatch(PeerId peer, MessageType type, std::span<const std::byte> body) {
auto handler = m_handlers.find(type);
if(handler == m_handlers.end()) {
return false;
}
handler->second(peer, body);
return true;
}
};
}
@@ -0,0 +1,110 @@
#pragma once
#include "message_protocol/MessageConnection.hpp"
#include "message_protocol/MessageDispatcher.hpp"
#include "message_protocol/MessageError.hpp"
#include "message_protocol/MessageType.hpp"
#include "message_protocol/PeerId.hpp"
#include <tl/expected.hpp>
#include <cstddef>
#include <functional>
#include <memory>
#include <span>
#include <string>
#include <unordered_map>
#include <vector>
namespace tw::net::quicr {
class QuicrEndpoint;
class QuicrConnectionListener;
}
namespace tw::msg {
/**
* Owns the connections to every peer and the handlers shared between them.
*
* An endpoint created with bind() accepts incoming peers; either kind may
* connect() outwards, so one endpoint can serve peers and reach out to others
* at the same time.
*
* update() must be called regularly. Nothing is received and no request ever
* times out between calls.
*/
class MessageEndpoint {
std::unique_ptr<net::quicr::QuicrEndpoint> m_endpoint;
std::unique_ptr<net::quicr::QuicrConnectionListener> m_listener;
MessageDispatcher m_dispatcher;
std::unordered_map<PeerId, std::unique_ptr<MessageConnection>> m_peers;
std::function<void(PeerId)> m_on_peer_connected;
PeerId m_next_peer_id = 1;
std::vector<std::byte> m_receive_buffer;
uint64_t m_bytes_received = 0;
explicit MessageEndpoint(std::unique_ptr<net::quicr::QuicrEndpoint> endpoint);
MessageConnection* add_peer(net::quicr::QuicrConnection* connection);
void accept_peers();
void receive();
public:
~MessageEndpoint();
MessageEndpoint(const MessageEndpoint&) = delete;
MessageEndpoint& operator=(const MessageEndpoint&) = delete;
/** Creates an endpoint that only connects outwards. */
static tl::expected<std::unique_ptr<MessageEndpoint>, MessageError> create();
/** Creates an endpoint that also accepts peers on `port`. */
static tl::expected<std::unique_ptr<MessageEndpoint>, MessageError> bind(int port);
tl::expected<MessageConnection*, MessageError> connect(const std::string& host, int port);
/** Registers `handler` for every peer. */
void set_handler(MessageType type, MessageDispatcher::Handler handler) {
m_dispatcher.set_handler(type, std::move(handler));
}
MessageDispatcher& dispatcher() {
return m_dispatcher;
}
/** Receives pending messages, accepts new peers and times out requests. */
void update();
MessageConnection* peer(PeerId id);
/**
* Calls `handler` for each peer that connects or is accepted, before any
* of that peer's messages are dispatched.
*/
void set_on_peer_connected(std::function<void(PeerId)> handler) {
m_on_peer_connected = std::move(handler);
}
std::vector<MessageConnection*> peers() const;
void broadcast(MessageType type, std::span<const std::byte> body, bool reliable = false);
/** Bytes received since the endpoint was created. */
uint64_t bytes_received() const {
return m_bytes_received;
}
/** Bytes handed to every peer for sending since the endpoint was created. */
uint64_t bytes_sent() const;
/** Messages handed to every peer for sending since the endpoint was created. */
uint64_t messages_sent() const;
/** Messages routed from every peer since the endpoint was created. */
uint64_t messages_received() const;
};
}
@@ -0,0 +1,37 @@
#pragma once
#include <string>
#include <utility>
namespace tw::msg {
enum class MessageErrorType {
NotConnected,
SendFailed,
BindFailed,
ConnectFailed,
};
struct MessageError {
MessageErrorType type;
std::string detail;
explicit MessageError(MessageErrorType type, std::string detail = {}) :
type(type),
detail(std::move(detail)) {
}
std::string message() const {
std::string text;
switch(type) {
case MessageErrorType::NotConnected: text = "Not connected to the peer"; break;
case MessageErrorType::SendFailed: text = "Failed to send the message"; break;
case MessageErrorType::BindFailed: text = "Failed to bind the endpoint"; break;
case MessageErrorType::ConnectFailed: text = "Failed to connect to the peer"; break;
}
return detail.empty() ? text : text + ": " + detail;
}
};
}
@@ -0,0 +1,45 @@
#pragma once
#include "message_protocol/MessageType.hpp"
#include <cstdint>
#include <cstring>
#include <optional>
#include <span>
namespace tw::msg {
/**
* Fixed-size prefix carried by every message.
*
* `seq` correlates a reply with the request that produced it. SEQ_NONE marks a
* message that expects no reply, which is the common case.
*/
struct MessageHeader {
static constexpr uint32_t SEQ_NONE = 0;
static constexpr size_t SIZE = sizeof(MessageType) + sizeof(uint32_t);
MessageType type = 0;
uint32_t seq = SEQ_NONE;
/** Writes the header at the start of `target`, which must hold SIZE bytes. */
void encode(std::span<std::byte> target) const {
std::memcpy(target.data(), &type, sizeof(type));
std::memcpy(target.data() + sizeof(type), &seq, sizeof(seq));
}
/** Reads a header from the start of `source`, or nothing if it is too short. */
static std::optional<MessageHeader> decode(std::span<const std::byte> source) {
if(source.size() < SIZE) {
return {};
}
MessageHeader header;
std::memcpy(&header.type, source.data(), sizeof(header.type));
std::memcpy(&header.seq, source.data() + sizeof(header.type), sizeof(header.seq));
return header;
}
};
}
@@ -0,0 +1,13 @@
#pragma once
#include <cstdint>
namespace tw::msg {
/**
* Address a message is delivered to. Concrete values are assigned by the
* application.
*/
using MessageType = uint32_t;
}
@@ -0,0 +1,13 @@
#pragma once
#include <cstdint>
namespace tw::msg {
/**
* Identifies a remote peer. Assigned when the peer connects or is accepted and
* stable until it disconnects.
*/
using PeerId = uint64_t;
}
@@ -0,0 +1,176 @@
#include "message_protocol/MessageConnection.hpp"
#include "message_protocol/MessageDispatcher.hpp"
#include "quicr/QuicrConnection.hpp"
#include <spdlog/spdlog.h>
#include <cstring>
namespace tw::msg {
namespace {
constexpr size_t INITIAL_SEND_BUFFER_SIZE = 64 * 1024;
}
MessageConnection::MessageConnection(PeerId peer_id,
net::quicr::QuicrConnection* connection,
MessageDispatcher* dispatcher) :
m_connection(connection),
m_dispatcher(dispatcher),
m_peer_id(peer_id),
m_send_buffer(INITIAL_SEND_BUFFER_SIZE) {
}
bool MessageConnection::is_established() const {
return m_connection->state() == net::quicr::QuicrConnectionState::Established;
}
uint32_t MessageConnection::next_seq() {
uint32_t seq = m_next_seq++;
if(m_next_seq == MessageHeader::SEQ_NONE) {
m_next_seq = 1;
}
return seq;
}
tl::expected<void, MessageError> MessageConnection::send_impl(MessageType type,
std::span<const std::byte> body,
uint32_t seq,
bool reliable) {
const size_t size = MessageHeader::SIZE + body.size();
if(m_send_buffer.size() < size) {
m_send_buffer.resize(size);
}
MessageHeader{ type, seq }.encode(m_send_buffer);
std::memcpy(m_send_buffer.data() + MessageHeader::SIZE, body.data(), body.size());
auto send_r = m_connection->send_message(std::span(m_send_buffer).subspan(0, size), reliable);
if(!send_r) {
return tl::make_unexpected(MessageError(MessageErrorType::SendFailed, send_r.error().message()));
}
m_bytes_sent += size;
m_messages_sent++;
return {};
}
tl::expected<void, MessageError> MessageConnection::send(MessageType type,
std::span<const std::byte> body,
bool reliable) {
return send_impl(type, body, MessageHeader::SEQ_NONE, reliable);
}
tl::expected<void, MessageError> MessageConnection::send_framed(std::span<const std::byte> message,
bool reliable) {
if(message.size() < MessageHeader::SIZE) {
return tl::make_unexpected(
MessageError(MessageErrorType::SendFailed, "the message is too short to hold a header"));
}
// send_message takes a writable span, so the bytes are staged in the send
// buffer rather than sent straight from the caller's buffer.
if(m_send_buffer.size() < message.size()) {
m_send_buffer.resize(message.size());
}
std::memcpy(m_send_buffer.data(), message.data(), message.size());
auto send_r = m_connection->send_message(std::span(m_send_buffer).subspan(0, message.size()), reliable);
if(!send_r) {
return tl::make_unexpected(MessageError(MessageErrorType::SendFailed, send_r.error().message()));
}
m_bytes_sent += message.size();
m_messages_sent++;
return {};
}
tl::expected<void, MessageError> MessageConnection::request(MessageType type,
std::span<const std::byte> body,
ReplyHandler on_reply,
std::chrono::milliseconds timeout,
std::function<void()> on_timeout,
bool reliable) {
const uint32_t seq = next_seq();
auto send_r = send_impl(type, body, seq, reliable);
if(!send_r) {
return send_r;
}
m_pending.emplace(seq,
PendingRequest{ std::move(on_reply),
std::move(on_timeout),
std::chrono::steady_clock::now() + timeout });
return {};
}
size_t MessageConnection::receive(std::span<std::byte> scratch) {
size_t total = 0;
while(true) {
auto read_r = m_connection->read_into(scratch);
if(!read_r) {
spdlog::error("Failed to read from peer {}: {}", m_peer_id, read_r.error().message());
break;
}
if(*read_r == 0) {
break;
}
total += *read_r;
m_messages_received++;
on_message(scratch.subspan(0, *read_r));
}
return total;
}
void MessageConnection::on_message(std::span<const std::byte> message) {
auto header = MessageHeader::decode(message);
if(!header) {
spdlog::warn("Dropped a message of {} bytes, too short to hold a header", message.size());
return;
}
auto body = message.subspan(MessageHeader::SIZE);
if(header->seq != MessageHeader::SEQ_NONE) {
auto pending = m_pending.find(header->seq);
if(pending != m_pending.end()) {
auto on_reply = std::move(pending->second.on_reply);
m_pending.erase(pending);
on_reply(body);
return;
}
}
if(!m_dispatcher->dispatch(m_peer_id, header->type, body)) {
spdlog::warn("No handler for message type {}", header->type);
}
}
void MessageConnection::expire_requests(std::chrono::steady_clock::time_point now) {
std::erase_if(m_pending, [&](auto& entry) {
if(entry.second.expires_at > now) {
return false;
}
spdlog::warn("Request {} timed out", entry.first);
if(entry.second.on_timeout) {
entry.second.on_timeout();
}
return true;
});
}
}
@@ -0,0 +1,162 @@
#include "message_protocol/MessageEndpoint.hpp"
#include "quicr/QuicrAddress.hpp"
#include "quicr/QuicrConnection.hpp"
#include "quicr/QuicrConnectionListener.hpp"
#include "quicr/QuicrEndpoint.hpp"
#include <spdlog/spdlog.h>
namespace tw::msg {
namespace {
constexpr size_t RECEIVE_BUFFER_SIZE = 64 * 1024;
}
MessageEndpoint::MessageEndpoint(std::unique_ptr<net::quicr::QuicrEndpoint> endpoint) :
m_endpoint(std::move(endpoint)),
m_receive_buffer(RECEIVE_BUFFER_SIZE) {
}
MessageEndpoint::~MessageEndpoint() = default;
tl::expected<std::unique_ptr<MessageEndpoint>, MessageError> MessageEndpoint::create() {
auto endpoint_r = net::quicr::QuicrEndpoint::create();
if(!endpoint_r) {
return tl::make_unexpected(
MessageError(MessageErrorType::BindFailed, endpoint_r.error().message()));
}
return std::unique_ptr<MessageEndpoint>(new MessageEndpoint(std::move(endpoint_r.value())));
}
tl::expected<std::unique_ptr<MessageEndpoint>, MessageError> MessageEndpoint::bind(int port) {
auto endpoint_r = create();
if(!endpoint_r) {
return endpoint_r;
}
auto& endpoint = endpoint_r.value();
auto bind_r = endpoint->m_endpoint->bind(port);
if(!bind_r) {
return tl::make_unexpected(MessageError(MessageErrorType::BindFailed, bind_r.error().message()));
}
auto listener_r = net::quicr::QuicrConnectionListener::listen(endpoint->m_endpoint.get());
if(!listener_r) {
return tl::make_unexpected(
MessageError(MessageErrorType::BindFailed, listener_r.error().message()));
}
endpoint->m_listener = std::move(listener_r.value());
return endpoint_r;
}
MessageConnection* MessageEndpoint::add_peer(net::quicr::QuicrConnection* connection) {
const PeerId id = m_next_peer_id++;
auto peer = std::make_unique<MessageConnection>(id, connection, &m_dispatcher);
auto* raw = peer.get();
m_peers.emplace(id, std::move(peer));
if(m_on_peer_connected) {
m_on_peer_connected(id);
}
return raw;
}
tl::expected<MessageConnection*, MessageError> MessageEndpoint::connect(const std::string& host, int port) {
auto connection_r = m_endpoint->connect(net::quicr::QuicrAddress(host, port));
if(!connection_r) {
return tl::make_unexpected(
MessageError(MessageErrorType::ConnectFailed, connection_r.error().message()));
}
return add_peer(connection_r.value());
}
void MessageEndpoint::accept_peers() {
if(!m_listener) {
return;
}
while(net::quicr::QuicrConnection* connection = m_listener->listen()) {
add_peer(connection);
}
}
void MessageEndpoint::receive() {
for(auto& [id, peer] : m_peers) {
m_bytes_received += peer->receive(m_receive_buffer);
}
}
void MessageEndpoint::update() {
m_endpoint->poll();
accept_peers();
receive();
const auto now = std::chrono::steady_clock::now();
for(auto& [id, peer] : m_peers) {
peer->expire_requests(now);
}
}
MessageConnection* MessageEndpoint::peer(PeerId id) {
auto peer = m_peers.find(id);
return peer != m_peers.end() ? peer->second.get() : nullptr;
}
std::vector<MessageConnection*> MessageEndpoint::peers() const {
std::vector<MessageConnection*> result;
result.reserve(m_peers.size());
for(const auto& [id, peer] : m_peers) {
result.push_back(peer.get());
}
return result;
}
void MessageEndpoint::broadcast(MessageType type, std::span<const std::byte> body, bool reliable) {
for(auto& [id, peer] : m_peers) {
auto send_r = peer->send(type, body, reliable);
if(!send_r) {
spdlog::error("Failed to send to peer {}: {}", id, send_r.error().message());
}
}
}
uint64_t MessageEndpoint::bytes_sent() const {
uint64_t total = 0;
for(const auto& [id, peer] : m_peers) {
total += peer->bytes_sent();
}
return total;
}
uint64_t MessageEndpoint::messages_sent() const {
uint64_t total = 0;
for(const auto& [id, peer] : m_peers) {
total += peer->messages_sent();
}
return total;
}
uint64_t MessageEndpoint::messages_received() const {
uint64_t total = 0;
for(const auto& [id, peer] : m_peers) {
total += peer->messages_received();
}
return total;
}
}
@@ -0,0 +1,24 @@
project(tw_message_protocol_tests)
file(GLOB FILES
./*.cpp
)
add_executable(${PROJECT_NAME} ${FILES})
# tw::quicr is listed explicitly because CMake does not propagate the object
# files of an OBJECT library through another OBJECT library.
target_link_libraries(${PROJECT_NAME}
PRIVATE
tw::message_protocol
tw::quicr
Catch2::Catch2WithMain
tl::expected
)
list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras)
include(CTest)
include(Catch)
catch_discover_tests(${PROJECT_NAME})
@@ -0,0 +1,73 @@
#include "message_protocol/MessageDispatcher.hpp"
#include <catch2/catch_test_macros.hpp>
#include <array>
#include <string>
using namespace tw::msg;
namespace {
std::span<const std::byte> as_bytes(const std::array<std::byte, 2>& body) {
return { body.data(), body.size() };
}
}
TEST_CASE("Dispatch reaches the handler bound to the address", "[message_dispatcher]") {
MessageDispatcher dispatcher;
PeerId seen_peer = 0;
size_t seen_size = 0;
dispatcher.set_handler(7, [&](PeerId peer, std::span<const std::byte> body) {
seen_peer = peer;
seen_size = body.size();
});
std::array<std::byte, 2> body{};
REQUIRE(dispatcher.dispatch(99, 7, as_bytes(body)));
REQUIRE(seen_peer == 99);
REQUIRE(seen_size == 2);
}
TEST_CASE("Dispatch to an unbound address reports failure", "[message_dispatcher]") {
MessageDispatcher dispatcher;
std::array<std::byte, 2> body{};
REQUIRE_FALSE(dispatcher.dispatch(1, 7, as_bytes(body)));
}
TEST_CASE("Addresses are routed independently", "[message_dispatcher]") {
MessageDispatcher dispatcher;
std::string called;
dispatcher.set_handler(1, [&](PeerId, std::span<const std::byte>) { called = "first"; });
dispatcher.set_handler(2, [&](PeerId, std::span<const std::byte>) { called = "second"; });
std::array<std::byte, 2> body{};
dispatcher.dispatch(1, 2, as_bytes(body));
REQUIRE(called == "second");
dispatcher.dispatch(1, 1, as_bytes(body));
REQUIRE(called == "first");
}
TEST_CASE("Rebinding an address replaces the handler", "[message_dispatcher]") {
MessageDispatcher dispatcher;
std::string called;
dispatcher.set_handler(7, [&](PeerId, std::span<const std::byte>) { called = "first"; });
dispatcher.set_handler(7, [&](PeerId, std::span<const std::byte>) { called = "second"; });
std::array<std::byte, 2> body{};
dispatcher.dispatch(1, 7, as_bytes(body));
REQUIRE(called == "second");
}
@@ -0,0 +1,45 @@
#include "message_protocol/MessageHeader.hpp"
#include <catch2/catch_test_macros.hpp>
#include <array>
using namespace tw::msg;
TEST_CASE("Header survives a round trip", "[message_header]") {
std::array<std::byte, MessageHeader::SIZE> buffer{};
MessageHeader{ 42, 7 }.encode(buffer);
auto decoded = MessageHeader::decode(buffer);
REQUIRE(decoded.has_value());
REQUIRE(decoded->type == 42);
REQUIRE(decoded->seq == 7);
}
TEST_CASE("Header defaults to expecting no reply", "[message_header]") {
std::array<std::byte, MessageHeader::SIZE> buffer{};
MessageHeader{ 3 }.encode(buffer);
auto decoded = MessageHeader::decode(buffer);
REQUIRE(decoded.has_value());
REQUIRE(decoded->seq == MessageHeader::SEQ_NONE);
}
TEST_CASE("Decoding a message shorter than a header fails", "[message_header]") {
std::array<std::byte, MessageHeader::SIZE - 1> buffer{};
REQUIRE_FALSE(MessageHeader::decode(buffer).has_value());
}
TEST_CASE("Encoding only writes the header", "[message_header]") {
std::array<std::byte, MessageHeader::SIZE + 4> buffer{};
buffer[MessageHeader::SIZE] = std::byte{ 0xAB };
MessageHeader{ 1, 2 }.encode(buffer);
REQUIRE(buffer[MessageHeader::SIZE] == std::byte{ 0xAB });
}