#1 - quicr module

modules/chat_service/chat_client/CMakeLists.txt

@@ -26,8 +26,8 @@ target_link_libraries(${PROJECT_NAME}
     PUBLIC
         tw::network
         tl::expected
-        tw::messaging
         tw::protocol
+        tw::message_protocol
         protobuf::libprotobuf
         tw::chat::lib
         spdlog::spdlog

modules/chat_service/chat_client/include/ChatClient.hpp

@@ -1,16 +1,20 @@
 #pragma once
 
-#include "MessageSession.hpp"
+#include "ProtobufMessages.hpp"
 #include "SendChatMessage.hpp"
 #include "ChatClientError.hpp"
 #include "models/ChatMessage.hpp"
+#include "message_protocol/MessageEndpoint.hpp"
 #include <functional>
+#include <memory>
 #include <string>
 
 namespace tw::chat {
 
 class ChatClient {
-    tw::MessageSession m_session;
+    std::unique_ptr<msg::MessageEndpoint> m_endpoint;
+    msg::MessageConnection*               m_server;
+    ProtobufMessages                      m_messages;
 
     std::function<void(ChatMessage)>                                      m_on_message;
     std::function<void(tl::expected<void, ChatClientError>)>              m_on_send_response;

modules/chat_service/chat_client/src/ChatClient.cpp

@@ -22,45 +22,68 @@ tl::expected<void, tw::chat::ChatClientError> from_error_code(mmo::chat::ChatErr
 
 namespace tw::chat {
 
+namespace {
+
+std::unique_ptr<tw::msg::MessageEndpoint> create_endpoint() {
+    auto endpoint_r = tw::msg::MessageEndpoint::create();
+    if (!endpoint_r) {
+        throw std::runtime_error("Chat client failed to create an endpoint: " +
+                                 endpoint_r.error().message());
+    }
+
+    return std::move(endpoint_r.value());
+}
+
+tw::msg::MessageConnection* connect_to_server(tw::msg::MessageEndpoint* endpoint,
+                                              const std::string&        server_address,
+                                              int16_t                   port) {
+    auto server_r = endpoint->connect(server_address, port);
+    if (!server_r) {
+        throw std::runtime_error("Chat client failed to connect: " + server_r.error().message());
+    }
+
+    return server_r.value();
+}
+
+}
+
 ChatClient::ChatClient(const std::string& server_address, int16_t port)
-    : m_session(net::Address(server_address, port))
+    : m_endpoint(create_endpoint())
+    , m_server(connect_to_server(m_endpoint.get(), server_address, port))
+    , m_messages(m_endpoint.get())
 {
-    m_session.set_handler(CHAT_MESSAGE_BROADCAST_REQUEST, [this](std::span<const std::byte> data) {
-        if (!m_on_message) return;
-        mmo::chat::ChatMessageBroadcastRequest proto;
-        if (!proto.ParseFromArray(data.data(), static_cast<int>(data.size()))) return;
-        ChatMessage msg;
-        msg.channel_id = proto.channel_id();
-        msg.client_id  = proto.sender_id();
-        msg.message    = proto.message();
-        msg.timestamp  = ChatMessage::Clock::now();
-        m_on_message(std::move(msg));
-    });
+    m_messages.set_handler<mmo::chat::ChatMessageBroadcastRequest>(
+        [this](msg::PeerId, const mmo::chat::ChatMessageBroadcastRequest& proto) {
+            if (!m_on_message) return;
+            ChatMessage msg;
+            msg.channel_id = proto.channel_id();
+            msg.client_id  = proto.sender_id();
+            msg.message    = proto.message();
+            msg.timestamp  = ChatMessage::Clock::now();
+            m_on_message(std::move(msg));
+        });
 
-    m_session.set_handler(CHAT_SEND_MESSAGE_RESPONSE, [this](std::span<const std::byte> data) {
-        if (!m_on_send_response) return;
-        mmo::chat::SendChatMessageResponse proto;
-        if (!proto.ParseFromArray(data.data(), static_cast<int>(data.size()))) return;
-        m_on_send_response(from_error_code(proto.error()));
-    });
+    m_messages.set_handler<mmo::chat::SendChatMessageResponse>(
+        [this](msg::PeerId, const mmo::chat::SendChatMessageResponse& proto) {
+            if (!m_on_send_response) return;
+            m_on_send_response(from_error_code(proto.error()));
+        });
 
-    m_session.set_handler(CHAT_JOIN_CHANNEL_RESPONSE, [this](std::span<const std::byte> data) {
-        if (!m_on_join_response) return;
-        mmo::chat::JoinChannelResponse proto;
-        if (!proto.ParseFromArray(data.data(), static_cast<int>(data.size()))) return;
-        m_on_join_response(static_cast<uint32_t>(proto.channel_id()), from_error_code(proto.error()));
-    });
+    m_messages.set_handler<mmo::chat::JoinChannelResponse>(
+        [this](msg::PeerId, const mmo::chat::JoinChannelResponse& proto) {
+            if (!m_on_join_response) return;
+            m_on_join_response(static_cast<uint32_t>(proto.channel_id()), from_error_code(proto.error()));
+        });
 
-    m_session.set_handler(CHAT_LEAVE_CHANNEL_RESPONSE, [this](std::span<const std::byte> data) {
-        if (!m_on_leave_response) return;
-        mmo::chat::LeaveChannelResponse proto;
-        if (!proto.ParseFromArray(data.data(), static_cast<int>(data.size()))) return;
-        m_on_leave_response(static_cast<uint32_t>(proto.channel_id()), from_error_code(proto.error()));
-    });
+    m_messages.set_handler<mmo::chat::LeaveChannelResponse>(
+        [this](msg::PeerId, const mmo::chat::LeaveChannelResponse& proto) {
+            if (!m_on_leave_response) return;
+            m_on_leave_response(static_cast<uint32_t>(proto.channel_id()), from_error_code(proto.error()));
+        });
 }
 
 void ChatClient::update() {
-    m_session.update();
+    m_endpoint->update();
 }
 
 tl::expected<void, ChatClientError> ChatClient::send_mesg(SendChatMessage message) {
@@ -68,11 +91,7 @@ tl::expected<void, ChatClientError> ChatClient::send_mesg(SendChatMessage messag
     mesg.set_channel_id(message.channel_id);
     mesg.set_message(message.message);
 
-    std::vector<std::byte> buf(mesg.ByteSizeLong());
-    (void)mesg.SerializeToArray(buf.data(), static_cast<int>(buf.size()));
-
-    auto send_r = m_session.send(Message<mmo::chat::SendChatMessageRequest>::value,
-                                 std::span(buf), true);
+    auto send_r = m_messages.send(m_server, mesg, true);
     if (!send_r)
         return tl::make_unexpected(ChatClientError::PermissionDenied);
     return {};

modules/chat_service/chat_server_exe/CMakeLists.txt

@@ -13,8 +13,9 @@ target_include_directories(${PROJECT_NAME}
 target_link_libraries(${PROJECT_NAME}
     PRIVATE
         tw::chat::service
-        tw::messaging
         tw::protocol
+        tw::message_protocol
         tw::network
+        tw::quicr
         spdlog::spdlog
 )

modules/chat_service/chat_server_exe/src/ChatServerController.cpp

@@ -3,7 +3,7 @@
 #include "Chat.pb.h"
 #include "MessageRegistry.hpp"
 #include <spdlog/spdlog.h>
-#include <cstring>
+#include <stdexcept>
 
 namespace tw::chat {
 
@@ -16,108 +16,68 @@ static mmo::chat::ChatErrorCode to_error_code(tl::expected<void, ChatServerError
     return mmo::chat::CHAT_ERROR_CODE_CHANNEL_NOT_FOUND;
 }
 
-template<typename T>
-static std::vector<std::byte> serialize(const T& msg) {
-    std::vector<std::byte> buf(msg.ByteSizeLong());
-    (void)msg.SerializeToArray(buf.data(), static_cast<int>(buf.size()));
-    return buf;
+static std::unique_ptr<msg::MessageEndpoint> bind_endpoint(int port) {
+    auto endpoint_r = msg::MessageEndpoint::bind(port);
+    if (!endpoint_r) {
+        throw std::runtime_error("Chat server failed to bind to port " + std::to_string(port) +
+                                 ": " + endpoint_r.error().message());
+    }
+
+    return std::move(endpoint_r.value());
 }
 
 ChatServerController::ChatServerController(int port)
-    : m_endpoint(net::quicr::QuicrEndpoint::create_and_bind(port).value())
-    , m_listener(net::quicr::QuicrConnectionListener::listen(m_endpoint.get()).value())
+    : m_endpoint(bind_endpoint(port))
+    , m_messages(m_endpoint.get())
     , m_service([this](uint64_t id, const ChatMessage& msg) { broadcast(id, msg); })
 {
+    m_endpoint->set_on_peer_connected([](msg::PeerId client_id) {
+        spdlog::info("Chat client connected: {}", client_id);
+    });
+
     register_handlers();
     spdlog::info("Chat server listening on port {}", port);
 }
 
 void ChatServerController::register_handlers() {
-    m_handlers[Message<mmo::chat::SendChatMessageRequest>::value] =
-        [this](uint64_t client_id, std::span<const std::byte> data) {
-            mmo::chat::SendChatMessageRequest msg;
-            msg.ParseFromArray(data.data(), static_cast<int>(data.size()));
+    m_messages.set_handler<mmo::chat::SendChatMessageRequest>(
+        [this](msg::PeerId client_id, const mmo::chat::SendChatMessageRequest& msg) {
             mmo::chat::SendChatMessageResponse r;
             r.set_channel_id(msg.channel_id());
             r.set_error(to_error_code(m_service.send_message(client_id, msg.channel_id(), msg.message())));
-            send_to(client_id, Message<mmo::chat::SendChatMessageResponse>::value, serialize(r));
-        };
+            (void)m_messages.send_to(client_id, r, false);
+        });
 
-    m_handlers[Message<mmo::chat::JoinChannelRequest>::value] =
-        [this](uint64_t client_id, std::span<const std::byte> data) {
-            mmo::chat::JoinChannelRequest msg;
-            msg.ParseFromArray(data.data(), static_cast<int>(data.size()));
+    m_messages.set_handler<mmo::chat::JoinChannelRequest>(
+        [this](msg::PeerId client_id, const mmo::chat::JoinChannelRequest& msg) {
             m_service.join_channel(client_id, msg.channel_id());
             mmo::chat::JoinChannelResponse r;
             r.set_channel_id(msg.channel_id());
             r.set_error(mmo::chat::CHAT_ERROR_CODE_OK);
-            send_to(client_id, Message<mmo::chat::JoinChannelResponse>::value, serialize(r));
-        };
+            (void)m_messages.send_to(client_id, r, false);
+        });
 
-    m_handlers[Message<mmo::chat::LeaveChannelRequest>::value] =
-        [this](uint64_t client_id, std::span<const std::byte> data) {
-            mmo::chat::LeaveChannelRequest msg;
-            msg.ParseFromArray(data.data(), static_cast<int>(data.size()));
+    m_messages.set_handler<mmo::chat::LeaveChannelRequest>(
+        [this](msg::PeerId client_id, const mmo::chat::LeaveChannelRequest& msg) {
             m_service.leave_channel(client_id, msg.channel_id());
             mmo::chat::LeaveChannelResponse r;
             r.set_channel_id(msg.channel_id());
             r.set_error(mmo::chat::CHAT_ERROR_CODE_OK);
-            send_to(client_id, Message<mmo::chat::LeaveChannelResponse>::value, serialize(r));
-        };
+            (void)m_messages.send_to(client_id, r, false);
+        });
 }
 
 void ChatServerController::update() {
-    m_endpoint->poll();
-
-    net::quicr::QuicrConnection* conn = nullptr;
-    while ((conn = m_listener->listen())) {
-        m_connections.emplace(conn->self_id(), conn);
-        spdlog::info("Chat client connected: {}", conn->self_id());
-    }
-
-    for (auto& [client_id, conn] : m_connections) {
-        auto r = conn->read_into(m_recv_buf);
-        if (!r || *r == 0) continue;
-        dispatch(client_id, std::span(m_recv_buf.data(), *r));
-    }
+    m_endpoint->update();
 }
 
-void ChatServerController::dispatch(uint64_t client_id, std::span<const std::byte> data) {
-    constexpr size_t HEADER = sizeof(uint32_t) * 2;
-    if (data.size() < HEADER) {
-        spdlog::warn("ChatServerController: dropped short datagram ({} bytes)", data.size());
-        return;
-    }
-    uint32_t type{};
-    std::memcpy(&type, data.data(), sizeof(type));
-
-    if (type >= m_handlers.size() || !m_handlers[type]) {
-        spdlog::warn("ChatServerController: no handler for type {}", type);
-        return;
-    }
-    m_handlers[type](client_id, data.subspan(HEADER));
-}
-
-void ChatServerController::send_to(uint64_t client_id, uint32_t type,
-                                    std::span<const std::byte> payload, bool reliable) {
-    auto it = m_connections.find(client_id);
-    if (it == m_connections.end()) return;
-
-    constexpr uint32_t SEQ_NONE = 0;
-    std::vector<std::byte> buf(sizeof(type) + sizeof(SEQ_NONE) + payload.size());
-    std::memcpy(buf.data(),                    &type,     sizeof(type));
-    std::memcpy(buf.data() + sizeof(type),     &SEQ_NONE, sizeof(SEQ_NONE));
-    std::memcpy(buf.data() + sizeof(type) + sizeof(SEQ_NONE), payload.data(), payload.size());
-    (void)it->second->send_message(std::span(buf), reliable);
-}
-
-void ChatServerController::broadcast(uint64_t client_id, const ChatMessage& msg) {
+void ChatServerController::broadcast(msg::PeerId client_id, const ChatMessage& msg) {
     mmo::chat::ChatMessageBroadcastRequest bcast;
     bcast.set_channel_id(msg.channel_id);
     bcast.set_sender_id(msg.client_id);
     bcast.set_message(msg.message);
-    send_to(client_id, Message<mmo::chat::ChatMessageBroadcastRequest>::value,
-            serialize(bcast), true);
+
+    (void)m_messages.send_to(client_id, bcast, true);
 }
 
 } // namespace tw::chat

modules/chat_service/chat_server_exe/src/ChatServerController.hpp

@@ -1,28 +1,18 @@
 #pragma once
 
 #include "ChatService.hpp"
-#include "protocol/quicr/QuicrEndpoint.hpp"
-#include "protocol/quicr/QuicrConnectionListener.hpp"
+#include "ProtobufMessages.hpp"
+#include "message_protocol/MessageEndpoint.hpp"
 
-#include <array>
 #include <cstdint>
-#include <functional>
-#include <span>
-#include <unordered_map>
-#include <vector>
+#include <memory>
 
 namespace tw::chat {
 
 class ChatServerController {
-    static constexpr size_t MAX_TYPES = 32;
-
-    std::unique_ptr<net::quicr::QuicrEndpoint>           m_endpoint;
-    std::unique_ptr<net::quicr::QuicrConnectionListener> m_listener;
-    std::unordered_map<uint64_t, net::quicr::QuicrConnection*> m_connections;
-    std::vector<std::byte>              m_recv_buf{64 * 1024};
-    ChatService                         m_service;
-
-    std::array<std::function<void(uint64_t, std::span<const std::byte>)>, MAX_TYPES> m_handlers{};
+    std::unique_ptr<msg::MessageEndpoint> m_endpoint;
+    ProtobufMessages                      m_messages;
+    ChatService                           m_service;
 
 public:
     explicit ChatServerController(int port = CHAT_DEFAULT_PORT);
@@ -31,10 +21,7 @@ public:
 
 private:
     void register_handlers();
-    void dispatch(uint64_t client_id, std::span<const std::byte> data);
-    void send_to(uint64_t client_id, uint32_t type, std::span<const std::byte> payload,
-                 bool reliable = false);
-    void broadcast(uint64_t client_id, const ChatMessage& msg);
+    void broadcast(msg::PeerId client_id, const ChatMessage& msg);
 };
 
 } // namespace tw::chat

modules/chat_service/chat_service/tests/chat_mock_client/CMakeLists.txt

@@ -5,7 +5,8 @@ add_executable(${PROJECT_NAME} ChatMockClient.cpp)
 target_link_libraries(${PROJECT_NAME}
     PRIVATE
         tw::protocol
-        tw::messaging
+        tw::message_protocol
         tw::network
+        tw::quicr
         spdlog::spdlog
 )

modules/chat_service/chat_service/tests/chat_mock_client/ChatMockClient.cpp

@@ -1,7 +1,8 @@
 #include "Address.hpp"
-#include "MessageSession.hpp"
+#include "ProtobufMessages.hpp"
 #include "MessageRegistry.hpp"
 #include "Chat.pb.h"
+#include "message_protocol/MessageEndpoint.hpp"
 
 #include <spdlog/spdlog.h>
 #include <atomic>
@@ -56,30 +57,42 @@ int main(int argc, char* argv[]) {
         return 1;
     }
 
-    tw::MessageSession session(tw::net::Address{std::string{host}, port});
+    auto endpoint_r = tw::msg::MessageEndpoint::create();
+    if (!endpoint_r) {
+        spdlog::error("Failed to create an endpoint: {}", endpoint_r.error().message());
+        return 1;
+    }
+    auto& endpoint = endpoint_r.value();
+
+    auto server_r = endpoint->connect(host, port);
+    if (!server_r) {
+        spdlog::error("Failed to connect: {}", server_r.error().message());
+        return 1;
+    }
+    auto* server = server_r.value();
+
+    tw::ProtobufMessages messages(endpoint.get());
 
     spdlog::info("Connecting to {}:{}...", host, port);
     const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5);
-    while (!session.is_established()) {
+    while (!server->is_established()) {
         if (std::chrono::steady_clock::now() > deadline) {
             spdlog::error("Connection timed out");
             return 1;
         }
-        session.update();
+        endpoint->update();
         std::this_thread::sleep_for(std::chrono::milliseconds(10));
     }
     spdlog::info("Connected. Joining channel {}...", channel_id);
 
-    session.set_handler(tw::Message<mmo::chat::ChatMessageBroadcastRequest>::value,
-        [](std::span<const std::byte> data) {
-            mmo::chat::ChatMessageBroadcastRequest bcast;
-            bcast.ParseFromArray(data.data(), static_cast<int>(data.size()));
+    messages.set_handler<mmo::chat::ChatMessageBroadcastRequest>(
+        [](tw::msg::PeerId, const mmo::chat::ChatMessageBroadcastRequest& bcast) {
             std::println("[ch:{}] <{}> {}", bcast.channel_id(), bcast.sender_id(), bcast.message());
         });
 
     mmo::chat::JoinChannelRequest join;
     join.set_channel_id(channel_id);
-    (void)session.request(
+    (void)server->request(
         tw::Message<mmo::chat::JoinChannelRequest>::value,
         serialize(join),
         [channel_id](std::span<const std::byte> data) {
@@ -102,7 +115,7 @@ int main(int argc, char* argv[]) {
                 mmo::chat::SendChatMessageRequest msg;
                 msg.set_channel_id(channel_id);
                 msg.set_message(line);
-                (void)session.request(
+                (void)server->request(
                     tw::Message<mmo::chat::SendChatMessageRequest>::value,
                     serialize(msg),
                     [channel_id](std::span<const std::byte> data) {
@@ -113,7 +126,7 @@ int main(int argc, char* argv[]) {
             }
         }
 
-        session.update();
+        endpoint->update();
     }
 
     return 0;

modules/client/CMakeLists.txt

@@ -10,6 +10,7 @@ file(GLOB FILES
     src/draw/*.cpp
     src/draw/RenderPasses/*.cpp
     src/debug/*.cpp
+    src/debug/metrics/*.cpp
     src/debug/tools/*.cpp
 )
 
@@ -20,11 +21,14 @@ target_link_libraries(${PROJECT_NAME}
         PUBLIC
         towards
         tw::network
+        tw::metrics
+        tw::quicr
         loft::common
 		loft::base
 		loft_window
 		loft::render_graph
 		tw::protocol
+		tw::message_protocol
 		tw::serialization
 		tw::gui
         imgui::imgui

modules/client/src/debug/metrics/NetworkMetrics.hpp

@@ -0,0 +1,159 @@
+#pragma once
+
+#include "metrics/MetricSeries.hpp"
+
+#include <chrono>
+#include <cstdint>
+
+namespace tw::dbg {
+
+/**
+ * Per-second history of what the client sends, receives and waits for.
+ *
+ * Traffic arrives as running totals, one reading per tick: sample() keeps the
+ * change since the previous reading, so a bucket sums to the traffic of that
+ * second and its extremes are the quietest and busiest tick within it.
+ * Durations are recorded as they are measured.
+ */
+class NetworkMetrics {
+public:
+    using Interval = std::chrono::seconds;
+    using Series   = metrics::MetricSeries<Interval>;
+
+    /** How many seconds of history are kept. */
+    static constexpr size_t DEFAULT_HISTORY = 300;
+
+    /** Running totals as of one tick. */
+    struct Totals {
+        uint64_t bytes_sent        = 0;
+        uint64_t bytes_received    = 0;
+        uint64_t messages_sent     = 0;
+        uint64_t messages_received = 0;
+    };
+
+private:
+    Series m_bytes_out;
+    Series m_bytes_in;
+    Series m_messages_out;
+    Series m_messages_in;
+    Series m_response_ms;
+    Series m_update_ms;
+
+    Series m_rollbacks;
+    Series m_correction_distance;
+    Series m_ack_lag_frames;
+    Series m_replayed_frames;
+
+    Totals m_previous;
+    bool   m_has_previous = false;
+
+    static uint64_t delta(uint64_t current, uint64_t previous) {
+        return current > previous ? current - previous : 0;
+    }
+
+    static double to_millis(std::chrono::nanoseconds elapsed) {
+        return std::chrono::duration<double, std::milli>(elapsed).count();
+    }
+
+public:
+    explicit NetworkMetrics(size_t history_in_seconds = DEFAULT_HISTORY) :
+        m_bytes_out(history_in_seconds),
+        m_bytes_in(history_in_seconds),
+        m_messages_out(history_in_seconds),
+        m_messages_in(history_in_seconds),
+        m_response_ms(history_in_seconds),
+        m_update_ms(history_in_seconds),
+        m_rollbacks(history_in_seconds),
+        m_correction_distance(history_in_seconds),
+        m_ack_lag_frames(history_in_seconds),
+        m_replayed_frames(history_in_seconds)
+    { }
+
+    /**
+     * Records how much `totals` grew since the previous call. The first call
+     * only remembers where the counters started.
+     */
+    void sample(const Totals& totals) {
+        if(m_has_previous) {
+            m_bytes_out.push((double)delta(totals.bytes_sent, m_previous.bytes_sent));
+            m_bytes_in.push((double)delta(totals.bytes_received, m_previous.bytes_received));
+            m_messages_out.push((double)delta(totals.messages_sent, m_previous.messages_sent));
+            m_messages_in.push((double)delta(totals.messages_received, m_previous.messages_received));
+        }
+
+        m_previous     = totals;
+        m_has_previous = true;
+    }
+
+    /** Time between sending an input and seeing the answer to it. */
+    void record_response_time(std::chrono::nanoseconds elapsed) {
+        m_response_ms.push(to_millis(elapsed));
+    }
+
+    /** Time one tick spent moving messages in and out, handlers included. */
+    void record_update_time(std::chrono::nanoseconds elapsed) {
+        m_update_ms.push(to_millis(elapsed));
+    }
+
+    const Series& bytes_out() const {
+        return m_bytes_out;
+    }
+
+    const Series& bytes_in() const {
+        return m_bytes_in;
+    }
+
+    const Series& messages_out() const {
+        return m_messages_out;
+    }
+
+    const Series& messages_in() const {
+        return m_messages_in;
+    }
+
+    const Series& response_ms() const {
+        return m_response_ms;
+    }
+
+    const Series& update_ms() const {
+        return m_update_ms;
+    }
+
+    /** Records a rollback event (one sample per rollback). */
+    void record_rollback() {
+        m_rollbacks.push(1.0);
+    }
+
+    /** Records the distance in meters of a position correction. */
+    void record_correction_distance(double meters) {
+        m_correction_distance.push(meters);
+    }
+
+    /** Records how many frames behind the ack is trailing the current frame. */
+    void record_ack_lag(uint32_t frames) {
+        m_ack_lag_frames.push((double)frames);
+    }
+
+    /** Records how many frames were replayed during a rollback. */
+    void record_replayed_frames(uint32_t frames) {
+        m_replayed_frames.push((double)frames);
+    }
+
+    const Series& rollbacks() const {
+        return m_rollbacks;
+    }
+
+    const Series& correction_distance() const {
+        return m_correction_distance;
+    }
+
+    const Series& ack_lag_frames() const {
+        return m_ack_lag_frames;
+    }
+
+    const Series& replayed_frames() const {
+        return m_replayed_frames;
+    }
+};
+
+}

modules/client/src/debug/tools/MetricWidget.hpp

@@ -2,95 +2,109 @@
 
 #include <imgui.h>
 #include <implot.h>
-#include <implot_internal.h>
 
-#include "metrics/BucketMetric.hpp"
+#include "metrics/MetricSeries.hpp"
+
+#include <algorithm>
+#include <chrono>
+#include <string>
+#include <vector>
 
 namespace tw::dbg::tools {
 
-template<typename T, typename Interval, typename Op = net::SumOp<T>>
+/**
+ * Plots one series against the seconds behind now, ending at the last second
+ * that has fully elapsed.
+ *
+ * The line follows whichever statistic the series is read with. Reading an
+ * average also shades the quietest and busiest value of each second behind it;
+ * a total has no such range to show, since its buckets already hold every value
+ * of that second added together.
+ */
 class MetricWidget {
-    std::string m_name;
-
-    net::BucketMetric<T, Interval, Op>& m_metric;
-
-    using Self = MetricWidget<T, Interval, Op>;
-
-    struct {
-        T constraint_from;
-        T constraint_to;
-
-        T from;
-        T to;
-    } y_axis;
-    
-    bool m_is_scrolling = true;
-
 public:
-    MetricWidget(
-            const std::string& name,
-            net::BucketMetric<T, Interval, Op>& metric
-    ) :
-        m_name(name),
-        m_metric(metric)
-    {
-        y_axis = {
-            .constraint_from = 0,
-            .constraint_to = 500,
-            .from = 0,
-            .to = 250
-        };
+    using Series = metrics::MetricSeries<std::chrono::seconds>;
+
+private:
+    std::string                m_name;
+    std::string                m_unit;
+    const Series*              m_series;
+    metrics::MetricField       m_field;
+
+    /**
+     * The second in progress is left out: it only holds the part of itself
+     * that has elapsed, so drawing it makes the newest point drop and climb
+     * back once a second.
+     */
+    static constexpr size_t SKIP_IN_PROGRESS = 1;
+
+    std::vector<double> m_ages;
+    std::vector<double> m_values;
+    std::vector<double> m_lows;
+    std::vector<double> m_highs;
+
+    bool has_range() const {
+        return m_field == metrics::MetricField::Avg;
     }
 
-    Self& set_x_axis_limits_contraints(T from, T to) {
-        ImPlot::SetupAxisLimits(ImAxis_X1, from, to);
-        return *this;
-    }
-
-    Self& set_y_axis_limits(double from, double to) {
-        return *this;
-    }
-
-    Self& enable_scrolling() {
-        m_is_scrolling = true;
-    }
-
-    Self& disable_scrolling() {
-        m_is_scrolling = true;
-    }
-
-    
-    void draw() {
-        ImGui::PushID(m_name.c_str());
-
-        auto head = m_metric.get_head();
-        auto head_timeline = m_metric.get_head_timeline();
-
-        auto tail = m_metric.get_tail();
-        auto tail_timeline = m_metric.get_tail_timeline();
-
-        static float m_metric_history = 10.0f;
-        ImGui::Checkbox("Is Scrolling", &m_is_scrolling);
-        if(m_is_scrolling) {
-            ImGui::SliderFloat("History", &m_metric_history,1,30,"%.1f s");
+    /** Describes what is drawn, so the numbers always match the line. */
+    void draw_summary() const {
+        if(m_values.empty()) {
+            ImGui::TextUnformatted("no samples yet");
+            return;
         }
 
-        ImGui::Text("Min: %i", m_metric.min());
-        ImGui::Text("Max: %i", m_metric.max());
+        auto [low, high] = std::minmax_element(m_values.begin(), m_values.end());
 
-        if(ImPlot::BeginPlot(m_name.c_str())) {
-            auto from = tail_timeline.empty() ? *(head_timeline.end() - 1) : *(tail_timeline.end() - 1);
+        double total = 0.0;
+        for(double value : m_values) {
+            total += value;
+        }
 
-            ImPlot::SetupAxes("Time", m_name.c_str(), ImPlotAxisFlags_None, ImPlotAxisFlags_None);
-            if(m_is_scrolling) {
-                ImPlot::SetupAxisLimits(ImAxis_X1, from - m_metric_history, from, ImGuiCond_Always);
+        ImGui::Text("min %.1f %s   avg %.1f %s   max %.1f %s",
+            *low, m_unit.c_str(),
+            total / (double)m_values.size(), m_unit.c_str(),
+            *high, m_unit.c_str());
+    }
+
+public:
+    MetricWidget(std::string name, std::string unit, const Series& series, metrics::MetricField field) :
+        m_name(std::move(name)),
+        m_unit(std::move(unit)),
+        m_series(&series),
+        m_field(field)
+    { }
+
+    /** Draws the last `history_in_seconds` seconds of the series. */
+    void draw(size_t history_in_seconds) {
+        ImGui::PushID(m_name.c_str());
+
+        m_series->linearize(m_ages, m_values, m_field, history_in_seconds, SKIP_IN_PROGRESS);
+
+        if(has_range()) {
+            m_series->linearize(m_ages, m_lows, metrics::MetricField::Min,
+                history_in_seconds, SKIP_IN_PROGRESS);
+            m_series->linearize(m_ages, m_highs, metrics::MetricField::Max,
+                history_in_seconds, SKIP_IN_PROGRESS);
+        }
+
+        draw_summary();
+
+        if(ImPlot::BeginPlot(m_name.c_str(), ImVec2(-1.0f, 150.0f))) {
+            ImPlot::SetupAxes("seconds ago", m_unit.c_str(),
+                ImPlotAxisFlags_None, ImPlotAxisFlags_AutoFit);
+            ImPlot::SetupAxisLimits(ImAxis_X1, -(double)history_in_seconds, 0.0, ImGuiCond_Always);
+
+            const int count = (int)m_values.size();
+
+            if(has_range() && count > 0) {
+                ImPlot::PlotShaded("range", m_ages.data(), m_lows.data(), m_highs.data(), count);
             }
 
-            ImPlot::SetupAxisLimits(ImAxis_Y1, 0, m_metric.max() * 2, ImGuiCond_Always);
-            // ImPlot::SetupAxisLimitsConstraints(ImAxis_Y1, 0, 10000);
+            if(count > 0) {
+                ImPlot::PlotLine(m_name.c_str(), m_ages.data(), m_values.data(), count);
+            }
 
-            ImPlot::PlotLine(m_name.c_str(), head_timeline.data(), head.data(), head.size());
-            ImPlot::PlotLine(m_name.c_str(), tail_timeline.data(), tail.data(), tail.size());
             ImPlot::EndPlot();
         }
 

modules/client/src/debug/tools/NetworkStatsGui.hpp

@@ -1,59 +1,68 @@
 #pragma once
 
-#include "metrics/BucketMetric.hpp"
-#include "metrics/NetworkStatsLogger.hpp"
+#include "debug/metrics/NetworkMetrics.hpp"
 #include "debug/tools/MetricWidget.hpp"
 
-#include <implot.h>
-#include <implot_internal.h>
+#include <imgui.h>
 
 namespace tw::dbg::tools {
 
+/**
+ * Panel over everything the client measured about its traffic.
+ *
+ * Traffic is shown as the total of each second, since that is the rate the
+ * connection actually carried. Durations are shown as the average of each
+ * second, with the range behind them.
+ */
 class NetworkStatsGui {
-private:
-    // MetricWidget<uint32_t, std::chrono::seconds, net::AverageOp<uint32_t>> m_ping_widget;
-    // MetricWidget<uint32_t, std::chrono::seconds> m_outgoing_widget;
-    // MetricWidget<uint32_t, std::chrono::seconds> m_incoming_widget;
+    MetricWidget m_response;
+    MetricWidget m_update;
+    MetricWidget m_bytes_in;
+    MetricWidget m_bytes_out;
+    MetricWidget m_messages_in;
+    MetricWidget m_messages_out;
+
+    MetricWidget m_rollbacks;
+    MetricWidget m_correction_distance;
+    MetricWidget m_ack_lag_frames;
+    MetricWidget m_replayed_frames;
+
+    int m_history_in_seconds = 30;
 
 public:
-    // NetworkStatsGui() :
-    //     m_ping_widget("Ping", net::NetworkStatsLogger::instance()->ping()),
-    //     m_outgoing_widget("Outgoing", net::NetworkStatsLogger::instance()->outgoing()),
-    //     m_incoming_widget("Incoming", net::NetworkStatsLogger::instance()->incoming())
-    // {
-    // }
+    explicit NetworkStatsGui(const NetworkMetrics& metrics) :
+        m_response("Response", "ms", metrics.response_ms(), metrics::MetricField::Avg),
+        m_update("Network update", "ms", metrics.update_ms(), metrics::MetricField::Avg),
+        m_bytes_in("Bytes in", "B/s", metrics.bytes_in(), metrics::MetricField::Sum),
+        m_bytes_out("Bytes out", "B/s", metrics.bytes_out(), metrics::MetricField::Sum),
+        m_messages_in("Messages in", "1/s", metrics.messages_in(), metrics::MetricField::Sum),
+        m_messages_out("Messages out", "1/s", metrics.messages_out(), metrics::MetricField::Sum),
+        m_rollbacks("Rollbacks", "1/s", metrics.rollbacks(), metrics::MetricField::Sum),
+        m_correction_distance("Correction distance", "m", metrics.correction_distance(), metrics::MetricField::Avg),
+        m_ack_lag_frames("Ack lag", "frames", metrics.ack_lag_frames(), metrics::MetricField::Avg),
+        m_replayed_frames("Replayed frames", "frames", metrics.replayed_frames(), metrics::MetricField::Avg)
+    { }
 
     void draw() {
-        // auto* instance = net::NetworkStatsLogger::instance();
-        // auto& ping = instance->ping();
-
         ImGui::Begin("Network Stats");
 
-        /* auto head = ping.get_head();
-        auto head_timeline = ping.get_head_timeline();
+        ImGui::SliderInt("History", &m_history_in_seconds, 5, 300, "%d s");
 
-        auto tail = ping.get_tail();
-        auto tail_timeline = ping.get_tail_timeline();
+        const size_t history = (size_t)m_history_in_seconds;
 
-        static float ping_history = 10.0f;
-        ImGui::SliderFloat("Ping History", &ping_history,1,30,"%.1f s");
+        m_response.draw(history);
+        m_update.draw(history);
+        m_bytes_in.draw(history);
+        m_bytes_out.draw(history);
+        m_messages_in.draw(history);
+        m_messages_out.draw(history);
 
-        if(ImPlot::BeginPlot("Ping")) {
-            auto from = tail_timeline.empty() ? *(head_timeline.end() - 1) : *(tail_timeline.end() - 1);
-
-            ImPlot::SetupAxes("FrameIdx","FPS", ImPlotAxisFlags_None, ImPlotAxisFlags_None);
-            ImPlot::SetupAxisLimits(ImAxis_X1, from - ping_history, from, ImGuiCond_Always);
-            ImPlot::SetupAxisLimits(ImAxis_Y1, 0, 120);
-            ImPlot::SetupAxisLimitsConstraints(ImAxis_Y1, 0, 10000);
-
-            ImPlot::PlotLine("Ping", head_timeline.data(), head.data(), head.size());
-            ImPlot::PlotLine("Ping", tail_timeline.data(), tail.data(), tail.size());
-            ImPlot::EndPlot();
-        } */
-
-        // m_ping_widget.draw();
-        // m_outgoing_widget.draw();
-        // m_incoming_widget.draw();
+        ImGui::Separator();
+        ImGui::TextUnformatted("Prediction");
+        m_rollbacks.draw(history);
+        m_correction_distance.draw(history);
+        m_ack_lag_frames.draw(history);
+        m_replayed_frames.draw(history);
 
         ImGui::End();
     }

modules/client/src/debug/tools/PacketBacklogGui.hpp

@@ -1,57 +0,0 @@
-#pragma once
-
-#include <format>
-#include <vector>
-
-#include "imgui.h"
-
-#include "metrics/NetworkStatsLogger.hpp"
-
-namespace tw::dbg::tools {
-
-class PacketBacklogGui {
-private:
-    std::vector<uint32_t> m_buckets;
-    uint32_t m_last_backlog_idx;
-
-public:
-    void draw() {
-        // auto* instance = net::NetworkStatsLogger::instance();
-        // size_t size = instance->get_size();
-
-        // if(ImGui::BeginTable("Network Packets", 5)) {
-        //     ImGui::TableSetupColumn("Message Type");
-        //     ImGui::TableSetupColumn("Time");
-        //     ImGui::TableSetupColumn("Is From Us");
-        //     ImGui::TableSetupColumn("Target");
-        //     ImGui::TableSetupColumn("Size");
-
-        //     for(int32_t i = size-1; i >= 0; i--) {
-        //         auto& item = instance->get_item(i);
-        //         ImGui::PushID(item.timepoint.time_since_epoch().count());
-
-        //         ImGui::TableNextRow();
-
-        //         ImGui::TableNextColumn();
-        //         ImGui::Text("%i", item.message_type);
-
-        //         ImGui::TableNextColumn();
-        //         ImGui::Text(std::format("{}", item.timepoint.time_since_epoch()).c_str());
-
-        //         ImGui::TableNextColumn();
-        //         ImGui::Checkbox("is_sent_from_us", &item.is_sent_by_us);
-
-        //         ImGui::TableNextColumn();
-        //         ImGui::Text(item.target.to_string().c_str());
-
-        //         ImGui::TableNextColumn();
-        //         ImGui::Text("%ld", item.buffer.size());
-
-        //         ImGui::PopID();
-        //     }
-        //     ImGui::EndTable();
-        // }
-    }
-};
-
-}

modules/client/src/debug/tools/PacketLogger.hpp

@@ -2,7 +2,6 @@
 
 #include "Address.hpp"
 #include "ByteBuffer.hpp"
-#include "packets/Packet.hpp"
 #include <cstring>
 
 namespace tw::dbg::tools {

modules/client/src/network/PlayerReconciler.cpp

@@ -0,0 +1,122 @@
+#include "PlayerReconciler.hpp"
+
+#include "world/JoltPhysicsWorld.hpp"
+#include "world/CharacterBody.hpp"
+#include "world/Transform.hpp"
+#include <spdlog/spdlog.h>
+#include <glm/glm.hpp>
+#include <glm/gtx/norm.hpp>
+
+namespace tw::net {
+
+PlayerReconciler::PlayerReconciler(JoltPhysicsWorld* physics)
+    : m_physics(physics), m_last_reconciled_ack(0), m_rollback_count(0),
+      m_last_correction_distance(0.0f), m_last_replayed_frames(0), m_last_ack_frame(0)
+{
+    for (auto& record : m_records) {
+        record.frame = 0;
+        record.valid = false;
+        record.input = glm::vec3(0.0f);
+        record.predicted_position = glm::vec3(0.0f);
+    }
+}
+
+void PlayerReconciler::record_input(uint32_t frame, glm::vec3 input) {
+    size_t idx = frame % RING_SIZE;
+    m_records[idx].frame = frame;
+    m_records[idx].valid = true;
+    m_records[idx].input = input;
+}
+
+void PlayerReconciler::record_prediction(uint32_t frame, glm::vec3 position) {
+    size_t idx = frame % RING_SIZE;
+    if (m_records[idx].frame == frame && m_records[idx].valid) {
+        m_records[idx].predicted_position = position;
+    }
+}
+
+bool PlayerReconciler::reconcile(uint32_t ack_frame, glm::vec3 authoritative_position,
+                                  entt::entity player, entt::registry* registry,
+                                  uint32_t current_frame)
+{
+    if (ack_frame == 0 || ack_frame <= m_last_reconciled_ack || ack_frame >= current_frame) {
+        return false;
+    }
+
+    m_last_reconciled_ack = ack_frame;
+    m_last_ack_frame = ack_frame;
+
+    Record& record = m_records[ack_frame % RING_SIZE];
+    const bool has_prediction = record.valid && record.frame == ack_frame;
+
+    // With a prediction to compare against, an answer that already matches costs
+    // nothing further. This is the case almost every frame.
+    if (has_prediction) {
+        float distance = glm::distance(record.predicted_position, authoritative_position);
+        m_last_correction_distance = distance;
+
+        if (distance < kPositionEpsilon) {
+            return false;
+        }
+    }
+
+    // Restoring the frame the answer describes keeps everything the simulation
+    // derived from it, so only the character has to be moved. Without a stored
+    // frame there is nothing to restore and the answer is taken as it stands.
+    const bool restored = has_prediction && m_physics->rollback(ack_frame);
+
+    place_character(player, registry, authoritative_position, !restored);
+    replay_from(ack_frame, player, registry, current_frame);
+
+    m_last_replayed_frames = current_frame - 1 - ack_frame;
+    m_rollback_count++;
+
+    spdlog::debug("Corrected at frame {}: distance {}, replayed {}, restored {}",
+                  ack_frame, m_last_correction_distance, m_last_replayed_frames, restored);
+
+    return true;
+}
+
+void PlayerReconciler::place_character(entt::entity player, entt::registry* registry,
+                                       glm::vec3 position, bool clear_velocity) {
+    CharacterBody* body = registry->try_get<CharacterBody>(player);
+    if (!body) {
+        return;
+    }
+
+    body->m_character->SetPosition(JPH::RVec3(position.x, position.y, position.z));
+
+    if (clear_velocity) {
+        body->m_character->SetLinearVelocity(JPH::Vec3::sZero());
+        body->m_desired_velocity = JPH::Vec3::sZero();
+    }
+}
+
+void PlayerReconciler::replay_from(uint32_t from_frame, entt::entity player,
+                                   entt::registry* registry, uint32_t current_frame) {
+    for (uint32_t f = from_frame + 1; f < current_frame; ++f) {
+        m_physics->step(f, tw::JoltPhysicsWorld::FIXED_DELTA_TIME, true);
+
+        Transform* transform = registry->try_get<Transform>(player);
+        if (!transform) {
+            continue;
+        }
+
+        // Frames the ring never saw still need an entry, or the answer to them
+        // arrives with nothing to compare against and forces another correction.
+        Record& replayed = m_records[f % RING_SIZE];
+        replayed.frame = f;
+        replayed.valid = true;
+        replayed.predicted_position = transform->position();
+    }
+}
+
+void PlayerReconciler::reset_at(uint32_t frame) {
+    m_last_reconciled_ack = frame;
+
+    for (auto& record : m_records) {
+        record.valid = false;
+    }
+}
+
+}

modules/client/src/network/PlayerReconciler.hpp

@@ -0,0 +1,72 @@
+#pragma once
+
+#include <cstdint>
+#include <array>
+#include <glm/glm.hpp>
+#include <entt/entt.hpp>
+
+namespace tw {
+class JoltPhysicsWorld;
+}
+
+namespace tw::net {
+
+class PlayerReconciler {
+private:
+    struct Record {
+        uint32_t frame;
+        bool valid;
+        glm::vec3 input;
+        glm::vec3 predicted_position;
+    };
+
+    static constexpr size_t RING_SIZE = 64;
+    static constexpr float kPositionEpsilon = 0.05f;
+
+    tw::JoltPhysicsWorld* m_physics;
+    std::array<Record, RING_SIZE> m_records;
+
+    uint32_t m_last_reconciled_ack = 0;
+    uint64_t m_rollback_count = 0;
+    float m_last_correction_distance = 0.0f;
+    uint32_t m_last_replayed_frames = 0;
+    uint32_t m_last_ack_frame = 0;
+
+public:
+    PlayerReconciler(tw::JoltPhysicsWorld* physics);
+
+    void record_input(uint32_t frame, glm::vec3 input);
+    void record_prediction(uint32_t frame, glm::vec3 position);
+
+    bool reconcile(uint32_t ack_frame, glm::vec3 authoritative_position,
+                   entt::entity player, entt::registry* registry, uint32_t current_frame);
+
+private:
+    /**
+     * Re-simulates `from_frame + 1` up to the newest frame, refreshing the stored
+     * prediction for each. The inputs come from the character itself, so this
+     * works even for frames this ring never recorded.
+     */
+    void replay_from(uint32_t from_frame, entt::entity player,
+                     entt::registry* registry, uint32_t current_frame);
+
+    /** Places the character at `position` without disturbing the stored frames. */
+    void place_character(entt::entity player, entt::registry* registry,
+                         glm::vec3 position, bool clear_velocity);
+
+public:
+
+    /**
+     * Drops every stored frame and treats `frame` as already answered. Used when
+     * the player is placed outright, where nothing recorded before the placement
+     * describes where it now is.
+     */
+    void reset_at(uint32_t frame);
+
+    uint64_t rollback_count() const { return m_rollback_count; }
+    float last_correction_distance() const { return m_last_correction_distance; }
+    uint32_t last_replayed_frames() const { return m_last_replayed_frames; }
+    uint32_t last_ack_frame() const { return m_last_ack_frame; }
+};
+
+}

modules/client/src/runtime.cpp

@@ -3,7 +3,6 @@
 #include "Address.hpp"
 #include "SDLWindow.h"
 #include "debug/tools/NetworkStatsGui.hpp"
-#include "debug/tools/PacketBacklogGui.hpp"
 #include "entt/entity/fwd.hpp"
 #include "io/InputState.hpp"
 #include "debug/tools/EntityManagerGui.hpp"
@@ -52,7 +51,8 @@ Runtime::Runtime(int argc, char** argv) :
     m_physics_world(&m_world),
     m_world_renderer("towards", m_window.get(), &m_world, &m_files),
     m_input_manager(m_window.get()),
-    m_world_controller(&m_input_manager, &m_world, &m_physics_world, &m_world_renderer, { "127.0.0.1", get_port_from_args(argc, argv) }),
+    m_network_metrics(),
+    m_world_controller(&m_input_manager, &m_world, &m_physics_world, &m_world_renderer, { "127.0.0.1", get_port_from_args(argc, argv) }, &m_network_metrics),
     m_lockstep(60)
 {
 }
@@ -95,8 +95,7 @@ Runtime::Runtime(int argc, char** argv) :
 void Runtime::run() {
     dbg::tools::EntityManagerGui entity_manager(&m_world);
     dbg::tools::PerformanceStatsGui perf_stats(m_lockstep);
-    dbg::tools::PacketBacklogGui packet_backlog;
-    dbg::tools::NetworkStatsGui network_stats;
+    dbg::tools::NetworkStatsGui network_stats(m_network_metrics);
 
     ImPlot::CreateContext();
     m_is_running = true;
@@ -147,7 +146,6 @@ void Runtime::run() {
         m_world_controller.update(m_lockstep.delta_time());
 
         perf_stats.draw();
-        packet_backlog.draw();
         network_stats.draw();
         tw::dbg::ComponentGui<tw::io::InputManager>().draw(&m_input_manager);
 

modules/client/src/runtime.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "debug/metrics/NetworkMetrics.hpp"
 #include "draw/WorldRenderer.hpp"
 #include "io/InputState.hpp"
 #include "runtime/LockStep.hpp"
@@ -28,6 +29,8 @@ private:
 
     tw::io::InputManager m_input_manager;
 
+    tw::dbg::NetworkMetrics m_network_metrics;
+
     tw::ClientWorldController m_world_controller;
 
     tw::LockStep m_lockstep;

modules/client/src/world/Camera.hpp

@@ -14,8 +14,7 @@ struct CameraData {
     CameraData(glm::mat4 projection, Transform view) :
         projection(projection),
         view(view)
-    {
-    }
+    { }
 };
 
 class Camera {

modules/client/src/world/ClientWorldController.cpp

@@ -12,9 +12,7 @@
 #include "PlayerMove.pb.h"
 
 #include "entt/entity/entity.hpp"
-#include "messenger/MessageHandler.hpp"
-#include "messenger/Messenger.hpp"
-#include "TcpStream.hpp"
+#include "entt/entity/fwd.hpp"
 #include "messages/PlayerMoveMessage.hpp"
 #include "metrics/HistoryBuffer.hpp"
 #include "world/CharacterBody.hpp"
@@ -22,6 +20,7 @@
 #include "world/JoltPhysicsWorld.hpp"
 #include "world/WorldEntity.hpp"
 #include "tw/serial/WorldStateWriter.hpp"
+#include "network/EntityInterpolation.hpp"
 
 namespace tw {
 
@@ -83,16 +82,26 @@ ClientWorldController::create_entity(const std::string& name, glm::vec3 position
     return entity;
 }
 
-net::TcpStream create_stream(tw::net::Address address) {
-    auto stream = net::TcpStream::connect(address);
-    if(!stream.has_value()) {
-        spdlog::error("Failed to connect to server");
+static std::unique_ptr<msg::MessageEndpoint> create_endpoint() {
+    auto endpoint_r = msg::MessageEndpoint::create();
+    if(!endpoint_r) {
+        spdlog::error("Failed to create the endpoint: {}", endpoint_r.error().message());
+        throw std::runtime_error("Failed to create the endpoint");
+    }
+
+    return std::move(endpoint_r.value());
+}
+
+static msg::MessageConnection* connect_to_server(msg::MessageEndpoint* endpoint, net::Address address) {
+    auto server_r = endpoint->connect(address.ip_string(), address.port());
+    if(!server_r) {
+        spdlog::error("Failed to connect to server: {}", server_r.error().message());
         throw std::runtime_error("Failed to connect to server");
     }
 
-    stream.value().set_non_blocking();
+    spdlog::info("Connected to server at {}", address.to_string());
 
-    return std::move(stream.value());
+    return server_r.value();
 }
 
 std::optional<entt::entity> ClientWorldController::map_from_server_entity(int id) {
@@ -134,33 +143,49 @@ ClientWorldController::ClientWorldController(
         World* world,
         JoltPhysicsWorld* physics_world,
         drw::WorldRenderer* world_renderer,
-        tw::net::Address address
+        tw::net::Address address,
+        dbg::NetworkMetrics* network_metrics
 ) :
     m_input_manager(inputs),
     m_world(world),
     m_physics_world(physics_world),
     m_world_renderer(world_renderer),
-    m_player_entity(create_player_entity(world, physics_world, world_renderer)),
+    // m_player_entity(/* create_player_entity(world, physics_world, world_renderer) */),
     m_player_controller(&world_renderer->camera(), glm::vec3()),
-    m_messenger{address},
+    m_endpoint(create_endpoint()),
+    m_server(connect_to_server(m_endpoint.get(), address)),
+    m_messages(m_endpoint.get()),
+    m_network_metrics(network_metrics),
     m_tick_step(20),
     m_is_connected(false),
+    m_input_send_times(INPUT_SEND_TIME_COUNT),
     m_position_history_exporter("/home/martin/output.csv"),
-    m_entity_interpolator(&m_world->registry(), m_player_entity, 300)
+    m_entity_interpolator(&m_world->registry(), (entt::entity)0, 300),
+    m_reconciler(physics_world)
 {
-    m_messenger->set_handler<mmo::LoginResponse>(
-        [&](mmo::LoginResponse* mesg) {
+    m_messages.set_handler<mmo::LoginResponse>(
+        [this](msg::PeerId, const mmo::LoginResponse& mesg) {
             if(!m_is_connected) {
-                spdlog::info("Joined the game!");
+                spdlog::info("Logged in!");
             }
         });
 
-    m_messenger->set_raw_handler(Message<mmo::WorldStateMessage>::value,
-        [&](std::span<std::byte> data) -> tl::expected<void, net::NetworkError> {
+    m_messages.set_handler<mmo::SetControlledEntity>(
+        [this](msg::PeerId, const mmo::SetControlledEntity& mesg) {
+            spdlog::info("Setting controlled entity from server id {}", mesg.entity_id());
+            m_controlled_server_id = mesg.entity_id();
+            try_bind_player_entity();
+        });
+
+    m_endpoint->set_handler(Message<mmo::WorldStateMessage>::value,
+        [this](msg::PeerId, std::span<const std::byte> data) {
+
             serial::WorldStateReader reader(data);
 
             auto header = reader.read_header();
 
+            measure_response_time(header.frame_idx);
+
             while(reader.has_spawn()) {
                 auto spawn = reader.read_spawn();
                 auto entity = create_entity("test", glm::vec3());
@@ -168,7 +193,11 @@ ClientWorldController::ClientWorldController(
 
                 map_server_entity(spawn, entity);
 
-                m_entity_interpolator.register_entity(entity);
+                if(m_controlled_server_id.has_value() && m_controlled_server_id.value() == spawn) {
+                    try_bind_player_entity();
+                } else {
+                    m_entity_interpolator.register_entity(entity);
+                }
             }
 
             while(reader.has_entity()) {
@@ -182,7 +211,57 @@ ClientWorldController::ClientWorldController(
                 }
 
                 glm::vec3 p = {entity_r.position.x, entity_r.position.y, entity_r.position.z};
-                m_entity_interpolator.add_position_for_entity(entity.value(), p);
+
+                if(m_player_entity.has_value() && entity.value() == m_player_entity.value()) {
+                    // The entity was created before its position was known, so the
+                    // body sits at the origin until the server places it. There is
+                    // no predicted history to reconcile against yet.
+                    if(!m_player_position_initialized) {
+                        m_player_position_initialized = true;
+                        snap_player_to(entity.value(), p);
+                        continue;
+                    }
+
+                    glm::vec3 position_before = glm::vec3(0.0f);
+                    Transform* player_transform = m_world->registry().try_get<Transform>(entity.value());
+                    if(player_transform) {
+                        position_before = player_transform->position();
+                    }
+
+                    bool reconcile_happened = m_reconciler.reconcile(header.frame_idx, p, entity.value(), &m_world->registry(), m_frame_idx);
+
+                    if(reconcile_happened) {
+                        // Where the replay actually ended up, which is ahead of the
+                        // acked position by the frames that were re-simulated.
+                        glm::vec3 position_after = player_transform
+                            ? player_transform->position()
+                            : p;
+                        glm::vec3 correction_delta = position_before - position_after;
+                        float correction_magnitude = glm::length(correction_delta);
+                        if(correction_magnitude > 5.0f) {
+                            correction_delta = glm::normalize(correction_delta) * 5.0f;
+                        }
+                        m_visual_error += correction_delta;
+
+                        m_render_curr_position = position_after;
+                        m_render_prev_position = position_after;
+                        m_tick_accumulator = 0.0;
+
+                        m_network_metrics->record_rollback();
+                        m_network_metrics->record_correction_distance(m_reconciler.last_correction_distance());
+                        m_network_metrics->record_replayed_frames(m_reconciler.last_replayed_frames());
+                    }
+
+                    if(header.frame_idx != 0) {
+                        uint32_t ack_lag = 0;
+                        if(m_frame_idx >= header.frame_idx) {
+                            ack_lag = m_frame_idx - header.frame_idx;
+                        }
+                        m_network_metrics->record_ack_lag(ack_lag);
+                    }
+                } else {
+                    m_entity_interpolator.add_position_for_entity(entity.value(), p);
+                }
 
                 EntityPositionHistory* history = m_world->registry().try_get<EntityPositionHistory>(entity.value());
 
@@ -193,33 +272,111 @@ ClientWorldController::ClientWorldController(
             }
 
             // apply_entity_positions();
-
-
-            return {};
         });
 
-    m_messenger->set_handler<mmo::EntitySpawnMessage>(
-        [&](mmo::EntitySpawnMessage* mesg) {
-            auto entity = create_entity(mesg->name(), glm::vec3());
+    m_messages.set_handler<mmo::EntitySpawnMessage>(
+        [this](msg::PeerId, const mmo::EntitySpawnMessage& mesg) {
+            auto entity = create_entity(mesg.name(), glm::vec3());
 
-            map_server_entity(mesg->entity_id(), entity);
+            map_server_entity(mesg.entity_id(), entity);
 
-            m_entity_interpolator.register_entity(entity);
+            if(m_controlled_server_id.has_value() && m_controlled_server_id.value() == mesg.entity_id()) {
+                try_bind_player_entity();
+            } else {
+                m_entity_interpolator.register_entity(entity);
+            }
         });
 }
 
 ClientWorldController::~ClientWorldController() {
 }
 
+void ClientWorldController::try_bind_player_entity() {
+    if(!m_controlled_server_id.has_value()) {
+        return;
+    }
+
+    auto local_entity = map_from_server_entity(m_controlled_server_id.value());
+    if(!local_entity.has_value()) {
+        return;
+    }
+
+    if(m_player_entity.has_value() && m_player_entity.value() == local_entity.value()) {
+        return;
+    }
+
+    entt::entity entity = local_entity.value();
+    spdlog::info("Binding player entity");
+
+    m_player_entity = entity;
+
+    Transform* transform = m_world->registry().try_get<Transform>(entity);
+    glm::vec3 position = transform ? transform->position() : glm::vec3(0.0f);
+
+    m_world->registry().emplace<CharacterController>(entity, 20.0f);
+    m_world->registry().emplace<CharacterBody>(entity, m_physics_world->create_character(
+        new JPH::BoxShape(JPH::Vec3Arg(0.5f, 0.5f, 0.5f)),
+        position
+    ));
+
+    if(m_world->registry().all_of<net::EntityPositionInterpolation>(entity)) {
+        m_world->registry().remove<net::EntityPositionInterpolation>(entity);
+    }
+}
+
+void ClientWorldController::snap_player_to(entt::entity entity, glm::vec3 position) {
+    CharacterBody* body = m_world->registry().try_get<CharacterBody>(entity);
+    if(body) {
+        body->m_character->SetPosition(JPH::RVec3(position.x, position.y, position.z));
+        body->m_character->SetLinearVelocity(JPH::Vec3::sZero());
+        body->m_desired_velocity = JPH::Vec3::sZero();
+    }
+
+    Transform* transform = m_world->registry().try_get<Transform>(entity);
+    if(transform) {
+        transform->set_position(position);
+    }
+
+    m_render_prev_position = position;
+    m_render_curr_position = position;
+    m_tick_accumulator     = 0.0;
+    m_visual_error         = glm::vec3(0.0f);
+
+    // Frames simulated before the player was placed describe a position it never
+    // actually had, so answers to them must not be reconciled against.
+    m_reconciler.reset_at(m_frame_idx);
+}
+
 void ClientWorldController::export_entity_history() {
 }
 
+void ClientWorldController::measure_response_time(uint32_t frame_idx) {
+    // Snapshots carry frame zero until the server has an input to answer, and
+    // repeat the same frame whenever no newer one arrived in between.
+    if(frame_idx == 0 || frame_idx <= m_last_measured_frame) {
+        return;
+    }
+
+    // Anything the send times no longer cover, including a frame we never sent,
+    // which underflows into a large distance.
+    if(m_frame_idx - frame_idx >= INPUT_SEND_TIME_COUNT) {
+        return;
+    }
+
+    m_last_measured_frame = frame_idx;
+
+    auto sent_at = m_input_send_times[frame_idx % INPUT_SEND_TIME_COUNT];
+    m_network_metrics->record_response_time(Clock::now() - sent_at);
+}
+
 void ClientWorldController::update(double delta_time) {
     m_player_controller.update(m_input_manager, delta_time);
 
     ImGui::Begin("Player Controller");
+    if(m_player_entity.has_value()) {
+        ImGui::Text("Player entity ID: %d", (uint32_t)m_player_entity.value());
+    }
 
-    ImGui::Text("Player entity ID: %d", (uint32_t)m_player_entity);
     ImGui::Text("Player count: %ld", m_entity_mapping.size());
     for(auto mapping : m_entity_mapping) {
         ImGui::Text("%d -> %d", (uint32_t)mapping.first, mapping.second);
@@ -228,34 +385,57 @@ void ClientWorldController::update(double delta_time) {
     ImGui::End();
 
     if(m_tick_step.update()) {
-        m_messenger->update();
+        auto network_start = Clock::now();
+        m_endpoint->update();
+        m_network_metrics->record_update_time(Clock::now() - network_start);
+
+        m_network_metrics->sample({
+            .bytes_sent        = m_endpoint->bytes_sent(),
+            .bytes_received    = m_endpoint->bytes_received(),
+            .messages_sent     = m_endpoint->messages_sent(),
+            .messages_received = m_endpoint->messages_received()
+        });
 
         if(!m_is_connected && false) {
             return;
         } else {
+            glm::vec3 input = m_player_controller.input();
 
-            // CharacterController& character = m_world->registry().get<CharacterController>(m_player_entity);
-            // character.set_input(m_frame_idx, m_player_controller.input());
+            if(m_player_entity.has_value()) {
+                CharacterController* controller = m_world->registry().try_get<CharacterController>(m_player_entity.value());
+                if(controller) {
+                    controller->set_input(m_frame_idx, input);
+                    m_reconciler.record_input(m_frame_idx, input);
+                }
+            }
+
+            m_physics_world->step(m_frame_idx, JoltPhysicsWorld::FIXED_DELTA_TIME, true);
+
+            if(m_player_entity.has_value()) {
+                Transform* player_transform = m_world->registry().try_get<Transform>(m_player_entity.value());
+                if(player_transform) {
+                    glm::vec3 true_position = player_transform->position();
+                    m_reconciler.record_prediction(m_frame_idx, true_position);
+
+                    m_render_prev_position = m_render_curr_position;
+                    m_render_curr_position = true_position;
+                    m_tick_accumulator = 0.0;
+                }
+            }
 
             mmo::PlayerMoveMessage player_move_message = {};
 
             player_move_message.set_frame_idx(m_frame_idx);
             mmo::PlayerInput* player_input = new mmo::PlayerInput();
-            player_input->set_x(m_player_controller.input().x);
-            player_input->set_y(m_player_controller.input().y);
-            player_input->set_z(m_player_controller.input().z);
+            player_input->set_x(input.x);
+            player_input->set_y(input.y);
+            player_input->set_z(input.z);
 
             player_move_message.set_allocated_input(player_input);
-            auto r = m_messenger->send(player_move_message);
+            auto r = m_messages.send(m_server, player_move_message, false);
 
-            // CharacterBody& ts = m_world->registry().get<CharacterBody>(m_player_entity);
-            // auto position = ts.m_character->GetPosition();
-            // character.position_history().set(m_frame_idx, glm::vec3(position[0], position[1], position[2]));
-            // EntityInterpolation& interpolation = m_world->registry().get<EntityInterpolation>(m_player_entity);
-            // interpolation.push(std::chrono::steady_clock::now(), glm::vec3(position[0], position[1], position[2]));
+            m_input_send_times[m_frame_idx % INPUT_SEND_TIME_COUNT] = Clock::now();
 
-            // m_player_controller.set_target(glm::vec3(position[0], position[1], position[2]));
-            //
             export_entity_history();
 
             m_frame_idx++;
@@ -268,9 +448,27 @@ void ClientWorldController::update(double delta_time) {
         }
     }
 
-    m_physics_world->step(m_frame_idx, delta_time);
-
     m_entity_interpolator.update();
+
+    if(m_player_entity.has_value()) {
+        Transform* player_transform = m_world->registry().try_get<Transform>(m_player_entity.value());
+        if(player_transform) {
+            m_visual_error *= std::exp(-delta_time * kVisualErrorDecayRate);
+            if(glm::length(m_visual_error) < 0.001f) {
+                m_visual_error = glm::vec3(0.0f);
+            }
+
+            m_tick_accumulator += delta_time;
+            float alpha = glm::clamp(
+                static_cast<float>(m_tick_accumulator / JoltPhysicsWorld::FIXED_DELTA_TIME),
+                0.0f, 1.0f
+            );
+            glm::vec3 smoothed_position = glm::mix(m_render_prev_position, m_render_curr_position, alpha) + m_visual_error;
+            player_transform->set_position(smoothed_position);
+
+            m_player_controller.set_target(smoothed_position);
+        }
+    }
 }
 
 }

modules/client/src/world/ClientWorldController.hpp

@@ -4,8 +4,9 @@
 #include <glm/gtx/io.hpp>
 
 #include "Address.hpp"
-#include "TcpStream.hpp"
-#include "messenger/MessageHandler.hpp"
+#include "ProtobufMessages.hpp"
+#include "debug/metrics/NetworkMetrics.hpp"
+#include "message_protocol/MessageEndpoint.hpp"
 #include "entt/entity/fwd.hpp"
 #include "io/InputState.hpp"
 #include "metrics/HistoryBufferExporter.hpp"
@@ -15,6 +16,7 @@
 #include "draw/WorldRenderer.hpp"
 #include "world/ThirdPersonPlayerController.hpp"
 #include "network/EntityPositionInterpolator.hpp"
+#include "network/PlayerReconciler.hpp"
 
 
 namespace tw {
@@ -35,10 +37,15 @@ class ClientWorldController {
     drw::WorldRenderer* m_world_renderer;
     JoltPhysicsWorld* m_physics_world;
 
-    entt::entity m_player_entity;
+    std::optional<entt::entity> m_player_entity;
+    std::optional<uint32_t> m_controlled_server_id;
     ThirdPersonPlayerController m_player_controller;
 
-    std::optional<tw::net::MessageHandler> m_messenger;
+    std::unique_ptr<msg::MessageEndpoint> m_endpoint;
+    msg::MessageConnection*               m_server;
+    ProtobufMessages                      m_messages;
+
+    dbg::NetworkMetrics* m_network_metrics;
 
     LockStep m_tick_step;
 
@@ -52,10 +59,59 @@ class ClientWorldController {
     std::optional<drw::Mesh> m_mesh;
 
     using Clock = std::chrono::steady_clock;
+
+    void try_bind_player_entity();
+
+    /**
+     * Places the player at an authoritative position outright, clearing the
+     * predicted state that led there. Used for the first position the server
+     * sends, which the local simulation has no history to reconcile against.
+     */
+    void snap_player_to(entt::entity entity, glm::vec3 position);
+
+    /**
+     * Whether the server has placed the player at least once. Entities are
+     * created before their position arrives, so the body starts at the origin
+     * and has to be moved once the first position shows up.
+     */
+    bool m_player_position_initialized = false;
+
+    /**
+     * When each input was sent, indexed by its frame. Holds the most recent
+     * INPUT_SEND_TIME_COUNT frames; an answer that takes longer than that goes
+     * unmeasured.
+     */
+    static constexpr size_t INPUT_SEND_TIME_COUNT = 256;
+
+    std::vector<Clock::time_point> m_input_send_times;
+    uint32_t                       m_last_measured_frame = 0;
+
+    /**
+     * Records how long the answer to `frame_idx` took to arrive, ignoring
+     * frames that were already measured or are too old to still have a send
+     * time.
+     */
+    void measure_response_time(uint32_t frame_idx);
+
     HistoryBufferExporter<long, glm::vec3> m_position_history_exporter;
 
     net::EntityPositionInterpolator m_entity_interpolator;
 
+    net::PlayerReconciler m_reconciler;
+
+    /**
+     * Visual smoothing for render-rate interpolation between 20 Hz ticks.
+     */
+    glm::vec3 m_render_prev_position{0.0f};
+    glm::vec3 m_render_curr_position{0.0f};
+    double m_tick_accumulator = 0.0;
+
+    /**
+     * Visual error from reconciliation corrections, decays over time.
+     */
+    glm::vec3 m_visual_error{0.0f};
+    static constexpr double kVisualErrorDecayRate = 12.0;
+
     /**
      * Mapping from the server entity_id to local entity_id
      * Server might have the same entity under different name
@@ -65,8 +121,6 @@ class ClientWorldController {
 
     entt::entity create_entity(const std::string& name, glm::vec3 position);
 
-    net::MessageHandler create_messenger();
-
     std::optional<entt::entity> map_from_server_entity(int id);
 
     void map_server_entity(int server_id, entt::entity local_id);
@@ -86,7 +140,8 @@ public:
             World* world,
             JoltPhysicsWorld* physics_world,
             drw::WorldRenderer* world_renderer,
-            tw::net::Address address
+            tw::net::Address address,
+            dbg::NetworkMetrics* network_metrics
     );
 
     ~ClientWorldController();

modules/io/CMakeLists.txt

@@ -0,0 +1,26 @@
+project(tw_io)
+
+file(GLOB HEADERS
+    include/*.hpp
+    include/io/*.hpp
+    include/bytebuffer/*.hpp
+    include/exception/*.hpp
+)
+
+add_library(${PROJECT_NAME} INTERFACE)
+add_library(tw::io ALIAS ${PROJECT_NAME})
+target_sources(${PROJECT_NAME}
+    INTERFACE FILE_SET HEADERS
+    BASE_DIRS include
+    FILES ${HEADERS})
+
+target_include_directories(${PROJECT_NAME}
+    INTERFACE
+    ${PROJECT_SOURCE_DIR}/include/
+)
+
+target_link_libraries(${PROJECT_NAME}
+    INTERFACE
+    spdlog::spdlog
+    tl::expected
+)

modules/io/README.md

@@ -0,0 +1,18 @@
+# io
+
+Transport-agnostic I/O primitives shared by the networking modules.
+
+Header-only (`tw::io`), namespace `tw::net`.
+
+| Path | Contents |
+| --- | --- |
+| `io/Read.hpp`, `io/Write.hpp` | `Read<T>` / `Write<T>` interfaces every stream implements (`read_into`, `write`, `flush`, plus `read_exact` helpers) |
+| `io/BufferReader.hpp`, `io/BufferWriter.hpp` | Buffering decorators that wrap another `Read<T>` / `Write<T>` |
+| `bytebuffer/ByteBuffer.hpp` | `RingByteBuffer` — circular buffer over a caller-owned `std::span` |
+| `bytebuffer/ByteBufferReader.hpp`, `ByteBufferWriter.hpp` | Linear cursor read/write over a `std::span` |
+| `bytebuffer/ByteBufferCodec.hpp`, `ByteBufferEncoder.hpp`, `ByteBufferDecoder.hpp` | Typed push/pop on top of `RingByteBuffer`, specialize `ByteBufferCodec<T>` for custom types |
+| `NetworkError.hpp` | errno-backed error type returned by `Read` / `Write` |
+
+This module owns no sockets and links no transport — it sits below `tw::network`
+and `tw::quicr` so both can share buffers and stream interfaces without depending
+on each other.

modules/message_protocol/CMakeLists.txt

@@ -0,0 +1,33 @@
+project(tw_message_protocol)
+
+file(GLOB FILES
+    src/*.cpp
+)
+
+file(GLOB HEADERS
+    include/message_protocol/*.hpp
+)
+
+add_library(${PROJECT_NAME} OBJECT ${FILES})
+add_library(tw::message_protocol ALIAS ${PROJECT_NAME})
+target_sources(${PROJECT_NAME}
+    PUBLIC FILE_SET HEADERS
+    BASE_DIRS include
+    FILES ${HEADERS})
+
+set_target_properties(${PROJECT_NAME} PROPERTIES POSITION_INDEPENDENT_CODE 1)
+
+target_include_directories(${PROJECT_NAME}
+    PUBLIC
+    ${PROJECT_SOURCE_DIR}/include/
+)
+
+target_link_libraries(${PROJECT_NAME}
+    PUBLIC
+    tw::io
+    tw::quicr
+    tl::expected
+    spdlog::spdlog
+)
+
+add_subdirectory(tests)

modules/message_protocol/include/message_protocol/MessageConnection.hpp

@@ -0,0 +1,127 @@
+#pragma once
+
+#include "message_protocol/MessageError.hpp"
+#include "message_protocol/MessageHeader.hpp"
+#include "message_protocol/MessageType.hpp"
+#include "message_protocol/PeerId.hpp"
+
+#include <tl/expected.hpp>
+
+#include <chrono>
+#include <cstddef>
+#include <functional>
+#include <span>
+#include <unordered_map>
+#include <vector>
+
+namespace tw::net::quicr {
+class QuicrConnection;
+}
+
+namespace tw::msg {
+
+class MessageDispatcher;
+
+/**
+ * Sends messages to one peer and routes the ones it sends back.
+ *
+ * Owned by the endpoint that created it and valid until the peer disconnects.
+ * Handlers for message addresses are registered on the endpoint and shared by
+ * every peer; a connection only holds the reply handlers for requests it made
+ * itself.
+ */
+class MessageConnection {
+public:
+    using ReplyHandler = std::function<void(std::span<const std::byte>)>;
+
+private:
+    struct PendingRequest {
+        ReplyHandler                          on_reply;
+        std::function<void()>                 on_timeout;
+        std::chrono::steady_clock::time_point expires_at;
+    };
+
+    net::quicr::QuicrConnection* m_connection;
+    MessageDispatcher*           m_dispatcher;
+    PeerId                       m_peer_id;
+
+    std::vector<std::byte>                       m_send_buffer;
+    std::unordered_map<uint32_t, PendingRequest> m_pending;
+    uint32_t                                     m_next_seq = 1;
+    uint64_t                                     m_bytes_sent = 0;
+    uint64_t                                     m_messages_sent = 0;
+    uint64_t                                     m_messages_received = 0;
+
+    uint32_t next_seq();
+
+    tl::expected<void, MessageError> send_impl(MessageType                type,
+                                               std::span<const std::byte> body,
+                                               uint32_t                   seq,
+                                               bool                       reliable);
+
+public:
+    MessageConnection(PeerId peer_id,
+                      net::quicr::QuicrConnection* connection,
+                      MessageDispatcher* dispatcher);
+
+    MessageConnection(const MessageConnection&)            = delete;
+    MessageConnection& operator=(const MessageConnection&) = delete;
+
+    PeerId peer_id() const {
+        return m_peer_id;
+    }
+
+    uint64_t bytes_sent() const {
+        return m_bytes_sent;
+    }
+
+    /** Messages handed over for sending since the connection was created. */
+    uint64_t messages_sent() const {
+        return m_messages_sent;
+    }
+
+    /** Messages routed from the peer since the connection was created. */
+    uint64_t messages_received() const {
+        return m_messages_received;
+    }
+
+    bool is_established() const;
+
+    tl::expected<void, MessageError> send(MessageType                type,
+                                          std::span<const std::byte> body,
+                                          bool                       reliable = false);
+
+    /**
+     * Sends a message the caller has already written a header into, for
+     * callers that build the whole message in a buffer of their own.
+     */
+    tl::expected<void, MessageError> send_framed(std::span<const std::byte> message,
+                                                 bool                       reliable = false);
+
+    /**
+     * Sends `body` and calls `on_reply` with the reply carrying the same
+     * sequence number, or `on_timeout` if no reply arrives in time.
+     */
+    tl::expected<void, MessageError> request(
+        MessageType                type,
+        std::span<const std::byte> body,
+        ReplyHandler               on_reply,
+        std::chrono::milliseconds  timeout    = std::chrono::seconds(5),
+        std::function<void()>      on_timeout = nullptr,
+        bool                       reliable   = true);
+
+    /**
+     * Reads everything the peer has sent, routing each message, and returns
+     * how many bytes were read. `scratch` is used to hold one message at a
+     * time and may be reused between peers.
+     */
+    size_t receive(std::span<std::byte> scratch);
+
+    /** Routes one received message to its reply handler, or to the dispatcher. */
+    void on_message(std::span<const std::byte> message);
+
+    /** Fails every request whose reply did not arrive before `now`. */
+    void expire_requests(std::chrono::steady_clock::time_point now);
+};
+
+}

modules/message_protocol/include/message_protocol/MessageDispatcher.hpp

@@ -0,0 +1,47 @@
+#pragma once
+
+#include "message_protocol/MessageType.hpp"
+#include "message_protocol/PeerId.hpp"
+
+#include <functional>
+#include <span>
+#include <unordered_map>
+
+namespace tw::msg {
+
+/**
+ * Routes a message body to the handler registered for its address.
+ *
+ * Never inspects the body, so how it is encoded is entirely the caller's
+ * concern. At most one handler may be registered per address.
+ */
+class MessageDispatcher {
+public:
+    using Handler = std::function<void(PeerId, std::span<const std::byte>)>;
+
+private:
+    std::unordered_map<MessageType, Handler> m_handlers;
+
+public:
+    /** Registers `handler` for `type`, replacing any handler already there. */
+    void set_handler(MessageType type, Handler handler) {
+        m_handlers[type] = std::move(handler);
+    }
+
+    bool has_handler(MessageType type) const {
+        return m_handlers.contains(type);
+    }
+
+    /** Invokes the handler for `type`. Returns false if there is none. */
+    bool dispatch(PeerId peer, MessageType type, std::span<const std::byte> body) {
+        auto handler = m_handlers.find(type);
+        if(handler == m_handlers.end()) {
+            return false;
+        }
+
+        handler->second(peer, body);
+        return true;
+    }
+};
+
+}

modules/message_protocol/include/message_protocol/MessageEndpoint.hpp

@@ -0,0 +1,110 @@
+#pragma once
+
+#include "message_protocol/MessageConnection.hpp"
+#include "message_protocol/MessageDispatcher.hpp"
+#include "message_protocol/MessageError.hpp"
+#include "message_protocol/MessageType.hpp"
+#include "message_protocol/PeerId.hpp"
+
+#include <tl/expected.hpp>
+
+#include <cstddef>
+#include <functional>
+#include <memory>
+#include <span>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace tw::net::quicr {
+class QuicrEndpoint;
+class QuicrConnectionListener;
+}
+
+namespace tw::msg {
+
+/**
+ * Owns the connections to every peer and the handlers shared between them.
+ *
+ * An endpoint created with bind() accepts incoming peers; either kind may
+ * connect() outwards, so one endpoint can serve peers and reach out to others
+ * at the same time.
+ *
+ * update() must be called regularly. Nothing is received and no request ever
+ * times out between calls.
+ */
+class MessageEndpoint {
+    std::unique_ptr<net::quicr::QuicrEndpoint>           m_endpoint;
+    std::unique_ptr<net::quicr::QuicrConnectionListener> m_listener;
+
+    MessageDispatcher m_dispatcher;
+
+    std::unordered_map<PeerId, std::unique_ptr<MessageConnection>> m_peers;
+    std::function<void(PeerId)>                                    m_on_peer_connected;
+    PeerId                                                         m_next_peer_id = 1;
+
+    std::vector<std::byte> m_receive_buffer;
+    uint64_t               m_bytes_received = 0;
+
+    explicit MessageEndpoint(std::unique_ptr<net::quicr::QuicrEndpoint> endpoint);
+
+    MessageConnection* add_peer(net::quicr::QuicrConnection* connection);
+    void               accept_peers();
+    void               receive();
+
+public:
+    ~MessageEndpoint();
+
+    MessageEndpoint(const MessageEndpoint&)            = delete;
+    MessageEndpoint& operator=(const MessageEndpoint&) = delete;
+
+    /** Creates an endpoint that only connects outwards. */
+    static tl::expected<std::unique_ptr<MessageEndpoint>, MessageError> create();
+
+    /** Creates an endpoint that also accepts peers on `port`. */
+    static tl::expected<std::unique_ptr<MessageEndpoint>, MessageError> bind(int port);
+
+    tl::expected<MessageConnection*, MessageError> connect(const std::string& host, int port);
+
+    /** Registers `handler` for every peer. */
+    void set_handler(MessageType type, MessageDispatcher::Handler handler) {
+        m_dispatcher.set_handler(type, std::move(handler));
+    }
+
+    MessageDispatcher& dispatcher() {
+        return m_dispatcher;
+    }
+
+    /** Receives pending messages, accepts new peers and times out requests. */
+    void update();
+
+    MessageConnection* peer(PeerId id);
+
+    /**
+     * Calls `handler` for each peer that connects or is accepted, before any
+     * of that peer's messages are dispatched.
+     */
+    void set_on_peer_connected(std::function<void(PeerId)> handler) {
+        m_on_peer_connected = std::move(handler);
+    }
+
+    std::vector<MessageConnection*> peers() const;
+
+    void broadcast(MessageType type, std::span<const std::byte> body, bool reliable = false);
+
+    /** Bytes received since the endpoint was created. */
+    uint64_t bytes_received() const {
+        return m_bytes_received;
+    }
+
+    /** Bytes handed to every peer for sending since the endpoint was created. */
+    uint64_t bytes_sent() const;
+
+    /** Messages handed to every peer for sending since the endpoint was created. */
+    uint64_t messages_sent() const;
+
+    /** Messages routed from every peer since the endpoint was created. */
+    uint64_t messages_received() const;
+};
+
+}

modules/message_protocol/include/message_protocol/MessageError.hpp

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <string>
+#include <utility>
+
+namespace tw::msg {
+
+enum class MessageErrorType {
+    NotConnected,
+    SendFailed,
+    BindFailed,
+    ConnectFailed,
+};
+
+struct MessageError {
+    MessageErrorType type;
+    std::string      detail;
+
+    explicit MessageError(MessageErrorType type, std::string detail = {}) :
+        type(type),
+        detail(std::move(detail)) {
+    }
+
+    std::string message() const {
+        std::string text;
+        switch(type) {
+            case MessageErrorType::NotConnected:  text = "Not connected to the peer"; break;
+            case MessageErrorType::SendFailed:    text = "Failed to send the message"; break;
+            case MessageErrorType::BindFailed:    text = "Failed to bind the endpoint"; break;
+            case MessageErrorType::ConnectFailed: text = "Failed to connect to the peer"; break;
+        }
+
+        return detail.empty() ? text : text + ": " + detail;
+    }
+};
+
+}

modules/message_protocol/include/message_protocol/MessageHeader.hpp

@@ -0,0 +1,45 @@
+#pragma once
+
+#include "message_protocol/MessageType.hpp"
+
+#include <cstdint>
+#include <cstring>
+#include <optional>
+#include <span>
+
+namespace tw::msg {
+
+/**
+ * Fixed-size prefix carried by every message.
+ *
+ * `seq` correlates a reply with the request that produced it. SEQ_NONE marks a
+ * message that expects no reply, which is the common case.
+ */
+struct MessageHeader {
+    static constexpr uint32_t SEQ_NONE = 0;
+    static constexpr size_t   SIZE     = sizeof(MessageType) + sizeof(uint32_t);
+
+    MessageType type = 0;
+    uint32_t    seq  = SEQ_NONE;
+
+    /** Writes the header at the start of `target`, which must hold SIZE bytes. */
+    void encode(std::span<std::byte> target) const {
+        std::memcpy(target.data(), &type, sizeof(type));
+        std::memcpy(target.data() + sizeof(type), &seq, sizeof(seq));
+    }
+
+    /** Reads a header from the start of `source`, or nothing if it is too short. */
+    static std::optional<MessageHeader> decode(std::span<const std::byte> source) {
+        if(source.size() < SIZE) {
+            return {};
+        }
+
+        MessageHeader header;
+        std::memcpy(&header.type, source.data(), sizeof(header.type));
+        std::memcpy(&header.seq, source.data() + sizeof(header.type), sizeof(header.seq));
+
+        return header;
+    }
+};
+
+}

modules/message_protocol/include/message_protocol/MessageType.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+#include <cstdint>
+
+namespace tw::msg {
+
+/**
+ * Address a message is delivered to. Concrete values are assigned by the
+ * application.
+ */
+using MessageType = uint32_t;
+
+}

modules/message_protocol/include/message_protocol/PeerId.hpp

@@ -0,0 +1,13 @@
+#pragma once
+
+#include <cstdint>
+
+namespace tw::msg {
+
+/**
+ * Identifies a remote peer. Assigned when the peer connects or is accepted and
+ * stable until it disconnects.
+ */
+using PeerId = uint64_t;
+
+}

modules/message_protocol/src/MessageConnection.cpp

@@ -0,0 +1,176 @@
+#include "message_protocol/MessageConnection.hpp"
+
+#include "message_protocol/MessageDispatcher.hpp"
+#include "quicr/QuicrConnection.hpp"
+
+#include <spdlog/spdlog.h>
+
+#include <cstring>
+
+namespace tw::msg {
+
+namespace {
+constexpr size_t INITIAL_SEND_BUFFER_SIZE = 64 * 1024;
+}
+
+MessageConnection::MessageConnection(PeerId                       peer_id,
+                                     net::quicr::QuicrConnection* connection,
+                                     MessageDispatcher*           dispatcher) :
+    m_connection(connection),
+    m_dispatcher(dispatcher),
+    m_peer_id(peer_id),
+    m_send_buffer(INITIAL_SEND_BUFFER_SIZE) {
+}
+
+bool MessageConnection::is_established() const {
+    return m_connection->state() == net::quicr::QuicrConnectionState::Established;
+}
+
+uint32_t MessageConnection::next_seq() {
+    uint32_t seq = m_next_seq++;
+    if(m_next_seq == MessageHeader::SEQ_NONE) {
+        m_next_seq = 1;
+    }
+
+    return seq;
+}
+
+tl::expected<void, MessageError> MessageConnection::send_impl(MessageType                type,
+                                                              std::span<const std::byte> body,
+                                                              uint32_t                   seq,
+                                                              bool                       reliable) {
+    const size_t size = MessageHeader::SIZE + body.size();
+
+    if(m_send_buffer.size() < size) {
+        m_send_buffer.resize(size);
+    }
+
+    MessageHeader{ type, seq }.encode(m_send_buffer);
+    std::memcpy(m_send_buffer.data() + MessageHeader::SIZE, body.data(), body.size());
+
+    auto send_r = m_connection->send_message(std::span(m_send_buffer).subspan(0, size), reliable);
+    if(!send_r) {
+        return tl::make_unexpected(MessageError(MessageErrorType::SendFailed, send_r.error().message()));
+    }
+
+    m_bytes_sent += size;
+    m_messages_sent++;
+
+    return {};
+}
+
+tl::expected<void, MessageError> MessageConnection::send(MessageType                type,
+                                                         std::span<const std::byte> body,
+                                                         bool                       reliable) {
+    return send_impl(type, body, MessageHeader::SEQ_NONE, reliable);
+}
+
+tl::expected<void, MessageError> MessageConnection::send_framed(std::span<const std::byte> message,
+                                                                bool                       reliable) {
+    if(message.size() < MessageHeader::SIZE) {
+        return tl::make_unexpected(
+            MessageError(MessageErrorType::SendFailed, "the message is too short to hold a header"));
+    }
+
+    // send_message takes a writable span, so the bytes are staged in the send
+    // buffer rather than sent straight from the caller's buffer.
+    if(m_send_buffer.size() < message.size()) {
+        m_send_buffer.resize(message.size());
+    }
+
+    std::memcpy(m_send_buffer.data(), message.data(), message.size());
+
+    auto send_r = m_connection->send_message(std::span(m_send_buffer).subspan(0, message.size()), reliable);
+    if(!send_r) {
+        return tl::make_unexpected(MessageError(MessageErrorType::SendFailed, send_r.error().message()));
+    }
+
+    m_bytes_sent += message.size();
+    m_messages_sent++;
+
+    return {};
+}
+
+tl::expected<void, MessageError> MessageConnection::request(MessageType                type,
+                                                            std::span<const std::byte> body,
+                                                            ReplyHandler               on_reply,
+                                                            std::chrono::milliseconds  timeout,
+                                                            std::function<void()>      on_timeout,
+                                                            bool                       reliable) {
+    const uint32_t seq = next_seq();
+
+    auto send_r = send_impl(type, body, seq, reliable);
+    if(!send_r) {
+        return send_r;
+    }
+
+    m_pending.emplace(seq,
+                      PendingRequest{ std::move(on_reply),
+                                      std::move(on_timeout),
+                                      std::chrono::steady_clock::now() + timeout });
+
+    return {};
+}
+
+size_t MessageConnection::receive(std::span<std::byte> scratch) {
+    size_t total = 0;
+
+    while(true) {
+        auto read_r = m_connection->read_into(scratch);
+        if(!read_r) {
+            spdlog::error("Failed to read from peer {}: {}", m_peer_id, read_r.error().message());
+            break;
+        }
+
+        if(*read_r == 0) {
+            break;
+        }
+
+        total += *read_r;
+        m_messages_received++;
+        on_message(scratch.subspan(0, *read_r));
+    }
+
+    return total;
+}
+
+void MessageConnection::on_message(std::span<const std::byte> message) {
+    auto header = MessageHeader::decode(message);
+    if(!header) {
+        spdlog::warn("Dropped a message of {} bytes, too short to hold a header", message.size());
+        return;
+    }
+
+    auto body = message.subspan(MessageHeader::SIZE);
+
+    if(header->seq != MessageHeader::SEQ_NONE) {
+        auto pending = m_pending.find(header->seq);
+        if(pending != m_pending.end()) {
+            auto on_reply = std::move(pending->second.on_reply);
+            m_pending.erase(pending);
+            on_reply(body);
+            return;
+        }
+    }
+
+    if(!m_dispatcher->dispatch(m_peer_id, header->type, body)) {
+        spdlog::warn("No handler for message type {}", header->type);
+    }
+}
+
+void MessageConnection::expire_requests(std::chrono::steady_clock::time_point now) {
+    std::erase_if(m_pending, [&](auto& entry) {
+        if(entry.second.expires_at > now) {
+            return false;
+        }
+
+        spdlog::warn("Request {} timed out", entry.first);
+        if(entry.second.on_timeout) {
+            entry.second.on_timeout();
+        }
+
+        return true;
+    });
+}
+
+}

modules/message_protocol/src/MessageEndpoint.cpp

@@ -0,0 +1,162 @@
+#include "message_protocol/MessageEndpoint.hpp"
+
+#include "quicr/QuicrAddress.hpp"
+#include "quicr/QuicrConnection.hpp"
+#include "quicr/QuicrConnectionListener.hpp"
+#include "quicr/QuicrEndpoint.hpp"
+
+#include <spdlog/spdlog.h>
+
+namespace tw::msg {
+
+namespace {
+constexpr size_t RECEIVE_BUFFER_SIZE = 64 * 1024;
+}
+
+MessageEndpoint::MessageEndpoint(std::unique_ptr<net::quicr::QuicrEndpoint> endpoint) :
+    m_endpoint(std::move(endpoint)),
+    m_receive_buffer(RECEIVE_BUFFER_SIZE) {
+}
+
+MessageEndpoint::~MessageEndpoint() = default;
+
+tl::expected<std::unique_ptr<MessageEndpoint>, MessageError> MessageEndpoint::create() {
+    auto endpoint_r = net::quicr::QuicrEndpoint::create();
+    if(!endpoint_r) {
+        return tl::make_unexpected(
+            MessageError(MessageErrorType::BindFailed, endpoint_r.error().message()));
+    }
+
+    return std::unique_ptr<MessageEndpoint>(new MessageEndpoint(std::move(endpoint_r.value())));
+}
+
+tl::expected<std::unique_ptr<MessageEndpoint>, MessageError> MessageEndpoint::bind(int port) {
+    auto endpoint_r = create();
+    if(!endpoint_r) {
+        return endpoint_r;
+    }
+
+    auto& endpoint = endpoint_r.value();
+
+    auto bind_r = endpoint->m_endpoint->bind(port);
+    if(!bind_r) {
+        return tl::make_unexpected(MessageError(MessageErrorType::BindFailed, bind_r.error().message()));
+    }
+
+    auto listener_r = net::quicr::QuicrConnectionListener::listen(endpoint->m_endpoint.get());
+    if(!listener_r) {
+        return tl::make_unexpected(
+            MessageError(MessageErrorType::BindFailed, listener_r.error().message()));
+    }
+
+    endpoint->m_listener = std::move(listener_r.value());
+
+    return endpoint_r;
+}
+
+MessageConnection* MessageEndpoint::add_peer(net::quicr::QuicrConnection* connection) {
+    const PeerId id = m_next_peer_id++;
+
+    auto peer = std::make_unique<MessageConnection>(id, connection, &m_dispatcher);
+    auto* raw = peer.get();
+
+    m_peers.emplace(id, std::move(peer));
+
+    if(m_on_peer_connected) {
+        m_on_peer_connected(id);
+    }
+
+    return raw;
+}
+
+tl::expected<MessageConnection*, MessageError> MessageEndpoint::connect(const std::string& host, int port) {
+    auto connection_r = m_endpoint->connect(net::quicr::QuicrAddress(host, port));
+    if(!connection_r) {
+        return tl::make_unexpected(
+            MessageError(MessageErrorType::ConnectFailed, connection_r.error().message()));
+    }
+
+    return add_peer(connection_r.value());
+}
+
+void MessageEndpoint::accept_peers() {
+    if(!m_listener) {
+        return;
+    }
+
+    while(net::quicr::QuicrConnection* connection = m_listener->listen()) {
+        add_peer(connection);
+    }
+}
+
+void MessageEndpoint::receive() {
+    for(auto& [id, peer] : m_peers) {
+        m_bytes_received += peer->receive(m_receive_buffer);
+    }
+}
+
+void MessageEndpoint::update() {
+    m_endpoint->poll();
+
+    accept_peers();
+    receive();
+
+    const auto now = std::chrono::steady_clock::now();
+    for(auto& [id, peer] : m_peers) {
+        peer->expire_requests(now);
+    }
+}
+
+MessageConnection* MessageEndpoint::peer(PeerId id) {
+    auto peer = m_peers.find(id);
+    return peer != m_peers.end() ? peer->second.get() : nullptr;
+}
+
+std::vector<MessageConnection*> MessageEndpoint::peers() const {
+    std::vector<MessageConnection*> result;
+    result.reserve(m_peers.size());
+
+    for(const auto& [id, peer] : m_peers) {
+        result.push_back(peer.get());
+    }
+
+    return result;
+}
+
+void MessageEndpoint::broadcast(MessageType type, std::span<const std::byte> body, bool reliable) {
+    for(auto& [id, peer] : m_peers) {
+        auto send_r = peer->send(type, body, reliable);
+        if(!send_r) {
+            spdlog::error("Failed to send to peer {}: {}", id, send_r.error().message());
+        }
+    }
+}
+
+uint64_t MessageEndpoint::bytes_sent() const {
+    uint64_t total = 0;
+    for(const auto& [id, peer] : m_peers) {
+        total += peer->bytes_sent();
+    }
+
+    return total;
+}
+
+uint64_t MessageEndpoint::messages_sent() const {
+    uint64_t total = 0;
+    for(const auto& [id, peer] : m_peers) {
+        total += peer->messages_sent();
+    }
+
+    return total;
+}
+
+uint64_t MessageEndpoint::messages_received() const {
+    uint64_t total = 0;
+    for(const auto& [id, peer] : m_peers) {
+        total += peer->messages_received();
+    }
+
+    return total;
+}
+
+}

modules/message_protocol/tests/CMakeLists.txt

@@ -0,0 +1,24 @@
+project(tw_message_protocol_tests)
+
+file(GLOB FILES
+    ./*.cpp
+)
+
+add_executable(${PROJECT_NAME} ${FILES})
+
+# tw::quicr is listed explicitly because CMake does not propagate the object
+# files of an OBJECT library through another OBJECT library.
+target_link_libraries(${PROJECT_NAME}
+    PRIVATE
+    tw::message_protocol
+    tw::quicr
+    Catch2::Catch2WithMain
+    tl::expected
+)
+
+list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras)
+
+include(CTest)
+include(Catch)
+
+catch_discover_tests(${PROJECT_NAME})

modules/message_protocol/tests/MessageDispatcherTests.cpp

@@ -0,0 +1,73 @@
+#include "message_protocol/MessageDispatcher.hpp"
+
+#include <catch2/catch_test_macros.hpp>
+
+#include <array>
+#include <string>
+
+using namespace tw::msg;
+
+namespace {
+
+std::span<const std::byte> as_bytes(const std::array<std::byte, 2>& body) {
+    return { body.data(), body.size() };
+}
+
+}
+
+TEST_CASE("Dispatch reaches the handler bound to the address", "[message_dispatcher]") {
+    MessageDispatcher dispatcher;
+
+    PeerId seen_peer = 0;
+    size_t seen_size = 0;
+
+    dispatcher.set_handler(7, [&](PeerId peer, std::span<const std::byte> body) {
+        seen_peer = peer;
+        seen_size = body.size();
+    });
+
+    std::array<std::byte, 2> body{};
+
+    REQUIRE(dispatcher.dispatch(99, 7, as_bytes(body)));
+    REQUIRE(seen_peer == 99);
+    REQUIRE(seen_size == 2);
+}
+
+TEST_CASE("Dispatch to an unbound address reports failure", "[message_dispatcher]") {
+    MessageDispatcher dispatcher;
+
+    std::array<std::byte, 2> body{};
+
+    REQUIRE_FALSE(dispatcher.dispatch(1, 7, as_bytes(body)));
+}
+
+TEST_CASE("Addresses are routed independently", "[message_dispatcher]") {
+    MessageDispatcher dispatcher;
+
+    std::string called;
+
+    dispatcher.set_handler(1, [&](PeerId, std::span<const std::byte>) { called = "first"; });
+    dispatcher.set_handler(2, [&](PeerId, std::span<const std::byte>) { called = "second"; });
+
+    std::array<std::byte, 2> body{};
+
+    dispatcher.dispatch(1, 2, as_bytes(body));
+    REQUIRE(called == "second");
+
+    dispatcher.dispatch(1, 1, as_bytes(body));
+    REQUIRE(called == "first");
+}
+
+TEST_CASE("Rebinding an address replaces the handler", "[message_dispatcher]") {
+    MessageDispatcher dispatcher;
+
+    std::string called;
+
+    dispatcher.set_handler(7, [&](PeerId, std::span<const std::byte>) { called = "first"; });
+    dispatcher.set_handler(7, [&](PeerId, std::span<const std::byte>) { called = "second"; });
+
+    std::array<std::byte, 2> body{};
+    dispatcher.dispatch(1, 7, as_bytes(body));
+
+    REQUIRE(called == "second");
+}

modules/message_protocol/tests/MessageHeaderTests.cpp

@@ -0,0 +1,45 @@
+#include "message_protocol/MessageHeader.hpp"
+
+#include <catch2/catch_test_macros.hpp>
+
+#include <array>
+
+using namespace tw::msg;
+
+TEST_CASE("Header survives a round trip", "[message_header]") {
+    std::array<std::byte, MessageHeader::SIZE> buffer{};
+
+    MessageHeader{ 42, 7 }.encode(buffer);
+
+    auto decoded = MessageHeader::decode(buffer);
+
+    REQUIRE(decoded.has_value());
+    REQUIRE(decoded->type == 42);
+    REQUIRE(decoded->seq == 7);
+}
+
+TEST_CASE("Header defaults to expecting no reply", "[message_header]") {
+    std::array<std::byte, MessageHeader::SIZE> buffer{};
+
+    MessageHeader{ 3 }.encode(buffer);
+
+    auto decoded = MessageHeader::decode(buffer);
+
+    REQUIRE(decoded.has_value());
+    REQUIRE(decoded->seq == MessageHeader::SEQ_NONE);
+}
+
+TEST_CASE("Decoding a message shorter than a header fails", "[message_header]") {
+    std::array<std::byte, MessageHeader::SIZE - 1> buffer{};
+
+    REQUIRE_FALSE(MessageHeader::decode(buffer).has_value());
+}
+
+TEST_CASE("Encoding only writes the header", "[message_header]") {
+    std::array<std::byte, MessageHeader::SIZE + 4> buffer{};
+    buffer[MessageHeader::SIZE] = std::byte{ 0xAB };
+
+    MessageHeader{ 1, 2 }.encode(buffer);
+
+    REQUIRE(buffer[MessageHeader::SIZE] == std::byte{ 0xAB });
+}

modules/messaging/CMakeLists.txt

@@ -1,14 +0,0 @@
-project(tw_messaging)
-
-add_library(${PROJECT_NAME} INTERFACE)
-add_library(tw::messaging ALIAS ${PROJECT_NAME})
-
-target_include_directories(${PROJECT_NAME} INTERFACE ./include/)
-
-target_link_libraries(${PROJECT_NAME}
-    INTERFACE
-        tw::network
-        tw::protocol
-        tl::expected
-        protobuf::libprotobuf
-)

modules/messaging/include/MessageSession.hpp

@@ -1,158 +0,0 @@
-#pragma once
-
-#include "Address.hpp"
-#include "protocol/quicr/QuicrEndpoint.hpp"
-#include "protocol/quicr/QuicrConnection.hpp"
-#include "protocol/quicr/QuicrError.hpp"
-
-#include <tl/expected.hpp>
-#include <spdlog/spdlog.h>
-
-#include <chrono>
-#include <cstring>
-#include <functional>
-#include <span>
-#include <unordered_map>
-#include <vector>
-
-namespace tw {
-
-// Bidirectional typed messaging layer. Owns its QuicrEndpoint and connection.
-// Wire format: [uint32_t type][uint32_t seq][payload bytes]
-// seq == 0 means fire-and-forget; non-zero seq correlates a reply to a request().
-// Protocol-agnostic: callers are responsible for serialising/deserialising payloads.
-// update() polls the endpoint and dispatches inbound data automatically.
-class MessageSession {
-    static constexpr size_t   MAX_TYPES = 32;
-    static constexpr uint32_t SEQ_NONE  = 0;
-
-    struct PendingRequest {
-        std::function<void(std::span<const std::byte>)> handler;
-        std::chrono::steady_clock::time_point expires_at;
-        std::function<void()> on_timeout;
-    };
-
-    std::unique_ptr<net::quicr::QuicrEndpoint>    m_endpoint;
-    net::quicr::QuicrConnection* m_conn;
-    std::vector<std::byte>       m_recv_buf{64 * 1024};
-    std::vector<std::function<void(std::span<const std::byte>)>> m_handlers{MAX_TYPES};
-    std::unordered_map<uint32_t, PendingRequest> m_pending{};
-    uint32_t m_next_seq = 1;
-
-public:
-    // Creates a QuicrEndpoint, connects to the given address, and owns both.
-    // Throws on failure (via tl::expected::value()).
-    explicit MessageSession(net::Address address)
-        : m_endpoint(std::make_unique<net::quicr::QuicrEndpoint>(net::quicr::QuicrEndpoint::create().value()))
-        , m_conn(m_endpoint->connect(address).value())
-    {}
-
-    MessageSession(const MessageSession&) = delete;
-    MessageSession& operator=(const MessageSession&) = delete;
-    MessageSession(MessageSession&&) = default;
-    MessageSession& operator=(MessageSession&&) = default;
-
-    // Register a permanent handler for the given type ID.
-    void set_handler(uint32_t type, std::function<void(std::span<const std::byte>)> fn) {
-        if (type >= m_handlers.size()) {
-            spdlog::warn("MessageSession: type {} exceeds MAX_TYPES", type);
-            return;
-        }
-        m_handlers[type] = std::move(fn);
-    }
-
-    // Send a request with a one-shot response handler matched by sequence number.
-    // Call update() each game tick to evict timed-out requests.
-    tl::expected<void, net::quicr::QuicrError> request(
-        uint32_t type,
-        std::span<const std::byte> payload,
-        std::function<void(std::span<const std::byte>)> on_response,
-        std::chrono::milliseconds timeout = std::chrono::seconds(5),
-        std::function<void()> on_timeout = nullptr,
-        bool reliable = true
-    ) {
-        const uint32_t seq = m_next_seq++;
-        if (m_next_seq == SEQ_NONE) m_next_seq = 1;
-
-        m_pending.emplace(seq, PendingRequest{
-            std::move(on_response),
-            std::chrono::steady_clock::now() + timeout,
-            std::move(on_timeout)
-        });
-        return send_impl(type, payload, seq, reliable);
-    }
-
-    // Expire timed-out pending requests, poll the endpoint, and dispatch any
-    // inbound datagrams. Call once per game tick.
-    void update() {
-        const auto now = std::chrono::steady_clock::now();
-        std::erase_if(m_pending, [&](auto& kv) {
-            if (kv.second.expires_at > now) return false;
-            spdlog::warn("MessageSession: request seq={} timed out", kv.first);
-            if (kv.second.on_timeout) kv.second.on_timeout();
-            return true;
-        });
-
-        m_endpoint->poll();
-        while (true) {
-            auto r = m_conn->read_into(m_recv_buf);
-            if (!r || *r == 0) break;
-            dispatch(std::span(m_recv_buf.data(), *r));
-        }
-    }
-
-    // Decode one framed datagram: [uint32_t type][uint32_t seq][payload].
-    // Non-zero seq routes to a pending one-shot handler; seq==0 routes by type.
-    void dispatch(std::span<const std::byte> data) {
-        constexpr size_t HEADER = sizeof(uint32_t) * 2;
-        if (data.size() < HEADER) {
-            spdlog::warn("MessageSession: dropped short datagram ({} bytes)", data.size());
-            return;
-        }
-        uint32_t type{}, seq{};
-        std::memcpy(&type, data.data(),                    sizeof(type));
-        std::memcpy(&seq,  data.data() + sizeof(uint32_t), sizeof(seq));
-
-        const auto payload = data.subspan(HEADER);
-
-        if (seq != SEQ_NONE) {
-            if (auto it = m_pending.find(seq); it != m_pending.end()) {
-                auto handler = std::move(it->second.handler);
-                m_pending.erase(it);
-                handler(payload);
-                return;
-            }
-        }
-
-        if (type >= m_handlers.size() || !m_handlers[type]) {
-            spdlog::warn("MessageSession: no handler for type {}", type);
-            return;
-        }
-        m_handlers[type](payload);
-    }
-
-    // Send a fire-and-forget message.
-    tl::expected<void, net::quicr::QuicrError> send(uint32_t type,
-                                                     std::span<const std::byte> payload,
-                                                     bool reliable = false) {
-        return send_impl(type, payload, SEQ_NONE, reliable);
-    }
-
-    bool is_established() const {
-        return m_conn->state() == net::quicr::QuicrConnectionState::Established;
-    }
-
-private:
-    tl::expected<void, net::quicr::QuicrError> send_impl(uint32_t type,
-                                                          std::span<const std::byte> payload,
-                                                          uint32_t seq,
-                                                          bool reliable) {
-        std::vector<std::byte> buf(sizeof(type) + sizeof(seq) + payload.size());
-        std::memcpy(buf.data(),                    &type, sizeof(type));
-        std::memcpy(buf.data() + sizeof(type),     &seq,  sizeof(seq));
-        std::memcpy(buf.data() + sizeof(type) + sizeof(seq), payload.data(), payload.size());
-        return m_conn->send_message(std::span(buf), reliable);
-    }
-};
-
-} // namespace tw

modules/metrics/CMakeLists.txt

@@ -0,0 +1,19 @@
+project(tw_metrics)
+
+add_subdirectory(tests)
+
+file(GLOB HEADERS
+    include/metrics/*.hpp
+)
+
+add_library(${PROJECT_NAME} INTERFACE)
+add_library(tw::metrics ALIAS ${PROJECT_NAME})
+target_sources(${PROJECT_NAME}
+    INTERFACE FILE_SET HEADERS
+    BASE_DIRS include
+    FILES ${HEADERS})
+
+target_include_directories(${PROJECT_NAME}
+    INTERFACE
+    ${PROJECT_SOURCE_DIR}/include/
+)

modules/metrics/include/metrics/MetricSample.hpp

@@ -0,0 +1,86 @@
+#pragma once
+
+#include <algorithm>
+#include <cstdint>
+
+namespace tw::metrics {
+
+/**
+ * Aggregate of every value that fell into one bucket.
+ *
+ * Answers sum, average, minimum and maximum without keeping the individual
+ * values. A sample nothing was added to reports zero for all of them, so gaps
+ * read as zero rather than as an unset extreme.
+ */
+struct MetricSample {
+    uint32_t count = 0;
+    double   sum   = 0.0;
+    double   min   = 0.0;
+    double   max   = 0.0;
+
+    void add(double value) {
+        if(count == 0) {
+            min = value;
+            max = value;
+        } else {
+            min = std::min(min, value);
+            max = std::max(max, value);
+        }
+
+        sum += value;
+        count++;
+    }
+
+    /** Folds `other` in, as if its values had been added to this sample. */
+    void merge(const MetricSample& other) {
+        if(other.count == 0) {
+            return;
+        }
+
+        if(count == 0) {
+            *this = other;
+            return;
+        }
+
+        min = std::min(min, other.min);
+        max = std::max(max, other.max);
+
+        sum += other.sum;
+        count += other.count;
+    }
+
+    double avg() const {
+        return count == 0 ? 0.0 : sum / count;
+    }
+
+    bool is_empty() const {
+        return count == 0;
+    }
+
+    void reset() {
+        *this = {};
+    }
+};
+
+/** Statistic to read out of a sample. */
+enum class MetricField {
+    Avg,
+    Min,
+    Max,
+    Sum,
+    Count
+};
+
+inline double value_of(const MetricSample& sample, MetricField field) {
+    switch(field) {
+        case MetricField::Avg:   return sample.avg();
+        case MetricField::Min:   return sample.min;
+        case MetricField::Max:   return sample.max;
+        case MetricField::Sum:   return sample.sum;
+        case MetricField::Count: return (double)sample.count;
+    }
+
+    return 0.0;
+}
+
+}

modules/metrics/include/metrics/MetricSeries.hpp

@@ -0,0 +1,193 @@
+#pragma once
+
+#include "metrics/MetricSample.hpp"
+
+#include <chrono>
+#include <cstddef>
+#include <stdexcept>
+#include <vector>
+
+namespace tw::metrics {
+
+/**
+ * Ring of samples, one bucket per `Interval` of elapsed time.
+ *
+ * Values pushed during the same interval fold into one bucket, and intervals
+ * that pass without a value become empty buckets, so the distance between two
+ * buckets always matches the time between them. Once `capacity` buckets are
+ * held the oldest one is dropped.
+ */
+template<typename Interval, typename Clock = std::chrono::steady_clock>
+class MetricSeries {
+public:
+    using TimePoint = typename Clock::time_point;
+
+private:
+    std::vector<MetricSample> m_buckets;
+
+    /** Index of the newest bucket, in `Interval` units since the clock epoch. */
+    int64_t m_newest = 0;
+
+    /** Buckets holding data, counted back from the newest. */
+    size_t m_count = 0;
+
+    static int64_t bucket_of(TimePoint time) {
+        return (int64_t)std::chrono::floor<Interval>(time).time_since_epoch().count();
+    }
+
+    size_t slot_of(int64_t index) const {
+        int64_t size = (int64_t)m_buckets.size();
+        int64_t slot = index % size;
+
+        return (size_t)(slot < 0 ? slot + size : slot);
+    }
+
+    MetricSample& bucket_at(int64_t index) {
+        return m_buckets[slot_of(index)];
+    }
+
+    const MetricSample& bucket_at(int64_t index) const {
+        return m_buckets[slot_of(index)];
+    }
+
+    /**
+     * Moves the newest bucket up to `index`, emptying every bucket the gap
+     * covers. A gap wider than the ring empties all of it.
+     */
+    void advance_to(int64_t index) {
+        int64_t steps    = index - m_newest;
+        int64_t capacity = (int64_t)m_buckets.size();
+        int64_t to_clear = std::min(steps, capacity);
+
+        for(int64_t i = 0; i < to_clear; i++) {
+            bucket_at(index - i).reset();
+        }
+
+        m_newest = index;
+        m_count  = steps >= capacity
+            ? m_buckets.size()
+            : std::min(m_count + (size_t)steps, m_buckets.size());
+    }
+
+public:
+    explicit MetricSeries(size_t capacity) :
+        m_buckets(capacity)
+    {
+        if(capacity == 0) {
+            throw std::invalid_argument("`capacity` must hold at least one bucket");
+        }
+    }
+
+    size_t capacity() const {
+        return m_buckets.size();
+    }
+
+    size_t size() const {
+        return m_count;
+    }
+
+    bool is_empty() const {
+        return m_count == 0;
+    }
+
+    /** Index of the newest bucket, in `Interval` units since the clock epoch. */
+    int64_t newest_index() const {
+        return m_newest;
+    }
+
+    void push(double value) {
+        push(value, Clock::now());
+    }
+
+    /**
+     * Adds `value` to the bucket `at` falls into. A value older than every
+     * bucket still held is dropped.
+     */
+    void push(double value, TimePoint at) {
+        int64_t index = bucket_of(at);
+
+        if(m_count == 0) {
+            m_newest = index;
+            m_count  = 1;
+
+            bucket_at(index).reset();
+            bucket_at(index).add(value);
+            return;
+        }
+
+        if(index > m_newest) {
+            advance_to(index);
+        } else if(m_newest - index >= (int64_t)m_count) {
+            return;
+        }
+
+        bucket_at(index).add(value);
+    }
+
+    /** Newest first: age 0 is the bucket currently being filled. */
+    const MetricSample& at_age(size_t age) const {
+        return bucket_at(m_newest - (int64_t)age);
+    }
+
+    /** Aggregate of the newest `buckets` buckets. */
+    MetricSample window(size_t buckets) const {
+        MetricSample result;
+
+        size_t count = std::min(buckets, m_count);
+        for(size_t age = 0; age < count; age++) {
+            result.merge(at_age(age));
+        }
+
+        return result;
+    }
+
+    /** Aggregate of everything still held. */
+    MetricSample window() const {
+        return window(m_count);
+    }
+
+    /**
+     * Writes the newest `buckets` buckets into `xs` and `ys` oldest first, as
+     * two contiguous arrays. `xs` holds the age of each bucket in `Interval`
+     * units, so the bucket being filled sits at 0 and older ones run negative.
+     * Both vectors are resized to the number of points written.
+     *
+     * `skip_newest` leaves that many of the newest buckets out. The bucket
+     * being filled only holds the part of its interval that has elapsed, so
+     * reading it next to whole ones makes the newest point dip and recover;
+     * skipping it keeps every point covering the same span of time. Ages stay
+     * true, so a skipped bucket leaves a gap rather than shifting the rest.
+     */
+    size_t linearize(std::vector<double>& xs,
+                     std::vector<double>& ys,
+                     MetricField          field,
+                     size_t               buckets,
+                     size_t               skip_newest = 0) const {
+        size_t available = m_count > skip_newest ? m_count - skip_newest : 0;
+        size_t count     = std::min(buckets, available);
+
+        xs.resize(count);
+        ys.resize(count);
+
+        for(size_t i = 0; i < count; i++) {
+            size_t age = skip_newest + count - 1 - i;
+
+            xs[i] = -(double)age;
+            ys[i] = value_of(at_age(age), field);
+        }
+
+        return count;
+    }
+
+    size_t linearize(std::vector<double>& xs,
+                     std::vector<double>& ys,
+                     MetricField          field) const {
+        return linearize(xs, ys, field, m_count);
+    }
+
+    void clear() {
+        m_count = 0;
+    }
+};
+
+}

modules/metrics/tests/CMakeLists.txt

@@ -0,0 +1,24 @@
+project(tw_metrics_tests)
+
+set(LIBS
+    tw::metrics
+)
+
+file(GLOB FILES
+    ./*.cpp
+)
+
+add_executable(${PROJECT_NAME} ${FILES})
+
+target_link_libraries(${PROJECT_NAME}
+    PRIVATE
+    ${LIBS}
+    Catch2::Catch2WithMain
+)
+
+list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras)
+
+include(CTest)
+include(Catch)
+
+catch_discover_tests(${PROJECT_NAME})

modules/metrics/tests/MetricSampleTests.cpp

@@ -0,0 +1,105 @@
+#include "metrics/MetricSample.hpp"
+
+#include "catch2/catch_test_macros.hpp"
+
+using tw::metrics::MetricField;
+using tw::metrics::MetricSample;
+using tw::metrics::value_of;
+
+TEST_CASE("Empty sample reports zero", "[metric_sample]") {
+    MetricSample sample;
+
+    REQUIRE(sample.is_empty());
+    REQUIRE(sample.count == 0);
+    REQUIRE(sample.sum == 0.0);
+    REQUIRE(sample.min == 0.0);
+    REQUIRE(sample.max == 0.0);
+    REQUIRE(sample.avg() == 0.0);
+}
+
+TEST_CASE("Sample tracks sum, average and extremes", "[metric_sample]") {
+    MetricSample sample;
+
+    sample.add(4.0);
+    sample.add(1.0);
+    sample.add(7.0);
+
+    REQUIRE(sample.count == 3);
+    REQUIRE(sample.sum == 12.0);
+    REQUIRE(sample.min == 1.0);
+    REQUIRE(sample.max == 7.0);
+    REQUIRE(sample.avg() == 4.0);
+}
+
+TEST_CASE("First value sets both extremes", "[metric_sample]") {
+    MetricSample sample;
+
+    sample.add(-5.0);
+
+    REQUIRE(sample.min == -5.0);
+    REQUIRE(sample.max == -5.0);
+}
+
+TEST_CASE("Merge folds one sample into another", "[metric_sample]") {
+    MetricSample left;
+    left.add(2.0);
+    left.add(4.0);
+
+    MetricSample right;
+    right.add(10.0);
+    right.add(0.5);
+
+    left.merge(right);
+
+    REQUIRE(left.count == 4);
+    REQUIRE(left.sum == 16.5);
+    REQUIRE(left.min == 0.5);
+    REQUIRE(left.max == 10.0);
+}
+
+TEST_CASE("Merging with an empty sample changes nothing", "[metric_sample]") {
+    MetricSample sample;
+    sample.add(3.0);
+
+    sample.merge(MetricSample{});
+
+    REQUIRE(sample.count == 1);
+    REQUIRE(sample.min == 3.0);
+    REQUIRE(sample.max == 3.0);
+}
+
+TEST_CASE("Merging into an empty sample adopts the other", "[metric_sample]") {
+    MetricSample other;
+    other.add(3.0);
+    other.add(9.0);
+
+    MetricSample sample;
+    sample.merge(other);
+
+    REQUIRE(sample.count == 2);
+    REQUIRE(sample.sum == 12.0);
+    REQUIRE(sample.min == 3.0);
+    REQUIRE(sample.max == 9.0);
+}
+
+TEST_CASE("Field selects the statistic to read", "[metric_sample]") {
+    MetricSample sample;
+    sample.add(2.0);
+    sample.add(6.0);
+
+    REQUIRE(value_of(sample, MetricField::Avg) == 4.0);
+    REQUIRE(value_of(sample, MetricField::Min) == 2.0);
+    REQUIRE(value_of(sample, MetricField::Max) == 6.0);
+    REQUIRE(value_of(sample, MetricField::Sum) == 8.0);
+    REQUIRE(value_of(sample, MetricField::Count) == 2.0);
+}
+
+TEST_CASE("Reset empties the sample", "[metric_sample]") {
+    MetricSample sample;
+    sample.add(5.0);
+
+    sample.reset();
+
+    REQUIRE(sample.is_empty());
+    REQUIRE(sample.max == 0.0);
+}

modules/metrics/tests/MetricSeriesTests.cpp

@@ -0,0 +1,325 @@
+#include "metrics/MetricSeries.hpp"
+
+#include "catch2/catch_test_macros.hpp"
+
+#include <chrono>
+
+using namespace std::chrono_literals;
+
+using tw::metrics::MetricField;
+using tw::metrics::MetricSeries;
+
+using Clock  = std::chrono::steady_clock;
+using Series = MetricSeries<std::chrono::seconds>;
+
+/** Fixed origin so every test drives the series by hand. */
+static Clock::time_point at(int64_t seconds) {
+    return Clock::time_point{} + std::chrono::hours(1) + std::chrono::seconds(seconds);
+}
+
+TEST_CASE("Series starts empty", "[metric_series]") {
+    Series series(8);
+
+    REQUIRE(series.capacity() == 8);
+    REQUIRE(series.size() == 0);
+    REQUIRE(series.is_empty());
+    REQUIRE(series.window().is_empty());
+}
+
+TEST_CASE("Series rejects a zero capacity", "[metric_series]") {
+    REQUIRE_THROWS_AS(Series(0), std::invalid_argument);
+}
+
+TEST_CASE("Values in the same interval fold into one bucket", "[metric_series]") {
+    Series series(8);
+
+    series.push(1.0, at(0));
+    series.push(3.0, at(0));
+
+    REQUIRE(series.size() == 1);
+    REQUIRE(series.at_age(0).count == 2);
+    REQUIRE(series.at_age(0).avg() == 2.0);
+    REQUIRE(series.at_age(0).min == 1.0);
+    REQUIRE(series.at_age(0).max == 3.0);
+}
+
+TEST_CASE("Values in different intervals land in different buckets", "[metric_series]") {
+    Series series(8);
+
+    series.push(1.0, at(0));
+    series.push(5.0, at(1));
+
+    REQUIRE(series.size() == 2);
+    REQUIRE(series.at_age(0).sum == 5.0);
+    REQUIRE(series.at_age(1).sum == 1.0);
+}
+
+TEST_CASE("Intervals without a value become empty buckets", "[metric_series]") {
+    Series series(8);
+
+    series.push(1.0, at(0));
+    series.push(4.0, at(3));
+
+    REQUIRE(series.size() == 4);
+    REQUIRE(series.at_age(0).sum == 4.0);
+    REQUIRE(series.at_age(1).is_empty());
+    REQUIRE(series.at_age(2).is_empty());
+    REQUIRE(series.at_age(3).sum == 1.0);
+}
+
+TEST_CASE("Series never holds more than its capacity", "[metric_series]") {
+    Series series(4);
+
+    for(int64_t i = 0; i < 10; i++) {
+        series.push((double)i, at(i));
+    }
+
+    REQUIRE(series.size() == 4);
+    REQUIRE(series.at_age(0).sum == 9.0);
+    REQUIRE(series.at_age(3).sum == 6.0);
+}
+
+TEST_CASE("A gap wider than the ring leaves only the newest bucket filled", "[metric_series]") {
+    Series series(4);
+
+    series.push(1.0, at(0));
+    series.push(2.0, at(100));
+
+    REQUIRE(series.size() == 4);
+    REQUIRE(series.at_age(0).sum == 2.0);
+    REQUIRE(series.at_age(1).is_empty());
+    REQUIRE(series.at_age(2).is_empty());
+    REQUIRE(series.at_age(3).is_empty());
+}
+
+TEST_CASE("Buckets dropped by wrapping do not come back", "[metric_series]") {
+    Series series(4);
+
+    series.push(100.0, at(0));
+
+    for(int64_t i = 1; i < 5; i++) {
+        series.push(1.0, at(i));
+    }
+
+    REQUIRE(series.window().max == 1.0);
+}
+
+TEST_CASE("Window aggregates across buckets", "[metric_series]") {
+    Series series(8);
+
+    series.push(4.0, at(0));
+    series.push(1.0, at(1));
+    series.push(7.0, at(2));
+
+    auto window = series.window();
+
+    REQUIRE(window.count == 3);
+    REQUIRE(window.sum == 12.0);
+    REQUIRE(window.min == 1.0);
+    REQUIRE(window.max == 7.0);
+    REQUIRE(window.avg() == 4.0);
+}
+
+TEST_CASE("Window can be narrowed to the newest buckets", "[metric_series]") {
+    Series series(8);
+
+    series.push(4.0, at(0));
+    series.push(1.0, at(1));
+    series.push(7.0, at(2));
+
+    auto window = series.window(2);
+
+    REQUIRE(window.count == 2);
+    REQUIRE(window.min == 1.0);
+    REQUIRE(window.max == 7.0);
+}
+
+TEST_CASE("Empty buckets do not skew the window extremes", "[metric_series]") {
+    Series series(8);
+
+    series.push(5.0, at(0));
+    series.push(9.0, at(4));
+
+    auto window = series.window();
+
+    REQUIRE(window.count == 2);
+    REQUIRE(window.min == 5.0);
+    REQUIRE(window.max == 9.0);
+}
+
+TEST_CASE("A late value folds into the bucket it belongs to", "[metric_series]") {
+    Series series(8);
+
+    series.push(1.0, at(0));
+    series.push(2.0, at(2));
+    series.push(6.0, at(1));
+
+    REQUIRE(series.size() == 3);
+    REQUIRE(series.at_age(1).sum == 6.0);
+    REQUIRE(series.at_age(0).sum == 2.0);
+}
+
+TEST_CASE("A value older than every bucket held is dropped", "[metric_series]") {
+    Series series(4);
+
+    for(int64_t i = 0; i < 4; i++) {
+        series.push(1.0, at(i));
+    }
+
+    series.push(99.0, at(-10));
+
+    REQUIRE(series.size() == 4);
+    REQUIRE(series.window().max == 1.0);
+    REQUIRE(series.window().count == 4);
+}
+
+TEST_CASE("Linearize writes buckets oldest first", "[metric_series]") {
+    Series series(8);
+
+    series.push(1.0, at(0));
+    series.push(2.0, at(1));
+    series.push(3.0, at(2));
+
+    std::vector<double> xs;
+    std::vector<double> ys;
+
+    size_t count = series.linearize(xs, ys, MetricField::Sum);
+
+    REQUIRE(count == 3);
+    REQUIRE(xs == std::vector<double>{-2.0, -1.0, 0.0});
+    REQUIRE(ys == std::vector<double>{1.0, 2.0, 3.0});
+}
+
+TEST_CASE("Linearize can be limited to the newest buckets", "[metric_series]") {
+    Series series(8);
+
+    series.push(1.0, at(0));
+    series.push(2.0, at(1));
+    series.push(3.0, at(2));
+
+    std::vector<double> xs;
+    std::vector<double> ys;
+
+    size_t count = series.linearize(xs, ys, MetricField::Sum, 2);
+
+    REQUIRE(count == 2);
+    REQUIRE(xs == std::vector<double>{-1.0, 0.0});
+    REQUIRE(ys == std::vector<double>{2.0, 3.0});
+}
+
+TEST_CASE("Linearize can leave out the newest buckets", "[metric_series]") {
+    Series series(8);
+
+    series.push(1.0, at(0));
+    series.push(2.0, at(1));
+    series.push(3.0, at(2));
+
+    std::vector<double> xs;
+    std::vector<double> ys;
+
+    size_t count = series.linearize(xs, ys, MetricField::Sum, 8, 1);
+
+    REQUIRE(count == 2);
+    REQUIRE(ys == std::vector<double>{1.0, 2.0});
+}
+
+TEST_CASE("Skipping the newest bucket keeps the ages of the rest", "[metric_series]") {
+    Series series(8);
+
+    series.push(1.0, at(0));
+    series.push(2.0, at(1));
+    series.push(3.0, at(2));
+
+    std::vector<double> xs;
+    std::vector<double> ys;
+
+    series.linearize(xs, ys, MetricField::Sum, 8, 1);
+
+    REQUIRE(xs == std::vector<double>{-2.0, -1.0});
+}
+
+TEST_CASE("Skipping more buckets than are held writes nothing", "[metric_series]") {
+    Series series(8);
+
+    series.push(1.0, at(0));
+
+    std::vector<double> xs;
+    std::vector<double> ys;
+
+    size_t count = series.linearize(xs, ys, MetricField::Sum, 8, 4);
+
+    REQUIRE(count == 0);
+    REQUIRE(xs.empty());
+    REQUIRE(ys.empty());
+}
+
+TEST_CASE("A limit counts buckets that were not skipped", "[metric_series]") {
+    Series series(8);
+
+    for(int64_t i = 0; i < 5; i++) {
+        series.push((double)i, at(i));
+    }
+
+    std::vector<double> xs;
+    std::vector<double> ys;
+
+    size_t count = series.linearize(xs, ys, MetricField::Sum, 2, 1);
+
+    REQUIRE(count == 2);
+    REQUIRE(ys == std::vector<double>{2.0, 3.0});
+}
+
+TEST_CASE("Linearize reports empty buckets as zero", "[metric_series]") {
+    Series series(8);
+
+    series.push(5.0, at(0));
+    series.push(9.0, at(2));
+
+    std::vector<double> xs;
+    std::vector<double> ys;
+
+    series.linearize(xs, ys, MetricField::Max);
+
+    REQUIRE(ys == std::vector<double>{5.0, 0.0, 9.0});
+}
+
+TEST_CASE("Linearize resizes the vectors it is given", "[metric_series]") {
+    Series series(8);
+
+    series.push(1.0, at(0));
+
+    std::vector<double> xs(64, 7.0);
+    std::vector<double> ys(64, 7.0);
+
+    series.linearize(xs, ys, MetricField::Avg);
+
+    REQUIRE(xs.size() == 1);
+    REQUIRE(ys.size() == 1);
+}
+
+TEST_CASE("Clear empties the series but keeps its capacity", "[metric_series]") {
+    Series series(8);
+
+    series.push(1.0, at(0));
+    series.clear();
+
+    REQUIRE(series.is_empty());
+    REQUIRE(series.capacity() == 8);
+
+    series.push(2.0, at(1));
+
+    REQUIRE(series.size() == 1);
+    REQUIRE(series.at_age(0).sum == 2.0);
+}
+
+TEST_CASE("A coarser interval folds more values together", "[metric_series]") {
+    MetricSeries<std::chrono::minutes> series(4);
+
+    series.push(1.0, at(0));
+    series.push(2.0, at(30));
+    series.push(3.0, at(90));
+
+    REQUIRE(series.size() == 2);
+    REQUIRE(series.at_age(1).count == 2);
+    REQUIRE(series.at_age(0).sum == 3.0);
+}

modules/mock_client/CMakeLists.txt

@@ -10,7 +10,9 @@ target_link_libraries(${PROJECT_NAME}
         PUBLIC
         towards
         tw::network
+        tw::quicr
         tw::protocol
+        tw::message_protocol
         glm::glm
         EnTT::EnTT
         Jolt

modules/mock_client/src/main.cpp

@@ -1,24 +1,43 @@
 #include "Address.hpp"
 #include "Login.pb.h"
 #include "PlayerMove.pb.h"
-#include "TcpStream.hpp"
+#include "ProtobufMessages.hpp"
 #include "WorldState.pb.h"
-#include "messenger/MessageHandler.hpp"
-#include "messenger/Messenger.hpp"
+#include "message_protocol/MessageEndpoint.hpp"
 #include "runtime/LockStep.hpp"
 
 #include <chrono>
 #include <cstdlib>
+#include <memory>
 #include <semaphore>
 #include <thread>
+#include <vector>
 #include <glm/glm.hpp>
 
-tw::net::MessageHandler create_messenger(tw::net::Address& address) {
-    return tw::net::MessageHandler(address);
+static std::unique_ptr<tw::msg::MessageEndpoint> create_endpoint() {
+    auto endpoint_r = tw::msg::MessageEndpoint::create();
+    if(!endpoint_r) {
+        throw std::runtime_error("Failed to create the endpoint: " + endpoint_r.error().message());
+    }
+
+    return std::move(endpoint_r.value());
+}
+
+static tw::msg::MessageConnection* connect_to_server(tw::msg::MessageEndpoint* endpoint,
+                                                     tw::net::Address          address) {
+    auto server_r = endpoint->connect(address.ip_string(), address.port());
+    if(!server_r) {
+        throw std::runtime_error("Failed to connect to server: " + server_r.error().message());
+    }
+
+    return server_r.value();
 }
 
 class MockClient {
-    tw::net::MessageHandler m_handler;
+    std::unique_ptr<tw::msg::MessageEndpoint> m_endpoint;
+    tw::msg::MessageConnection*               m_server;
+    tw::ProtobufMessages                      m_messages;
+
     tw::LockStep m_lock_step;
 
     uint32_t m_frame_idx;
@@ -33,28 +52,30 @@ class MockClient {
 
 public:
     MockClient(tw::net::Address address, const std::string& name) :
-        m_handler(create_messenger(address)),
+        m_endpoint(create_endpoint()),
+        m_server(connect_to_server(m_endpoint.get(), address)),
+        m_messages(m_endpoint.get()),
         m_lock_step(20),
         m_is_running(true),
         m_is_connected(false),
         m_connected_semaphore(0)
     {
-        m_handler.set_handler<mmo::WorldStateMessage>(
-            [&](mmo::WorldStateMessage* mesg) { });
+        m_messages.set_handler<mmo::WorldStateMessage>(
+            [this](tw::msg::PeerId, const mmo::WorldStateMessage& mesg) { });
 
-        m_handler.set_handler<mmo::LoginResponse>(
-            [&](mmo::LoginResponse* mesg) {
+        m_messages.set_handler<mmo::LoginResponse>(
+            [this](tw::msg::PeerId, const mmo::LoginResponse& mesg) {
                 if(!m_is_connected) {
                     m_is_connected = true;
-                    m_entity_id = mesg->entity_id();
+                    m_entity_id = mesg.entity_id();
                     m_connected_semaphore.release();
                 }
             });
     }
 
     void run() {
-        while(!m_handler.is_connected()) {
-            m_handler.update();
+        while(!m_server->is_established()) {
+            m_endpoint->update();
             std::this_thread::sleep_for(std::chrono::milliseconds(10));
         }
 
@@ -67,7 +88,7 @@ public:
                 continue;
             }
 
-            m_handler.update();
+            m_endpoint->update();
             value += (float)std::rand() / RAND_MAX;
 
             m_velocity.x = glm::sin(value);
@@ -81,7 +102,7 @@ public:
             player_input->set_y(0);
             player_input->set_z(m_velocity.z);
             player_move_mesg.set_allocated_input(player_input);
-            auto r = m_handler.send(player_move_mesg);
+            auto r = m_messages.send(m_server, player_move_mesg, false);
         }
     }
 };

modules/network/CMakeLists.txt

@@ -5,7 +5,6 @@ add_subdirectory(tests)
 
 file(GLOB FILES
     src/*.cpp
-    src/messenger/*.cpp
     src/protocol/quicr/*.cpp
     src/frames/*.cpp
 )
@@ -13,9 +12,6 @@ file(GLOB FILES
 file(GLOB HEADERS
     include/*.hpp
     include/exception/*.hpp
-    include/io/*.hpp
-    include/messenger/*.hpp
-    include/packets/*.hpp
     include/metrics/*.hpp
     include/protocol/quicr/*.hpp
 )
@@ -37,7 +33,10 @@ target_include_directories(${PROJECT_NAME}
 target_link_libraries(${PROJECT_NAME}
     PUBLIC
     spdlog::spdlog
+    tw::io
     tw::protocol
+    tw::message_protocol
+    tw::quicr
     tl::expected
     Tracy::TracyClient
     TracyClient

modules/network/include/frames/Frame.hpp

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "common.hpp"
-#include "protocol/quicr/QuicrFrameType.hpp"
+#include "quicr/QuicrFrameType.hpp"
 
 #include <cstddef>
 #include <vector>

modules/network/include/messenger/MessageHandler.hpp

@@ -1,168 +0,0 @@
-#pragma once
-
-#include <concepts>
-#include <functional>
-#include <google/protobuf/message.h>
-#include <span>
-
-#include "Address.hpp"
-#include "Messenger.hpp"
-#include "NetworkError.hpp"
-#include "TcpStream.hpp"
-#include "packets/Packet.hpp"
-#include "packets/LoginPacket.hpp"
-#include "protocol/quicr/QuicrConnection.hpp"
-
-namespace tw::net {
-
-/**
- * Contains handlers for each message type. Calls this handler when message comes in.
- */
-class MessageHandler {
-private:
-    // std::optional<Messenger<std::byte, quicr::QuicrConnection>> m_quicr_messenger;
-    // Messenger<std::byte, TcpStream> m_server_messenger;
-    std::unique_ptr<quicr::QuicrEndpoint> m_quicr_endpoint;
-    quicr::QuicrConnection* m_quicr_connection;
-
-    std::vector<std::function<tl::expected<void, NetworkError>(std::span<std::byte>)>> m_handlers;
-
-    std::unique_ptr<quicr::QuicrEndpoint> create_endpoint() {
-        auto endpoint_r = quicr::QuicrEndpoint::create();
-        if(!endpoint_r) {
-            spdlog::error("Failed to create QuicrEndpoint: {}", endpoint_r.error().message());
-            throw std::runtime_error("Failed to create QuicrEndpoint");
-        }
-
-        return std::make_unique<quicr::QuicrEndpoint>(std::move(endpoint_r.value()));
-    }
-
-public:
-    const bool is_connected() const {
-        return m_quicr_connection->state() == quicr::QuicrConnectionState::Established;
-    }
-
-    MessageHandler(MessageHandler&& m)
-        // : m_server_messenger{std::move(m.m_server_messenger)},
-      :
-      m_handlers(std::move(m.m_handlers)),
-      m_quicr_endpoint(std::move(m.m_quicr_endpoint)),
-      m_quicr_connection(m.m_quicr_connection) {
-
-    }
-
-    MessageHandler(Address address) :
-        m_quicr_endpoint(create_endpoint()),
-        m_quicr_connection(m_quicr_endpoint->connect(address).value()),
-        m_handlers(100) {
-        spdlog::info("Connected to server at {}", address.to_string());
-    }
-
-
-    // MessageHandler(Messenger<std::byte, TcpStream>&& server_messenger) :
-    //     // m_server_messenger{std::move(server_messenger)},
-    //     m_quicr_connection(std::move(server_messenger.connection())),
-    //     m_handlers(100) {
-
-    // }
-
-    template<typename T>
-    constexpr void set_handler(const std::function<void(T*)> handler) {
-        PacketType type = Message<T>::value;
-        m_handlers[type] = [handler, this](std::span<std::byte> data) -> tl::expected<void, NetworkError> {
-            T result = {};
-
-            result.ParseFromArray(data.data(), data.size());
-            // spdlog::info("Deserialized message [{}]: {}", (int32_t)Message<T>::value, result.DebugString());
-
-            handler(&result);
-            // if(m_server_messenger.peek().has_value() && m_server_messenger.peek().value() == Message<T>::value) {
-            //     tl::expected<T, NetworkError> mesg = m_server_messenger.pop<T>(nullptr);
-            //     if(!mesg.has_value()) {
-            //         return tl::make_unexpected(mesg.error());
-            //     }
-
-            //     handler(&mesg.value());
-            // }
-
-
-            return {};
-        };
-    }
-
-    constexpr void set_raw_handler(uint32_t type, const std::function<tl::expected<void, NetworkError>(std::span<std::byte>)> handler) {
-        m_handlers[type] = handler;
-    }
-
-    void update() {
-        m_quicr_endpoint->poll();
-        while(true) {
-            std::vector<std::byte> buffer(64 * 1024);
-            auto read_r = m_quicr_connection->read_into(buffer);
-
-            if(!read_r) {
-                spdlog::error("Failed to read from QUICr stream: {}", read_r.error().message());
-                break;
-            }
-
-            if(*read_r == 0) {
-                break;
-            }
-
-            uint32_t type = reinterpret_cast<uint32_t*>(buffer.data())[0];
-            if(m_handlers[type] == nullptr) {
-                spdlog::warn("Unknown message type: {}", type);
-                throw std::runtime_error("Unknown message type: {}");
-                break;
-            }
-
-            auto handler_r = m_handlers[type](std::span<std::byte>(buffer.data(), *read_r).subspan(sizeof(uint32_t)));
-            if(!handler_r) {
-                spdlog::error("Handler error");
-                break;
-            }
-        }
-        // while(m_server_messenger.peek().has_value() && m_server_messenger.peek().value().has_value()) {
-        //     std::optional<PacketType> type = m_server_messenger.peek().value();
-        //     if(type >= m_handlers.size() || m_handlers[type.value()] == nullptr) {
-        //         spdlog::warn("Unknown message type: {}", (int)type.value());
-        //         break;
-        //     }
-
-        //     auto r = m_handlers[type.value()]();
-        //     if(!r) {
-        //         spdlog::error("Failed to handle message: {}", r.error().message());
-        //     }
-        // }
-    }
-
-    template<std::derived_from<google::protobuf::Message> T>
-    tl::expected<size_t, NetworkError> send(T& mesg) {
-        std::string payload;
-        if(!mesg.SerializeToString(&payload)) {
-            spdlog::error("Failed to serialize message");
-            return 0;
-        }
-
-        int32_t length = payload.length();
-        if(length == 0) {
-            return 0;
-        }
-
-        std::vector<std::byte> bytes(length + sizeof(uint32_t));
-
-        uint32_t type = Message<T>::value;
-        auto payload_bytes = std::as_writable_bytes(std::span(payload));
-        memcpy(bytes.data(), &type, sizeof(type));
-        memcpy(bytes.data() + sizeof(uint32_t), payload_bytes.data(), payload_bytes.size());
-
-        auto send_r = m_quicr_connection->send_message(bytes, false);
-        if(!send_r) {
-            spdlog::error("Failed to send message: {}", send_r.error().message());
-            return 0;
-        }
-        return payload_bytes.size();
-    }
-};
-
-}

modules/network/include/messenger/Messenger.hpp

@@ -1,209 +0,0 @@
-#pragma once
-
-#include <immintrin.h>
-#include <optional>
-#include <span>
-#include <google/protobuf/message.h>
-#include <google/protobuf/io/zero_copy_stream_impl.h>
-#include <tracy/Tracy.hpp>
-
-#include "NetworkError.hpp"
-#include "packets/Packet.hpp"
-#include "MessageRegistry.hpp"
-#include "protocol/quicr/QuicrFrameType.hpp"
-#include "tl/expected.hpp"
-
-namespace tw::net {
-
-template<typename TData, std::derived_from<Write<TData>> TOutput>
-class Messenger {
-private:
-    const uint32_t MAX_MESG_BODY_SIZE = 65536;
-    const uint32_t MESG_MAGIC = 0x1DEADBEE;
-
-    TOutput m_stream;
-
-    std::optional<PacketType> m_next_packet_type;
-
-    bool m_is_skipping;
-    uint32_t m_buffered_size;
-
-    size_t m_mesg_size;
-    size_t m_read_head;
-    std::vector<std::byte> m_input_buffer;
-
-public:
-    Messenger(Messenger && m) :
-      m_stream(std::move(m.m_stream)),
-      m_next_packet_type(m.m_next_packet_type),
-      m_input_buffer(std::move(m.m_input_buffer)),
-      m_buffered_size(m.m_buffered_size),
-      m_is_skipping(m.m_is_skipping),
-      m_mesg_size(m.m_mesg_size),
-      m_read_head(m.m_read_head)
-    {
-        // m_stream.set_non_blocking();
-    }
-
-    Messenger(TOutput&& stream) :
-        m_stream(std::move(stream)),
-        m_input_buffer(MAX_MESG_BODY_SIZE),
-        m_buffered_size(0),
-        m_is_skipping(false),
-        m_mesg_size(0),
-        m_read_head(0)
-    {
-        // m_stream.set_non_blocking();
-    }
-
-    Messenger<TData, TOutput> operator=(const Messenger<TData, TOutput>&) = delete;
-
-    Messenger<TData, TOutput> operator=(Messenger<TData, TOutput>&& m) {
-        m_stream = std::move(m.m_stream);
-        m_next_packet_type = m.m_next_packet_type;
-        m_input_buffer = std::move(m.m_input_buffer);
-        m_buffered_size = m.m_buffered_size;
-        m_is_skipping = m.m_is_skipping;
-        m_mesg_size = m.m_mesg_size;
-        m_read_head = m.m_read_head;
-    }
-
-    template <std::derived_from<google::protobuf::Message> T>
-    tl::expected<size_t, NetworkError> send(T &content) {
-        ZoneScopedN("Messenger::send");
-        auto id = (int32_t)Message<T>::value;
-
-        std::string payload;
-        if(!content.SerializeToString(&payload)) {
-            spdlog::error("Failed to serialize message");
-            throw std::runtime_error("Serialization failed");
-        }
-
-        // spdlog::info("Sending {}: {}", (int)Message<T>::value, content.DebugString());
-
-        int32_t length = payload.length();
-        if(length == 0) {
-            return 0;
-        }
-
-        // append encoded id & length before payload and write it to the stream
-        //
-        const uint32_t HEADER_SIZE = 4 + 4 + 4 + 4;
-
-        std::string message;
-        message.resize(HEADER_SIZE + payload.length());
-
-        const uint32_t magic = 0xDEADBEEF;
-        const uint32_t frame_type = quicr::FrameType::StreamBase;
-
-        std::memcpy(message.data(), &magic, sizeof(magic));
-        std::memcpy(message.data() + sizeof(magic), &frame_type, sizeof(frame_type));
-        std::memcpy(message.data() + sizeof(frame_type) + sizeof(magic), &length, sizeof(length));
-        std::memcpy(message.data() + sizeof(frame_type) + sizeof(magic) + sizeof(length), &id, sizeof(id));
-        // std::memcpy(message.data() + sizeof(id) + sizeof(length), &MESG_MAGIC, sizeof(MESG_MAGIC));
-        std::memcpy(message.data() + HEADER_SIZE, payload.data(), payload.length());
-
-        auto write_result = m_stream.write(std::as_writable_bytes(std::span(message)));
-        if(!write_result.has_value()) {
-            return tl::make_unexpected(write_result.error());
-        }
-
-        return write_result.value();
-    }
-
-    int32_t m_packet_peek_size = 0;
-
-    tl::expected<std::optional<PacketType>, NetworkError> peek() {
-        ZoneScopedN("Messenger::peek");
-        if(m_next_packet_type.has_value()) {
-            return m_next_packet_type;
-        }
-
-        if(m_read_head < 4) {
-            auto result = m_stream.read_into(std::as_writable_bytes(std::span{(char*)m_input_buffer.data(), sizeof(PacketType) - m_read_head}));
-
-            if(!result.has_value()) {
-                return tl::make_unexpected(result.error());
-            }
-
-            m_read_head += result.value();
-
-            if(m_read_head < 4) {
-                return {};
-            }
-        }
-
-        if(m_read_head < 8) {
-            auto result = m_stream.read_into(std::as_writable_bytes(std::span{(char*)m_input_buffer.data() + m_read_head, 8 - m_read_head}));
-
-            if(!result.has_value()) {
-                return tl::make_unexpected(result.error());
-            }
-
-            m_read_head += result.value();
-
-            if(m_read_head < 8) {
-                return {};
-            }
-
-            m_mesg_size = *reinterpret_cast<uint32_t*>(m_input_buffer.data() + 4);
-        }
-
-        if(m_read_head < m_mesg_size + 8) {
-            if(m_mesg_size + 8 > m_input_buffer.size()) {
-                return tl::make_unexpected(NetworkError(NetworkErrorType::NOT_ENOUGH_MEMORY));
-            }
-
-            auto result = m_stream.read_into(std::as_writable_bytes(std::span{(char*)m_input_buffer.data() + m_read_head, m_mesg_size + 8 - m_read_head}));
-
-            if(!result.has_value()) {
-                return tl::make_unexpected(result.error());
-            }
-
-            m_read_head += result.value();
-
-            if(m_read_head < m_mesg_size + 8) {
-                return {};
-            }
-        }
-
-        m_next_packet_type = (PacketType)(*reinterpret_cast<int32_t*>(m_input_buffer.data()));
-        return m_next_packet_type;
-    }
-
-    template<typename T>
-    tl::expected<T, NetworkError> pop(size_t* out_size) {
-        ZoneScopedN("Messenger::pop");
-        T result = {};
-
-        result.ParseFromArray(m_input_buffer.data() + 8, m_mesg_size);
-        // spdlog::info("Received {}: {}", (int)m_next_packet_type.value(), result.DebugString());
-
-        m_read_head = 0;
-        m_mesg_size = 0;
-        m_next_packet_type = {};
-
-
-        return result;
-    }
-
-    void skip() {
-
-    }
-
-    void clear() {
-        m_next_packet_type = std::nullopt;
-
-        int message_length = 0;
-        int size = sizeof(message_length);
-
-        // m_stream.read_exact(std::as_writable_bytes(std::span{&message_length, 1}));
-
-        std::vector<char> data(message_length);
-        // m_stream.read_exact(std::as_writable_bytes(std::span{data.data(), (size_t)message_length}));
-
-        // m_input_buffer.reset();
-    }
-};
-
-}

modules/network/include/messenger/MessengerDebugLog.hpp

@@ -1,29 +0,0 @@
-#pragma once
-
-#include "MessageRegistry.hpp"
-#include <spdlog/spdlog.h>
-#include <string>
-#include <fstream>
-
-class MessengerDebugLog {
-public:
-    MessengerDebugLog(MessengerDebugLog&& m) :
-        m_log_file(std::move(m.m_log_file))
-    { }
-
-    MessengerDebugLog(const std::string& log_file_path);
-    ~MessengerDebugLog();
-
-    template<typename T>
-    void log_send(const T& message) {
-        spdlog::info("Sending [{}]: {}", (int)tw::Message<T>::value, message.DebugString());
-    }
-
-    template<typename T>
-    void log_recv(const T& message) {
-        spdlog::info("Received [{}]: {}", (int)tw::Message<T>::value, message.DebugString());
-    }
-
-private:
-    std::ofstream m_log_file;
-};

modules/network/include/metrics/BucketMetric.hpp

@@ -1,157 +0,0 @@
-#pragma once
-
-#include <chrono>
-#include <cstdint>
-#include <stdexcept>
-#include <vector>
-#include <print>
-
-namespace tw::net {
-
-template<typename T>
-struct AverageOp {
-    uint32_t count;
-    T sum;
-
-    AverageOp() :
-    count(0),
-    sum{} {
-    }
-
-    void add(const T value) {
-        sum += value;
-        count++;
-    }
-
-    T result() const {
-        return count == 0 ? 0 : sum / count;
-    }
-};
-
-template<typename T>
-struct SumOp {
-    T sum;
-
-    void add(const T value) {
-        sum += value;
-    }
-
-    T result() const {
-        return sum;
-    }
-};
-
-template<typename T, typename Interval,
-    typename Operation = SumOp<T>,
-    typename Clock = std::chrono::steady_clock>
-class BucketMetric {
-
-    T m_min, m_max;
-
-    std::vector<T> m_metric;
-    std::vector<uint32_t> m_bucket_idx;
-
-    Operation m_op;
-
-    uint32_t m_offset;
-    uint32_t m_right, m_left;
-
-    std::string m_format;
-
-    const uint32_t get_bucket(Clock::time_point time_point) const {
-        return std::chrono::floor<Interval>(time_point).time_since_epoch().count() - m_offset;
-    }
-
-public:
-    BucketMetric(std::string format, uint32_t size) : 
-        m_metric(size),
-        m_bucket_idx(size),
-        m_right(0), m_left(0),
-        m_offset(0),
-        m_format(format)
-    {
-        m_offset = get_bucket(Clock::now());
-    }
-
-    const T max() const {
-        return m_max;
-    }
-
-    const T min() const {
-        return m_min;
-    }
-
-    const std::string& format() const {
-        return m_format;
-    }
-
-    size_t max_size() const {
-        return m_metric.size();
-    }
-
-    void push(T value) {
-        auto time = Clock::now();
-        size_t bucket = get_bucket(time) % m_metric.size();
-        size_t idx = get_bucket(time);
-
-        // set result to correct bucket
-        if(m_right != bucket) {
-            m_metric[m_right] = m_op.result();
-            m_min = std::min(m_min, m_op.result());
-            m_max = std::max(m_max, m_op.result());
-
-            m_bucket_idx[m_right] = idx++;
-            m_right++;
-            m_op = {};
-        }
-
-        // reset all buckets until the required one
-        for(; m_right != bucket; m_right = (m_right + 1) % m_metric.size()) {
-            m_metric[m_right] = {};
-            m_bucket_idx[m_right] = idx++;
-            if(m_right == m_left) {
-                m_left = (m_left + 1) % m_metric.size();
-            }
-        }
-
-        m_op.add(value);
-    }
-
-    const size_t get_size() const {
-        return m_right - m_left + (m_left > m_right ? m_metric.size() : 0);
-    }
-
-    const T get(uint32_t idx) const {
-        if(idx > get_size()) {
-            throw std::invalid_argument("`idx` cannot be higher than buffer size");
-        }
-
-        return m_metric[m_left + idx].result();
-    }
-
-    std::span<T> get_head() {
-        return std::span(m_metric).subspan(m_left, (m_right > m_left ? m_right : m_metric.size()));
-    }
-
-    std::span<uint32_t> get_head_timeline() {
-        return std::span(m_bucket_idx).subspan(m_left, (m_right > m_left ? m_right : m_bucket_idx.size()));
-    }
-
-    std::span<T> get_tail() {
-        if(m_right > m_left) {
-            return std::span<T>();
-        }
-
-        return std::span(m_metric).subspan(0, m_right);
-    }
-
-    std::span<uint32_t> get_tail_timeline() {
-        if(m_right > m_left) {
-            return std::span<T>();
-        }
-
-        return std::span(m_bucket_idx).subspan(0, m_right);
-    }
-};
-
-}

modules/network/include/metrics/HistoryBuffer.hpp

@@ -42,7 +42,7 @@ public:
     }
 
     std::optional<const TValue*> get(TKey key) const {
-        for(size_t i = m_tail; i != m_head; (i++) % max_size()) {
+        for(size_t i = m_tail; i != m_head; i = (i + 1) % max_size()) {
             if(m_buffer[i].first > key) {
                 return {};
             }
@@ -56,24 +56,25 @@ public:
     }
 
     bool set(TKey key, const TValue& value) {
-        if(key < m_buffer.at(m_tail).first) {
+        // If buffer has entries and key is older than the oldest, reject it
+        if(m_tail != m_head && key < m_buffer.at(m_tail).first) {
             return false;
         }
 
-        int i = m_tail + 1;
+        size_t i = (m_tail + 1) % max_size();
         if(m_tail != m_head) {
-            for(i = m_tail + 1; i != m_head; i++) {
+            for(i = (m_tail + 1) % max_size(); i != m_head; i = (i + 1) % max_size()) {
                 if(m_buffer.at(i).first > key) {
-                    m_buffer[(i - 1) % max_size()] = std::make_pair(key, value);
-                    m_head++;
+                    m_buffer[(i - 1 + max_size()) % max_size()] = std::make_pair(key, value);
+                    m_head = (m_head + 1) % max_size();
                     return true;
                 } else {
-                    m_buffer[(i - 1) % max_size()] = m_buffer[i];
+                    m_buffer[(i - 1 + max_size()) % max_size()] = m_buffer[i];
                 }
             }
         }
 
-        m_buffer[(i - 1) % max_size()] = std::make_pair(key, value);
+        m_buffer[(i - 1 + max_size()) % max_size()] = std::make_pair(key, value);
         m_head = (m_head + 1) % max_size();
         return true;
     }

modules/network/include/metrics/NetworkStatsLogger.hpp

@@ -1,134 +0,0 @@
-#pragma once
-
-#include <cstdint>
-#include <chrono>
-#include <filesystem>
-#include <cassert>
-
-#include "Address.hpp"
-#include "BucketMetric.hpp"
-#include "packets/Packet.hpp"
-#include "MessageRegistry.hpp"
-
-namespace tw::net {
-
-using Clock = std::chrono::steady_clock;
-using TimePoint = Clock::time_point;
-
-struct NetworkSendInfo {
-    PacketType message_type;
-    bool is_sent_by_us;
-    Address target;
-    TimePoint timepoint;
-    std::span<uint8_t> buffer;
-
-    NetworkSendInfo(
-            PacketType message_type,
-            bool is_sent_by_us,
-            const Address& target,
-            const std::span<uint8_t> buffer
-    ) :
-        message_type(message_type),
-        is_sent_by_us(is_sent_by_us),
-        target(target),
-        timepoint(std::chrono::steady_clock::now()),
-        buffer(buffer)
-    {
-    }
-};
-
-class NetworkStatsLogger {
-private:
-
-
-    std::vector<NetworkSendInfo> m_backlog;
-
-    std::vector<uint8_t> m_buffer;
-    size_t m_left, m_right;
-
-    std::optional<std::ostream> m_output;
-
-    using Interval = std::chrono::seconds;
-
-    BucketMetric<uint32_t, Interval, AverageOp<uint32_t>> m_ping_metric;
-    BucketMetric<uint32_t, Interval, SumOp<uint32_t>> m_outgoing;
-    BucketMetric<uint32_t, Interval, SumOp<uint32_t>> m_incoming;
-
-public:
-
-    NetworkStatsLogger() :
-        m_backlog(10000, {MESSAGE_PACKET, false, Address({}, 0), {}}),
-        m_buffer(1000000),
-        m_left(0), m_right(0),
-        m_ping_metric("ms", 1000),
-        m_outgoing("b/s", 1000),
-        m_incoming("b/s", 1000)
-    { }
-
-    void set_file_output(std::filesystem::path path);
-
-    size_t get_size() {
-        return m_right - m_left + (m_right < m_left ? m_backlog.size() : 0);
-    }
-
-    NetworkSendInfo& get_item(uint32_t idx) {
-        return m_backlog[(m_left + idx) % m_backlog.size()];
-    }
-
-    std::span<uint8_t> allocate_memory_for_buffer(size_t size) {
-        uint32_t start = m_right;
-        if(m_buffer.size() - m_right < size) {
-            // throw away packets from the start to make space
-            for(; m_backlog[m_left].buffer.data() < m_buffer.data() + start + size &&
-                    m_left != m_right; m_left = (m_left + 1) % m_buffer.size()) { }
-
-            start = 0;
-        }
-
-        uint32_t end = start + size;
-        return std::span<uint8_t>(m_buffer.begin() + start, m_buffer.begin() + end);
-    }
-
-    // constexpr void log(PacketType message_type, bool is_sent, const Address& target, const ByteBuffer& content) {
-    //     std::span<uint8_t> span = allocate_memory_for_buffer(content.size());
-
-    //     memcpy(span.data(), content.data().data(), content.size());
-
-    //     m_right = (m_right + 1) % m_backlog.size();
-    //     m_backlog[m_right] = NetworkSendInfo(message_type, is_sent, target, span);
-    // }
-
-    // constexpr void log_receive(
-    //         PacketType message_type,
-    //         const Address& from
-    // ) {
-    //     log(message_type, false, from, content);
-    //     m_incoming.push(content.size());
-    // }
-
-    // constexpr void log_send(
-    //         PacketType message_type,
-    //         const Address& to
-    // ) {
-    //     log(message_type, true, to, content);
-    //     m_outgoing.push(content.size());
-    // }
-
-    void log_ping(uint32_t ping) {
-        m_ping_metric.push(ping);
-    }
-
-    BucketMetric<uint32_t, Interval, AverageOp<uint32_t>>& ping(){
-        return m_ping_metric;
-    }
-
-    BucketMetric<uint32_t, Interval>& outgoing(){
-        return m_outgoing;
-    }
-
-    BucketMetric<uint32_t, Interval>& incoming(){
-        return m_incoming;
-    }
-};
-
-}

modules/network/include/packets/LoginPacket.hpp

@@ -1,67 +0,0 @@
-#pragma once
-
-#include "Entity.pb.h"
-#include "PlayerMove.pb.h"
-#include "WorldState.pb.h"
-#include "Login.pb.h"
-
-#include "Packet.hpp"
-#include "Serialization.hpp"
-
-const int MAX_USERNAME_LENGTH = 128;
-
-struct LoginPacket {
-    uint32_t username_length;
-    char username[MAX_USERNAME_LENGTH];
-};
-
-// template<>
-// class Message<LoginPacket> {
-// public:
-//     static constexpr PacketType value = LOGIN_REQUEST_MSG;
-// };
-
-
-
-
-template<>
-class tw::net::Serializer<LoginPacket> final {
-public:
-    static bool serialize(Serialization& buffer, LoginPacket& value) {
-        buffer.serialize(&value.username_length);
-        buffer.serialize(value.username, value.username_length);
-        return true;
-    }
-};
-
-// inline void to_json(json& j, const LoginPacket& value) {
-//     j = json{
-//         {"username_length", value.username_length},
-//         {"username", std::string(value.username, value.username_length)}
-//     };
-// }
-
-// inline void from_json(const json& j, LoginPacket& value) {
-//     j.at("username_length").get_to(value.username_length);
-//     j.at("username").get_to(value.username);
-// }
-
-struct LoginStatusPacket {
-    bool is_okay;
-
-    LoginStatusPacket() {
-    }
-
-    LoginStatusPacket(bool is_okay) :
-        is_okay(is_okay)
-    {
-    }
-};
-
-template<>
-class tw::net::Serializer<LoginStatusPacket> final {
-public:
-    static bool serialize(Serialization& buffer, LoginStatusPacket& value) {
-        return buffer.serialize(&value.is_okay);
-    }
-};

modules/network/include/packets/Packet.hpp

@@ -1,26 +0,0 @@
-#pragma once
-
-#include "Serializers.hpp"
-
-// #define PACKET(name) struct #name {
-
-
-
-
-// template<>
-// class tw::net::Serializer<const PacketType> final {
-// public:
-//     static bool serialize(tw::net::Serialization& buffer, const PacketType& value) {
-//         uint32_t v = value;
-//         return buffer.serialize((uint32_t*)&v);
-//     }
-// };
-
-
-// template<>
-// class tw::net::Serializer<PacketType> final {
-// public:
-//     static bool serialize(tw::net::Serialization& buffer, PacketType& value) {
-//         return buffer.serialize((uint32_t*)&value);
-//     }
-// };

modules/network/include/protocol/quicr/QuicrError.hpp

@@ -1,34 +0,0 @@
-#pragma once
-
-#include <algorithm>
-#include <string>
-namespace tw::net::quicr {
-
-enum class QuicrErrorType {
-    ConnectionClosed
-};
-
-
-struct QuicrError {
-public:
-    QuicrError(QuicrErrorType type) : type_(type), message_(map_quicr_error_type(type)) {}
-    QuicrError(QuicrErrorType type, std::string message) : type_(type), message_(std::move(message)) {}
-    QuicrErrorType type() const { return type_; }
-
-    std::string message() const { return message_; }
-
-private:
-    static std::string map_quicr_error_type(QuicrErrorType type) {
-        switch (type) {
-            case QuicrErrorType::ConnectionClosed:
-                return "ConnectionClosed";
-            default:
-                return "Unknown";
-        }
-    }
-
-    std::string message_;
-    QuicrErrorType type_;
-};
-
-}

modules/network/include/protocol/quicr/QuicrStream.hpp

@@ -1,34 +0,0 @@
-
-#include "NetworkError.hpp"
-#include "protocol/quicr/QuicrConnection.hpp"
-#include "tl/expected.hpp"
-
-namespace tw::net::quicr {
-
-class QuicrStream : Write<std::byte>, Read<std::byte> {
-    QuicrConnection* m_connection;
-    bool m_is_reliable;
-
-public:
-    QuicrStream(QuicrConnection* connection, bool is_reliable);
-
-    tl::expected<size_t, NetworkError> write(std::span<std::byte> data) override {
-        auto send_r = m_connection->send_message(data, m_is_reliable);
-        if(!send_r) {
-            return tl::make_unexpected(NetworkError::from_errno(CONNECTION_RESET));
-        }
-
-        return *send_r;
-    }
-
-    tl::expected<size_t, NetworkError> read_into(std::span<std::byte> target) override {
-        auto read_r = m_connection->read_into(target);
-        if(!read_r) {
-            return tl::make_unexpected(NetworkError::from_errno(CONNECTION_RESET));
-        }
-
-        return *read_r;
-    }
-};
-
-}

modules/network/src/NetworkStatsLogger.cpp

@@ -1,6 +0,0 @@
-#include "metrics/NetworkStatsLogger.hpp"
-
-namespace tw::net {
-
-
-}

modules/network/src/messenger/Messenger.cpp

@@ -1,6 +0,0 @@
-#include "messenger/Messenger.hpp"
-
-namespace tw::net {
-
-
-}

modules/network/src/messenger/MessengerDebugLog.cpp

@@ -1,12 +0,0 @@
-#include "messenger/MessengerDebugLog.hpp"
-#include <nlohmann/json.hpp>
-
-MessengerDebugLog::MessengerDebugLog(const std::string& log_file_path) : m_log_file(log_file_path) {
-    if (!m_log_file.is_open()) {
-        throw std::runtime_error("Failed to open log file");
-    }
-}
-
-MessengerDebugLog::~MessengerDebugLog() {
-    m_log_file.close();
-}

modules/network/tests/CMakeLists.txt

@@ -18,29 +18,6 @@ target_link_libraries(${PROJECT_NAME}_sources
 )
 
 add_executable(${PROJECT_NAME})
-add_executable(QuicrOverloadTest ./quicr/QuicrOverloadTests.cpp)
-add_executable(QuicrBenchmarks ./quicr/QuicrBenchmarks.cpp)
-
-target_link_libraries(QuicrBenchmarks
-    PRIVATE
-    ${LIBS}
-    ${PROJECT_NAME}_sources
-    Tracy::TracyClient
-    Catch2::Catch2WithMain
-    tl::expected
-    EnTT::EnTT
-)
-
-target_link_libraries(QuicrOverloadTest
-    PRIVATE
-    ${LIBS}
-    ${PROJECT_NAME}_sources
-    Tracy::TracyClient
-    TracyClient
-    Catch2::Catch2WithMain
-    tl::expected
-    EnTT::EnTT
-)
 
 target_link_libraries(${PROJECT_NAME}
     PRIVATE

modules/network/tests/FrameEncoderTests.cpp

@@ -2,7 +2,7 @@
 #include "bytebuffer/ByteBufferDecoder.hpp"
 #include "catch2/catch_test_macros.hpp"
 #include "frames/FrameCodec.hpp"
-#include "protocol/quicr/QuicrFrameType.hpp"
+#include "quicr/QuicrFrameType.hpp"
 
 using namespace tw::net;
 

modules/network/tests/MessengerTests.cpp

@@ -1,46 +0,0 @@
-#include <catch2/catch_test_macros.hpp>
-
-// #include "io/Read.hpp"
-// #include "messenger/Messenger.hpp"
-
-// class MockReader : public Read<std::byte> {
-//     size_t m_cursor;
-//     std::string m_content;
-
-// public:
-//     MockReader(const std::string& content) :
-//         m_cursor(0),
-//         m_content(content) {
-
-//     }
-
-//     size_t read(std::span<std::byte> data) override {
-//         size_t read_len = std::min(data.size(), m_content.size() - m_cursor);
-//         if(read_len == 0) {
-//             return 0;
-//         }
-
-//         std::copy(m_content.begin() + m_cursor, m_content.begin() + m_cursor + read_len, data.begin());
-//         m_cursor += read_len;
-//         return read_len;
-//     }
-// };
-
-// class MockWriter : public Write<std::byte> {
-
-// public:
-//     MockWriter() {
-
-//     }
-
-//     size_t write(std::span<const std::byte> data) override {
-//     }
-// };
-
-// TEST_CASE("Test01", "[Messenger_Test]") {
-//     MockReader reader("0Hello, World!");
-//     MockWriter writer;
-//     tw::net::Messenger messenger(&writer, &reader);
-
-//     REQUIRE(messenger.peek() == '0');
-// }

modules/network/tests/UdpStreamTests.cpp

@@ -5,9 +5,9 @@
 
 #include "UdpStream.hpp"
 #include "Address.hpp"
-#include "protocol/quicr/QuicrConnection.hpp"
-#include "protocol/quicr/QuicrConnectionListener.hpp"
-#include "protocol/quicr/QuicrEndpoint.hpp"
+#include "quicr/QuicrConnection.hpp"
+#include "quicr/QuicrConnectionListener.hpp"
+#include "quicr/QuicrEndpoint.hpp"
 
 TEST_CASE("Start two sockets and send message", "[udp]") {
     std::barrier create_sync_point(2);

modules/peer_to_peer/CMakeLists.txt

@@ -24,7 +24,9 @@ target_link_libraries(tw_peer_to_peer_lib
     PUBLIC
         towards
         tw::network
+        tw::quicr
         tw::protocol
+        tw::message_protocol
         glm::glm
         EnTT::EnTT
         spdlog::spdlog

modules/peer_to_peer/src/QuicrPeerLink.cpp

@@ -1,6 +1,6 @@
 #include "QuicrPeerLink.hpp"
-#include "protocol/quicr/QuicrConnectionListener.hpp"
-#include "protocol/quicr/QuicrEndpoint.hpp"
+#include "quicr/QuicrConnectionListener.hpp"
+#include "quicr/QuicrEndpoint.hpp"
 
 #include <cstring>
 #include <spdlog/spdlog.h>
@@ -15,7 +15,7 @@ QuicrPeerLink::QuicrPeerLink(uint32_t self_id, uint16_t port)
 {}
 
 void QuicrPeerLink::connect_to(uint32_t peer_id, const tw::net::Address& addr) {
-    auto r = m_endpoint->connect(addr);
+    auto r = m_endpoint->connect(net::quicr::QuicrAddress(addr.ip_string(), addr.port()));
     if (!r) {
         spdlog::warn("QuicrPeerLink[{}]: connect to peer {} failed", m_self_id, peer_id);
         return;

modules/peer_to_peer/src/QuicrPeerLink.hpp

@@ -1,9 +1,9 @@
 #pragma once
 
 #include "PeerLink.hpp"
-#include "protocol/quicr/QuicrConnection.hpp"
-#include "protocol/quicr/QuicrConnectionListener.hpp"
-#include "protocol/quicr/QuicrEndpoint.hpp"
+#include "quicr/QuicrConnection.hpp"
+#include "quicr/QuicrConnectionListener.hpp"
+#include "quicr/QuicrEndpoint.hpp"
 
 #include <list>
 #include <optional>

modules/protocol/CMakeLists.txt

@@ -24,6 +24,7 @@ target_link_libraries(${PROJECT_NAME}
     PUBLIC
     glm::glm
     protobuf::libprotobuf
+    tw::message_protocol
 )
 
 target_include_directories(${PROJECT_NAME}

modules/protocol/proto/Entity.proto

@@ -11,3 +11,7 @@ message EntitySpawnMessage {
 message EntityDespawnMessage {
     uint32 entity_id = 1;
 }
+
+message SetControlledEntity {
+    uint32 entity_id = 1;
+}

modules/protocol/src/MessageRegistry.hpp

@@ -7,10 +7,17 @@
 #include "PlayerMove.pb.h"
 #include "Entity.pb.h"
 
+#include "message_protocol/MessageType.hpp"
+
 #include <spdlog/fmt/fmt.h>
 #include <string_view>
 
-enum PacketType {
+namespace tw {
+
+/**
+ * Every address this application assigns, and the message that travels to it.
+ */
+enum MessageType : msg::MessageType {
     MESSAGE_PACKET,
     LOGIN_REQUEST_MSG,
     LOGIN_RESPONSE_PACKET,
@@ -33,9 +40,11 @@ enum PacketType {
 
     CLUSTER_ZONE_HELLO,
     CLUSTER_ZONE_BYE,
+
+    SET_CONTROLLED_ENTITY_MSG,
 };
 
-namespace tw {
+
 template<typename Msg>
 class Message;
 
@@ -43,99 +52,107 @@ class Message;
 template<>
 class Message<mmo::LoginRequest> {
 public:
-    static constexpr PacketType value = LOGIN_REQUEST_MSG;
+    static constexpr MessageType value = LOGIN_REQUEST_MSG;
 };
 
 template<>
 class Message<mmo::LoginResponse> {
 public:
-    static constexpr PacketType value = LOGIN_RESPONSE_PACKET;
+    static constexpr MessageType value = LOGIN_RESPONSE_PACKET;
 };
 
 template<>
 class Message<mmo::WorldStateMessage> {
 public:
-    static constexpr PacketType value = WORLD_STATE_PACKET;
+    static constexpr MessageType value = WORLD_STATE_PACKET;
 };
 
 template<>
 class Message<mmo::PlayerMoveMessage> {
 public:
-    static constexpr PacketType value = PLAYER_UPDATE_MSG;
+    static constexpr MessageType value = PLAYER_UPDATE_MSG;
 };
 
 template<>
 class Message<mmo::EntitySpawnMessage> {
 public:
-    static constexpr PacketType value = ENTITY_SPAWN_MSG;
+    static constexpr MessageType value = ENTITY_SPAWN_MSG;
 };
 
 template<>
 class Message<mmo::EntityDespawnMessage> {
 public:
-    static constexpr PacketType value = ENTITY_DESPAWN_MSG;
+    static constexpr MessageType value = ENTITY_DESPAWN_MSG;
+};
+
+template<>
+class Message<mmo::SetControlledEntity> {
+public:
+    static constexpr MessageType value = SET_CONTROLLED_ENTITY_MSG;
 };
 
 
 template<>
 class Message<mmo::chat::SendChatMessageRequest> {
 public:
-    static constexpr PacketType value = CHAT_SEND_MESSAGE_REQUEST;
+    static constexpr MessageType value = CHAT_SEND_MESSAGE_REQUEST;
 };
 
 template<>
 class Message<mmo::chat::SendChatMessageResponse> {
 public:
-    static constexpr PacketType value = CHAT_SEND_MESSAGE_RESPONSE;
+    static constexpr MessageType value = CHAT_SEND_MESSAGE_RESPONSE;
 };
 
 template<>
 class Message<mmo::chat::JoinChannelRequest> {
 public:
-    static constexpr PacketType value = CHAT_JOIN_CHANNEL_REQUEST;
+    static constexpr MessageType value = CHAT_JOIN_CHANNEL_REQUEST;
 };
 
 template<>
 class Message<mmo::chat::JoinChannelResponse> {
 public:
-    static constexpr PacketType value = CHAT_JOIN_CHANNEL_RESPONSE;
+    static constexpr MessageType value = CHAT_JOIN_CHANNEL_RESPONSE;
 };
 
 template<>
 class Message<mmo::chat::LeaveChannelRequest> {
 public:
-    static constexpr PacketType value = CHAT_LEAVE_CHANNEL_REQUEST;
+    static constexpr MessageType value = CHAT_LEAVE_CHANNEL_REQUEST;
 };
 
 template<>
 class Message<mmo::chat::LeaveChannelResponse> {
 public:
-    static constexpr PacketType value = CHAT_LEAVE_CHANNEL_RESPONSE;
+    static constexpr MessageType value = CHAT_LEAVE_CHANNEL_RESPONSE;
 };
 
 template<>
 class Message<mmo::chat::ChatMessageBroadcastRequest> {
 public:
-    static constexpr PacketType value = CHAT_MESSAGE_BROADCAST_REQUEST;
+    static constexpr MessageType value = CHAT_MESSAGE_BROADCAST_REQUEST;
 };
 
 template<>
 class Message<mmo::cluster::ZoneHello> {
 public:
-    static constexpr PacketType value = CLUSTER_ZONE_HELLO;
+    static constexpr MessageType value = CLUSTER_ZONE_HELLO;
 };
 
 template<>
 class Message<mmo::cluster::ZoneBye> {
 public:
-    static constexpr PacketType value = CLUSTER_ZONE_BYE;
+    static constexpr MessageType value = CLUSTER_ZONE_BYE;
 };
 }
 
 
 template<>
-struct fmt::formatter<PacketType> : fmt::formatter<std::string_view> {
-    auto format(PacketType type, fmt::format_context& ctx) const {
+struct fmt::formatter<tw::MessageType> : fmt::formatter<std::string_view> {
+    auto format(tw::MessageType type, fmt::format_context& ctx) const {
+        using enum tw::MessageType;
+
         std::string_view name;
         switch (type) {
             case MESSAGE_PACKET:               name = "MESSAGE_PACKET";               break;

modules/protocol/src/ProtobufMessages.hpp

@@ -0,0 +1,94 @@
+#pragma once
+
+#include "MessageRegistry.hpp"
+
+#include "message_protocol/MessageConnection.hpp"
+#include "message_protocol/MessageEndpoint.hpp"
+
+#include <spdlog/spdlog.h>
+
+#include <functional>
+#include <span>
+#include <utility>
+#include <vector>
+
+namespace tw {
+
+/**
+ * Sends and receives protobuf messages over an endpoint.
+ *
+ * A view rather than an owner: several of these may share one endpoint, and
+ * messages encoded some other way can travel over it at the same time.
+ */
+class ProtobufMessages {
+    msg::MessageEndpoint* m_endpoint;
+
+    // Reused between sends so a steady stream of messages does not allocate.
+    std::vector<std::byte> m_buffer;
+
+    template<typename T>
+    bool serialize(const T& message) {
+        m_buffer.resize(message.ByteSizeLong());
+        return message.SerializeToArray(m_buffer.data(), static_cast<int>(m_buffer.size()));
+    }
+
+public:
+    explicit ProtobufMessages(msg::MessageEndpoint* endpoint) :
+        m_endpoint(endpoint) {
+    }
+
+    /** Calls `handler` for every T that arrives from any peer. */
+    template<typename T>
+    void set_handler(std::function<void(msg::PeerId, const T&)> handler) {
+        m_endpoint->set_handler(
+            Message<T>::value,
+            [handler = std::move(handler), message = T{}](msg::PeerId                  peer,
+                                                          std::span<const std::byte> body) mutable {
+                if(!message.ParseFromArray(body.data(), static_cast<int>(body.size()))) {
+                    spdlog::warn("Failed to parse a {} of {} bytes", Message<T>::value, body.size());
+                    return;
+                }
+
+                handler(peer, message);
+            });
+    }
+
+    template<typename T>
+    tl::expected<void, msg::MessageError> send(msg::MessageConnection* peer,
+                                               const T&                message,
+                                               bool                    reliable = true) {
+        if(!serialize(message)) {
+            return tl::make_unexpected(
+                msg::MessageError(msg::MessageErrorType::SendFailed, "failed to serialize the message"));
+        }
+
+        return peer->send(Message<T>::value, m_buffer, reliable);
+    }
+
+    template<typename T>
+    tl::expected<void, msg::MessageError> send_to(msg::PeerId id, const T& message, bool reliable = true) {
+        auto* peer = m_endpoint->peer(id);
+        if(peer == nullptr) {
+            return tl::make_unexpected(msg::MessageError(msg::MessageErrorType::NotConnected));
+        }
+
+        return send(peer, message, reliable);
+    }
+
+    template<typename T>
+    void broadcast(const T& message, bool reliable = false) {
+        if(!serialize(message)) {
+            spdlog::error("Failed to serialize a {} for broadcast", Message<T>::value);
+            return;
+        }
+
+        for(auto* peer : m_endpoint->peers()) {
+            auto send_r = peer->send(Message<T>::value, m_buffer, reliable);
+            if(!send_r) {
+                spdlog::error("Failed to send to peer {}: {}", peer->peer_id(), send_r.error().message());
+            }
+        }
+    }
+};
+
+}

modules/protocol/src/messages/PlayerMoveMessage.hpp

@@ -3,7 +3,6 @@
 #include <nlohmann/json.hpp>
 
 #include "Serialization.hpp"
-#include "packets/Packet.hpp"
 #include "Serializers.hpp"
 #include "GlmSerializers.hpp"
 #include <glm/glm.hpp>

modules/protocol/src/messages/PlayerUpdateMessage.hpp

@@ -4,7 +4,6 @@
 #include <glm/glm.hpp>
 
 #include "Serializers.hpp"
-#include "packets/Packet.hpp"
 
 struct PlayerUpdate {
     uint32_t id;

modules/quicr/CMakeLists.txt

@@ -0,0 +1,34 @@
+project(tw_quicr)
+
+# set(CMAKE_CXX_CLANG_TIDY "/usr/bin/clang-tidy;-checks=*")
+
+file(GLOB FILES
+    src/*.cpp
+)
+
+file(GLOB HEADERS
+    include/*.hpp
+)
+
+add_library(${PROJECT_NAME} OBJECT ${FILES})
+add_library(tw::quicr ALIAS ${PROJECT_NAME})
+target_sources(${PROJECT_NAME}
+    PUBLIC FILE_SET HEADERS
+    BASE_DIRS include
+    FILES ${HEADERS})
+
+
+target_include_directories(${PROJECT_NAME}
+        PUBLIC
+        ${PROJECT_SOURCE_DIR}/include/
+)
+
+target_link_libraries(${PROJECT_NAME}
+        PUBLIC
+        tw::io
+        tl::expected
+        Tracy::TracyClient
+        TracyClient
+)
+
+# add_subdirectory(./tests/)

modules/quicr/include/quicr/QuicrAddress.hpp

@@ -0,0 +1,182 @@
+#pragma once
+
+#include <arpa/inet.h>
+#include <spdlog/spdlog.h>
+#include <string>
+#include <cstring>
+#include <optional>
+#include <sys/socket.h>
+#include <format>
+
+namespace tw::net::quicr {
+
+/**
+ * IP Address
+ */
+struct QuicrAddress {
+private:
+    sockaddr_storage m_storage {};
+
+public:
+    QuicrAddress(const std::optional<std::string>& address, int port) {
+        std::memset((char*)&this->m_storage, 0, sizeof(this->m_storage));
+
+        auto& addr = reinterpret_cast<sockaddr_in&>(m_storage);
+        addr.sin_family = AF_INET;
+        addr.sin_port = htons(static_cast<uint16_t>(port));
+        addr.sin_addr.s_addr = address.has_value()
+            ? inet_addr(address->c_str())
+            : INADDR_ANY;
+    }
+
+
+    QuicrAddress(sockaddr_storage& storage)
+        : m_storage(storage)
+    { }
+
+    QuicrAddress(sockaddr_storage&& storage)
+        : m_storage(storage)
+    { }
+
+    /** Return a const pointer suitable for connect / sendto / bind. */
+    const struct sockaddr* sockaddr() const {
+        return reinterpret_cast<const struct sockaddr*>(&m_storage);
+    }
+
+    /** Return a mutable pointer suitable for recvfrom / accept. */
+    struct sockaddr* sockaddr_mut() {
+        return reinterpret_cast<struct sockaddr*>(&m_storage);
+    }
+
+    /** Return the size of the active address (depends on family). */
+    socklen_t socklen() const {
+        switch (m_storage.ss_family) {
+            case AF_INET:  return sizeof(sockaddr_in);
+            case AF_INET6: return sizeof(sockaddr_in6);
+            default:       return sizeof(sockaddr_storage);
+        }
+    }
+
+    /** Mutable reference to the raw storage — useful when you need to pass
+     *  a sockaddr_storage* to recvfrom together with a socklen_t. */
+    sockaddr_storage& storage() { return m_storage; }
+    const sockaddr_storage& storage() const { return m_storage; }
+
+    sa_family_t family() const { return m_storage.ss_family; }
+
+    /** Returns the raw network-order IPv4 address, or 0 if not AF_INET. */
+    uint32_t ipv4_addr_raw() const {
+        if (m_storage.ss_family == AF_INET) {
+            return reinterpret_cast<const sockaddr_in&>(m_storage).sin_addr.s_addr;
+        }
+        return 0;
+    }
+
+    /** Returns the raw network-order port (any family). */
+    uint16_t port_raw() const {
+        switch (m_storage.ss_family) {
+            case AF_INET:
+            return reinterpret_cast<const sockaddr_in&>(m_storage).sin_port;
+            case AF_INET6:
+            return reinterpret_cast<const sockaddr_in6&>(m_storage).sin6_port;
+            default:
+            return 0;
+        }
+    }
+
+    uint16_t port() const {
+        switch (m_storage.ss_family) {
+            case AF_INET:
+            return ntohs(reinterpret_cast<const sockaddr_in&>(m_storage).sin_port);
+            case AF_INET6:
+            return ntohs(reinterpret_cast<const sockaddr_in6&>(m_storage).sin6_port);
+            default:
+            return 0;
+        }
+    }
+
+    /** Return the IP portion only (no port). */
+    std::string ip_string() const {
+        char buf[INET6_ADDRSTRLEN]{};
+        switch (m_storage.ss_family) {
+            case AF_INET: {
+                const auto& v4 = reinterpret_cast<const sockaddr_in&>(m_storage);
+                inet_ntop(AF_INET, &v4.sin_addr, buf, sizeof(buf));
+                break;
+            }
+            case AF_INET6: {
+                const auto& v6 = reinterpret_cast<const sockaddr_in6&>(m_storage);
+                inet_ntop(AF_INET6, &v6.sin6_addr, buf, sizeof(buf));
+                break;
+            }
+            default:
+            return "<unknown>";
+        }
+        return std::string(buf);
+    }
+
+    /** Human-readable "ip:port" (or "[ip]:port" for IPv6). */
+    std::string to_string() const {
+        if (m_storage.ss_family == AF_INET6) {
+            return std::format("[{}]:{}", ip_string(), port());
+        }
+        return std::format("{}:{}", ip_string(), port());
+    }
+
+    bool operator==(const QuicrAddress& other) const {
+        if (m_storage.ss_family != other.m_storage.ss_family) return false;
+
+        switch (m_storage.ss_family) {
+            case AF_INET: {
+                const auto& a = reinterpret_cast<const sockaddr_in&>(m_storage);
+                const auto& b = reinterpret_cast<const sockaddr_in&>(other.m_storage);
+                return a.sin_port == b.sin_port
+                && a.sin_addr.s_addr == b.sin_addr.s_addr;
+            }
+            case AF_INET6: {
+                const auto& a = reinterpret_cast<const sockaddr_in6&>(m_storage);
+                const auto& b = reinterpret_cast<const sockaddr_in6&>(other.m_storage);
+                return a.sin6_port == b.sin6_port
+                && std::memcmp(&a.sin6_addr, &b.sin6_addr, sizeof(in6_addr)) == 0;
+            }
+            default:
+            return std::memcmp(&m_storage, &other.m_storage, sizeof(m_storage)) == 0;
+        }
+    }
+
+    bool operator!=(const QuicrAddress& other) const { return !(*this == other); }
+
+    /**
+        * Retained for source compatibility. Prefer operator==.
+        */
+    bool equals(const QuicrAddress& other) const { return *this == other; }
+};
+
+
+
+}
+template<>
+struct std::hash<tw::net::quicr::QuicrAddress> {
+    std::size_t operator()(const tw::net::quicr::QuicrAddress& addr) const noexcept {
+        // FNV-style combine of family + port + address bytes
+        std::size_t h = std::hash<uint16_t>{}(addr.family());
+        h ^= std::hash<uint16_t>{}(addr.port_raw()) + 0x9e3779b9 + (h << 6) + (h >> 2);
+
+        switch (addr.family()) {
+            case AF_INET:
+                h ^= std::hash<uint32_t>{}(addr.ipv4_addr_raw()) + 0x9e3779b9 + (h << 6) + (h >> 2);
+                break;
+            case AF_INET6: {
+                const auto& s = reinterpret_cast<const sockaddr_in6&>(addr.storage());
+                const auto* bytes = reinterpret_cast<const uint8_t*>(&s.sin6_addr);
+                for (int i = 0; i < 16; ++i) {
+                    h ^= std::hash<uint8_t>{}(bytes[i]) + 0x9e3779b9 + (h << 6) + (h >> 2);
+                }
+                break;
+            }
+            default:
+                break;
+        }
+        return h;
+    }
+};

modules/quicr/include/quicr/QuicrConnection.hpp

@@ -1,20 +1,20 @@
 #pragma once
 
-#include "Address.hpp"
-#include "NetworkError.hpp"
-#include "bytebuffer/ByteBuffer.hpp"
-#include "io/Read.hpp"
-#include "protocol/quicr/QuicrConnectionIdGenerator.hpp"
-#include "protocol/quicr/QuicrEndpoint.hpp"
-#include "protocol/quicr/QuicrError.hpp"
-#include "protocol/quicr/QuicrPacket.hpp"
-#include "protocol/quicr/QuicrReliability.hpp"
 #include <cstddef>
 #include <chrono>
 #include <deque>
 #include <sys/socket.h>
 #include <tl/expected.hpp>
 
+#include "quicr/QuicrAddress.hpp"
+#include "quicr/QuicrError.hpp"
+#include "bytebuffer/ByteBuffer.hpp"
+#include "quicr/QuicrConnectionIdGenerator.hpp"
+#include "quicr/QuicrEndpoint.hpp"
+#include "quicr/QuicrPacket.hpp"
+#include "quicr/QuicrReliability.hpp"
+
+
 namespace tw::net::quicr {
 
 const int TW_NET_HEARTBEAT_INTERVAL_IN_MILLIS = 5000;
@@ -74,14 +74,14 @@ class QuicrEndpoint;
 /**
  * Established QUICr connection.
  */
-class QuicrConnection : Read<std::byte> {
+class QuicrConnection {
     static constexpr int PROTOCOL_VERSION = 1;
     static constexpr int MAX_HELLO_RETRIES = 5;
     static constexpr int HELLO_RETRY_INTERVAL_MS = 200;
 
     using Clock = std::chrono::steady_clock;
 
-    Address m_peer_address;
+    QuicrAddress m_peer_address;
     QuicrEndpoint* m_endpoint;
 
     QuicrReliabilityUnit* m_reliability_unit;
@@ -109,10 +109,10 @@ class QuicrConnection : Read<std::byte> {
     /**
      * Builds and writes next datagram.
      */
-    tl::expected<size_t, NetworkError> write_datagram(std::span<std::byte> data);
+    tl::expected<size_t, QuicrError> write_datagram(std::span<std::byte> data);
 
 public:
-    QuicrConnection(uint64_t self_id, uint64_t peer_id, Address peer_address, QuicrEndpoint* endpoint) :
+    QuicrConnection(uint64_t self_id, uint64_t peer_id, QuicrAddress peer_address, QuicrEndpoint* endpoint) :
         m_peer_address{peer_address},
         m_endpoint{endpoint},
         m_self_id{generate_id()},
@@ -125,7 +125,7 @@ public:
 
     // static tl::expected<QuicrConnection, NetworkError> connect(const Address& address);
 
-    constexpr Address address() {
+    constexpr QuicrAddress address() {
         return m_peer_address;
     }
 
@@ -149,7 +149,7 @@ public:
         return m_last_heartbeat_received < std::chrono::steady_clock::now() - std::chrono::milliseconds(TW_NET_HEARTBEAT_INTERVAL_IN_MILLIS * 2);
     }
 
-    tl::expected<void, NetworkError> send_keep_alive();
+    tl::expected<void, QuicrError> send_keep_alive();
 
     void send_initial_hello();
 
@@ -184,6 +184,7 @@ public:
     void send_hello_ack_frame();
 
     bool process_hello_ack_frame(std::span<const std::byte> dgram, size_t& off);
+
     /*
      * Handshake Done Frame
      * - Protocol version
@@ -199,11 +200,7 @@ public:
 
     void process_datagram(std::span<std::byte> dgram);
 
-    // void update();
-
-    // void drain_socket();
-
-    tl::expected<size_t, NetworkError> read_into(std::span<std::byte> target) override;
+    tl::expected<size_t, QuicrError> read_into(std::span<std::byte> target);
 
     void on_tick(std::chrono::steady_clock::time_point now);
 

modules/quicr/include/quicr/QuicrConnectionListener.hpp

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "NetworkError.hpp"
+#include "quicr/QuicrError.hpp"
 #include "tl/expected.hpp"
 
 #include <memory>
@@ -23,7 +23,7 @@ public:
     QuicrConnectionListener(QuicrConnectionListener&&) = delete;
     QuicrConnectionListener& operator=(QuicrConnectionListener&&) = delete;
 
-    static tl::expected<std::unique_ptr<QuicrConnectionListener>, NetworkError>
+    static tl::expected<std::unique_ptr<QuicrConnectionListener>, QuicrError>
     listen(QuicrEndpoint* endpoint);
 
     QuicrConnection* listen();
@@ -35,7 +35,7 @@ public:
     /**
      * Receives single datagram.
      */
-    // tl::expected<size_t, NetworkError> recv_into(std::span<std::byte> buffer, Address* from) {
+    // tl::expected<size_t, QuicrError> recv_into(std::span<std::byte> buffer, Address* from) {
     //     struct sockaddr_storage sockaddr_from;
     //     socklen_t from_length = sizeof( sockaddr_from );
 

modules/quicr/include/quicr/QuicrEncoder.hpp

@@ -3,8 +3,8 @@
 #include "bytebuffer/ByteBuffer.hpp"
 #include "bytebuffer/ByteBufferReader.hpp"
 #include "bytebuffer/ByteBufferWriter.hpp"
-#include "protocol/quicr/QuicrFrame.hpp"
-#include "protocol/quicr/QuicrPacket.hpp"
+#include "quicr/QuicrFrame.hpp"
+#include "quicr/QuicrPacket.hpp"
 
 #include <cstddef>
 

modules/quicr/include/quicr/QuicrEndpoint.hpp

@@ -1,14 +1,16 @@
 #pragma once
 
-#include "Address.hpp"
-#include "NetworkError.hpp"
-#include "protocol/quicr/QuicrConnection.hpp"
-
-#include <tl/expected.hpp>
 #include <memory>
 #include <unordered_map>
 #include <unistd.h>
 
+#include <tl/expected.hpp>
+
+#include "quicr/QuicrAddress.hpp"
+#include "quicr/QuicrError.hpp"
+#include "quicr/QuicrConnection.hpp"
+
+
 namespace tw::net::quicr {
 
 class QuicrConnection;
@@ -22,7 +24,7 @@ class QuicrEndpoint {
 
     QuicrConnectionListener* m_new_connection_handler;
 
-    void process_datagram(std::span<std::byte> datagram, Address from);
+    void process_datagram(std::span<std::byte> datagram, QuicrAddress from);
 
     QuicrEndpoint(int socket_fd);
 
@@ -45,24 +47,24 @@ public:
         return result;
     }
 
-    static tl::expected<std::unique_ptr<QuicrEndpoint>, NetworkError> create();
+    static tl::expected<std::unique_ptr<QuicrEndpoint>, QuicrError> create();
 
     /**
      * Creates the QUICr endpoint and binds it to a port.
      */
-    static tl::expected<std::unique_ptr<QuicrEndpoint>, NetworkError> create_and_bind(int16_t port);
+    static tl::expected<std::unique_ptr<QuicrEndpoint>, QuicrError> create_and_bind(int16_t port);
 
     void assign_listener(QuicrConnectionListener* listener) {
         m_new_connection_handler = listener;
     }
 
-    tl::expected<void, NetworkError> bind(int port);
+    tl::expected<void, QuicrError> bind(int port);
 
-    tl::expected<QuicrConnection*, NetworkError> connect(Address address);
+    tl::expected<QuicrConnection*, QuicrError> connect(QuicrAddress address);
 
-    tl::expected<size_t, NetworkError> send_to(std::span<std::byte> data, Address to);
+    tl::expected<size_t, QuicrError> send_to(std::span<std::byte> data, QuicrAddress to);
 
-    tl::expected<size_t, NetworkError> read_from_into(std::span<std::byte> data, Address* out_from);
+    tl::expected<size_t, QuicrError> read_from_into(std::span<std::byte> data, QuicrAddress* out_from);
 
     void poll();
 };

modules/quicr/include/quicr/QuicrError.hpp

@@ -0,0 +1,109 @@
+#pragma once
+
+#include <cerrno>
+#include <cstdint>
+#include <cstring>
+#include <string>
+
+namespace tw::net::quicr {
+
+/**
+ * Error categories surfaced by the QUICr stack.
+ *
+ * The socket-level values are taken straight from <cerrno> so a QuicrError can
+ * be built directly from `errno` after a failed UDP syscall (see from_errno).
+ * The protocol-level values use a negative range to stay clear of the errno
+ * space; they describe QUICr conditions that have no errno equivalent.
+ */
+enum class QuicrErrorType : int32_t {
+    // Protocol-level errors (no errno equivalent).
+    ConnectionClosed = -1,
+    Unknown          = -2,
+
+    // Socket / errno-derived errors raised by UDP operations.
+    MessageTooLong             = EMSGSIZE,
+    AddressFamilyNotSupported  = EAFNOSUPPORT,
+    BadFileDescriptor          = EBADF,
+    ConnectionReset            = ECONNRESET,
+    WouldBlock                 = EWOULDBLOCK,
+    Interrupted                = EINTR,
+    InvalidArgument            = EINVAL,
+    NotConnected               = ENOTCONN,
+    NotSocket                  = ENOTSOCK,
+    OperationNotSupported      = EOPNOTSUPP,
+    TimedOut                   = ETIMEDOUT,
+    IoError                    = EIO,
+    NoBufferSpace              = ENOBUFS,
+    NotEnoughMemory            = ENOMEM,
+    DestinationAddressRequired = EDESTADDRREQ,
+    BrokenPipe                 = EPIPE,
+};
+
+struct QuicrError {
+public:
+    QuicrError(QuicrErrorType type)
+        : type_(type), message_(map_quicr_error_type(type)) {}
+    QuicrError(QuicrErrorType type, std::string message)
+        : type_(type), message_(std::move(message)) {}
+
+    QuicrErrorType type() const { return type_; }
+    std::string message() const { return message_; }
+
+    /**
+     * Builds a QuicrError from a raw errno value (as returned by UDP socket
+     * syscalls). Unknown codes still carry the errno through and fall back to
+     * strerror() for their message.
+     */
+    static QuicrError from_errno(int32_t err) {
+        return QuicrError(static_cast<QuicrErrorType>(err));
+    }
+
+private:
+    static std::string map_quicr_error_type(QuicrErrorType type) {
+        switch (type) {
+            case QuicrErrorType::ConnectionClosed:
+                return "The QUICr connection has been closed.";
+            case QuicrErrorType::MessageTooLong:
+                return "The message is larger than the maximum supported datagram size.";
+            case QuicrErrorType::AddressFamilyNotSupported:
+                return "The address family is not supported.";
+            case QuicrErrorType::BadFileDescriptor:
+                return "The socket is not a valid file descriptor.";
+            case QuicrErrorType::ConnectionReset:
+                return "The connection was forcibly closed by the peer.";
+            case QuicrErrorType::WouldBlock:
+                return "The operation would block on a non-blocking socket.";
+            case QuicrErrorType::Interrupted:
+                return "The operation was interrupted by a signal before any data was transferred.";
+            case QuicrErrorType::InvalidArgument:
+                return "An invalid argument was supplied.";
+            case QuicrErrorType::NotConnected:
+                return "The socket is not connected.";
+            case QuicrErrorType::NotSocket:
+                return "The operation was attempted on a non-socket.";
+            case QuicrErrorType::OperationNotSupported:
+                return "The operation is not supported for this socket type or protocol.";
+            case QuicrErrorType::TimedOut:
+                return "The operation timed out.";
+            case QuicrErrorType::IoError:
+                return "An I/O error occurred.";
+            case QuicrErrorType::NoBufferSpace:
+                return "Insufficient buffer space was available to complete the operation.";
+            case QuicrErrorType::NotEnoughMemory:
+                return "Insufficient memory was available to complete the operation.";
+            case QuicrErrorType::DestinationAddressRequired:
+                return "A destination address is required for this operation.";
+            case QuicrErrorType::BrokenPipe:
+                return "The write end of the socket has been closed.";
+            case QuicrErrorType::Unknown:
+                return "Unknown QUICr error.";
+            default:
+                return std::string(std::strerror(static_cast<int>(type)));
+        }
+    }
+
+    QuicrErrorType type_;
+    std::string message_;
+};
+
+}

modules/quicr/include/quicr/QuicrFrame.hpp

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "protocol/quicr/QuicrFrameType.hpp"
+#include "quicr/QuicrFrameType.hpp"
 
 #include <cstddef>
 #include <cstdint>

modules/quicr/include/quicr/QuicrPacket.hpp

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "QuicrFrame.hpp"
-#include "protocol/quicr/QuicrPacketType.hpp"
+#include "quicr/QuicrPacketType.hpp"
 
 #include <optional>