#1 - quicr module

modules/server/CMakeLists.txt

@@ -24,9 +24,12 @@ target_include_directories(tw_server_lib
 target_link_libraries(tw_server_lib
         PUBLIC
         towards
+        tw::io
         tw::network
         tw::protocol
+        tw::message_protocol
         tw::serialization
+        tw::quicr
         glm::glm
         EnTT::EnTT
         Jolt

modules/server/Dockerfile

@@ -0,0 +1,58 @@
+# Build the toolchain stage on its own and pass it back in to skip the apt step:
+#   docker build -f modules/server/Dockerfile --target toolchain -t tw_toolchain .
+#   docker build -f modules/server/Dockerfile --build-arg TOOLCHAIN_IMAGE=tw_toolchain .
+ARG TOOLCHAIN_IMAGE=toolchain
+
+# ── toolchain stage ──────────────────────────────────────────────────────────
+FROM ubuntu:24.04 AS toolchain
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    clang \
+    libstdc++-14-dev \
+    cmake \
+    ninja-build \
+    mold \
+    git \
+    ca-certificates \
+    pkg-config \
+    glslang-tools \
+    libvulkan-dev \
+    libsdl2-dev \
+    libprotobuf-dev protobuf-compiler \
+    libpq-dev \
+    libpqxx-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV CC=clang
+ENV CXX=clang++
+
+# ── build stage ──────────────────────────────────────────────────────────────
+FROM ${TOOLCHAIN_IMAGE} AS builder
+
+WORKDIR /src
+COPY . .
+
+# glm, entt, Jolt, spdlog, expected, Catch2 and tracy are cloned at configure time
+RUN cmake -B /build -G Ninja -DCMAKE_BUILD_TYPE=Release \
+    && cmake --build /build --target tw_server
+
+# Stage the shared libraries the binary was linked against; the runtime image has
+# no package manager. glibc and its loader stay behind, they come with that image.
+RUN mkdir -p /rootfs/usr/lib/x86_64-linux-gnu \
+    && ldd /build/modules/server/tw_server \
+        | awk '/=> \//{ print $3 }' \
+        | grep -vE '/(libc|libm|libdl|libpthread|librt|libresolv|libanl)\.so' \
+        | xargs -I{} cp -L {} /rootfs/usr/lib/x86_64-linux-gnu/
+
+# ── runtime stage ─────────────────────────────────────────────────────────────
+FROM gcr.io/distroless/cc-debian13:nonroot
+
+COPY --from=builder /rootfs/ /
+COPY --from=builder /build/modules/server/tw_server /usr/local/bin/tw_server
+
+# player connections, zone-server peering
+EXPOSE 8101/udp 8102/udp
+
+ENTRYPOINT ["/usr/local/bin/tw_server"]

modules/server/README.md

@@ -1,3 +1,101 @@
 # Server
 
-The authoritative server executable source code.
+The authoritative server executable.
+
+## Running
+
+```
+Usage: tw_server [--quicr-port <port>] [--cluster-port <port>]
+  --quicr-port    UDP port for player connections  (default: 8101)
+  --cluster-port  UDP port for zone-server peering (default: 8102)
+```
+
+Metrics reporting to TimescaleDB stays off until `TIMESCALEDB_HOST` is set:
+
+| Variable               | Default        |
+|------------------------|----------------|
+| `TIMESCALEDB_HOST`     | unset (off)    |
+| `TIMESCALEDB_PORT`     | `5432`         |
+| `TIMESCALEDB_DB`       | `mmo`          |
+| `TIMESCALEDB_USER`     | `mmo`          |
+| `TIMESCALEDB_PASSWORD` | empty          |
+| `TIMESCALEDB_TABLE`    | `zone_metrics` |
+
+## Building natively
+
+See the root `README.md`. The target is `tw_server` and the binary lands in
+`<build-dir>/modules/server/tw_server`.
+
+## Building the image
+
+`Dockerfile` has three stages: a toolchain stage holding the C++ build
+environment, a build stage that compiles `tw_server` with clang, and a runtime
+stage that carries only the binary and its shared libraries. The build context is
+the repository root, so run it from there:
+
+```bash
+$ docker build -f modules/server/Dockerfile -t tw_server .
+```
+
+The toolchain stage installs the compiler and tools (clang, cmake, ninja, mold,
+glslangValidator) and the development packages CMake looks for (Vulkan, SDL2,
+protobuf, libpq, libpqxx) from apt. It touches no source, so it only rebuilds
+when that package list changes.
+
+The build stage configures a Release build on top of it with `CC=clang` /
+`CXX=clang++`. glm, entt, Jolt, spdlog, expected, Catch2 and tracy are cloned
+while configuring, so the build needs network access. `.dockerignore` keeps the
+local build directories and those cloned sources out of the context.
+
+## Reusing the toolchain
+
+Layer caching already keeps apt out of a rebuild, but the cache is local and dies
+with `docker builder prune`. To pin the toolchain down, build that stage on its
+own and tag it:
+
+```bash
+$ docker build -f modules/server/Dockerfile --target toolchain -t tw_toolchain .
+```
+
+`TOOLCHAIN_IMAGE` then points the build stage at it, and the apt step is skipped
+outright rather than cache-hit:
+
+```bash
+$ docker build -f modules/server/Dockerfile \
+      --build-arg TOOLCHAIN_IMAGE=tw_toolchain -t tw_server .
+```
+
+The default is the in-file `toolchain` stage, so a plain build still works
+standalone. Any registry tag works too, which is the useful form on CI. To carry
+it between machines by hand:
+
+```bash
+$ docker save tw_toolchain | zstd -o tw_toolchain.tar.zst
+$ zstd -dc tw_toolchain.tar.zst | docker load
+```
+
+The runtime stage is `gcr.io/distroless/cc-debian13:nonroot` — glibc, libstdc++
+and a `nonroot` user, no shell and no package manager. Since nothing can be
+installed there, the build stage walks `ldd` over the binary and stages every
+shared library it resolved into `/rootfs`, which the runtime stage copies in
+whole. glibc and the loader are filtered out of that list: the binary is built
+against 2.39 and the runtime image ships 2.41, which runs it, but the two must
+not be mixed.
+
+Nothing in the image can be executed except the server, so `docker exec` and
+`docker run --entrypoint` are of no use for poking around. Swap the base for
+`gcr.io/distroless/cc-debian13:debug-nonroot` when a busybox shell is needed.
+
+Run it, publishing both UDP ports:
+
+```bash
+$ docker run --rm -p 8101:8101/udp -p 8102:8102/udp tw_server
+```
+
+Arguments after the image name reach the binary, and environment variables are
+passed as usual:
+
+```bash
+$ docker run --rm -p 9101:9101/udp -e TIMESCALEDB_HOST=timescale \
+      tw_server --quicr-port 9101
+```

modules/server/src/PlayerSession.hpp

@@ -1,6 +1,8 @@
 #pragma once
 
-#include "protocol/quicr/QuicrConnection.hpp"
+#include "network/SessionId.hpp"
+
+#include "message_protocol/MessageConnection.hpp"
 
 #include <cstdint>
 
@@ -8,16 +10,18 @@ namespace tw::net {
 
 struct PlayerSession {
 public:
-    uint32_t session_id;
+    SessionId session_id;
 
-    quicr::QuicrConnection* quicr_connection;
+    msg::MessageConnection* connection;
 
     uint32_t last_frame;
+    uint32_t acked_frame;
 
-    PlayerSession(uint32_t session_id, quicr::QuicrConnection* quicr_connection) :
+    PlayerSession(SessionId session_id, msg::MessageConnection* connection) :
         session_id(session_id),
-        quicr_connection(std::move(quicr_connection)),
-        last_frame(0)
+        connection(connection),
+        last_frame(0),
+        acked_frame(0)
     { }
 };
 

modules/server/src/ZoneClusterLink.cpp

@@ -1,42 +1,48 @@
 #include "ZoneClusterLink.hpp"
 
-#include "Address.hpp"
-
 #include <spdlog/spdlog.h>
 #include <stdexcept>
 
 namespace tw::net {
 
-ZoneClusterLink::ZoneClusterLink(int port)
-    : m_endpoint(quicr::QuicrEndpoint::create().value()),
-      m_listener(quicr::QuicrConnectionListener::listen(m_endpoint.get()).value())
-{
-    if (auto r = m_endpoint->bind(port); !r) {
-        throw std::runtime_error("ZoneClusterLink: failed to bind to port " + std::to_string(port));
+namespace {
+
+std::unique_ptr<msg::MessageEndpoint> bind_endpoint(int port) {
+    auto endpoint_r = msg::MessageEndpoint::bind(port);
+    if(!endpoint_r) {
+        throw std::runtime_error("ZoneClusterLink: failed to bind to port " + std::to_string(port) +
+                                 ": " + endpoint_r.error().message());
     }
+
+    return std::move(endpoint_r.value());
+}
+
+}
+
+ZoneClusterLink::ZoneClusterLink(int port) :
+    m_endpoint(bind_endpoint(port)),
+    m_messages(m_endpoint.get()) {
+    m_endpoint->set_on_peer_connected([](msg::PeerId peer) {
+        spdlog::info("Zone peer {} connected", peer);
+    });
+
     spdlog::info("ZoneClusterLink listening on port {}", port);
 }
 
 void ZoneClusterLink::update() {
-    m_endpoint->poll();
-
-    quicr::QuicrConnection* connection;
-    while ((connection = m_listener->listen())) {
-        spdlog::info("Zone peer connected from {}", connection->address().to_string());
-        m_peers.push_back(connection);
-    }
+    m_endpoint->update();
 }
 
-quicr::QuicrConnection* ZoneClusterLink::connect_to_peer(const std::string& host, int port) {
-    auto result = m_endpoint->connect(Address(host, port));
-    if (!result) {
-        spdlog::error("ZoneClusterLink: failed to connect to peer {}:{}", host, port);
+msg::MessageConnection* ZoneClusterLink::connect_to_peer(const std::string& host, int port) {
+    auto peer_r = m_endpoint->connect(host, port);
+    if (!peer_r) {
+        spdlog::error("ZoneClusterLink: failed to connect to peer {}:{}: {}",
+                      host, port, peer_r.error().message());
         return nullptr;
     }
-    quicr::QuicrConnection* conn = result.value();
-    m_peers.push_back(conn);
+
     spdlog::info("ZoneClusterLink: connected to peer {}:{}", host, port);
-    return conn;
+    return peer_r.value();
 }
 
 } // namespace tw::net

modules/server/src/ZoneClusterLink.hpp

@@ -1,12 +1,10 @@
 #pragma once
 
-#include "MessageRegistry.hpp"
-#include "protocol/quicr/QuicrConnection.hpp"
-#include "protocol/quicr/QuicrConnectionListener.hpp"
-#include "protocol/quicr/QuicrEndpoint.hpp"
+#include "ProtobufMessages.hpp"
+#include "message_protocol/MessageConnection.hpp"
+#include "message_protocol/MessageEndpoint.hpp"
 
 #include <memory>
-#include <span>
 #include <string>
 #include <vector>
 
@@ -14,51 +12,47 @@
 
 namespace tw::net {
 
-// Listens on a dedicated QUICr port for incoming zone-server peer connections
-// and opens outgoing connections to known peers.
+// Listens on a dedicated port for incoming zone-server peer connections and
+// opens outgoing connections to known peers.
 //
-// All connections (incoming and outgoing) are collected in m_peers so that
-// broadcast messages (ZoneHello, ZoneBye) reach every peer uniformly.
-//
-// Accepted QuicrConnection* pointers are owned by the endpoint and remain valid
-// for the lifetime of this object.
+// Incoming and outgoing peers are held together so that broadcast messages
+// (ZoneHello, ZoneBye) reach every peer uniformly.
 class ZoneClusterLink {
-    std::unique_ptr<quicr::QuicrEndpoint>           m_endpoint;
-    std::unique_ptr<quicr::QuicrConnectionListener> m_listener;
-    std::vector<quicr::QuicrConnection*>            m_peers;
+    std::unique_ptr<msg::MessageEndpoint> m_endpoint;
+    ProtobufMessages                      m_messages;
 
 public:
     explicit ZoneClusterLink(int port);
 
-    // Poll for datagrams and accept any pending peer connections.
+    // Poll for messages and accept any pending peer connections.
     void update();
 
     // Connect to a remote zone peer and register it in the peer list.
-    quicr::QuicrConnection* connect_to_peer(const std::string& host, int port);
+    msg::MessageConnection* connect_to_peer(const std::string& host, int port);
 
-    // Serialize and send a protobuf message to a single peer.
+    // Send a protobuf message to a single peer.
     template<typename T>
-    void send_mesg(quicr::QuicrConnection* conn, const T& msg) {
-        std::string payload;
-        if (!msg.SerializeToString(&payload)) {
-            spdlog::error("ZoneClusterLink: failed to serialize message");
-            return;
+    void send_mesg(msg::MessageConnection* peer, const T& msg) {
+        auto send_r = m_messages.send(peer, msg, false);
+        if(!send_r) {
+            spdlog::error("ZoneClusterLink: failed to send to peer {}: {}",
+                          peer->peer_id(), send_r.error().message());
         }
-        std::vector<std::byte> bytes(payload.size() + sizeof(uint32_t));
-        uint32_t type = tw::Message<T>::value;
-        memcpy(bytes.data(), &type, sizeof(type));
-        memcpy(bytes.data() + sizeof(uint32_t), payload.data(), payload.size());
-        conn->send_message(bytes, false);
     }
 
-    // Broadcast a protobuf message to all connected peers.
+    // Send a protobuf message to all connected peers.
     template<typename T>
     void broadcast(const T& msg) {
-        for (auto* peer : m_peers)
-            send_mesg(peer, msg);
+        m_messages.broadcast(msg);
     }
 
-    std::span<quicr::QuicrConnection* const> peers() const { return m_peers; }
+    std::vector<msg::MessageConnection*> peers() const {
+        return m_endpoint->peers();
+    }
+
+    msg::MessageEndpoint& endpoint() {
+        return *m_endpoint;
+    }
 };
 
 } // namespace tw::net

modules/server/src/ZoneManager.cpp

@@ -68,6 +68,8 @@ void ZoneManager::on_player_move(SessionId session_id, mmo::PlayerMoveMessage&&
     entt::entity entity = client_entity(session_id);
     if (entity == entt::null) return;
 
+    m_session_input_frame[session_id] = message.frame_idx();
+
     auto* controller = m_world->registry().try_get<CharacterController>(entity);
     if (controller) {
         controller->set_input(
@@ -97,6 +99,11 @@ const im::Interest* ZoneManager::get_interest(im::InterestId id) const {
     return m_interest_system->get_interest(id);
 }
 
+uint32_t ZoneManager::acked_input_frame(im::InterestId interest_id) const {
+    auto it = m_session_acked_frame.find(interest_id);
+    return it != m_session_acked_frame.end() ? it->second : 0;
+}
+
 // ── Private helpers ───────────────────────────────────────────────────────────
 
 void ZoneManager::check_neighbor_transfers() {
@@ -169,9 +176,25 @@ void ZoneManager::tick(uint32_t frame_idx, float delta_time) {
     m_world->step(delta_time);
     FrameMarkEnd("World step");
 
+    // Re-stamp each client entity's newest input onto the server's frame index.
+    // The input ring is keyed by client frame numbers (independent counter),
+    // but physics lookup uses the server's frame counter. By re-stamping onto
+    // the server frame, input(server_frame_idx) resolves by exact match rather
+    // than fallback, ensuring deterministic and correct input consumption.
+    for (const auto& [interest_id, entity] : m_client_entities) {
+        auto* controller = m_world->registry().try_get<CharacterController>(entity);
+        if (!controller) continue;
+        controller->set_input(frame_idx, controller->input());
+    }
+
     FrameMarkStart("Physics step");
-    m_physics_world.step(frame_idx, delta_time);
+    // Fixed step, not the measured interval: the simulation has to advance by the
+    // same amount every frame for a replay of the same inputs to land in the same
+    // place.
+    m_physics_world.step(frame_idx, JoltPhysicsWorld::FIXED_DELTA_TIME);
     FrameMarkEnd("Physics step");
+
+    m_session_acked_frame = m_session_input_frame;
 }
 
 } // namespace tw::net

modules/server/src/ZoneManager.hpp

@@ -40,6 +40,8 @@ class ZoneManager : public ZoneProxy {
 
     std::unordered_map<im::InterestId, entt::entity>         m_client_entities;
     std::unordered_map<entt::entity, im::InterestId>         m_entity_sessions;  // reverse map
+    std::unordered_map<im::InterestId, uint32_t>             m_session_input_frame;  // most recent input frame received
+    std::unordered_map<im::InterestId, uint32_t>             m_session_acked_frame;  // input frame consumed by last tick
     std::vector<NeighborEntry>                               m_neighbors;
     uint32_t                                                 m_next_neighbor_id = 0x80000000u;
 
@@ -83,6 +85,9 @@ public:
     // Returns the current interest state for any registered id, or nullptr.
     const im::Interest* get_interest(im::InterestId id) const;
 
+    // Returns the input frame that this session's entity was last simulated with, or 0 if unknown.
+    uint32_t acked_input_frame(im::InterestId interest_id) const;
+
     // Runs one tick: interest queries, neighbour transfer checks, world step, physics step.
     void tick(uint32_t frame_idx, float delta_time);
 };

modules/server/src/ZoneServer.cpp

@@ -25,7 +25,6 @@ ZoneServer::ZoneServer(ZoneServerConfiguration config)
     m_network_receiver(std::make_unique<NetworkReceiver>(
         m_player_session_registry.get(), config.quicr_port, m_metrics_reporter.get()
     )),
-    m_message_dispatcher(std::make_unique<MessageDispatcher>(m_network_receiver.get())),
     m_replicator(std::make_unique<StateReplicator<SpatialBackendType>>(
         m_player_session_registry.get(),
         m_network_receiver.get()
@@ -58,9 +57,16 @@ ZoneServer::ZoneServer(ZoneServerConfiguration config)
     }
 }
 
-void ZoneServer::player_update_handler(SessionId session_id, mmo::PlayerMoveMessage&& message) {
+void ZoneServer::player_update_handler(SessionId session_id, mmo::PlayerMoveMessage message) {
     auto it = m_session_zone.find(session_id);
     if (it == m_session_zone.end()) return;
+
+    // Echoed back in the next snapshot, so the client can tell how long its
+    // input took to come back.
+    if (auto* session = m_player_session_registry->session(session_id)) {
+        session->last_frame = message.frame_idx();
+    }
+
     it->second->on_player_move(session_id, std::move(message));
 }
 
@@ -80,7 +86,6 @@ static void register_signal_handler() {
 
 void ZoneServer::update_clients(uint32_t frame_idx) {
     m_network_receiver->update();
-    m_message_dispatcher->drain_queue();
 
     while (m_network_receiver->peek_new_session()) {
         auto session_id = m_network_receiver->pop_new_session();
@@ -96,6 +101,10 @@ void ZoneServer::update_clients(uint32_t frame_idx) {
         zone->add_client(session_id, entity);
         m_session_zone[session_id] = zone;
 
+        mmo::SetControlledEntity mesg = {};
+        mesg.set_entity_id((uint32_t)entity);
+        m_network_receiver->send_mesg(session_id, mesg);
+
         spdlog::info("Client {} connected", session_id);
     }
 }
@@ -106,13 +115,13 @@ void ZoneServer::run() {
 
     uint32_t frame_idx = 1;
 
-    m_message_dispatcher->set_handler<mmo::PlayerMoveMessage>(
-        [&](uint64_t session_id, mmo::PlayerMoveMessage mesg) {
-            player_update_handler(static_cast<SessionId>(session_id), std::move(mesg));
+    m_network_receiver->set_handler<mmo::PlayerMoveMessage>(
+        [this](SessionId session_id, const mmo::PlayerMoveMessage& mesg) {
+            player_update_handler(session_id, mesg);
         });
 
-    m_message_dispatcher->set_handler<mmo::chat::SendChatMessageRequest>(
-        [&](uint64_t session_id, mmo::chat::SendChatMessageRequest mesg) {
+    m_network_receiver->set_handler<mmo::chat::SendChatMessageRequest>(
+        [this](SessionId session_id, const mmo::chat::SendChatMessageRequest& mesg) {
         });
 
     while (!quit.load()) {
@@ -127,6 +136,16 @@ void ZoneServer::run() {
         for (auto& zone : m_zones) {
             zone->tick(frame_idx, lock_step.delta_time());
 
+            // Copy acked frames from zone to sessions before replication.
+            // This ensures snapshots carry the input frame actually consumed by this tick,
+            // not frames that arrive in the trailing network update.
+            for (const auto& [session_id, session_zone] : m_session_zone) {
+                if (session_zone != zone.get()) continue;
+                if (auto* session = m_player_session_registry->session(session_id)) {
+                    session->acked_frame = zone->acked_input_frame(session_id);
+                }
+            }
+
             m_replicator->replicate(zone->registry(), zone->interest());
         }
 

modules/server/src/ZoneServer.hpp

@@ -10,7 +10,6 @@
 #include "ZoneManager.hpp"
 #include "ZoneServerConfiguration.hpp"
 #include "monitoring/MetricsReporter.hpp"
-#include "network/MessageDispatcher.hpp"
 #include "network/NetworkReceiver.hpp"
 #include "network/PlayerSessionRegistry.hpp"
 #include "replication/StateReplicator.hpp"
@@ -21,7 +20,6 @@ class ZoneServer {
     std::unique_ptr<NetworkMetricsReporter>              m_metrics_reporter;
     std::unique_ptr<PlayerSessionRegistry>               m_player_session_registry;
     std::unique_ptr<NetworkReceiver>                     m_network_receiver;
-    std::unique_ptr<MessageDispatcher>                   m_message_dispatcher;
     std::unique_ptr<StateReplicator<SpatialBackendType>> m_replicator;
     ZoneClusterLink                                      m_cluster_link;
     ZoneCoordinator                                      m_coordinator;
@@ -30,7 +28,7 @@ class ZoneServer {
     std::unordered_map<SessionId, ZoneManager*> m_session_zone;
     uint32_t                                    m_own_zone_id = 0;
 
-    void player_update_handler(SessionId session_id, mmo::PlayerMoveMessage&& message);
+    void player_update_handler(SessionId session_id, mmo::PlayerMoveMessage message);
     void update_clients(uint32_t frame_idx);
 
 public:

modules/server/src/network/InboundMessage.hpp

@@ -1,17 +0,0 @@
-#pragma once
-
-#include <cstdint>
-#include <vector>
-
-namespace tw::net {
-
-class InboundMessage {
-public:
-    uint32_t session_id;
-    std::vector<std::byte> payload;
-
-    InboundMessage(uint32_t session_id, std::vector<std::byte> payload)
-        : session_id(session_id), payload(std::move(payload)) {}
-};
-
-}

modules/server/src/network/MessageDeserializer.hpp

@@ -1,21 +0,0 @@
-#pragma once
-
-#include <optional>
-#include <span>
-#include <spdlog/spdlog.h>
-
-namespace tw::net {
-
-class MessageDeserializer {
-public:
-    template<typename T>
-    static std::optional<T> deserialize(std::span<std::byte> data) {
-        T result = {};
-
-        result.ParseFromArray(data.data(), data.size());
-
-        return result;
-    }
-};
-
-}

modules/server/src/network/MessageDispatcher.hpp

@@ -1,67 +0,0 @@
-#pragma once
-
-#include "InboundMessage.hpp"
-#include "MessageQueue.hpp"
-#include "NetworkError.hpp"
-#include "MessageDeserializer.hpp"
-#include "monitoring/TimescaleDbMetricsReporter.hpp"
-#include "network/NetworkReceiver.hpp"
-#include "packets/Packet.hpp"
-
-#include <tl/expected.hpp>
-
-#include <functional>
-#include <tracy/Tracy.hpp>
-
-namespace tw::net {
-
-typedef std::function<tl::expected<void, NetworkError>(uint32_t, std::span<std::byte>)> MessageHandler;
-
-class MessageDispatcher {
-    NetworkReceiver* m_network;
-
-    std::unordered_map<uint32_t, MessageHandler>        m_handlers;
-
-public:
-    MessageDispatcher(NetworkReceiver* network) :
-        m_network(network) {
-    }
-
-    template<typename T>
-    void set_handler(std::function<void(uint32_t, T)> handler) {
-        m_handlers[static_cast<uint32_t>(Message<T>::value)] =
-            [handler, this](uint32_t session_id, std::span<std::byte> data) -> tl::expected<void, NetworkError> {
-                ZoneScopedN("Handling message");
-
-                size_t size = 0;
-                auto r = MessageDeserializer::deserialize<T>(data);
-                if(!r) {
-                    return {};
-                }
-
-                handler(session_id, *r);
-                return {};
-            };
-    }
-
-    void drain_queue() {
-        while(!m_network->inbound_queue()->is_empty()) {
-            auto msg = m_network->inbound_queue()->pop();
-            uint32_t message_type = *(uint32_t*)msg->payload.data();
-
-            if(m_handlers.find(message_type) == m_handlers.end()) {
-                spdlog::error("No handler for message type {}", message_type);
-                continue;
-            }
-
-            auto r = m_handlers[message_type](msg->session_id, std::span<std::byte>(msg->payload).subspan(sizeof(uint32_t)));
-            if(!r) {
-                spdlog::error("Failed to handle message of type {}: {}", message_type, r.error().message());
-                // TODO: Add session failure
-                continue;
-            }
-        }
-    }
-};
-
-}

modules/server/src/network/MessageQueue.hpp

@@ -1,32 +0,0 @@
-#pragma once
-
-#include <cstdint>
-#include <queue>
-
-namespace tw::net {
-
-template<typename T>
-class MessageQueue {
-    std::queue<T> m_queue;
-
-public:
-    MessageQueue() : m_queue() {
-
-    }
-
-    bool is_empty() {
-        return m_queue.empty();
-    }
-
-    void push(T mesg) {
-        m_queue.push(mesg);
-    }
-
-    T pop() {
-        auto mesg = m_queue.front();
-        m_queue.pop();
-        return mesg;
-    }
-};
-
-}

modules/server/src/network/NetworkReceiver.cpp

@@ -1,72 +1,77 @@
 #include "NetworkReceiver.hpp"
-#include "network/InboundMessage.hpp"
-#include "network/MessageQueue.hpp"
-#include "network/PlayerSessionRegistry.hpp"
-#include "protocol/quicr/QuicrConnectionListener.hpp"
+
+#include <stdexcept>
 
 namespace tw::net {
 
+namespace {
+
+std::unique_ptr<msg::MessageEndpoint> bind_endpoint(int32_t port) {
+    auto endpoint_r = msg::MessageEndpoint::bind(port);
+    if(!endpoint_r) {
+        throw std::runtime_error("NetworkReceiver: failed to bind to port " + std::to_string(port) +
+                                 ": " + endpoint_r.error().message());
+    }
+
+    return std::move(endpoint_r.value());
+}
+
+}
+
 NetworkReceiver::NetworkReceiver(
     PlayerSessionRegistry* session_registry,
     int32_t udp_port,
     NetworkMetricsReporter* metrics_reporter
 ) :
     m_session_registry(session_registry),
-    m_inbound_queue(new MessageQueue<InboundMessage*>),
-    m_quicr_endpoint(quicr::QuicrEndpoint::create().value()),
-    m_quicr_listener(quicr::QuicrConnectionListener::listen(m_quicr_endpoint.get()).value()),
+    m_endpoint(bind_endpoint(udp_port)),
+    m_messages(m_endpoint.get()),
     m_metrics_reporter(metrics_reporter)
 {
-    m_quicr_endpoint->bind(udp_port);
+    // Registering the session here rather than after update() means a peer
+    // already has one by the time its first message is dispatched.
+    m_endpoint->set_on_peer_connected([this](msg::PeerId peer) {
+        auto session_id = m_session_registry->register_session(peer, m_endpoint->peer(peer));
+        m_new_sessions.push_back(session_id);
+    });
 
     spdlog::info("Running on port: {}", udp_port);
 }
 
-bool NetworkReceiver::listen_quicr() {
-    quicr::QuicrConnection* connection = nullptr;
-    while((connection = m_quicr_listener->listen())) {
-        spdlog::info("Quicr client tries to connect");
+void NetworkReceiver::report_traffic() {
+    const uint64_t received = m_endpoint->bytes_received();
+    const uint64_t sent     = m_endpoint->bytes_sent();
 
-        auto session_id = m_session_registry->register_session(connection);
+    m_metrics_reporter->add_inbound(received - m_reported_bytes_received);
+    m_metrics_reporter->add_outbound(sent - m_reported_bytes_sent);
 
-        m_new_sessions.push_back(session_id);
-    }
-
-    return true;
-}
-
-void NetworkReceiver::listen() {
-    listen_quicr();
-}
-
-void NetworkReceiver::process_streams() {
-    m_quicr_endpoint->poll();
-
-    std::vector<std::byte> buffer(64 * 1024);
-    for(auto& client : m_session_registry->sessions()) {
-        auto read_r = client->quicr_connection->read_into(buffer);
-        if(!read_r) {
-            spdlog::error("Failed to read from QUICr stream");
-            continue;
-        }
-
-        if(*read_r == 0) {
-            continue;
-        }
-
-        m_metrics_reporter->add_inbound(*read_r);
-
-        auto data = std::vector<std::byte>(buffer.begin(), buffer.begin() + *read_r);
-        m_inbound_queue->push(new InboundMessage(client->session_id, data));
-    }
+    m_reported_bytes_received = received;
+    m_reported_bytes_sent     = sent;
 }
 
 void NetworkReceiver::update() {
-    m_quicr_endpoint->poll();
+    m_endpoint->update();
 
-    listen();
+    report_traffic();
+}
 
-    process_streams();
+size_t NetworkReceiver::send_framed(SessionId session_id, std::span<const std::byte> message) {
+    if(message.empty()) {
+        return 0;
+    }
+
+    auto* session = m_session_registry->session(session_id);
+    if(session == nullptr) {
+        return 0;
+    }
+
+    auto send_r = session->connection->send_framed(message, false);
+    if(!send_r) {
+        spdlog::error("Failed to send to session {}: {}", session_id, send_r.error().message());
+        return 0;
+    }
+
+    return message.size();
 }
 
 }

modules/server/src/network/NetworkReceiver.hpp

@@ -1,35 +1,40 @@
 #pragma once
 
-#include "MessageQueue.hpp"
-#include "InboundMessage.hpp"
-#include "MessageRegistry.hpp"
+#include "ProtobufMessages.hpp"
+#include "SessionId.hpp"
 #include "monitoring/MetricsReporter.hpp"
-#include "monitoring/TimescaleDbMetricsReporter.hpp"
 #include "network/PlayerSessionRegistry.hpp"
-#include "protocol/quicr/QuicrConnectionListener.hpp"
-#include "protocol/quicr/QuicrEndpoint.hpp"
 
+#include "message_protocol/MessageEndpoint.hpp"
+
+#include <spdlog/spdlog.h>
+
+#include <cstdint>
+#include <deque>
+#include <functional>
+#include <memory>
 #include <span>
 
 namespace tw::net {
 
+/**
+ * Accepts player connections and routes their messages to the handlers the
+ * zone server registers, translating peers into sessions on the way.
+ */
 class NetworkReceiver {
-    PlayerSessionRegistry *m_session_registry;
+    PlayerSessionRegistry* m_session_registry;
 
-    std::unique_ptr<quicr::QuicrEndpoint> m_quicr_endpoint;
-    std::unique_ptr<quicr::QuicrConnectionListener> m_quicr_listener;
-
-    MessageQueue<InboundMessage*>* m_inbound_queue;
+    std::unique_ptr<msg::MessageEndpoint> m_endpoint;
+    ProtobufMessages                      m_messages;
 
     std::deque<SessionId> m_new_sessions;
 
     NetworkMetricsReporter* m_metrics_reporter;
 
-    bool listen_quicr();
+    uint64_t m_reported_bytes_received = 0;
+    uint64_t m_reported_bytes_sent     = 0;
 
-    void listen();
-
-    void process_streams();
+    void report_traffic();
 
 public:
     NetworkReceiver(
@@ -38,8 +43,22 @@ public:
         NetworkMetricsReporter* metrics_reporter
     );
 
-    MessageQueue<InboundMessage*>* inbound_queue() {
-        return m_inbound_queue;
+    /**
+     * Calls `handler` for every T that arrives, along with the session that
+     * sent it.
+     */
+    template<typename T>
+    void set_handler(std::function<void(SessionId, const T&)> handler) {
+        m_messages.set_handler<T>(
+            [this, handler = std::move(handler)](msg::PeerId peer, const T& message) {
+                const SessionId session_id = m_session_registry->session_for_peer(peer);
+                if(session_id == 0) {
+                    spdlog::warn("Dropped a message from peer {}, which has no session", peer);
+                    return;
+                }
+
+                handler(session_id, message);
+            });
     }
 
     bool peek_new_session() const {
@@ -47,67 +66,35 @@ public:
     }
 
     SessionId pop_new_session() {
-        auto session = m_new_sessions.front();
+        auto session_id = m_new_sessions.front();
         m_new_sessions.pop_front();
-        return session;
+        return session_id;
     }
 
     void update();
 
     template<typename T>
     size_t send_mesg(SessionId session_id, const T& mesg) {
-        std::string payload;
-        if(!mesg.SerializeToString(&payload)) {
-            spdlog::error("Failed to serialize message");
+        auto* session = m_session_registry->session(session_id);
+        if(session == nullptr) {
+            spdlog::warn("Cannot send to session {}, it is not registered", session_id);
             return 0;
         }
 
-        int32_t length = payload.length();
-        if(length == 0) {
+        auto send_r = m_messages.send(session->connection, mesg, true);
+        if(!send_r) {
+            spdlog::error("Failed to send to session {}: {}", session_id, send_r.error().message());
             return 0;
         }
 
-        std::vector<std::byte> bytes(length + sizeof(uint32_t));
-
-        uint32_t type = Message<T>::value;
-        auto payload_bytes = std::as_writable_bytes(std::span(payload));
-        memcpy(bytes.data(), &type, sizeof(type));
-        memcpy(bytes.data() + sizeof(uint32_t), payload_bytes.data(), payload_bytes.size());
-
-        auto session = m_session_registry->session(session_id);
-        m_metrics_reporter->add_outbound(bytes.size());
-
-        session->quicr_connection->send_message(bytes, false);
-        return bytes.size();
+        return msg::MessageHeader::SIZE + mesg.ByteSizeLong();
     }
 
     /**
-     * Send a pre-serialised raw byte payload directly, with no additional
-     * framing.  The caller is responsible for including any type tag and
-     * length prefix in the payload (as tw::serial::WorldStateWriter does).
-     *
-     * This avoids the Protobuf SerializeToString heap allocation entirely —
-     * the span points into the caller's BinaryBuffer, which is reused every
-     * frame.
+     * Sends a message the caller has already framed, so that a writer holding
+     * its own buffer does not have to be copied through an encoder.
      */
-    size_t send_raw(SessionId session_id, std::span<const std::byte> payload) {
-        if(payload.empty()) {
-            return 0;
-        }
-
-        auto* session = m_session_registry->session(session_id);
-        if(!session) {
-            return 0;
-        }
-
-        // QuicrConnection::send_message takes a std::vector<std::byte>.
-        // We copy the span here.  If the QUICr layer is ever refactored to
-        // accept a span we can remove this copy entirely.
-        std::vector<std::byte> bytes(payload.begin(), payload.end());
-        m_metrics_reporter->add_outbound(bytes.size());
-        session->quicr_connection->send_message(bytes, false);
-        return bytes.size();
-    }
+    size_t send_framed(SessionId session_id, std::span<const std::byte> message);
 };
 
 }

modules/server/src/network/OutboundMessage.hpp

@@ -1,14 +0,0 @@
-#pragma once
-
-#include <vector>
-
-namespace tw::net {
-
-struct OutboundMessage {
-  std::vector<std::byte> payload;
-
-  OutboundMessage(std::vector<std::byte> payload)
-      : payload(std::move(payload)) {}
-};
-
-} // namespace tw::net

modules/server/src/network/PlayerSessionRegistry.cpp

@@ -1,6 +1,5 @@
 #include "PlayerSessionRegistry.hpp"
 #include "PlayerSession.hpp"
-#include "protocol/quicr/QuicrConnection.hpp"
 
 namespace tw::net {
 
@@ -9,13 +8,16 @@ SessionId PlayerSessionRegistry::generate_session_id() {
     return next_id++;
 }
 
-SessionId PlayerSessionRegistry::register_session(quicr::QuicrConnection* quicr_connection) {
-    auto session = new PlayerSession(quicr_connection->self_id(), quicr_connection);
-    m_session_vec.emplace_back(session);
+SessionId PlayerSessionRegistry::register_session(msg::PeerId peer, msg::MessageConnection* connection) {
+    const SessionId session_id = generate_session_id();
 
-    m_session_map.emplace(quicr_connection->self_id(), session);
+    auto session = std::make_unique<PlayerSession>(session_id, connection);
+    m_session_vec.emplace_back(session.get());
 
-    return quicr_connection->self_id();
+    m_session_map.emplace(session_id, std::move(session));
+    m_peer_sessions.emplace(peer, session_id);
+
+    return session_id;
 }
 
 }

modules/server/src/network/PlayerSessionRegistry.hpp

@@ -2,7 +2,10 @@
 
 #include "PlayerSession.hpp"
 #include "SessionId.hpp"
-#include "protocol/quicr/QuicrConnection.hpp"
+
+#include "message_protocol/MessageConnection.hpp"
+#include "message_protocol/PeerId.hpp"
+
 #include <memory>
 #include <unordered_map>
 #include <vector>
@@ -14,6 +17,10 @@ class PlayerSessionRegistry {
 
     std::unordered_map<SessionId, std::unique_ptr<PlayerSession>> m_session_map;
 
+    // Sessions are identified by a dense id of their own, so the wider peer id
+    // the network layer assigns stays at the boundary.
+    std::unordered_map<msg::PeerId, SessionId> m_peer_sessions;
+
     SessionId generate_session_id();
 
 public:
@@ -26,7 +33,13 @@ public:
         return session != m_session_map.end() ? session->second.get() : nullptr;
     }
 
-    SessionId register_session(quicr::QuicrConnection* quicr_connection);
+    /** The session belonging to a peer, or zero if the peer has none. */
+    SessionId session_for_peer(msg::PeerId peer) const {
+        auto session = m_peer_sessions.find(peer);
+        return session != m_peer_sessions.end() ? session->second : 0;
+    }
+
+    SessionId register_session(msg::PeerId peer, msg::MessageConnection* connection);
 
     void unregister_session(SessionId);
 };

modules/server/src/replication/StateReplicator.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "MessageRegistry.hpp"
 #include "network/NetworkReceiver.hpp"
 #include "network/PlayerSessionRegistry.hpp"
 #include "systems/Interest.hpp"
@@ -34,8 +35,9 @@ class StateReplicator {
     // Per-client backing buffers reused every frame.
     std::vector<tw::serial::BinaryBuffer> m_frames;
 
-    // Header(12) + spawn_hdr(4) + despawn_hdr(4) + 512 entities × 16 bytes
-    static constexpr std::size_t kInitialCapacity = 20 + 512 * 16;
+    // Header(16) + spawn_hdr(4) + despawn_hdr(4) + 512 entities × 16 bytes
+    static constexpr std::size_t kHeaderCapacity  = 24;
+    static constexpr std::size_t kInitialCapacity = kHeaderCapacity + 512 * 16;
 
 public:
     StateReplicator(
@@ -87,14 +89,14 @@ public:
                 if (!state) continue;
 
                 const std::size_t needed =
-                    20
+                    kHeaderCapacity
                     + state->spawn().size()    * 4
                     + state->despawn().size()  * 4
                     + state->interest().size() * 16;
 
                 m_frames[i].reserve(needed);
                 writers[i].reset();
-                writers[i].begin(session->last_frame);
+                writers[i].begin(session->acked_frame, Message<mmo::WorldStateMessage>::value);
                 writers[i].write_spawns(state->spawn());
                 writers[i].write_despawns(state->despawn());
             }
@@ -118,10 +120,9 @@ public:
             ZoneScopedN("Sending messages");
 
             for (std::size_t i = 0; i < session_count; ++i) {
-                spdlog::info("Sending");
                 if (!client_states[i]) continue;
                 writers[i].end();
-                m_network->send_raw(sessions[i]->session_id, writers[i].view());
+                m_network->send_framed(sessions[i]->session_id, writers[i].view());
             }
         }
     }