HTTP/3 improvements, threading fixes, ngtcp2 example code removal

.devcontainer/Dockerfile

@@ -2,7 +2,7 @@
 # The dependencies of this project have been moved into another Dockerfile that can be built
 # separately. This image can be pushed to the HUB, speeding up codespace creation considerably.
 #
-FROM docker.io/psedoc/anyhttp:0.27
+FROM docker.io/psedoc/anyhttp:0.28
 
 #
 # install some more interactive utils in the devcontainer

.devcontainer/base/Dockerfile

@@ -2,7 +2,7 @@
 # https://github.com/microsoft/vscode-dev-containers/tree/main/containers/debian
 # https://github.com/pgit/cpp-devcontainer
 #
-FROM docker.io/psedoc/cpp-devcontainer:0.27
+FROM docker.io/psedoc/cpp-devcontainer:0.28
 
 #
 # GDB libc++ pretty printer (works for string, but not for map with llvm-18)
@@ -66,10 +66,15 @@ ENV LD_LIBRARY_PATH=/usr/local/lib
 # With debian trixie, we now have OpenSSL 3 with QUIC support. Whether or not that is any good,
 # seems to be up for debate: https://www.haproxy.com/blog/state-of-ssl-stacks
 #
-ARG AWS_LC_VERSION=v5.5.0
+ARG AWS_LC_VERSION=v5.6.0
 RUN git clone --depth 1 -b ${AWS_LC_VERSION} https://github.com/aws/aws-lc && \
     cd aws-lc && \
-    cmake -B build -DDISABLE_GO=ON --install-prefix=/opt/boringssl && \
+    cmake -B build \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DDISABLE_GO=ON \
+      -DBUILD_TESTING=OFF \
+      -DBUILD_TOOL=OFF \
+      --install-prefix=/opt/boringssl && \
     make -j$(nproc) -C build && \
     cmake --install build && \
     cd .. && \
@@ -86,6 +91,14 @@ RUN git clone --recursive --depth 1 --branch ${NGHTTP3_VERSION} https://github.c
 #
 # nghtcp2 is built with both 'boringssl' (actually, aws-lc) and 'ossl' support
 #
+# The example client/server are built as well. There is no --enable-examples switch: they are
+# enabled implicitly by configure as long as we don't pass --enable-lib-only and nghttp3, libev
+# and a C++23 compiler are present -- see the 'Examples:' line in the configure summary.
+#
+# They are noinst_PROGRAMS, so 'make install' ignores them and we have to install them by hand.
+# 'osslclient'/'osslserver' are the OpenSSL variants, replacing the copy of the example sources
+# that used to live in src/ngtcp2/ as 'ngtcp-client'/'ngtcp-server'.
+#
 ARG NGTCP2_VERSION=v1.25.0
 RUN git clone --recursive --depth 1 --branch ${NGTCP2_VERSION} https://github.com/ngtcp2/ngtcp2 && \
     cd ngtcp2 && \
@@ -95,12 +108,15 @@ RUN git clone --recursive --depth 1 --branch ${NGTCP2_VERSION} https://github.co
        BORINGSSL_LIBS="-L/opt/boringssl/lib -lssl -lcrypto" \
        --with-boringssl --with-ossl && \
     make -j$(nproc) check && make install && \
+    make -j$(nproc) -C examples osslclient osslserver && \
+    install -m 755 -t /usr/local/bin examples/osslclient examples/osslserver && \
     cd third-party/urlparse && \
     autoreconf -i && \
     ./configure --prefix=/usr/local && \
     make -j$(nproc) && make install && \
     cd ../.. && \
-    cd .. && rm -rf ngctp2
+    cd .. && rm -rf ngtcp2 && \
+    ldconfig
 
 # ==================================================================================================
 

.github/instructions/anyhttp.instructions.md

@@ -2,7 +2,7 @@
 applyTo: '**'
 ---
 AnyHTTP is a type-erased interface for implementing HTTP Servers and Clients. It offers common
-interfaces for HTTP/1.1, HTTP/2 and HTTP/3 (QUIC, not implemented yet).
+interfaces for HTTP/1.1, HTTP/2 and HTTP/3 (QUIC).
 
 AnyHTTP is completely asynchronous and intended to be used with C++20 coroutines.
 It uses Boost ASIO as the underlying async runtime and tries to adhere to it's

.github/workflows/release.yml

@@ -15,7 +15,7 @@ jobs:
     name: RelWithDebInfo
     runs-on: ubuntu-latest
     container:
-      image: docker.io/psedoc/anyhttp:0.27
+      image: docker.io/psedoc/anyhttp:0.28
 
     steps:
     - &checkout
@@ -37,7 +37,7 @@ jobs:
     name: ASAN
     runs-on: ubuntu-latest
     container:
-      image: docker.io/psedoc/anyhttp:0.27
+      image: docker.io/psedoc/anyhttp:0.28
 
     steps:
       - *checkout
@@ -56,7 +56,7 @@ jobs:
     name: TSAN
     runs-on: ubuntu-latest
     container:
-      image: docker.io/psedoc/anyhttp:0.27
+      image: docker.io/psedoc/anyhttp:0.28
 
     steps:
       - *checkout
@@ -75,7 +75,7 @@ jobs:
     name: Coverage
     runs-on: ubuntu-latest
     container:
-      image: docker.io/psedoc/anyhttp:0.27
+      image: docker.io/psedoc/anyhttp:0.28
     permissions:
       contents: read
       pages: write

CMakeLists.txt

@@ -42,6 +42,14 @@ add_link_options(-stdlib=libc++ -lstdc++fs)
 # use std::filesystem instead of boost::filesystem
 add_compile_definitions(BOOST_PROCESS_USE_STD_FS)
 
+# Boost.Process v2's posix launcher calls asio's execution_context::notify_fork() in the forked
+# child, between fork() and execve(). That walks the io_context's services and locks their
+# mutexes -- and with more than one thread running the context, another thread may well have
+# been holding one of them at the moment of the fork, so the child inherits it locked and hangs
+# there forever, before it ever gets to dup2() its stdio or exec. The child does nothing but
+# exec, so it has no use for the notification in the first place.
+add_compile_definitions(BOOST_PROCESS_V2_DISABLE_NOTIFY_FORK)
+
 # we don't use fmtlib any more
 add_compile_definitions(SPDLOG_USE_STD_FORMAT)
 
@@ -86,21 +94,6 @@ find_package(spdlog REQUIRED)
 
 # --------------------------------------------------------------------------------------------------
 
-# Checks for header files, used by ngtcp2 code.
-include(CheckIncludeFile)
-check_include_file("arpa/inet.h"   HAVE_ARPA_INET_H)
-check_include_file("netinet/in.h"  HAVE_NETINET_IN_H)
-check_include_file("netinet/ip.h"  HAVE_NETINET_IP_H)
-check_include_file("unistd.h"      HAVE_UNISTD_H)
-check_include_file("sys/endian.h"  HAVE_SYS_ENDIAN_H)
-check_include_file("endian.h"      HAVE_ENDIAN_H)
-check_include_file("byteswap.h"    HAVE_BYTESWAP_H)
-check_include_file("asm/types.h"   HAVE_ASM_TYPES_H)
-check_include_file("linux/netlink.h"   HAVE_LINUX_NETLINK_H)
-check_include_file("linux/rtnetlink.h" HAVE_LINUX_RTNETLINK_H)
-
-# --------------------------------------------------------------------------------------------------
-
 enable_testing(false)
 include(cmake/pki.cmake)
 add_subdirectory(src)

HTTP3.md

@@ -1,7 +1,7 @@
 # HTTP/3 Manual Testing
 
 ```sh
-cmake --build build && build/src/ngtcp2/ngtcp-server ::1 8080 pki/out/server-key.pem pki/out/server-chain.pem
+osslserver ::1 8080 pki/out/server-key.pem pki/out/server-chain.pem
 ```
 
 ```sh

README.md

@@ -13,7 +13,7 @@ None of those protocols are implemented from scratch. Instead, it is a wrapper a
 
 * Boost Beast
 * nghttp2
-* nghttp3 - not done yet.
+* ngtcp2/nghttp3
 
 ## Synopsis
 ### Server

include/anyhttp/literals.hpp

@@ -1 +1,7 @@
-#include "../src/ngtcp2/template.h"
+#pragma once
+
+//
+// User-defined literals for K and M (powers of 1024), as in the ngtcp2 examples.
+//
+constexpr unsigned long long operator""_k(unsigned long long k) { return k * 1024; }
+constexpr unsigned long long operator""_m(unsigned long long m) { return m * 1024 * 1024; }

include/anyhttp/server_impl.hpp

@@ -53,6 +53,7 @@ class Response::Impl : public impl::Writer
 
 struct Endpoint;
 class Http3Session;
+struct QuicBatch;
 
 class Server::Impl : public std::enable_shared_from_this<Server::Impl>
 {
@@ -69,7 +70,7 @@ class Server::Impl : public std::enable_shared_from_this<Server::Impl>
    const Config& config() const { return m_config; }
    boost::asio::any_io_executor get_executor() const noexcept { return m_executor; }
 
-   asio::awaitable<void> listen_loop();
+   asio::awaitable<void> tcp_listen_loop();
    asio::awaitable<void> handleConnection(asio::ip::tcp::socket socket);
 
    asio::ip::tcp::endpoint local_endpoint() const
@@ -83,10 +84,13 @@ class Server::Impl : public std::enable_shared_from_this<Server::Impl>
 
    asio::awaitable<void> udp_receive_loop();
    int udp_on_read(Endpoint& ep);
+   void process_quic_batch(const std::shared_ptr<Http3Session>& session, QuicBatch&& batch);
 
    //
    // QUIC connection-ID demux table. Populated by QuicHandler as new source CIDs are minted,
-   // consulted by udp_on_read() to route packets to the right connection.
+   // consulted by udp_on_read() to route packets to the right connection. Guarded by
+   // m_quicMutex: the receive loop reads it while sessions mutate it from their own strands
+   // (get_new_connection_id/remove_connection_id callbacks, close timers).
    //
    void associate_quic_cid(const ngtcp2_cid& cid, Http3Session* session);
    void dissociate_quic_cid(const ngtcp2_cid& cid);
@@ -102,10 +106,11 @@ class Server::Impl : public std::enable_shared_from_this<Server::Impl>
    std::mutex m_sessionMutex;
    std::set<std::shared_ptr<Session::Impl>> m_sessions;
 
+   std::mutex m_quicMutex;
    std::unordered_map<std::string, std::shared_ptr<Http3Session>> m_quic_handlers;
 
    RequestHandler m_requestHandler;
-   bool m_stopped = false;
+   bool m_destroyed = false;
 };
 
 // =================================================================================================

include/anyhttp/tls.hpp

@@ -0,0 +1,22 @@
+#pragma once
+
+#include <string>
+
+struct ssl_st;
+
+// =================================================================================================
+
+namespace anyhttp
+{
+
+/**
+ * One-line summary of a completed TLS handshake, in the spirit of what h2load prints:
+ * protocol version, cipher, key exchange group and the negotiated ALPN protocol.
+ *
+ * Used for both TLS over TCP and QUIC, so that the log looks the same for all protocols.
+ */
+std::string tls_handshake_info(ssl_st* ssl);
+
+} // namespace anyhttp
+
+// =================================================================================================

src/CMakeLists.txt

@@ -4,8 +4,7 @@ include_directories(${CMAKE_SOURCE_DIR}/include)
 # https://github.com/andrewmd5/libwtf/blob/main/CMakeLists.txt
 
 #
-# QUIC / HTTP3 libraries -- shared between the anyhttp library below and the
-# stand-alone ngtcp2 example binaries in the ngtcp2/ subdirectory.
+# QUIC / HTTP3 libraries
 #
 pkg_check_modules(NGTCP2              REQUIRED IMPORTED_TARGET libngtcp2)
 pkg_check_modules(NGHTTP3             REQUIRED IMPORTED_TARGET libnghttp3)
@@ -22,14 +21,16 @@ target_sources(anyhttp PRIVATE ${anyhttp_sources})
 target_include_directories(anyhttp PUBLIC ../include)
 
 target_include_directories(anyhttp PRIVATE "/opt/nghttp3/build/include")
-target_sources(anyhttp PRIVATE ngtcp2/shared.cc)
+#
+# Small helpers taken from the ngtcp2 examples (see ngtcp2/README.md).
+#
+target_sources(anyhttp PRIVATE ngtcp2/shared.cc ngtcp2/util.cc)
 
 target_link_libraries(anyhttp PRIVATE Threads::Threads)
 target_link_libraries(anyhttp PRIVATE OpenSSL::SSL)
 target_link_libraries(anyhttp PRIVATE Boost::thread Boost::atomic Boost::url Boost::filesystem)
 target_link_libraries(anyhttp PRIVATE spdlog::spdlog_header_only)
 target_link_libraries(anyhttp PRIVATE PkgConfig::NGHTTP2)
-target_link_libraries(anyhttp PRIVATE ngtcp2_common)
 target_link_libraries(anyhttp PRIVATE PkgConfig::NGTCP2 PkgConfig::NGHTTP3 PkgConfig::NGTCP2_CRYPTO_OSSL)
 
 #
@@ -53,14 +54,3 @@ target_link_libraries(client PRIVATE anyhttp)
 #
 add_executable(beast_issue_3032)
 target_sources(beast_issue_3032 PRIVATE "beast_issue_3032.cpp")
-
-#
-# ngtcp-test
-#
-add_executable(ngtcp-test)
-target_sources(ngtcp-test PRIVATE "ngtcp-test_main.cpp")
-target_include_directories(ngtcp-test PUBLIC "/opt/boringssl/include")
-target_link_directories(ngtcp-test PRIVATE "/opt/boringssl/lib")
-target_link_libraries(ngtcp-test ssl crypto ev ngtcp2 libngtcp2_crypto_boringssl.a)
-
-add_subdirectory(ngtcp2)

src/client_impl_udp.cpp

@@ -16,6 +16,7 @@
 #include "anyhttp/formatter.hpp" // IWYU pragma: keep
 #include "anyhttp/literals.hpp"
 #include "anyhttp/session_impl.hpp"
+#include "anyhttp/tls.hpp"
 
 #include <boost/asio.hpp>
 #include <boost/asio/any_io_executor.hpp>
@@ -95,15 +96,6 @@ struct TlsClientContext
       static constexpr unsigned char alpn[] = "\x02h3";
       SSL_CTX_set_alpn_protos(ctx, alpn, sizeof(alpn) - 1);
 
-      //
-      // Same order as the server, so AES-128 GCM is also picked against peers that leave the
-      // choice to the client.
-      //
-      if (SSL_CTX_set_ciphersuites(ctx, "TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:"
-                                        "TLS_CHACHA20_POLY1305_SHA256") != 1)
-         throw std::runtime_error(std::string{"SSL_CTX_set_ciphersuites: "} +
-                                  ERR_error_string(ERR_get_error(), nullptr));
-
       //
       // TODO: verify the server certificate (e.g. against pki/out/root.pem) instead of accepting
       // anything.
@@ -183,8 +175,8 @@ class Http3ClientStream;
 //
 // Bound on how much of the caller's async_write() buffer we copy into write_chunk at a time (see
 // Http3ClientStream's write_* members) -- copying is paced by how much nghttp3/ngtcp2 actually
-// drains, rather than copying a huge caller buffer (e.g. 50MB) in one synchronous allocation+memcpy,
-// mirroring nghttp2's own per-call copy into its frame buffer.
+// drains, rather than copying a huge caller buffer (e.g. 50MB) in one synchronous
+// allocation+memcpy, mirroring nghttp2's own per-call copy into its frame buffer.
 //
 inline constexpr size_t kWriteChunkSize = 16 * 1024;
 
@@ -1016,7 +1008,6 @@ void Http3ClientStream::finish_active_write()
 
    if (handler)
       swap_and_invoke(handler, boost::system::error_code{});
-
 }
 
 // -------------------------------------------------------------------------------------------------
@@ -1346,6 +1337,17 @@ awaitable<void> Http3ClientSession::do_session(Buffer&&)
 
       if (on_read({buf.data(), n}) != 0)
          break; // handle_error() already tore things down.
+
+      //
+      // close() may have run from inside on_read(): handing a response chunk or EOF to the
+      // application resumes its coroutine, which may drop the last reference to the Session
+      // right there. Its socket_.cancel() then found no receive pending -- we are between two
+      // of them -- so nothing would stop us from arming a fresh one that no peer will ever
+      // complete. The server, already draining because it got our CONNECTION_CLOSE, does not
+      // even answer it.
+      //
+      if (closed_)
+         break;
    }
 
    //
@@ -1674,7 +1676,8 @@ int Http3ClientSession::handle_error(int /*rv*/)
 int Http3ClientSession::cb_handshake_completed(ngtcp2_conn*, void* user)
 {
    auto self = static_cast<Http3ClientSession*>(user);
-   logi("[{}] TLS handshake complete", self->log_prefix_);
+   logi("[{}] TLS handshake completed: {}", self->log_prefix_,
+        tls_handshake_info(ngtcp2_crypto_ossl_ctx_get_ssl(self->ossl_ctx_)));
    if (!self->h3_ && self->setup_http3() != 0)
       return NGTCP2_ERR_CALLBACK_FAILURE;
    return 0;