diff --git a/pq-tls-benchmark-framework/emulation-exp/code/cquiche/Makefile b/pq-tls-benchmark-framework/emulation-exp/code/cquiche/Makefile deleted file mode 100644 index 8a96a88c4e7e7ae926fa7e58de3155bb2c266a46..0000000000000000000000000000000000000000 --- a/pq-tls-benchmark-framework/emulation-exp/code/cquiche/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -OPENSSL_INSTALL=tmp/.local/openssl - -build_with_pkg_config: - cd cquiche_s_timer && PKG_CONFIG_PATH=${OPENSSL_INSTALL}/lib64/pkgconfig LD_LIBRARY_PATH=${OPENSSL_INSTALL}/lib64 cargo run - -ld_trace_loaded_objects: - LD_TRACE_LOADED_OBJECTS=1 ./your_executable diff --git a/pq-tls-benchmark-framework/emulation-exp/code/cquiche/cquiche_s_timer/.gitignore b/pq-tls-benchmark-framework/emulation-exp/code/cquiche/cquiche_s_timer/.gitignore deleted file mode 100644 index 2f7896d1d1365eafb0da03d9fe456fac81408487..0000000000000000000000000000000000000000 --- a/pq-tls-benchmark-framework/emulation-exp/code/cquiche/cquiche_s_timer/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target/ diff --git a/pq-tls-benchmark-framework/emulation-exp/code/cquiche/install.sh b/pq-tls-benchmark-framework/emulation-exp/code/cquiche/install.sh deleted file mode 100755 index d721dcf7dda54ce6650928f8823a8dcc29310366..0000000000000000000000000000000000000000 --- a/pq-tls-benchmark-framework/emulation-exp/code/cquiche/install.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/bash -set -ex - -# Make sure to have a recent version of openssl installed by default. For example by having an up to date os version. - -sudo apt update -sudo apt install -y git \ - build-essential \ - autoconf \ - automake \ - libtool \ - ninja-build \ - libssl-dev \ - libpcre3-dev \ - wget - -NGINX_VERSION=1.26.1 -CMAKE_VERSION=3.30 -CMAKE_BUILD=0 - -mkdir -p tmp -cd tmp -ROOT=$(pwd) -INSTALL_DIR=${ROOT}/.local -OPENSSL_INSTALL=${ROOT}/.local/openssl - -# Fetch all the files we need -wget https://cmake.org/files/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}.${CMAKE_BUILD}-linux-x86_64.sh -# git clone --no-checkout --single-branch --branch openssl-3.4 https://github.com/openssl/openssl.git -# (cd openssl && git switch --detach tags/openssl-3.4.0) -# care wrong version of quictls -git clone --no-checkout --single-branch --branch openssl-3.3.0+quic https://github.com/quictls/openssl.git quictls/openssl -(cd quictls/openssl && git switch --detach openssl-3.3.0+quic) -git clone --recursive https://github.com/cloudflare/quiche -git clone --no-checkout --single-branch --branch 0.10.1-release https://github.com/open-quantum-safe/liboqs.git -(cd liboqs && git switch --detach tags/0.10.1) -git clone --no-checkout --single-branch --branch main https://github.com/open-quantum-safe/oqs-provider.git -(cd oqs-provider && git switch --detach tags/0.6.1) -wget nginx.org/download/nginx-${NGINX_VERSION}.tar.gz && tar -zxvf nginx-${NGINX_VERSION}.tar.gz - -# Install the latest CMake -mkdir -p ${INSTALL_DIR}/cmake -sh cmake-${CMAKE_VERSION}.${CMAKE_BUILD}-linux-x86_64.sh --skip-license --prefix=${INSTALL_DIR}/cmake - - -# # Patch openssl to have a large CRYPTO_RECV_BUF -# patch ${ROOT}/openssl/ssl/quic/quic_channel.c < ${ROOT}/../patches/openssl/quic_channel.c.patch - -# Build OpenSSL so 'libcrypto.so' is avaiable for the build of liboqs. With Ubuntu 22.04 not longer needed. -( - cd quictls/openssl - # adds a runtime path to the executable, so it can find the libcrypto.so - LDFLAGS="-Wl,-rpath -Wl,${OPENSSL_INSTALL}/lib64" ./Configure --prefix=${OPENSSL_INSTALL} --openssldir=${OPENSSL_INSTALL}/ssl - # Commented out are the commands from https://github.com/open-quantum-safe/oqs-provider/blob/main/scripts/fullbuild.sh - # export OSSL_PREFIX=`pwd`/.local - # LDFLAGS="-Wl,-rpath -Wl,${OSSL_PREFIX}/lib64" ./config --prefix=$OSSL_PREFIX - - make && make install_sw install_ssldirs - - # NOTE maybe create a softlink if issue arises, something like - # ln -s lib64 lib -) - -# build liboqs -( - cd liboqs - # It needs the libcrypto library, either in .a or .so format, in 'openssl' it is .so and in 'openssl-source' it is .a - # -- Found OpenSSL: /absolute-path-to/tmp/openssl/lib64/libcrypto.so (found suitable version "3.0.2", minimum required is "1.1.1") - # NOTE here the enabled algorithms could be specified - ${INSTALL_DIR}/cmake/bin/cmake -GNinja -DOPENSSL_ROOT_DIR=${OPENSSL_INSTALL} -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/liboqs -S . -B build - cd build - ninja && ninja install -) - -( - cd oqs-provider - liboqs_DIR=${INSTALL_DIR}/liboqs ${INSTALL_DIR}/cmake/bin/cmake -DOPENSSL_ROOT_DIR=${OPENSSL_INSTALL} -S . -B build && ${INSTALL_DIR}/cmake/bin/cmake --build build - # next command does not work, but is not needed maybe cuz we just copy the library??? - # maybe use --install-prefix for next command - # cmake --install build --prefix ${ROOT}/oqs-provider/install - cp build/lib/oqsprovider.so ${OPENSSL_INSTALL}/lib64/ossl-modules/ - # can also be installed to system - # sudo cp /home/bebbo/own/master/benchmarking-pqc-in-quic/pq-tls-benchmark-framework/emulation-exp/code/tmp/oqs-provider/build/lib/oqsprovider.so /lib/x86_64-linux-gnu/ossl-modules -) - -sed -i "s/default = default_sect/default = default_sect\noqsprovider = oqsprovider_sect/g" ${OPENSSL_INSTALL}/ssl/openssl.cnf -sed -i "s/\[default_sect\]/\[default_sect\]\nactivate = 1\n\[oqsprovider_sect\]\nactivate = 1\n/g" ${OPENSSL_INSTALL}/ssl/openssl.cnf - -${OPENSSL_INSTALL}/bin/openssl version -d - -if [[ $(${OPENSSL_INSTALL}/bin/openssl version -d) != *"${OPENSSL_INSTALL}/ssl"* ]]; then - echo "The output of 'openssl version -d' does not include the string ${OPENSSL_INSTALL}/ssl" - exit 1 -fi -if [[ $(${OPENSSL_INSTALL}/bin/openssl list -providers) != *"OpenSSL OQS Provider"* ]]; then - echo "The output of 'openssl list -providers' does not include the string 'OpenSSL OQS Provider'" - exit 1 -fi -echo "Openssl seems to be installed correctly" - - -# build cquiche -( - cd quiche - # give cargo the context of the custom openssl (the pkgconfig file), so that it can find the correct libs - PKG_CONFIG_PATH=${OPENSSL_INSTALL}/lib64/pkgconfig cargo build --features=openssl - - # how to find out if openssl is used or boringssl? -) - - - - - -# # for nginx build -# # export OPENSSL_CONF=${OPENSSL_INSTALL}/ssl/openssl.cnf - -# # apply patches to nginx source -# patch ${ROOT}/nginx-${NGINX_VERSION}/src/event/quic/ngx_event_quic_openssl_compat.c < ${ROOT}/../patches/nginx-${NGINX_VERSION}-patches/ngx_event_quic_openssl_compat.c.diff - -# # build nginx with dynamically linked custom openssl -# # NOTE it may be that the first time nginx is built it is not linked correctly to the custom openssl, but just do it again, it should work -# ( -# cd nginx-${NGINX_VERSION} -# # NOTE why --without-http_gzip_module -# # -rpath with --enable-new-dtags makes it use RUNPATH which is evaluated after LD_LIBRARY_PATH (which is evaluated after RPATH) -# # export LD_LIBRARY_PATH=${OPENSSL_INSTALL}/lib64:${LD_LIBRARY_PATH} -# ./configure --prefix=${INSTALL_DIR}/nginx \ -# --with-debug \ -# --with-http_v2_module \ -# --with-http_v3_module \ -# --without-http_gzip_module \ -# --with-http_ssl_module \ -# --with-ld-opt="-Wl,--enable-new-dtags,-rpath,${OPENSSL_INSTALL}/lib64" -# # The --with-openssl builds openssl again, but we just want to link against it -# # --with-openssl=${ROOT}/openssl \ -# # --with-cc-opt="-I ${OPENSSL_INSTALL}/include/openssl" \ -# # --with-ld-opt="-L ${OPENSSL_INSTALL}/lib64" -# # sed -i 's/libcrypto.a/libcrypto.a -loqs/g' objs/Makefile; -# # NOTE why change this? -# # sed -i 's/EVP_MD_CTX_create/EVP_MD_CTX_new/g; s/EVP_MD_CTX_destroy/EVP_MD_CTX_free/g' src/event/ngx_event_openssl.c -# make && make install -# ) - -# # NOTE check with nginx -V and ldd if the openssl is linked correctly -# # it should show the path to the custom openssl -# # This only shows the correct path (if RUNPATH is not used), if the LD_LIBRARY_PATH env var is set correctly -# ${INSTALL_DIR}/nginx/sbin/nginx -V -# ldd ${INSTALL_DIR}/nginx/sbin/nginx -# readelf -d ${INSTALL_DIR}/nginx/sbin/nginx | grep 'R.*PATH' - -# echo "You should see that nginx is linked against the custom openssl in tmp/.local/openssl" diff --git a/pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/.gitignore b/pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..4d498c4f03f1947145bfcc7a4cad3648640cc98a --- /dev/null +++ b/pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/.gitignore @@ -0,0 +1,2 @@ +target/ +qlogdir/ diff --git a/pq-tls-benchmark-framework/emulation-exp/code/cquiche/cquiche_s_timer/Cargo.lock b/pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/Cargo.lock similarity index 100% rename from pq-tls-benchmark-framework/emulation-exp/code/cquiche/cquiche_s_timer/Cargo.lock rename to pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/Cargo.lock diff --git a/pq-tls-benchmark-framework/emulation-exp/code/cquiche/cquiche_s_timer/Cargo.toml b/pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/Cargo.toml similarity index 100% rename from pq-tls-benchmark-framework/emulation-exp/code/cquiche/cquiche_s_timer/Cargo.toml rename to pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/Cargo.toml diff --git a/pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/Makefile b/pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..c2c4c3f2e598d8924d1bbd1d9bb3eab91bb36b14 --- /dev/null +++ b/pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/Makefile @@ -0,0 +1,16 @@ +# PKG_CONFIG_PATH and/or LD_LIBRARY_PATH must be absolute paths +OPENSSL_INSTALL=$(shell pwd)/../tmp/.local/quictls-openssl + +# NOTE RUSTFLAGS not yet used everywhere to add rpath +build_with_pkg_config_release_mode: + RUSTFLAGS="-C link-arg=-Wl,-rpath,${OPENSSL_INSTALL}/lib64" PKG_CONFIG_PATH=${OPENSSL_INSTALL}/lib64/pkgconfig LD_LIBRARY_PATH=${OPENSSL_INSTALL}/lib64 cargo build --release + +run_with_pkg_config: + PKG_CONFIG_PATH=${OPENSSL_INSTALL}/lib64/pkgconfig LD_LIBRARY_PATH=${OPENSSL_INSTALL}/lib64 cargo run --bin cquiche_s_timer mlkem768 1 + +# LD_LIBRARY_PATH must be given via env to ip netns exec, QLOGDIR not???? -> i did it here just to show the syntax for transporting multiple envs to netns exec +run_in_network_namespace: + sudo --preserve-env ip netns exec cli_ns_1 env LD_LIBRARY_PATH=${OPENSSL_INSTALL}/lib64 env QLOGDIR=qlogdir target/release/cquiche_s_timer mlkem512 20 bbr2 + +ld_trace_loaded_objects: + LD_TRACE_LOADED_OBJECTS=1 ./your_executable diff --git a/pq-tls-benchmark-framework/emulation-exp/code/cquiche/cquiche_s_timer/src/main.rs b/pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/src/bin/http-client.rs similarity index 99% rename from pq-tls-benchmark-framework/emulation-exp/code/cquiche/cquiche_s_timer/src/main.rs rename to pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/src/bin/http-client.rs index 3409efed7318416e484d445600afceaa4e73895e..6f1a824e1a26bb0827266d16644f28b23cc08329 100644 --- a/pq-tls-benchmark-framework/emulation-exp/code/cquiche/cquiche_s_timer/src/main.rs +++ b/pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/src/bin/http-client.rs @@ -85,7 +85,7 @@ fn main() { .unwrap(); config.verify_peer(true); config - .load_verify_locations_from_file("../../tmp/.local/nginx/conf/CA.crt") + .load_verify_locations_from_file("../tmp/.local/nginx/conf/CA.crt") .unwrap(); // config.log_keys(); // config.discover_pmtu(true); diff --git a/pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/src/main.rs b/pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..73f3ff2b89abe4e6312077d5778d653f3b206763 --- /dev/null +++ b/pq-tls-benchmark-framework/emulation-exp/code/cquiche_s_timer/src/main.rs @@ -0,0 +1,395 @@ +// Copyright (C) 2019, Cloudflare, Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// PKG_CONFIG_PATH=${OPENSSL_INSTALL}/lib64/pkgconfig LD_LIBRARY_PATH=${OPENSSL_INSTALL}/lib64 cargo run -- https://localhost:8443/ + +#[macro_use] +extern crate log; + +use quiche::Connection; +use ring::rand::*; + +// const MAX_DATAGRAM_SIZE: usize = 1350; // This value is in quiche/apps/src/client.rs +const MAX_DATAGRAM_SIZE: usize = 1200; // This value is used by openssl s_timer + +const SERVER_ADDRESS: &str = "https://10.0.0.1:8443"; +const SERVER_NAME: &str = "localhost"; + +// NOTE this version does not implement pacing, so it is comparable to openssl s_timer +// if pacing should be implemented, probably should work user spaced based, with using the event polling used here for simplicity + +fn main() { + let mut buf = [0; 65535]; + let mut out = [0; MAX_DATAGRAM_SIZE]; + + let (group, amount_of_measurments, ccalgo) = initialize_process(); + let (mut poll, mut events, peer_addr, bind_addr) = general_setup(); + + let mut measurements = Vec::with_capacity(amount_of_measurments); + let mut error_count = 0; + let mut i = 0; + while i < amount_of_measurments { + let (mut conn, socket, local_addr) = + prepare_handshake(&mut poll, peer_addr, &bind_addr, &group, &ccalgo); + + match time_handshake( + &mut poll, + &mut events, + &mut conn, + &socket, + &mut out, + &mut buf, + local_addr, + ) { + Some(duration) => { + info!("Handshake took {:?}", duration); + measurements.push(duration); + } + None => { + info!("Handshake failed"); + error_count += 1; + } + } + + i += 1; + } + + print!("{error_count};"); + let measurements: Vec<f64> = measurements + .iter() + .map(|m| m.as_nanos() as f64 / 1_000_000.0) + .collect(); + for i in 0..amount_of_measurments - 1 { + print!("{:.6},", measurements[i]); + } + print!("{:.6}", measurements[amount_of_measurments - 1]); +} + +fn initialize_process() -> (String, usize, String) { + env_logger::builder() + .filter_level(log::LevelFilter::Error) + .target(env_logger::Target::Stderr) + .init(); + + let mut args = std::env::args(); + let cmd = &args.next().unwrap(); + + // only looks at remaining args + if args.len() != 3 { + error!("Usage: {cmd} GROUP AMOUNT_OF_MEASUREMENTS CCALGORITHM"); + std::process::exit(1); + } + + let group = args.next().unwrap(); + let amount_of_measurments = args.next().unwrap().parse::<usize>().unwrap(); + let ccalgorithm = args.next().unwrap(); + return (group, amount_of_measurments, ccalgorithm); +} + +fn general_setup() -> (mio::Poll, mio::Events, std::net::SocketAddr, String) { + let url = url::Url::parse(SERVER_ADDRESS).unwrap(); + // Resolve server address. + let peer_addr = url.socket_addrs(|| None).unwrap()[0]; + // Bind to INADDR_ANY or IN6ADDR_ANY depending on the IP family of the + // server address. This is needed on macOS and BSD variants that don't + // support binding to IN6ADDR_ANY for both v4 and v6. + let bind_addr = match peer_addr { + std::net::SocketAddr::V4(_) => "0.0.0.0:0".to_string(), + std::net::SocketAddr::V6(_) => "[::]:0".to_string(), + }; + + // Setup the event loop. + let poll = mio::Poll::new().unwrap(); + let events = mio::Events::with_capacity(1024); + + (poll, events, peer_addr, bind_addr) +} + +fn prepare_handshake( + poll: &mut mio::Poll, + peer_addr: std::net::SocketAddr, + bind_addr: &String, + group: &String, + ccalgo: &String, +) -> ( + quiche::Connection, + mio::net::UdpSocket, + std::net::SocketAddr, +) { + // Create the UDP socket backing the QUIC connection, and register it with + // the event loop. + let mut socket = mio::net::UdpSocket::bind(bind_addr.parse().unwrap()).unwrap(); + poll.registry() + .register(&mut socket, mio::Token(0), mio::Interest::READABLE) + .unwrap(); + + let mut config = quiche::Config::new(quiche::PROTOCOL_VERSION).unwrap(); + config.set_groups(&group).unwrap(); + config + .set_application_protos(quiche::h3::APPLICATION_PROTOCOL) + .unwrap(); + config.verify_peer(true); + config + .load_verify_locations_from_file("../tmp/.local/nginx/conf/CA.crt") + .unwrap(); + // config.log_keys(); + // config.discover_pmtu(true); + + // NOTE think about how these can work in this implementation, they can only differ in what cwd they assume and in what timeouts they use + match ccalgo.as_str() { + "cubic" => config.set_cc_algorithm(quiche::CongestionControlAlgorithm::CUBIC), + "reno" => config.set_cc_algorithm(quiche::CongestionControlAlgorithm::Reno), + "bbr" => config.set_cc_algorithm(quiche::CongestionControlAlgorithm::BBR), + "bbr2" => config.set_cc_algorithm(quiche::CongestionControlAlgorithm::BBR2), + _ => { + error!("Unknown congestion control algorithm: {ccalgo}"); + std::process::exit(1); + } + } + + // Disable pacing? implicitly, cuz packets have to be sent later by our application + // see https://docs.rs/quiche/latest/quiche/index.html#pacing + config.enable_pacing(false); + + // TODO read about every option and evaluate if default is good and comparable to openssl's impl + config.set_max_idle_timeout(30000); + config.set_max_recv_udp_payload_size(MAX_DATAGRAM_SIZE); + config.set_max_send_udp_payload_size(MAX_DATAGRAM_SIZE); + config.set_initial_max_data(10_000_000); + config.set_initial_max_stream_data_bidi_local(1_000_000); + config.set_initial_max_stream_data_bidi_remote(1_000_000); + config.set_initial_max_stream_data_uni(1_000_000); + config.set_initial_max_streams_bidi(100); + config.set_initial_max_streams_uni(100); + // config.set_disable_active_migration(true); + + // Generate a random source connection ID for the connection. + let mut scid = [0; quiche::MAX_CONN_ID_LEN]; + SystemRandom::new().fill(&mut scid[..]).unwrap(); + + let scid = quiche::ConnectionId::from_ref(&scid); + + // Get local address. + let local_addr = socket.local_addr().unwrap(); + + // Create a QUIC connection and initiate handshake. + let mut conn = + quiche::connect(Some(SERVER_NAME), &scid, local_addr, peer_addr, &mut config).unwrap(); + + if let Some(dir) = std::env::var_os("QLOGDIR") { + let id = format!("{scid:?}"); + let writer = make_qlog_writer(&dir, "client", &id); + + conn.set_qlog_with_level( + std::boxed::Box::new(writer), + "cquiche_s_timer qlog".to_string(), + format!("{} id={}", "cquiche_s_timer qlog", id), + quiche::QlogLevel::Extra, + ); + } + + info!( + "prepared handshake to connect to {:} from {:} with scid {:?}", + peer_addr, + socket.local_addr().unwrap(), + &scid + ); + + return (conn, socket, local_addr); +} + +fn time_handshake( + poll: &mut mio::Poll, + events: &mut mio::Events, + conn: &mut quiche::Connection, + socket: &mio::net::UdpSocket, + out: &mut [u8; MAX_DATAGRAM_SIZE], + buf: &mut [u8], + local_addr: std::net::SocketAddr, +) -> Option<std::time::Duration> { + let req_start = std::time::Instant::now(); + initial_send(conn, socket, out); + + loop { + poll.poll(events, conn.timeout()).unwrap(); + + receive(conn, socket, events, buf, local_addr); + + if conn.is_closed() { + info!("connection closed, {:?}", conn.stats()); + return None; + } + + send(conn, socket, out); + + if conn.is_established() { + let elapsed_time = req_start.elapsed(); + conn.close(true, 0x00, b"kthxbye").unwrap(); + send(conn, socket, out); + return Some(elapsed_time); + } + } +} + +/// Initial send does not stop sending, if the socket is blocked and retries sending. +fn initial_send( + conn: &mut Connection, + socket: &mio::net::UdpSocket, + out: &mut [u8; MAX_DATAGRAM_SIZE], +) { + loop { + let (write, send_info) = match conn.send(out) { + Ok(v) => v, + Err(quiche::Error::Done) => { + debug!("done writing"); + break; + } + Err(e) => { + error!("send failed: {:?}", e); + conn.close(false, 0x1, b"send failed").ok(); + break; + } + }; + + while let Err(e) = socket.send_to(&out[..write], send_info.to) { + if e.kind() == std::io::ErrorKind::WouldBlock { + debug!("send() would block"); + continue; + } + + panic!("send() failed: {:?}", e); + } + + debug!("written {}", write); + } +} + +/// Send does not retry sending if the socket is blocked. +fn send(conn: &mut Connection, socket: &mio::net::UdpSocket, out: &mut [u8; MAX_DATAGRAM_SIZE]) { + loop { + let (write, send_info) = match conn.send(out) { + Ok(v) => v, + Err(quiche::Error::Done) => { + debug!("done writing"); + break; + } + Err(e) => { + error!("send failed: {:?}", e); + conn.close(false, 0x1, b"send failed").ok(); + break; + } + }; + + if let Err(e) = socket.send_to(&out[..write], send_info.to) { + if e.kind() == std::io::ErrorKind::WouldBlock { + debug!("send() would block"); + break; + } + + panic!("send() failed: {:?}", e); + } + + debug!("written {}", write); + } +} + +fn receive( + conn: &mut Connection, + socket: &mio::net::UdpSocket, + events: &mio::Events, + buf: &mut [u8], + local_addr: std::net::SocketAddr, +) { + // Read incoming UDP packets from the socket and feed them to quiche, + // until there are no more packets to read. + 'read: loop { + // If the event loop reported no events, it means that the timeout + // has expired, so handle it without attempting to read packets. We + // will then proceed with the send loop. + if events.is_empty() { + debug!("timed out"); + + conn.on_timeout(); + + break 'read; + } + + let (len, from) = match socket.recv_from(buf) { + Ok(v) => v, + + Err(e) => { + // There are no more UDP packets to read, so end the read + // loop. + if e.kind() == std::io::ErrorKind::WouldBlock { + debug!("recv() would block"); + break 'read; + } + + panic!("recv() failed: {:?}", e); + } + }; + + debug!("got {} bytes", len); + + let recv_info = quiche::RecvInfo { + to: local_addr, + from, + }; + + // Process potentially coalesced packets. + let read = match conn.recv(&mut buf[..len], recv_info) { + Ok(v) => v, + + Err(e) => { + error!("recv failed: {:?}", e); + continue 'read; + } + }; + + debug!("processed {} bytes", read); + } + + debug!("done reading"); +} + +/// Makes a buffered writer for a qlog. +fn make_qlog_writer( + dir: &std::ffi::OsStr, + role: &str, + id: &str, +) -> std::io::BufWriter<std::fs::File> { + let mut path = std::path::PathBuf::from(dir); + let filename = format!("{role}-{id}.sqlog"); + path.push(filename); + + match std::fs::File::create(&path) { + Ok(f) => std::io::BufWriter::new(f), + Err(e) => panic!( + "Error creating qlog file attempted path was {:?}: {}", + path, e + ), + } +} diff --git a/pq-tls-benchmark-framework/emulation-exp/code/install-prereqs-ubuntu.sh b/pq-tls-benchmark-framework/emulation-exp/code/install-prereqs-ubuntu.sh index c77447f5861b9519bfbf816782606821c7e4b42c..11fac2bdd2572f2dbf80e7729d7fd2a911bdd211 100755 --- a/pq-tls-benchmark-framework/emulation-exp/code/install-prereqs-ubuntu.sh +++ b/pq-tls-benchmark-framework/emulation-exp/code/install-prereqs-ubuntu.sh @@ -35,6 +35,7 @@ git clone --no-checkout --single-branch --branch openssl-3.4 https://github.com/ git clone --no-checkout --single-branch --branch openssl-3.1.7+quic https://github.com/quictls/openssl.git quictls-openssl (cd quictls-openssl && git switch --detach openssl-3.1.7+quic) git clone --recursive https://github.com/cloudflare/quiche +(cd quiche && git switch --detach tags/0.22.0) git clone --no-checkout --single-branch --branch 0.11.0-release https://github.com/open-quantum-safe/liboqs.git (cd liboqs && git switch --detach tags/0.11.0) # git clone --no-checkout --single-branch --branch main https://github.com/open-quantum-safe/oqs-provider.git @@ -114,7 +115,8 @@ echo "Openssl seems to be installed correctly" # give cargo the context of the custom openssl (the pkgconfig file), so that it can find the correct libs PKG_CONFIG_PATH=${QUICTLS_OPENSSL_INSTALL}/lib64/pkgconfig cargo build --features=openssl - # how to find out if openssl is used or boringssl? + patch ${ROOT}/quiche/quiche/src/lib.rs < ${ROOT}/../patches/cquiche/lib.rs.diff + patch ${ROOT}/quiche/quiche/src/tls/mod.rs < ${ROOT}/../patches/cquiche/mod.rs.diff ) # apply patches to nginx source diff --git a/pq-tls-benchmark-framework/emulation-exp/code/kex/.gitignore b/pq-tls-benchmark-framework/emulation-exp/code/kex/.gitignore index 832e38ab17bea6205f8e69e89f8d4cc01a038586..1a79bb01fe5303bd2816a7fffb26e7e120a7e992 100644 --- a/pq-tls-benchmark-framework/emulation-exp/code/kex/.gitignore +++ b/pq-tls-benchmark-framework/emulation-exp/code/kex/.gitignore @@ -1,6 +1,7 @@ quic_s_timer s_timer quic-client-block +cquiche_s_timer .venv/ diff --git a/pq-tls-benchmark-framework/emulation-exp/code/kex/nginx.conf b/pq-tls-benchmark-framework/emulation-exp/code/kex/nginx.conf index faf17122d81214fa42b01d80a4ebb1f58c9f9ba0..0ff3e393d8315baa0e9201e6fab7a2d9fd113195 100644 --- a/pq-tls-benchmark-framework/emulation-exp/code/kex/nginx.conf +++ b/pq-tls-benchmark-framework/emulation-exp/code/kex/nginx.conf @@ -51,7 +51,7 @@ http { # quic_retry on; # does not matter when only doing handshakes # ssl_early_data on; # does not matter when only doing handshakes - ssl_ecdh_curve x25519_kyber768:p256_kyber768:secp256r1:x25519:mlkem512:bikel1:hqc128:frodo640aes:frodo640shake:p256_mlkem512:x25519_mlkem512:p256_bikel1:x25519_bikel1:p256_hqc128:x25519_hqc128:p256_frodo640aes:x25519_frodo640aes:p256_frodo640shake:x25519_frodo640shake:secp384r1:x448:mlkem768:bikel3:hqc192:frodo976aes:frodo976shake:p384_mlkem768:x448_mlkem768:p384_bikel3:x448_bikel3:p384_hqc192:x448_hqc192:p384_frodo976aes:x448_frodo976aes:p384_frodo976shake:x448_frodo976shake:secp521r1:mlkem1024:bikel5:hqc256:frodo1344aes:frodo1344shake:p521_mlkem1024:p521_bikel5:p521_hqc256:p521_frodo1344aes:p521_frodo1344shake:x25519_mlkem768:p256_mlkem768:p384_mlkem1024; + ssl_ecdh_curve x25519_kyber768:p256_kyber768:secp256r1:x25519:mlkem512:bikel1:hqc128:frodo640aes:frodo640shake:p256_mlkem512:x25519_mlkem512:p256_bikel1:x25519_bikel1:p256_hqc128:x25519_hqc128:p256_frodo640aes:x25519_frodo640aes:p256_frodo640shake:x25519_frodo640shake:secp384r1:x448:mlkem768:bikel3:hqc192:frodo976aes:frodo976shake:p384_mlkem768:x448_mlkem768:p384_bikel3:x448_bikel3:p384_hqc192:x448_hqc192:p384_frodo976aes:x448_frodo976aes:p384_frodo976shake:x448_frodo976shake:secp521r1:mlkem1024:bikel5:hqc256:frodo1344aes:frodo1344shake:p521_mlkem1024:p521_bikel5:p521_hqc256:p521_frodo1344aes:p521_frodo1344shake:X25519MLKEM768:SecP256r1MLKEM768:p384_mlkem1024; # NOTE normally a server sends all certificates in the chanin but the root certificate ssl_certificate server.crt; diff --git a/pq-tls-benchmark-framework/emulation-exp/code/kex/quic_s_timer.c b/pq-tls-benchmark-framework/emulation-exp/code/kex/quic_s_timer.c index dc14a3872e58e34a67a582c2da83640a1113a761..76a7a06747ec2457ef9190ca0ee945b37843739a 100644 --- a/pq-tls-benchmark-framework/emulation-exp/code/kex/quic_s_timer.c +++ b/pq-tls-benchmark-framework/emulation-exp/code/kex/quic_s_timer.c @@ -179,7 +179,7 @@ SSL *new_ssl_builder(SSL_CTX *ssl_ctx) return ssl; } -SSL *do_tls_handshake(SSL_CTX *ssl_ctx, BIO_ADDRINFO *addr_info) +SSL *do_tls_handshake(SSL_CTX *ssl_ctx, BIO_ADDRINFO *addr_info, double *time_taken) { // NOTE maybe also use ipv6 BIO *conn = create_socket_bio_helper(hostname, port, AF_INET, addr_info); @@ -196,10 +196,15 @@ SSL *do_tls_handshake(SSL_CTX *ssl_ctx, BIO_ADDRINFO *addr_info) SSL_set_bio(ssl, conn, conn); + struct timespec start, finish; // The total other time than the handshake in the hot path takes on average around 0.05 to 0.06 ms /* ok, lets connect */ - if (SSL_connect(ssl) <= 0) - { + clock_gettime(CLOCK_MONOTONIC_RAW, &start); + int ret = SSL_connect(ssl); + clock_gettime(CLOCK_MONOTONIC_RAW, &finish); + *time_taken = ((finish.tv_sec - start.tv_sec) * MS_IN_S) + ((finish.tv_nsec - start.tv_nsec) / NS_IN_MS); + + if (ret <= 0) { ERR_print_errors_fp(stderr); SSL_free(ssl); return 0; @@ -245,14 +250,12 @@ int main(int argc, char *argv[]) // Start experiments SSL *ssl = NULL; size_t measurements = 0; - struct timespec start, finish; double *handshake_times_ms = malloc(measurements_to_make * sizeof(*handshake_times_ms)); int error_count = 0; while (measurements < measurements_to_make) { - clock_gettime(CLOCK_MONOTONIC_RAW, &start); - ssl = do_tls_handshake(ssl_ctx, addr_info); - clock_gettime(CLOCK_MONOTONIC_RAW, &finish); + double time_taken = -1.0; + ssl = do_tls_handshake(ssl_ctx, addr_info, &time_taken); if (!ssl) { @@ -278,7 +281,7 @@ int main(int argc, char *argv[]) SSL_free(ssl); // This also frees the associated BIO - handshake_times_ms[measurements] = ((finish.tv_sec - start.tv_sec) * MS_IN_S) + ((finish.tv_nsec - start.tv_nsec) / NS_IN_MS); + handshake_times_ms[measurements] = time_taken; measurements++; } diff --git a/pq-tls-benchmark-framework/emulation-exp/code/kex/s_timer.c b/pq-tls-benchmark-framework/emulation-exp/code/kex/s_timer.c index a522504edea3fcd48b3ae1e46c585945c4cef815..241eef392ed7926c4ede7543081000bc83643679 100644 --- a/pq-tls-benchmark-framework/emulation-exp/code/kex/s_timer.c +++ b/pq-tls-benchmark-framework/emulation-exp/code/kex/s_timer.c @@ -21,7 +21,7 @@ const char *host = "10.0.0.1:8443"; // const char *host = "localhost:8443"; // const char *host = "google.com:443"; -SSL* do_tls_handshake(SSL_CTX* ssl_ctx) +SSL* do_tls_handshake(SSL_CTX* ssl_ctx, double *time_taken) { BIO* conn; SSL* ssl; @@ -43,9 +43,14 @@ SSL* do_tls_handshake(SSL_CTX* ssl_ctx) SSL_set_bio(ssl, conn, conn); + struct timespec start, finish; // The total other time than the handshake in the hot path takes on average around 0.01 ms /* ok, lets connect */ + clock_gettime(CLOCK_MONOTONIC_RAW, &start); ret = SSL_connect(ssl); + clock_gettime(CLOCK_MONOTONIC_RAW, &finish); + *time_taken = ((finish.tv_sec - start.tv_sec) * MS_IN_S) + ((finish.tv_nsec - start.tv_nsec) / NS_IN_MS); + if (ret <= 0) { ERR_print_errors_fp(stderr); @@ -84,7 +89,6 @@ int main(int argc, char* argv[]) const SSL_METHOD* ssl_meth = TLS_client_method(); SSL* ssl = NULL; - struct timespec start, finish; double* handshake_times_ms = malloc(measurements_to_make * sizeof(*handshake_times_ms)); ssl_ctx = SSL_CTX_new(ssl_meth); @@ -136,9 +140,8 @@ int main(int argc, char* argv[]) int error_count = 0; while(measurements < measurements_to_make) { - clock_gettime(CLOCK_MONOTONIC_RAW, &start); - ssl = do_tls_handshake(ssl_ctx); - clock_gettime(CLOCK_MONOTONIC_RAW, &finish); + double time_taken = -1.0; + ssl = do_tls_handshake(ssl_ctx, &time_taken); if (!ssl) { /* Retry since at high packet loss rates, @@ -159,7 +162,7 @@ int main(int argc, char* argv[]) SSL_free(ssl); - handshake_times_ms[measurements] = ((finish.tv_sec - start.tv_sec) * MS_IN_S) + ((finish.tv_nsec - start.tv_nsec) / NS_IN_MS); + handshake_times_ms[measurements] = time_taken; measurements++; } diff --git a/pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/experiment.py b/pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/experiment.py index 5ba2cac613163d10c93e1a628b3b6a7770a24f8f..913f137d2ef9ec0e072fc50431fe45c066a920bf 100755 --- a/pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/experiment.py +++ b/pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/experiment.py @@ -9,22 +9,36 @@ import sys import pandas as pd # POOL_SIZE is the parallelism level. There should be the same amount of cores or more on your CPU. -POOL_SIZE = 4 +POOL_SIZE = 16 # There must be the same amount of network namespaces with each an nginx running as NETWORK_NAMESPACES. # The amount of network namespaces should be at least POOL_SIZE. The amount is specified by setup.sh's first argument. -NETWORK_NAMESPACES = 4 -# Ensure that TIMERS * MEASUREMENTS_PER_TIMER is at least 200 to get a good results. +NETWORK_NAMESPACES = 16 +# Ensure that TIMERS * MEASUREMENTS_PER_TIMER is at least 200 to get good results. # TIMERS should be a multiple of POOL_SIZE. Best is to have TIMERS = POOL_SIZE. -TIMERS = 4 -MEASUREMENTS_PER_TIMER = 50 - - -PROTOCOLS_TO_BENCH = ["quic", "tlstcp"] +TIMERS = 16 +MEASUREMENTS_PER_TIMER = 15 + + +PROTOCOLS_TO_BENCH = [ + "quic", + "tlstcp", + "cquiche-reno", + "cquiche-cubic", + "cquiche-bbr", + "cquiche-bbr2", +] DEFAULT_SCENARIO_TO_BENCH = "testscenarios/scenario_static.csv" ALGORITHMS_CSV_FILE = "algorithms.csv" MAIN_DIR = "results" S_TIMER = "./s_timer" QUIC_S_TIMER = "./quic_s_timer" +CQUICHE_S_TIMER = "./cquiche_s_timer" + +CQUICHE_RENO = "cquiche-reno" +CQUICHE_CUBIC = "cquiche-cubic" +CQUICHE_BBR = "cquiche-bbr" +CQUICHE_BBR2 = "cquiche-bbr2" + SRV_NS = "srv_ns" CLI_NS = "cli_ns" @@ -50,15 +64,28 @@ def main(): for protocol in PROTOCOLS_TO_BENCH: scenarios = pd.read_csv(scenariofile) testscenario_name = scenarios.columns[0] + if protocol in [ + CQUICHE_RENO, + CQUICHE_BBR, + CQUICHE_BBR2, + ] and testscenario_name not in ["static", "corrupt", "packetloss"]: + continue make_dirs(testscenario_name, protocol, algorithms_to_bench_dict) # get_emulated_rtt(scenarios) for _, parameters in scenarios.iterrows(): # set network parameters of scenario set_network_parameters(parameters) + print(protocol) + continue for algorithm_class, algorithms in algorithms_to_bench_dict.items(): for kem_alg in algorithms: + algorithm_identifier_for_openssl = kem_alg + if kem_alg == "x25519_mlkem768": + algorithm_identifier_for_openssl = "X25519MLKEM768" + elif kem_alg == "p256_mlkem768": + algorithm_identifier_for_openssl = "SecP256r1MLKEM768" path_to_results_csv_file = os.path.join( MAIN_DIR, testscenario_name, @@ -72,7 +99,7 @@ def main(): "a", ) as out: error_count, result = run_timers( - timer_pool, protocol, kem_alg + timer_pool, protocol, algorithm_identifier_for_openssl ) csv.writer(out).writerow([error_count, *result]) @@ -206,7 +233,27 @@ def time_handshake(protocol, kem_alg, measurements) -> list[float]: namespace_condition.notify() network_namespace = aquire_network_namespace() - program = QUIC_S_TIMER if protocol == "quic" else S_TIMER + # program = QUIC_S_TIMER if protocol == "quic" else S_TIMER + cc_algo = None + match protocol: + case "tlstcp": + program = S_TIMER + case "quic": + program = QUIC_S_TIMER + case "cquiche-reno": + cc_algo = "reno" + program = CQUICHE_S_TIMER + case "cquiche-cubic": + cc_algo = "cubic" + program = CQUICHE_S_TIMER + case "cquiche-bbr": + cc_algo = "bbr" + program = CQUICHE_S_TIMER + case "cquiche-bbr2": + cc_algo = "bbr2" + program = CQUICHE_S_TIMER + case _: + raise ValueError("Invalid protocol") command = [ "ip", "netns", @@ -216,6 +263,9 @@ def time_handshake(protocol, kem_alg, measurements) -> list[float]: kem_alg, str(measurements), ] + if cc_algo is not None: + command.append(cc_algo) + result = run_subprocess(command) release_network_namespace(network_namespace) error_count, result = result.split(";") diff --git a/pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/setup.sh b/pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/setup.sh index 954206af600bc00e02377cb7b926d46a57788020..0974cb45521ca7804e5d7737a8ea8e8f4ca1d947 100755 --- a/pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/setup.sh +++ b/pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/setup.sh @@ -27,9 +27,11 @@ NGINX_CONF_DIR=${ROOT}/tmp/.local/nginx/conf NGINX_LOGS_DIR=${ROOT}/tmp/.local/nginx/logs ########################## -# Build s_timer and quic_s_timer +# Build s_timer and quic_s_timer and cquiche_s_timer ########################## make s_timer quic_s_timer +( cd ../cquiche_s_timer && make build_with_pkg_config_release_mode ) +cp ../cquiche_s_timer/target/release/cquiche_s_timer cquiche_s_timer ########################## # Setup network namespaces @@ -69,6 +71,7 @@ cp nginx.conf ${NGINX_CONF_DIR}/nginx.conf # echo "EXITING EARLY NOW TO TEST LOCALLY" # Then you would have to start nginx yourself, but can start it outside the emulated network. # exit 0 + if [ "$DEBUG" == "true" ]; then NGINX_DEBUG_GLOBAL_DIRECTIVE="error_log logs/debug.log debug;" fi diff --git a/pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/teardown.sh b/pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/teardown.sh index c29831afba60d86bb9b016fee584c5754c4b37ad..53f36bdfa988b9c09369e3c5077e9298d9b3b430 100755 --- a/pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/teardown.sh +++ b/pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/teardown.sh @@ -26,7 +26,9 @@ pkill -f "tail -f -n0" ########################## rm -f prime256v1.pem \ s_timer \ - quic_s_timer + quic_s_timer \ + cquiche_s_timer +# rm -rf ../cquiche_s_timer/target # NOTE you could do that, if you want to cleanup all, but then the cache is gone sudo rm -rf scripts/__pycache__ ########################## diff --git a/pq-tls-benchmark-framework/emulation-exp/code/cquiche/README.md b/pq-tls-benchmark-framework/emulation-exp/code/patches/cquiche/README.md similarity index 100% rename from pq-tls-benchmark-framework/emulation-exp/code/cquiche/README.md rename to pq-tls-benchmark-framework/emulation-exp/code/patches/cquiche/README.md diff --git a/pq-tls-benchmark-framework/emulation-exp/code/patches/cquiche/lib.rs.diff b/pq-tls-benchmark-framework/emulation-exp/code/patches/cquiche/lib.rs.diff new file mode 100644 index 0000000000000000000000000000000000000000..1896fd5393b7b5b346f58f795eac488eecfa6abc --- /dev/null +++ b/pq-tls-benchmark-framework/emulation-exp/code/patches/cquiche/lib.rs.diff @@ -0,0 +1,16 @@ +--- lib.rs.old 2024-12-17 14:28:11.781539625 +0100 ++++ lib.rs 2024-12-17 14:29:12.866862148 +0100 +@@ -848,6 +848,13 @@ + Self::with_tls_ctx(version, tls::Context::from_boring(tls_ctx_builder)) + } + ++ /// Sets groups for TLS key exchange. ++ /// Own patch implementation ++ // #[cfg(feature = "openssl")] ++ pub fn set_groups(&mut self, groups: &str) -> Result<()> { ++ self.tls_ctx.set_groups(groups) ++ } ++ + fn with_tls_ctx(version: u32, tls_ctx: tls::Context) -> Result<Config> { + if !is_reserved_version(version) && !version_is_supported(version) { + return Err(Error::UnknownVersion); diff --git a/pq-tls-benchmark-framework/emulation-exp/code/patches/cquiche/mod.rs.diff b/pq-tls-benchmark-framework/emulation-exp/code/patches/cquiche/mod.rs.diff new file mode 100644 index 0000000000000000000000000000000000000000..c47a4cb0c3ed548b724f7eb2c072de0497c8fd1b --- /dev/null +++ b/pq-tls-benchmark-framework/emulation-exp/code/patches/cquiche/mod.rs.diff @@ -0,0 +1,27 @@ +--- mod.rs.old 2024-12-17 14:29:22.177857881 +0100 ++++ mod.rs 2024-12-17 14:40:02.831851449 +0100 +@@ -160,6 +160,15 @@ + ctx + } + ++ #[cfg(feature = "openssl")] ++ pub fn set_groups(&mut self, groups: &str) -> Result<()> { ++ let cstr = ffi::CString::new(groups).map_err(|_| Error::TlsFail)?; ++ const SSL_CTRL_SET_GROUPS_LIST: c_int = 92; ++ map_result(unsafe { ++ SSL_CTX_ctrl(self.as_mut_ptr(), SSL_CTRL_SET_GROUPS_LIST, 0,cstr.as_ptr()) as i32 ++ }) ++ } ++ + pub fn new_handshake(&mut self) -> Result<Handshake> { + unsafe { + let ssl = SSL_new(self.as_mut_ptr()); +@@ -1047,6 +1056,8 @@ + extern { + // Note: some vendor-specific methods are implemented by each vendor's + // submodule (openssl-quictls / boringssl). ++ ++ fn SSL_CTX_ctrl(ctx: *mut SSL_CTX, cmd: c_int, larg: i64, parg: *const c_char) -> i64; + + // SSL_METHOD + fn TLS_method() -> *const SSL_METHOD;