Before Patching Nginx QUIC initial congestion window

pq-tls-benchmark-framework/emulation-exp/code/kex/.gitignore

@@ -8,4 +8,4 @@ cquiche_s_timer
 /results*/
 /captures/
 plots*/
-feathers/
+/feathers/

pq-tls-benchmark-framework/emulation-exp/code/kex/Makefile

@@ -27,7 +27,7 @@ run_quic_s_timer: quic_s_timer
 	LD_LIBRARY_PATH=$(INSTALL_DIR)/openssl/lib64 ./quic_s_timer x25519 1
 
 run_quic_s_timer_in_netns: quic_s_timer
-	sudo ip netns exec cli_ns_1 env QLOGDIR=qlogdir ./quic_s_timer x25519 10
+	sudo ip netns exec cli_ns_1 env QLOGDIR=qlogdir ./quic_s_timer x25519 1
 
 quic-client-block: quic-client-block.c
 	$(CC) -g -Wall -Wextra -Wpedantic -I$(OPENSSL_INCLUDE) -I$(OQS_INCLUDE) $< -L$(INSTALL_DIR)/openssl/lib64 -L$(OQS_LIB) -lssl -lcrypto -ldl -loqs -o $@
@@ -46,8 +46,8 @@ _copy_plots_to_thesis:
 	cp -rf plots ../../../../../master-thesis-bartolomeo-mueller/gfx/plots/
 
 _change_netem_settings:
-	sudo ip netns exec srv_ns_1 tc qdisc change dev srv_ve root netem limit 1000 rate 500.0mbit delay 2.684ms 0.0ms loss 20.0% duplicate 0.0% corrupt 0.0% reorder 0.0%
-	sudo ip netns exec cli_ns_1 tc qdisc change dev cli_ve root netem limit 1000 rate 500.0mbit delay 2.684ms 0.0ms loss 20.0% duplicate 0.0% corrupt 0.0% reorder 0.0%
+	sudo ip netns exec srv_ns_1 tc qdisc change dev srv_ve root netem limit 1000 rate 500.0mbit delay 2.684ms 0.0ms loss 0.0% duplicate 0.0% corrupt 0.0% reorder 0.0%
+	sudo ip netns exec cli_ns_1 tc qdisc change dev cli_ve root netem limit 1000 rate 500.0mbit delay 2.684ms 0.0ms loss 0.0% duplicate 0.0% corrupt 0.0% reorder 0.0%
 
 _run_openssl_speed:
 	$(OPENSSL) speed -seconds 3 bikel1

pq-tls-benchmark-framework/emulation-exp/code/kex/saved/qlogdir/quic_s_timer_frodo976_nginx_not_respecting_initial_cwnd-before-patch.sqlog

+{"qlog_version":"0.3","qlog_format":"JSON-SEQ","trace":{"common_fields":{"time_format":"delta","protocol_type":["QUIC"],"system_info":{"process_id":404370}},"vantage_point":{"type":"client","name":"OpenSSL/3.4.0 (linux-x86_64)"}}}
+{"name":"transport:parameters_set","data":{"owner":"local","disable_active_migration":true,"initial_source_connection_id":"","max_idle_timeout":30000,"max_udp_payload_size":1200,"active_connection_id_limit":2,"max_ack_delay":25,"initial_max_data":786432,"initial_max_stream_data_bidi_local":524288,"initial_max_stream_data_bidi_remote":524288,"initial_max_stream_data_uni":524288,"initial_max_streams_bidi":100,"initial_max_streams_uni":100},"time":1738682548545}
+{"name":"connectivity:connection_state_updated","data":{"state":"attempted"},"time":0}
+{"name":"connectivity:connection_started","data":{"protocol":"quic","dst_cid":"3d9050eb4cdd024c"},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":0,"dcid":"3d9050eb4cdd024c","scid":""},"datagram_id":0,"frames":[{"frame_type":"crypto","offset":0,"payload_length":1158,"length":4}]},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":1,"dcid":"3d9050eb4cdd024c","scid":""},"datagram_id":1,"frames":[{"frame_type":"crypto","offset":1158,"payload_length":1157,"length":5}]},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":2,"dcid":"3d9050eb4cdd024c","scid":""},"datagram_id":2,"frames":[{"frame_type":"crypto","offset":2315,"payload_length":1157,"length":5}]},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":3,"dcid":"3d9050eb4cdd024c","scid":""},"datagram_id":3,"frames":[{"frame_type":"crypto","offset":3472,"payload_length":1157,"length":5}]},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":4,"dcid":"3d9050eb4cdd024c","scid":""},"datagram_id":4,"frames":[{"frame_type":"crypto","offset":4629,"payload_length":1157,"length":5}]},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":5,"dcid":"3d9050eb4cdd024c","scid":""},"datagram_id":5,"frames":[{"frame_type":"crypto","offset":5786,"payload_length":1157,"length":5}]},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":6,"dcid":"3d9050eb4cdd024c","scid":""},"datagram_id":6,"frames":[{"frame_type":"crypto","offset":6943,"payload_length":1157,"length":5}]},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":7,"dcid":"3d9050eb4cdd024c","scid":""},"datagram_id":7,"frames":[{"frame_type":"crypto","offset":8100,"payload_length":1157,"length":5}]},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":8,"dcid":"3d9050eb4cdd024c","scid":""},"datagram_id":8,"frames":[{"frame_type":"crypto","offset":9257,"payload_length":1157,"length":5}]},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":9,"dcid":"3d9050eb4cdd024c","scid":""},"datagram_id":9,"frames":[{"frame_type":"crypto","offset":10414,"payload_length":1157,"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":0,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":0,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0]],"length":5}]},"time":201}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":1,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":1,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,1]],"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":2,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":2,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,2]],"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":3,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":3,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,3]],"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":4,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":4,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,4]],"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":5,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":5,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,5]],"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":6,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":6,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,6]],"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":7,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":7,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,7]],"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":8,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":8,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,8]],"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":9,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":9,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,9]],"length":5}]},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":10,"dcid":"000000000000200165d6842f82def0c657007014","scid":""},"datagram_id":10,"frames":[{"frame_type":"crypto","offset":11571,"payload_length":1145,"length":5}]},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":11,"dcid":"000000000000200165d6842f82def0c657007014","scid":""},"datagram_id":11,"frames":[{"frame_type":"crypto","offset":12716,"payload_length":1145,"length":5}]},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":12,"dcid":"000000000000200165d6842f82def0c657007014","scid":""},"datagram_id":12,"frames":[{"frame_type":"crypto","offset":13861,"payload_length":1145,"length":5}]},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":13,"dcid":"000000000000200165d6842f82def0c657007014","scid":""},"datagram_id":13,"frames":[{"frame_type":"crypto","offset":15006,"payload_length":950,"length":5},{"frame_type":"padding","payload_length":195,"length":195}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":10,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":10,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,10]],"length":5}]},"time":204}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":11,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":11,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,11]],"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":12,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":12,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,12]],"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":13,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":13,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,13]],"length":5},{"frame_type":"crypto","offset":0,"payload_length":1144,"length":4}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":14,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":14,"frames":[{"frame_type":"crypto","offset":1144,"payload_length":1148,"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":15,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":15,"frames":[{"frame_type":"crypto","offset":2292,"payload_length":1148,"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":16,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":16,"frames":[{"frame_type":"crypto","offset":3440,"payload_length":1148,"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":17,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":17,"frames":[{"frame_type":"crypto","offset":4588,"payload_length":1148,"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":18,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":18,"frames":[{"frame_type":"crypto","offset":5736,"payload_length":1148,"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":19,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":19,"frames":[{"frame_type":"crypto","offset":6884,"payload_length":1148,"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":20,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":20,"frames":[{"frame_type":"crypto","offset":8032,"payload_length":1148,"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":21,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":21,"frames":[{"frame_type":"crypto","offset":9180,"payload_length":1148,"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":22,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":22,"frames":[{"frame_type":"crypto","offset":10328,"payload_length":1148,"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":23,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":23,"frames":[{"frame_type":"crypto","offset":11476,"payload_length":1148,"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":24,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":24,"frames":[{"frame_type":"crypto","offset":12624,"payload_length":1148,"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":25,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":25,"frames":[{"frame_type":"crypto","offset":13772,"payload_length":1148,"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"initial","packet_number":26,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":26,"frames":[{"frame_type":"crypto","offset":14920,"payload_length":882,"length":5}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"handshake","packet_number":0,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":26,"frames":[{"frame_type":"crypto","offset":0,"payload_length":216,"length":4}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"handshake","packet_number":1,"dcid":"","scid":"000000000000200165d6842f82def0c657007014"},"datagram_id":27,"frames":[{"frame_type":"crypto","offset":216,"payload_length":15,"length":4}]},"time":0}
+{"name":"transport:parameters_set","data":{"owner":"remote","original_destination_connection_id":"3d9050eb4cdd024c","original_source_connection_id":"3d9050eb4cdd024c","initial_max_data":8585216,"initial_max_stream_data_bidi_local":65536,"initial_max_stream_data_bidi_remote":65536,"initial_max_stream_data_uni":65536,"initial_max_streams_bidi":128,"initial_max_streams_uni":3,"ack_delay_exponent":3,"max_ack_delay":25,"max_udp_payload_size":65527,"max_idle_timeout":75000,"active_connection_id_limit":2,"stateless_reset_token":"44faebd8b9b2a92319c0626b82a002ed","disable_active_migration":false},"time":0}
+{"name":"connectivity:connection_state_updated","data":{"state":"handshake_complete"},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"initial","packet_number":14,"dcid":"000000000000200165d6842f82def0c657007014","scid":""},"datagram_id":14,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,26]],"length":5},{"frame_type":"padding","payload_length":1036,"length":1036}]},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"handshake","packet_number":0,"dcid":"000000000000200165d6842f82def0c657007014","scid":""},"datagram_id":14,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0,1]],"length":5},{"frame_type":"crypto","offset":0,"payload_length":52,"length":3}]},"time":0}
+{"name":"transport:packet_received","data":{"header":{"packet_type":"1RTT","packet_number":0,"dcid":""},"datagram_id":28,"frames":[{"frame_type":"crypto","offset":0,"payload_length":265,"length":4}]},"time":206}
+{"name":"connectivity:connection_state_updated","data":{"state":"handshake_confirmed"},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"1RTT","packet_number":0,"dcid":"0000000000002001fb3b11a06fa22c9ea0b6eb1a"},"datagram_id":15,"frames":[{"frame_type":"ack","ack_delay":0,"acked_ranges":[[0]],"length":5}]},"time":0}
+{"name":"connectivity:connection_closed","data":{"owner":"local","application_code":0,"reason":""},"time":0}
+{"name":"connectivity:connection_state_updated","data":{"state":"closing"},"time":0}
+{"name":"transport:packet_sent","data":{"header":{"packet_type":"1RTT","packet_number":1,"dcid":"0000000000002001fb3b11a06fa22c9ea0b6eb1a"},"datagram_id":16,"frames":[{"frame_type":"connection_close","error_space":"application","error_code_value":0,"error_code":0,"reason":"","length":3}]},"time":0}

pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/generate_graphs.py

@@ -12,6 +12,7 @@ import pandas as pd
 import scipy
 
 import helper_scripts.helper_functions as helper_functions
+import helper_scripts.performance_eval_of_oqs as performance_eval_of_oqs
 
 RESULTS_DIR = "saved/results-run-20250126-vm-p16"
 FILTER_RESULTS = []
@@ -802,9 +803,12 @@ def plot_lines(data):
             )
             plt.close()
 
-    def plot_median_of_single_algorithm(data):
+    def plot_qtls_of_single_algorithm(data):
+        print("plot_qtls_of_single_algorithms")
         os.makedirs(
-            f"{PLOTS_DIR}/median-of-single-algorithm", mode=0o777, exist_ok=True
+            f"{PLOTS_DIR}/lines/quantiles-of-single-algorithm",
+            mode=0o777,
+            exist_ok=True,
         )
 
         # get all combination of scenario, protocol, sec_level, kem_alg
@@ -820,25 +824,38 @@ def plot_lines(data):
                 kem_alg=row["kem_alg"],
             )
             # print(f"scenario: {row['scenario']}, protocol: {row['protocol']}, sec_level: {row['sec_level']}, kem_alg: {row['kem_alg']}")
-            y = filtered_data["median"]
-            x = get_x_axis(row["scenario"], filtered_data, len(y))
 
             plt.figure()
-            plt.fill_between(
-                x, filtered_data["qtl_25"], filtered_data["qtl_75"], alpha=0.5
-            )
-            plt.plot(x, y, linestyle="-", marker=".")
+            # plt.fill_between(
+            #     x, filtered_data["qtl_25"], filtered_data["qtl_75"], alpha=0.5
+            # )
+
+            QUANTILES = [1, 5, 10, 25, 40, 50, 60, 75, 90, 95, 99]
+            color_map = plt.get_cmap("viridis")
+            measurements = np.array(filtered_data["measurements"])
+            for qtl in QUANTILES:
+                color = color_map(1 - (qtl / 100))
+                y = [np.quantile(x, qtl / 100) for x in measurements]
+                x = get_x_axis(row["scenario"], filtered_data, len(y))
+                plt.plot(
+                    x, y, linestyle="-", marker=".", color=color, label=f"{qtl} qtl"
+                )
+
             plt.ylim(bottom=0)
             plt.xlim(left=0, right=x.max() + (x.max() / 50))
             plt.xlabel(row["scenario"])
             plt.ylabel(f"Time-to-first-byte (ms)")
             plt.grid()
-            plt.title(
-                f"Median of {row['scenario']} in {row['protocol']} in {row['sec_level']} with {row['kem_alg']}"
+            plt.legend(
+                bbox_to_anchor=(0.5, 1), loc="lower center", ncol=6, fontsize="small"
             )
+            plt.tight_layout()
+            # plt.title(
+            #     f"Median of {row['scenario']} in {row['protocol']} in {row['sec_level']} with {row['kem_alg']}"
+            # )
 
             plt.savefig(
-                f"{PLOTS_DIR}/median-of-single-algorithm/median-{row['scenario']}-{row['protocol']}-{row['sec_level']}-{row['kem_alg']}.pdf"
+                f"{PLOTS_DIR}/lines/quantiles-of-single-algorithm/quantiles-for-{row['scenario']}-{row['protocol']}-{row['sec_level']}-{row['kem_alg']}.pdf"
             )
             plt.close()
 
@@ -867,8 +884,7 @@ def plot_lines(data):
             data, line_type=statistical_measurement
         )
 
-    # plot_median_of_single_algorithm(data)
-    # plot_median_against_iqr(data)
+    plot_qtls_of_single_algorithm(data)
 
 
 # plots distributions of the individual data points
@@ -1320,6 +1336,362 @@ def plot_general_plots():
         helper_functions.get_kem_performance_data()
     )
 
+    def plot_oqs_performance():
+        def plot_liboqs_performance(liboqs_speed_data):
+            os.makedirs(
+                "plots/general/oqs-performance/liboqs", mode=0o777, exist_ok=True
+            )
+
+            # print(liboqs_speed_data.columns)
+            # print(
+            #     liboqs_speed_data[
+            #         [
+            #             "algorithm",
+            #             "keygen_mean_time_us",
+            #             "encaps_mean_time_us",
+            #             "decaps_mean_time_us",
+            #             "total_liboqs_performance_time_us",
+            #         ]
+            #     ]
+            # )
+            # print(
+            #     liboqs_speed_data[
+            #         [
+            #             "algorithm",
+            #             "total_liboqs_performance_cpu_cycles",
+            #             "total_expected_cycles",
+            #             "total_liboqs_performance_ratio",
+            #         ]
+            #     ]
+            # )
+
+            # figure with all algorithms with times for each operation
+            plt.figure()
+            fig, ax = plt.subplots()
+            bar_width = 0.5
+            index = np.arange(len(liboqs_speed_data))
+
+            color_map = plt.get_cmap("viridis")
+
+            bar1 = ax.bar(
+                index,
+                liboqs_speed_data["keygen_mean_time_us"],
+                bar_width,
+                label="Keygen",
+                color=color_map(0.1),
+            )
+            bar2 = ax.bar(
+                index,
+                liboqs_speed_data["encaps_mean_time_us"],
+                bar_width,
+                bottom=liboqs_speed_data["keygen_mean_time_us"],
+                label="Encaps",
+                color=color_map(0.4),
+            )
+            bar3 = ax.bar(
+                index,
+                liboqs_speed_data["decaps_mean_time_us"],
+                bar_width,
+                bottom=liboqs_speed_data["keygen_mean_time_us"]
+                + liboqs_speed_data["encaps_mean_time_us"],
+                label="Decaps",
+                color=color_map(0.7),
+            )
+
+            values = liboqs_speed_data["total_liboqs_performance_time_us"]
+            for idx, rect in enumerate(bar3):
+                ax.text(
+                    rect.get_x() + rect.get_width() / 2.0,
+                    values.iloc[idx],
+                    f"{values.iloc[idx]:.1f}",
+                    ha="center",
+                    va="bottom",
+                    fontsize=8,
+                )
+
+            # ax.set_xlabel("Algorithm")
+            ax.set_ylabel("Time (µs)")
+            # ax.set_title("Keygen, Encaps, and Decaps Time by Algorithm")
+            ax.set_xticks(index)
+            ax.set_xticklabels(liboqs_speed_data["algorithm"], rotation=60, ha="right")
+            ax.legend()
+
+            ax.grid()
+            ax.set_axisbelow(True)
+
+            ax.set_ylim(
+                0, liboqs_speed_data["total_liboqs_performance_time_us"].max() * 1.1
+            )
+
+            plt.tight_layout()
+            plt.savefig(
+                "plots/general/oqs-performance/liboqs/time-for-operations-for-each-algo.pdf"
+            )
+            plt.close()
+
+            # figure with all algorithms with cycles for keygen, encaps and decaps, next to it the expected cycles for each operation
+            liboqs_speed_data_bikel5_changed = liboqs_speed_data.copy()
+            liboqs_speed_data_bikel5_changed.loc[
+                liboqs_speed_data_bikel5_changed["algorithm"] == "BIKE-L5",
+                "keygen_expected_cycles",
+            ] = 0
+            liboqs_speed_data_bikel5_changed.loc[
+                liboqs_speed_data_bikel5_changed["algorithm"] == "BIKE-L5",
+                "encaps_expected_cycles",
+            ] = 0
+            liboqs_speed_data_bikel5_changed.loc[
+                liboqs_speed_data_bikel5_changed["algorithm"] == "BIKE-L5",
+                "decaps_expected_cycles",
+            ] = 0
+
+            plt.figure()
+            fig, ax = plt.subplots()
+            bar_width = 0.35
+            index = np.arange(len(liboqs_speed_data_bikel5_changed))
+
+            # bar1 is for measured cycles, bar2 is for expected cycles
+            bar11 = ax.bar(
+                index - 0.01 - bar_width / 2,
+                liboqs_speed_data_bikel5_changed["keygen_mean_cpu_cycles"],
+                bar_width,
+                label="Keygen",
+                color=color_map(0.1),
+            )
+            bar12 = ax.bar(
+                index - 0.01 - bar_width / 2,
+                liboqs_speed_data_bikel5_changed["encaps_mean_cpu_cycles"],
+                bar_width,
+                bottom=liboqs_speed_data_bikel5_changed["keygen_mean_cpu_cycles"],
+                label="Encaps",
+                color=color_map(0.4),
+            )
+            bar13 = ax.bar(
+                index - 0.01 - bar_width / 2,
+                liboqs_speed_data_bikel5_changed["decaps_mean_cpu_cycles"],
+                bar_width,
+                bottom=liboqs_speed_data_bikel5_changed["keygen_mean_cpu_cycles"]
+                + liboqs_speed_data_bikel5_changed["encaps_mean_cpu_cycles"],
+                label="Decaps",
+                color=color_map(0.7),
+            )
+
+            bar21 = ax.bar(
+                index + 0.01 + bar_width / 2,
+                liboqs_speed_data_bikel5_changed["keygen_expected_cycles"],
+                bar_width,
+                color=color_map(0.1),
+            )
+            bar22 = ax.bar(
+                index + 0.01 + bar_width / 2,
+                liboqs_speed_data_bikel5_changed["encaps_expected_cycles"],
+                bar_width,
+                bottom=liboqs_speed_data_bikel5_changed["keygen_expected_cycles"],
+                color=color_map(0.4),
+            )
+            bar23 = ax.bar(
+                index + 0.01 + bar_width / 2,
+                liboqs_speed_data_bikel5_changed["decaps_expected_cycles"],
+                bar_width,
+                bottom=liboqs_speed_data_bikel5_changed["keygen_expected_cycles"]
+                + liboqs_speed_data_bikel5_changed["encaps_expected_cycles"],
+                color=color_map(0.7),
+            )
+
+            values = liboqs_speed_data_bikel5_changed[
+                "total_liboqs_performance_cpu_cycles"
+            ]
+            for idx, rect in enumerate(bar13):
+                ax.text(
+                    rect.get_x() + rect.get_width() / 2.0,
+                    values.iloc[idx] + 100000,
+                    f"{int(values.iloc[idx])}",
+                    ha="center",
+                    va="bottom",
+                    fontsize=8,
+                    rotation=90,
+                )
+
+            values = liboqs_speed_data_bikel5_changed["total_expected_cycles"]
+            for idx, rect in enumerate(bar23):
+                if "BIKE-L5" == liboqs_speed_data_bikel5_changed["algorithm"].iloc[idx]:
+                    continue
+                ax.text(
+                    rect.get_x() + rect.get_width() / 2.0,
+                    values.iloc[idx] + 100000,
+                    f"{int(values.iloc[idx])}",
+                    ha="center",
+                    va="bottom",
+                    fontsize=8,
+                    rotation=90,
+                )
+
+            ax.set_ylabel("CPU Zyklen in Millionen")
+            ax.yaxis.set_major_formatter(
+                ticker.FuncFormatter(lambda x, _: f"{int(x/1_000_000)}")
+            )
+            ax.set_xticks(index)
+            ax.set_xticklabels(
+                liboqs_speed_data_bikel5_changed["algorithm"], rotation=60, ha="right"
+            )
+            ax.legend()
+
+            ax.grid()
+            ax.set_axisbelow(True)
+
+            ax.set_ylim(
+                0,
+                max(
+                    liboqs_speed_data_bikel5_changed[
+                        "total_liboqs_performance_cpu_cycles"
+                    ].max(),
+                    liboqs_speed_data_bikel5_changed["total_expected_cycles"].max(),
+                )
+                * 1.3,
+            )
+
+            plt.tight_layout()
+            plt.savefig(
+                "plots/general/oqs-performance/liboqs/cycles-for-operations-for-each-algo-next-to-expected-cycles.pdf"
+            )
+            plt.close()
+
+        def plot_openssl_performance(openssl_speed_data, secLevel=""):
+            os.makedirs(
+                "plots/general/oqs-performance/openssl", mode=0o777, exist_ok=True
+            )
+
+            # print(openssl_speed_data.columns)
+
+            color_map = plt.get_cmap("inferno")
+
+            plt.figure()
+            fig, ax = plt.subplots()
+            bar_width = 0.35
+            index = np.arange(len(openssl_speed_data))
+
+            # bar1 is for measured cycles, bar2 is for expected cycles
+            bar11 = ax.bar(
+                index - 0.01 - bar_width / 2,
+                openssl_speed_data["keygen_mean_time_s"],
+                bar_width,
+                label="Keygen",
+                color=color_map(0.1),
+            )
+            bar12 = ax.bar(
+                index - 0.01 - bar_width / 2,
+                openssl_speed_data["encaps_mean_time_s"],
+                bar_width,
+                bottom=openssl_speed_data["keygen_mean_time_s"],
+                label="Encaps",
+                color=color_map(0.4),
+            )
+            bar13 = ax.bar(
+                index - 0.01 - bar_width / 2,
+                openssl_speed_data["decaps_mean_time_s"],
+                bar_width,
+                bottom=openssl_speed_data["keygen_mean_time_s"]
+                + openssl_speed_data["encaps_mean_time_s"],
+                label="Decaps",
+                color=color_map(0.7),
+            )
+
+            bar21 = ax.bar(
+                index + 0.01 + bar_width / 2,
+                openssl_speed_data["keygen_mean_time_s_expected"],
+                bar_width,
+                color=color_map(0.1),
+            )
+            bar22 = ax.bar(
+                index + 0.01 + bar_width / 2,
+                openssl_speed_data["encaps_mean_time_s_expected"],
+                bar_width,
+                bottom=openssl_speed_data["keygen_mean_time_s_expected"],
+                color=color_map(0.4),
+            )
+            bar23 = ax.bar(
+                index + 0.01 + bar_width / 2,
+                openssl_speed_data["decaps_mean_time_s_expected"],
+                bar_width,
+                bottom=openssl_speed_data["keygen_mean_time_s_expected"]
+                + openssl_speed_data["encaps_mean_time_s_expected"],
+                color=color_map(0.7),
+            )
+
+            values = openssl_speed_data["total_actual_performance_time_s"]
+            for idx, rect in enumerate(bar13):
+                ax.text(
+                    rect.get_x() + rect.get_width() / 2.0,
+                    values.iloc[idx] + 0.0001,
+                    f"{values.iloc[idx]:.6f}",
+                    ha="center",
+                    va="bottom",
+                    fontsize=8,
+                    rotation=90,
+                )
+
+            values = openssl_speed_data["total_expected_performance_time_s"]
+            for idx, rect in enumerate(bar23):
+                ax.text(
+                    rect.get_x() + rect.get_width() / 2.0,
+                    values.iloc[idx] + 0.0001,
+                    f"{values.iloc[idx]:.6f}",
+                    ha="center",
+                    va="bottom",
+                    fontsize=8,
+                    rotation=90,
+                )
+
+            ax.set_ylabel("Zeit in Sekunden")
+            ax.set_xticks(index)
+            ax.set_xticklabels(openssl_speed_data["algorithm"], rotation=60, ha="right")
+            ax.legend()
+
+            ax.grid()
+            ax.set_axisbelow(True)
+
+            ax.set_ylim(
+                0,
+                max(
+                    openssl_speed_data["total_actual_performance_time_s"].max(),
+                    openssl_speed_data["total_expected_performance_time_s"].max(),
+                )
+                * 1.3,
+            )
+
+            plt.tight_layout()
+            plt.savefig(
+                f"plots/general/oqs-performance/openssl/time-vs-expected-time{secLevel}.pdf"
+            )
+            plt.close()
+
+        liboqs_speed_data = pd.read_feather(f"saved/feathers/liboqs_speed_vm.feather")
+        liboqs_speed_data = (
+            performance_eval_of_oqs.concat_liboqs_speed_data_with_expected_values(
+                liboqs_speed_data
+            )
+        )
+        plot_liboqs_performance(liboqs_speed_data)
+
+        openssl_speed_data = pd.read_feather(f"saved/feathers/openssl_speed_vm.feather")
+        openssl_speed_data = performance_eval_of_oqs.analyze_openssl_speed_data(
+            openssl_speed_data
+        )
+        openssl_speed_data = openssl_speed_data.copy()
+        openssl_speed_data = openssl_speed_data[
+            openssl_speed_data["algorithm"].str.contains("_")
+        ]
+        plot_openssl_performance(openssl_speed_data)
+        openssl_speed_data_secLevel1 = openssl_speed_data.copy()
+        openssl_speed_data_secLevel1 = openssl_speed_data_secLevel1[
+            openssl_speed_data_secLevel1["algorithm"].str.contains("p256_|x25519_")
+        ]
+        plot_openssl_performance(openssl_speed_data_secLevel1, secLevel="-1")
+        openssl_speed_data_secLevel35 = openssl_speed_data.copy()
+        openssl_speed_data_secLevel35 = openssl_speed_data_secLevel35[
+            openssl_speed_data_secLevel35["algorithm"].str.contains("p384_|p521_|x448_")
+        ]
+        plot_openssl_performance(openssl_speed_data_secLevel35, secLevel="-35")
+
     def plot_send_bytes_against_kem_performance(df, with_hybrids: bool):
         if not with_hybrids:
             # filter out all hybrids, otherwise the plot is too cluttered
@@ -1474,6 +1846,8 @@ def plot_general_plots():
         plt.savefig(f"{PLOTS_DIR}/general/median_against_iqr_hist2d.pdf")
         plt.close()
 
+    plot_oqs_performance()
+
     plot_send_bytes_against_kem_performance(df, with_hybrids=False)
     plot_send_bytes_against_kem_performance(df, with_hybrids=True)
 

pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/helper_scripts/helper_functions.py

@@ -201,7 +201,7 @@ def prepare_kem_performance_data_for_plotting(df):
     df["performance_us"] = df["decaps_us"] + df["encaps_us"] + df["keygen_us"]
     df["bytes_sent"] = df["length_public_key"] + df["length_ciphertext"]
 
-    print(df.drop(columns=["length_public_key", "length_ciphertext"]))
+    # print(df.drop(columns=["length_public_key", "length_ciphertext"]))
     return df
 
 

pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/helper_scripts/performance_eval_of_oqs.py

@@ -79,6 +79,9 @@ OPENSSL_ALGORITHM_LIST = [
 
 
 def main():
+    print("VALUES FOR LOCAL DATA - NOT FOR PAPER")
+    print()
+
     os.makedirs(FEATHERS_DIR, exist_ok=True)
     if os.path.exists(f"{FEATHERS_DIR}/liboqs_speed.feather"):
         liboqs_speed_data = pd.read_feather(f"{FEATHERS_DIR}/liboqs_speed.feather")
@@ -187,14 +190,18 @@ def run_openssl_speed():
 
 
 def analyze_openssl_speed_data(data):
-    data["keygen_mean_time_s_expected"] = -1
-    data["encaps_mean_time_s_expected"] = -1
-    data["decaps_mean_time_s_expected"] = -1
+    # rename X25519MLKEM768 to x25519_mlkem768, SecP256r1MLKEM768 to p256_mlkem768
+    data["algorithm"] = data["algorithm"].replace("X25519MLKEM768", "x25519_mlkem768")
+    data["algorithm"] = data["algorithm"].replace("SecP256r1MLKEM768", "p256_mlkem768")
+
     data["total_actual_performance_time_s"] = (
         data["keygen_mean_time_s"]
         + data["encaps_mean_time_s"]
         + data["decaps_mean_time_s"]
     )
+    data["keygen_mean_time_s_expected"] = float("-inf")
+    data["encaps_mean_time_s_expected"] = float("-inf")
+    data["decaps_mean_time_s_expected"] = float("-inf")
 
     for index, row in data.iterrows():
         algorithm = row["algorithm"]
@@ -247,28 +254,62 @@ def analyze_openssl_speed_data(data):
         + data["decaps_mean_time_s_expected"]
     )
 
+    data["keygen_performance_difference"] = (
+        data["keygen_mean_time_s"] - data["keygen_mean_time_s_expected"]
+    )
+    data["encaps_performance_difference"] = (
+        data["encaps_mean_time_s"] - data["encaps_mean_time_s_expected"]
+    )
+    data["decaps_performance_difference"] = (
+        data["decaps_mean_time_s"] - data["decaps_mean_time_s_expected"]
+    )
+
     data["total_performance_difference"] = (
         data["total_actual_performance_time_s"]
         - data["total_expected_performance_time_s"]
     )
 
-    print(data)
+    pd.set_option(
+        "display.float_format", lambda x: "%.6f" % x
+    )  # there aint more digits measured
+    print(
+        data[
+            [
+                "algorithm",
+                "keygen_performance_difference",
+                "encaps_performance_difference",
+                "decaps_performance_difference",
+                "total_performance_difference",
+            ]
+        ]
+    )
+
+    # print(data)
 
     # filter data for all algorithms which have x25519 in their name
-    data_x25519 = data[data["algorithm"].str.contains("x25519")]
+    print("Statistics for x25519:")
+    data_x25519 = data[data["algorithm"].str.contains("x25519_")]
+    print(data_x25519["algorithm"])
     print(data_x25519["total_performance_difference"].describe())
-    data_x448 = data[data["algorithm"].str.contains("x448")]
+    print("Statistics for x448:")
+    data_x448 = data[data["algorithm"].str.contains("x448_")]
     print(data_x448["total_performance_difference"].describe())
 
     print()
 
-    data_p256 = data[data["algorithm"].str.contains("p256")]
+    print("Statistics for p256:")
+    data_p256 = data[data["algorithm"].str.contains("p256_")]
+    print(data_p256["algorithm"])
     print(data_p256["total_performance_difference"].describe())
-    data_p384 = data[data["algorithm"].str.contains("p384")]
+    print("Statistics for p384:")
+    data_p384 = data[data["algorithm"].str.contains("p384_")]
     print(data_p384["total_performance_difference"].describe())
-    data_p521 = data[data["algorithm"].str.contains("p521")]
+    print("Statistics for p521:")
+    data_p521 = data[data["algorithm"].str.contains("p521_")]
     print(data_p521["total_performance_difference"].describe())
 
+    return data
+
 
 def run_liboqs_speed():
     def run_subprocess_for(algorithm):
@@ -349,8 +390,9 @@ def concat_liboqs_speed_data_with_expected_values(data):
 
     # print(data)
     data["cycles_per_us"] = data["keygen_mean_cpu_cycles"] / data["keygen_mean_time_us"]
-    print("Statistics for cycles_per_us:")
-    print(data["cycles_per_us"].describe())
+
+    # print("Statistics for cycles_per_us:")
+    # print(data["cycles_per_us"].describe())
 
     data["keygen_expected_cycles"] = float("-inf")
     data["encaps_expected_cycles"] = float("-inf")
@@ -464,6 +506,24 @@ def concat_liboqs_speed_data_with_expected_values(data):
         data["decaps_mean_cpu_cycles"] / data["decaps_expected_cycles"]
     )
 
+    data["total_liboqs_performance_time_us"] = (
+        data["keygen_mean_time_us"]
+        + data["encaps_mean_time_us"]
+        + data["decaps_mean_time_us"]
+    )
+
+    data["total_liboqs_performance_cpu_cycles"] = (
+        data["keygen_mean_cpu_cycles"]
+        + data["encaps_mean_cpu_cycles"]
+        + data["decaps_mean_cpu_cycles"]
+    )
+
+    data["total_expected_cycles"] = (
+        data["keygen_expected_cycles"]
+        + data["encaps_expected_cycles"]
+        + data["decaps_expected_cycles"]
+    )
+
     data["total_liboqs_performance_ratio"] = (
         data["keygen_mean_cpu_cycles"]
         + data["encaps_mean_cpu_cycles"]
@@ -487,7 +547,8 @@ def concat_liboqs_speed_data_with_expected_values(data):
         )
     ) / data["cycles_per_us"]
 
-    print(data)
+    # print(data)
+    return data
 
 
 if __name__ == "__main__":

pq-tls-benchmark-framework/emulation-exp/code/kex/scripts/queries.py

@@ -10,7 +10,8 @@ def main():
     data = pd.read_feather(f"{FEATHERS_DIR}/data.feather")
     # data = pd.read_feather(f"{FEATHERS_DIR}/data_run_20241028.feather")
 
-    median_of_all_static_runs_per_algorithm(data)
+    static_scenario_statistical_analysis(data)
+    # median_of_all_static_runs_per_algorithm(data)
     # stats_of_qtl95_of_packetloss(data)
     # error_count_and_rate(data)
     # measurements_with_negative_skewness(data)
@@ -19,6 +20,33 @@ def main():
     # print_kem_ids()
 
 
+def static_scenario_statistical_analysis(data):
+    ldata = data
+    print("Static scenario statistical analysis")
+    ldata = gg.filter_data(
+        ldata,
+        scenario="static",
+        protocol="quic",
+        sec_level=["secLevel1", "secLevel1_hybrid"],
+    )
+
+    means_of_medians = []
+    stdevs_of_medians = []
+
+    kem_alg_names = ldata["kem_alg"].unique()
+    for kem_alg_name in kem_alg_names:
+        kem_alg_data = ldata.query(f"kem_alg == '{kem_alg_name}'")
+        medians = kem_alg_data["median"]
+        # print(kem_alg_name, medians.mean(), medians.std())
+        means_of_medians.append(medians.mean())
+        stdevs_of_medians.append(medians.std())
+
+    print("Mean of stdevs of medians")
+    print(pd.Series(stdevs_of_medians).mean())
+    print("Stdev of stdevs of medians")
+    print(pd.Series(stdevs_of_medians).std())
+
+
 def median_of_all_static_runs_per_algorithm(data):
     ldata = data
     print("Median of all static runs per algorithm")