improved testing capabilities and fixed bug that pre ended chunk sending

This commit is contained in:
Theo Tappe
2026-06-10 22:11:12 +02:00
parent 1e5eeb704f
commit f3e93d3fd9
10 changed files with 186 additions and 1715 deletions
-1611
View File
File diff suppressed because it is too large Load Diff
+25 -27
View File
@@ -7,6 +7,7 @@
#include "config.h" #include "config.h"
#include "log.h" #include "log.h"
#include "multiprocessing.h" #include "multiprocessing.h"
#include "pipeline.h"
#include "queue.h" #include "queue.h"
#include "scanner.h" #include "scanner.h"
#include "socket.h" #include "socket.h"
@@ -24,9 +25,6 @@ int send_chunk(Client *client, Chunk *chunk, bool use_compression) {
send_str(client->file_descriptor, file->path); send_str(client->file_descriptor, file->path);
send_data(client->file_descriptor, file->data, file->stats.st_size); send_data(client->file_descriptor, file->data, file->stats.st_size);
} }
send_status(client->file_descriptor, FINISHED);
if (receive_status(client->file_descriptor) != OK)
return -1;
} }
return 0; return 0;
} }
@@ -105,12 +103,14 @@ int send_files(Config *config) {
DirectoryScanner *scanner = directory_scanner_create(config->send_directory); DirectoryScanner *scanner = directory_scanner_create(config->send_directory);
Chunk *current_chunk; Chunk *current_chunk;
while ((current_chunk = directory_scanner_next(scanner)) != NULL) { while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
chunk_print(current_chunk);
for (int i = 0; i < current_chunk->element_count; i++) for (int i = 0; i < current_chunk->element_count; i++)
file_load_data(current_chunk->items[i]); file_load_data(current_chunk->items[i]);
send_chunk(client, current_chunk, config->use_compression); send_chunk(client, current_chunk, config->use_compression);
chunk_destroy(current_chunk); chunk_destroy(current_chunk);
} }
send_status(client->file_descriptor, FINISHED);
if (receive_status(client->file_descriptor) != OK)
return -1;
printf("FINISHED"); printf("FINISHED");
directory_scanner_destroy(scanner); directory_scanner_destroy(scanner);
client_disconnect(client); client_disconnect(client);
@@ -149,42 +149,40 @@ int send_files_multithreaded(Config *config) {
return 0; return 0;
} }
int send_files_multiprocessed(Config *config) { // int send_files_multiprocessed(Config *config) {
int cores = sysconf(_SC_NPROCESSORS_ONLN); // int cores = sysconf(_SC_NPROCESSORS_ONLN);
int pid = fork(); // int pid = fork();
if (pid == -1) { // if (pid == -1) {
perror("Error Forking!"); // perror("Error Forking!");
return 1; // return 1;
} else if (pid == 0) { // } else if (pid == 0) {
} // }
for (int i = 0; i < cores; i++) { // for (int i = 0; i < cores; i++) {
int pid = fork(); // int pid = fork();
if (pid == -1) { // if (pid == -1) {
perror("Error forking!"); // perror("Error forking!");
return 1; // return 1;
} else if (pid == 0) { // } else if (pid == 0) {
} // }
} // }
return 0; // return 0;
} // }
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
Config *config = config_create( Config *config = config_create(
str_dup("1.0.0"), str_dup("/home/taptap/Nextcloud/Uni/moodle/MINT-Raum"), str_dup("1.0.0"), str_dup("/home/taptap/Nextcloud/Uni/moodle/MINT-Raum"),
str_dup("./data_copied"), false, false, false, false, false, 1); str_dup("./data_copied"), false, false, false, false, 1);
for (int i = 1; i < argc; i++) { for (int i = 1; i < argc; i++) {
handle_arg(argv[i], "-m", &config->use_multithreading, handle_arg(argv[i], "-m", &config->use_multithreading,
"Enabled Multithreading"); "Enabled Multithreading");
handle_arg(argv[i], "-s", &config->use_chunk_serialization, handle_arg(argv[i], "-s", &config->use_chunk_serialization,
"Enabled Chunk Serialization"); "Enabled Chunk Serialization");
handle_arg(argv[i], "-c", &config->use_compression, "Enabled Compression"); handle_arg(argv[i], "-c", &config->use_compression, "Enabled Compression");
handle_arg(argv[i], "-p", &config->use_multiprocessing,
"Enabled Multiprocessing");
} }
if (config->use_multithreading) if (config->use_multithreading)
return send_files_multithreaded(config); return send_files_multithreaded(config);
else if (config->use_multiprocessing) // else if (config->use_multiprocessing)
return send_files_multiprocessed(config); // return send_files_multiprocessed(config);
return send_files(config); return send_files(config);
} }
-1
View File
@@ -44,7 +44,6 @@ Chunk *directory_scanner_next(DirectoryScanner *scanner) {
perror("Could not open directory!"); perror("Could not open directory!");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
printf("%s", path);
while ((entry = readdir(dir)) != NULL) { while ((entry = readdir(dir)) != NULL) {
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) { if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) {
continue; continue;
-1
View File
@@ -11,7 +11,6 @@
FileReceive *receive_file_receive(int file_descriptor) { FileReceive *receive_file_receive(int file_descriptor) {
char *path = (char *)receive_str(file_descriptor); char *path = (char *)receive_str(file_descriptor);
printf("%s\n", path);
DataFragment *file_data_fragment = receive_data(file_descriptor); DataFragment *file_data_fragment = receive_data(file_descriptor);
FileReceive *file = file_receive_create(path, file_data_fragment); FileReceive *file = file_receive_create(path, file_data_fragment);
return file; return file;
+1 -5
View File
@@ -7,8 +7,7 @@
Config *config_create(char *version, char *send_directory, Config *config_create(char *version, char *send_directory,
char *receive_directory, bool save_to_disk, char *receive_directory, bool save_to_disk,
bool use_multithreading, bool use_chunk_serialization, bool use_multithreading, bool use_chunk_serialization,
bool use_compression, bool use_multiprocessing, bool use_compression, int num_connections) {
int num_connections) {
Config *config = malloc(sizeof(Config)); Config *config = malloc(sizeof(Config));
config->version = version; config->version = version;
@@ -18,7 +17,6 @@ Config *config_create(char *version, char *send_directory,
config->use_multithreading = use_multithreading; config->use_multithreading = use_multithreading;
config->use_chunk_serialization = use_chunk_serialization; config->use_chunk_serialization = use_chunk_serialization;
config->use_compression = use_compression; config->use_compression = use_compression;
config->use_multiprocessing = use_multiprocessing;
config->num_connections = num_connections; config->num_connections = num_connections;
return config; return config;
} }
@@ -38,7 +36,6 @@ void config_send(int file_descriptor, Config *config) {
send_int(file_descriptor, config->use_multithreading); send_int(file_descriptor, config->use_multithreading);
send_int(file_descriptor, config->use_chunk_serialization); send_int(file_descriptor, config->use_chunk_serialization);
send_int(file_descriptor, config->use_compression); send_int(file_descriptor, config->use_compression);
send_int(file_descriptor, config->use_multiprocessing);
send_int(file_descriptor, config->num_connections); send_int(file_descriptor, config->num_connections);
if (receive_status(file_descriptor) != OK) { if (receive_status(file_descriptor) != OK) {
perror("Error transmitting config!"); perror("Error transmitting config!");
@@ -55,7 +52,6 @@ Config *config_receive(int file_descriptor) {
config->use_multithreading = receive_int(file_descriptor); config->use_multithreading = receive_int(file_descriptor);
config->use_chunk_serialization = receive_int(file_descriptor); config->use_chunk_serialization = receive_int(file_descriptor);
config->use_compression = receive_int(file_descriptor); config->use_compression = receive_int(file_descriptor);
config->use_multiprocessing = receive_int(file_descriptor);
config->num_connections = receive_int(file_descriptor); config->num_connections = receive_int(file_descriptor);
send_status(file_descriptor, OK); send_status(file_descriptor, OK);
return config; return config;
+1 -3
View File
@@ -11,15 +11,13 @@ typedef struct Config {
bool use_multithreading; bool use_multithreading;
bool use_chunk_serialization; bool use_chunk_serialization;
bool use_compression; bool use_compression;
bool use_multiprocessing;
int num_connections; int num_connections;
} Config; } Config;
Config *config_create(char *version, char *send_directory, Config *config_create(char *version, char *send_directory,
char *receive_directory, bool save_to_disk, char *receive_directory, bool save_to_disk,
bool use_multithreading, bool use_chunk_serialization, bool use_multithreading, bool use_chunk_serialization,
bool use_compression, bool use_multiprocessing, bool use_compression, int num_connections);
int num_connections);
void config_delete(Config *config); void config_delete(Config *config);
void config_send(int file_descriptor, Config *config); void config_send(int file_descriptor, Config *config);
Config *config_receive(int file_descriptor); Config *config_receive(int file_descriptor);
-6
View File
@@ -1,6 +0,0 @@
#ifndef PIPELINE_H
#define PIPELINE_H
struct
#endif
+17 -2
View File
@@ -198,14 +198,29 @@ int receive_int(int file_descriptor) {
return data; return data;
} }
const char *status_to_string(Status status) {
switch (status) {
case OK:
return "OK";
case ERROR:
return "ERROR";
case FINISHED:
return "FINISHED";
case NEXT:
return "NEXT";
default:
return "UNKNOWN";
}
}
void send_status(int file_descriptor, Status status) { void send_status(int file_descriptor, Status status) {
send_n_data(file_descriptor, &status, sizeof(Status)); send_n_data(file_descriptor, &status, sizeof(Status));
log_message(LOG_LEVEL_DEBUG, "Send Status: %d", status); log_message(LOG_LEVEL_DEBUG, "Send Status: %s", status_to_string(status));
} }
Status receive_status(int file_descriptor) { Status receive_status(int file_descriptor) {
Status data; Status data;
receive_n_data(file_descriptor, &data, sizeof(Status)); receive_n_data(file_descriptor, &data, sizeof(Status));
log_message(LOG_LEVEL_DEBUG, "Received Status: %d", data); log_message(LOG_LEVEL_DEBUG, "Received Status: %s", status_to_string(data));
return data; return data;
} }
+124 -43
View File
@@ -2,90 +2,171 @@ import subprocess
import time import time
# --- Configuration --- # --- Configuration ---
SERVER_CMD = ["./server"] SERVER_CMD = ["./build/server"]
#original_client_cmd = ["./client"] base_client_cmd = ["./build/client"]
original_client_cmd = ["./client" , "-m"]
# --- Resource Limit Configuration --- # --- Resource Limit Configuration ---
# 💾 Disk throttling settings # 💾 Disk throttling settings
DISK_DEVICE = "/dev/nvme0n1p5" # IMPORTANT: Change this to your disk (e.g., /dev/nvme0n1) DISK_DEVICE = (
"/dev/nvme0n1p5" # IMPORTANT: Change this to your disk (e.g., /dev/nvme0n1)
)
READ_BPS_MAX = "15M" # Max read speed (M for megabytes) READ_BPS_MAX = "15M" # Max read speed (M for megabytes)
WRITE_BPS_MAX = "10M" # Max write speed WRITE_BPS_MAX = "10M" # Max write speed
# 🐢 Network throttling settings (Linux tc) # 🐢 Network throttling settings (Linux tc)
NET_LIMIT = "2mbit" NET_LIMIT = "100mbit"
NET_DELAY = "100ms" NET_DELAY = "100ms"
NETWORK_INTERFACE = "lo" NETWORK_INTERFACE = "lo"
NET_LIMIT_CMD = f"sudo tc qdisc add dev {NETWORK_INTERFACE} root netem rate {NET_LIMIT} delay {NET_DELAY}".split() NET_LIMIT_CMD = f"sudo tc qdisc add dev {NETWORK_INTERFACE} root netem rate {NET_LIMIT} delay {NET_DELAY}".split()
NET_RESET_CMD = f"sudo tc qdisc del dev {NETWORK_INTERFACE} root".split() NET_RESET_CMD = f"sudo tc qdisc del dev {NETWORK_INTERFACE} root".split()
# --- Build the final client command with throttling --- # --- Build the client command prefix with throttling ---
# This uses systemd-run to wrap the original client command with I/O limits. # This uses systemd-run to wrap the original client command with I/O limits.
# The entire command must be run with sudo. # The entire command must be run with sudo.
CLIENT_CMD = [ CLIENT_CMD_PREFIX = [
"sudo", "systemd-run", "sudo",
"systemd-run",
"--scope", "--scope",
"-p", f"IOReadBandwidthMax={DISK_DEVICE} {READ_BPS_MAX}", "-p",
"-p", f"IOWriteBandwidthMax={DISK_DEVICE} {WRITE_BPS_MAX}", f"IOReadBandwidthMax={DISK_DEVICE} {READ_BPS_MAX}",
# The command to run is placed at the end "-p",
] + original_client_cmd f"IOWriteBandwidthMax={DISK_DEVICE} {WRITE_BPS_MAX}",
]
# --- Test Cases ---
TEST_CASES = [
{"name": "Standard (Single-threaded)", "flags": []},
{"name": "Multithreading (-m)", "flags": ["-m"]},
{"name": "Compression (-c)", "flags": ["-c"]},
{"name": "Chunk Serialization (-s)", "flags": ["-s"]},
{"name": "Multithreading + Compression (-m -c)", "flags": ["-m", "-c"]},
{"name": "Multithreading + Chunk Serialization (-m -s)", "flags": ["-m", "-s"]},
{"name": "Compression + Chunk Serialization (-c -s)", "flags": ["-c", "-s"]},
{
"name": "Multithreading + Compression + Chunk Serialization (-m -c -s)",
"flags": ["-m", "-c", "-s"],
},
]
results = []
# --- Benchmarking ---
server_process = None
print("🚀 Starting benchmark...") print("🚀 Starting benchmark...")
print(f"Limiting Disk I/O: Reads <= {READ_BPS_MAX}, Writes <= {WRITE_BPS_MAX}") print(f"Limiting Disk I/O: Reads <= {READ_BPS_MAX}, Writes <= {WRITE_BPS_MAX}")
print("Limiting Network: Simulating low bandwidth and high latency") print("Limiting Network: Simulating low bandwidth and high latency")
try: try:
# SETUP: Apply network limit # # SETUP: Apply network limit
print("Applying network limits...")
subprocess.run(NET_LIMIT_CMD, check=True) subprocess.run(NET_LIMIT_CMD, check=True)
for case in TEST_CASES:
name = case["name"]
flags = case["flags"]
print(f"\n--- Running: {name} ---")
server_process = None
try:
# 1. Start the server # 1. Start the server
print(" Starting server...") print(" Starting server...")
server_process = subprocess.Popen(SERVER_CMD) server_process = subprocess.Popen(
time.sleep(1) SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None
)
time.sleep(5) # Allow server to bind to port
# 2. Build and run the client
client_cmd = CLIENT_CMD_PREFIX + base_client_cmd + flags
print(f" Running client: {' '.join(client_cmd)}")
# 2. Run the client with all limits applied
print("Running client under network AND disk constraints...")
start_time = time.monotonic() start_time = time.monotonic()
client_result = subprocess.run(client_cmd, text=True, capture_output=True)
# The CLIENT_CMD now includes the sudo and systemd-run wrapper
client_result = subprocess.run(CLIENT_CMD, capture_output=True, text=True)
end_time = time.monotonic() end_time = time.monotonic()
# 3. Calculate and print duration
duration = end_time - start_time duration = end_time - start_time
print("-" * 30)
print(f"✅ Client execution time: {duration:.4f} seconds")
print("-" * 30)
if client_result.returncode != 0: if client_result.returncode == 0:
print(f"⚠️ Client exited with an error (code: {client_result.returncode}).") results.append(
print("--- Client STDERR ---") {
print(client_result.stderr) "name": name,
print("-" * 21) "status": "Success",
"time": f"{duration:.4f}s",
"error": "",
}
)
else:
print(f" ⚠️ Failed (code: {client_result.returncode})")
err_msg = (
client_result.stderr.strip().split("\n")[0]
if client_result.stderr
else (
client_result.stdout.strip().split("\n")[0]
if client_result.stdout
else "No output"
)
)
results.append(
{
"name": name,
"status": "Failed",
"time": "N/A",
"error": f"Exit code {client_result.returncode}: {err_msg[:40]}",
}
)
except subprocess.TimeoutExpired:
print(" ⚠️ Timeout (exceeded 15s)")
results.append(
{
"name": name,
"status": "Timeout",
"time": "N/A",
"error": "Exceeded 15 seconds",
}
)
except Exception as e:
print(f" ❌ Error: {e}")
results.append(
{"name": name, "status": "Error", "time": "N/A", "error": str(e)}
)
finally:
# Clean up the server for this test case
if server_process:
print(" Stopping server...")
try:
server_process.terminate()
server_process.wait(timeout=5)
except subprocess.TimeoutExpired:
server_process.kill()
server_process.wait()
except FileNotFoundError as e:
print(f"❌ Error: Command not found - {e.filename}. Is the path correct?")
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
print(f"❌ Error running command: {' '.join(e.cmd)}") print(f"❌ Error running system limit command: {' '.join(e.cmd)}")
print("Are you running this script with 'sudo'?") print("Are you running this script with 'sudo'?")
finally: finally:
# TEARDOWN: Remove network limits and shut down the server # TEARDOWN: Remove network limits
# No disk cleanup is needed because systemd-run handles it! print("\nCleaning up...")
print("Cleaning up...")
try: try:
print("Removing network limit...") print("Removing network limit...")
subprocess.run(NET_RESET_CMD, check=True, capture_output=True) subprocess.run(NET_RESET_CMD, check=True, capture_output=True)
except Exception as e: except Exception as e:
print(f"⚠️ Could not reset network settings: {e}") print(f"⚠️ Could not reset network settings: {e}")
if server_process:
print("Shutting down server...")
server_process.terminate()
server_process.wait()
print("Cleanup complete.") print("Cleanup complete.")
# Print comparison table
print("\n" + "=" * 80)
print(f"{'fastSync BENCHMARK RESULTS':^80}")
print("=" * 80)
print(f"{'Configuration':<45} | {'Status':<10} | {'Time':<10} | {'Details/Error':<20}")
print("-" * 80)
for res in results:
status_symbol = (
""
if res["status"] == "Success"
else ("" if res["status"] == "Timeout" else "")
)
status_str = f"{status_symbol} {res['status']}"
print(
f"{res['name']:<45} | {status_str:<10} | {res['time']:<10} | {res['error']:<20}"
)
print("=" * 80)
+7 -5
View File
@@ -2,12 +2,13 @@
#include "config.h" #include "config.h"
#include "multiprocessing.h" #include "multiprocessing.h"
#include "queue.h" #include "queue.h"
#include "utils.h"
#include "test_utils.h" #include "test_utils.h"
#include "utils.h"
#include <stdlib.h> #include <stdlib.h>
static void test_config_lifecycle() { static void test_config_lifecycle() {
Config *cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("/dst"), true, true, false, false, false, 4); Config *cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("/dst"),
true, true, false, false, 4);
EXPECT_NOT_NULL(cfg); EXPECT_NOT_NULL(cfg);
EXPECT_EQ_STR(cfg->version, "1.0"); EXPECT_EQ_STR(cfg->version, "1.0");
EXPECT_EQ_STR(cfg->send_directory, "/src"); EXPECT_EQ_STR(cfg->send_directory, "/src");
@@ -16,13 +17,13 @@ static void test_config_lifecycle() {
EXPECT_TRUE(cfg->use_multithreading); EXPECT_TRUE(cfg->use_multithreading);
EXPECT_FALSE(cfg->use_chunk_serialization); EXPECT_FALSE(cfg->use_chunk_serialization);
EXPECT_FALSE(cfg->use_compression); EXPECT_FALSE(cfg->use_compression);
EXPECT_FALSE(cfg->use_multiprocessing);
EXPECT_EQ_INT(cfg->num_connections, 4); EXPECT_EQ_INT(cfg->num_connections, 4);
config_delete(cfg); config_delete(cfg);
} }
static void test_pipeline_sender_lifecycle() { static void test_pipeline_sender_lifecycle() {
Config *cfg = config_create(str_dup("2.0"), str_dup("/src2"), str_dup("/dst2"), false, false, true, true, true, 8); Config *cfg = config_create(str_dup("2.0"), str_dup("/src2"),
str_dup("/dst2"), false, false, true, true, 8);
Queue *q1 = queue_create(5, NULL); Queue *q1 = queue_create(5, NULL);
Queue *q2 = queue_create(15, NULL); Queue *q2 = queue_create(15, NULL);
@@ -38,7 +39,8 @@ static void test_pipeline_sender_lifecycle() {
} }
static void test_pipeline_receiver_lifecycle() { static void test_pipeline_receiver_lifecycle() {
Config *cfg = config_create(str_dup("3.0"), str_dup("/src3"), str_dup("/dst3"), true, true, true, true, true, 2); Config *cfg = config_create(str_dup("3.0"), str_dup("/src3"),
str_dup("/dst3"), true, true, true, true, 2);
Queue *q = queue_create(20, NULL); Queue *q = queue_create(20, NULL);
PipelineContextReceiver *pcr = pipeline_context_receiver_create(cfg, q, 42); PipelineContextReceiver *pcr = pipeline_context_receiver_create(cfg, q, 42);