From 1905369c9fe595f8ac4b178317329be21595713d Mon Sep 17 00:00:00 2001 From: TapTap Date: Thu, 16 Jul 2026 15:38:48 +0200 Subject: [PATCH 1/6] Incremental sync: --incremental flag to skip unchanged files - New STATUS_CHECK protocol status (value 6) - Client sends path + size + mtime; server replies OK (skip) or NEXT (send) - config.h/c: use_incremental field sent/received over wire - file.c/h: file_send_single_calls_no_path helper for incremental path - client_cli.c: --incremental flag - client_send.c: per-file check before send in both single and chunk paths - server.c: STATUS_CHECK handling in receive_files() - multiprocessing.c: STATUS_CHECK handling in receive_thread() - test.py: incremental sync test case --- src/client/client_cli.c | 3 + src/client/client_send.c | 42 +++- src/server/server.c | 60 ++++- src/shared/config.c | 4 + src/shared/config.h | 1 + src/shared/file.c | 15 ++ src/shared/file.h | 1 + src/shared/multiprocessing.c | 53 +++- src/shared/protocol.c | 2 + src/shared/protocol.h | 2 +- test.py | 470 +++++++++++++++++------------------ 11 files changed, 406 insertions(+), 247 deletions(-) diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 44fca4b..714bd6e 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -34,6 +34,7 @@ static void print_usage(void) { printf(" --include Only include files matching pattern\n"); printf(" --max-size Skip files larger than n bytes\n"); printf(" --min-size Skip files smaller than n bytes\n"); + printf(" --incremental Skip files unchanged since last transfer\n"); printf(" -m Enable multithreading\n"); printf(" -s Enable chunk serialization\n"); printf(" -f Enable sendfile (TCP only, not with -c or -s)\n"); @@ -93,6 +94,8 @@ int main(int argc, char *argv[]) { config->max_size = strtoull(argv[++i], NULL, 10); } else if (strcmp(argv[i], "--min-size") == 0 && i + 1 < argc) { config->min_size = strtoull(argv[++i], NULL, 10); + } else if (strcmp(argv[i], "--incremental") == 0) { + config->use_incremental = true; } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) { config->use_compression = true; log_message(LOG_LEVEL_INFO, "Enabled Compression"); diff --git a/src/client/client_send.c b/src/client/client_send.c index 4c70204..255e834 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -32,17 +32,47 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) { data_destroy(data); } else if (config->use_sendfile && !config->use_compression) { for (int i = 0; i < chunk->element_count; i++) { - if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; + if (config->use_incremental) { + if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1; + if (!send_str(client->file_descriptor, chunk->items[i]->path)) return -1; + unsigned long long fsize = chunk->items[i]->data->size; + long long mtime = chunk->items[i]->metadata ? chunk->items[i]->metadata->mtime_sec : 0; + if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1; + if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1; + Status s; + if (!receive_status(client->file_descriptor, &s)) return -1; + if (s == STATUS_OK) continue; + if (s != STATUS_NEXT) return -1; + } else { + if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; + } if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata)) return -1; } } else { for (int i = 0; i < chunk->element_count; i++) { - if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; - if (!file_send_single_calls(chunk->items[i], client->file_descriptor, - config->use_metadata, - config->use_compression ? config->compression_level : 0)) - return -1; + if (config->use_incremental) { + if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1; + if (!send_str(client->file_descriptor, chunk->items[i]->path)) return -1; + unsigned long long fsize = chunk->items[i]->data->size; + long long mtime = chunk->items[i]->metadata ? chunk->items[i]->metadata->mtime_sec : 0; + if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1; + if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1; + Status s; + if (!receive_status(client->file_descriptor, &s)) return -1; + if (s == STATUS_OK) continue; + if (s != STATUS_NEXT) return -1; + if (!file_send_single_calls_no_path(chunk->items[i], client->file_descriptor, + config->use_metadata, + config->use_compression ? config->compression_level : 0)) + return -1; + } else { + if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; + if (!file_send_single_calls(chunk->items[i], client->file_descriptor, + config->use_metadata, + config->use_compression ? config->compression_level : 0)) + return -1; + } } } return 0; diff --git a/src/server/server.c b/src/server/server.c index a7edce5..c4e528e 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -16,12 +16,68 @@ #include #include #include +#include int receive_files(Config *config, int file_descriptor) { Status status; if (!receive_status(file_descriptor, &status)) return -1; - while (status == STATUS_NEXT || status == STATUS_CHUNK) { - if (status == STATUS_CHUNK) { + while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) { + if (status == STATUS_CHECK) { + char *check_path = receive_str(file_descriptor); + if (check_path == NULL) { send_status(file_descriptor, STATUS_ERROR); return -1; } + unsigned long long check_size; + long long check_mtime; + if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) || + !receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) { + free(check_path); + send_status(file_descriptor, STATUS_ERROR); + return -1; + } + char *full_path = path_cat(config->receive_root_directory, check_path); + struct stat st; + bool match = false; + if (full_path && stat(full_path, &st) == 0 && + (unsigned long long)st.st_size == check_size && + (long long)st.st_mtime == check_mtime) { + match = true; + } + free(full_path); + if (match) { + if (!send_status(file_descriptor, STATUS_OK)) { free(check_path); return -1; } + free(check_path); + } else { + if (!send_status(file_descriptor, STATUS_NEXT)) { free(check_path); return -1; } + File *file = file_create(check_path); + free(check_path); + if (file == NULL) { send_status(file_descriptor, STATUS_ERROR); return -1; } + if (config->use_metadata) { + file->metadata = metadata_receive(file_descriptor); + } + Data *file_data = receive_data(file_descriptor); + if (file_data == NULL) { + file_destroy(file); + send_status(file_descriptor, STATUS_ERROR); + return -1; + } + if (config->use_compression) { + Data *uncompressed = data_decompress(file_data); + data_destroy(file_data); + if (uncompressed == NULL) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return -1; } + file_data = uncompressed; + } + data_destroy(file->data); + file->data = file_data; + if (config->save_to_disk) { + char *disk_path = path_cat(config->receive_root_directory, file->path); + if (disk_path) { + to_disk(disk_path, file->data->data, file->data->size); + file_restore_metadata(disk_path, file->metadata); + free(disk_path); + } + } + file_destroy(file); + } + } else if (status == STATUS_CHUNK) { Data *chunk_data = receive_data(file_descriptor); if (chunk_data == NULL) { log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data"); diff --git a/src/shared/config.c b/src/shared/config.c index a22e489..1838d4d 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -38,6 +38,7 @@ Config *config_create(char *version, char *send_directory, config->include_count = 0; config->max_size = 0; config->min_size = 0; + config->use_incremental = false; return config; } @@ -89,6 +90,7 @@ bool config_send(int file_descriptor, Config *config) { if (!send_int(file_descriptor, (int)config->chunk_size)) return false; if (!send_int(file_descriptor, config->use_sendfile)) return false; if (!send_int(file_descriptor, config->use_delete)) return false; + if (!send_int(file_descriptor, config->use_incremental)) return false; Status status; if (!receive_status(file_descriptor, &status)) return false; if (status != STATUS_OK) { @@ -134,6 +136,8 @@ Config *config_receive(int file_descriptor) { config->use_sendfile = tmp; if (!receive_int(file_descriptor, &tmp)) goto error; config->use_delete = tmp; + if (!receive_int(file_descriptor, &tmp)) goto error; + config->use_incremental = tmp; config->show_progress = false; config->dry_run = false; config->ssh_port = 22; diff --git a/src/shared/config.h b/src/shared/config.h index b08f733..6af810b 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -32,6 +32,7 @@ typedef struct Config { int include_count; unsigned long long max_size; unsigned long long min_size; + bool use_incremental; } Config; #define PROTOCOL_VERSION "1.0.0" diff --git a/src/shared/file.c b/src/shared/file.c index 061343a..f04f2e2 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -96,6 +96,21 @@ bool file_load_data(File *file) { return true; } +bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level) { + if (compression_level > 0) { + Data *compressed_data = data_compress(file->data, compression_level); + if (compressed_data == NULL) { + log_message(LOG_LEVEL_ERROR, "Failed to compress file data"); + return false; + } + data_destroy(file->data); + file->data = compressed_data; + } + if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; + if (!send_data(file_descriptor, file->data)) return false; + return true; +} + bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) { if (compression_level > 0) { Data *compressed_data = data_compress(file->data, compression_level); diff --git a/src/shared/file.h b/src/shared/file.h index 11ec9fe..4f28d06 100644 --- a/src/shared/file.h +++ b/src/shared/file.h @@ -25,6 +25,7 @@ void file_destroy(void *item); bool file_load_data(File *file); File *file_receive(Config *config, int file_descriptor); bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level); +bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level); bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata); size_t file_content_to_buffer(File *file); FileMetadata *file_metadata_create(struct stat *stats); diff --git a/src/shared/multiprocessing.c b/src/shared/multiprocessing.c index ef99386..5e912d0 100644 --- a/src/shared/multiprocessing.c +++ b/src/shared/multiprocessing.c @@ -13,6 +13,7 @@ #include #include #include +#include #include PipelineContextSender *pipeline_context_sender_create(Config *config, @@ -126,8 +127,56 @@ int receive_thread(void *pipeline_context) { Status status; if (!receive_status(file_descriptor, &status)) return thrd_error; - while (status == STATUS_NEXT || status == STATUS_CHUNK) { - if (status == STATUS_CHUNK) { + while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) { + if (status == STATUS_CHECK) { + char *check_path = receive_str(file_descriptor); + if (check_path == NULL) return thrd_error; + unsigned long long check_size; + long long check_mtime; + if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) || + !receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) { + free(check_path); + return thrd_error; + } + char *full_path = path_cat(config->receive_root_directory, check_path); + struct stat st; + bool match = false; + if (full_path && stat(full_path, &st) == 0 && + (unsigned long long)st.st_size == check_size && + (long long)st.st_mtime == check_mtime) { + match = true; + } + free(full_path); + if (match) { + if (!send_status(file_descriptor, STATUS_OK)) { free(check_path); return thrd_error; } + free(check_path); + } else { + if (!send_status(file_descriptor, STATUS_NEXT)) { free(check_path); return thrd_error; } + File *file = file_create(check_path); + free(check_path); + if (file == NULL) { send_status(file_descriptor, STATUS_ERROR); return thrd_error; } + if (config->use_metadata) { + file->metadata = metadata_receive(file_descriptor); + } + Data *file_data = receive_data(file_descriptor); + if (file_data == NULL) { + file_destroy(file); + send_status(file_descriptor, STATUS_ERROR); + return thrd_error; + } + if (config->use_compression) { + Data *uncompressed = data_decompress(file_data); + data_destroy(file_data); + if (uncompressed == NULL) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return thrd_error; } + file_data = uncompressed; + } + data_destroy(file->data); + file->data = file_data; + queue_enqueue_multithreaded(context->queue, file, &context->mutex, + &context->condition_not_empty, + &context->condition_not_full); + } + } else if (status == STATUS_CHUNK) { receive_chunk_enqueue(file_descriptor, context); } else { File *file = file_receive(config, file_descriptor); diff --git a/src/shared/protocol.c b/src/shared/protocol.c index 4f2b82a..a95386e 100644 --- a/src/shared/protocol.c +++ b/src/shared/protocol.c @@ -111,6 +111,8 @@ static const char *status_to_string(Status status) { return "NEXT"; case STATUS_CHUNK: return "CHUNK"; + case STATUS_CHECK: + return "CHECK"; default: return "UNKNOWN"; } diff --git a/src/shared/protocol.h b/src/shared/protocol.h index 1880d24..83b19c4 100644 --- a/src/shared/protocol.h +++ b/src/shared/protocol.h @@ -6,7 +6,7 @@ #include typedef int Status; -enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST }; +enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST, STATUS_CHECK }; void io_set_fds(int read_fd, int write_fd); void io_set_bwlimit(unsigned long long bytes_per_sec); diff --git a/test.py b/test.py index e2efe31..3093ab1 100755 --- a/test.py +++ b/test.py @@ -50,7 +50,7 @@ CLIENT_CMD_PREFIX = [ BASE_CLIENT_FLAGS = ["--save-to-disk"] -TEST_CASES_FULL = [ +TEST_CASES = [ {"name": "Standard", "flags": []}, {"name": "Posix Args (no flags)", "flags": [], "posix": True}, {"name": "Standard (no metadata)", "flags": [], "use_metadata": False}, @@ -68,14 +68,7 @@ TEST_CASES_FULL = [ {"name": "Sendfile + Multithreading (-f -m)", "flags": ["-f", "-m"]}, ] -TEST_CASES_LIGHT = [ - {"name": "Standard", "flags": []}, - {"name": "Compression (-c)", "flags": ["-c"]}, - {"name": "Chunk Serialization (-s)", "flags": ["-s"]}, - {"name": "Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]}, -] - -SSH_CASES_FULL = [ +SSH_CASES = [ {"name": "SSH (localhost)", "flags": []}, {"name": "SSH Multithreading (-m)", "flags": ["-m"]}, {"name": "SSH Compression (-c)", "flags": ["-c"]}, @@ -86,17 +79,11 @@ SSH_CASES_FULL = [ {"name": "SSH Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]}, ] -SSH_CASES_LIGHT = [ - {"name": "SSH (localhost)", "flags": []}, -] - -RSYNC_CASES_FULL = [ +RSYNC_CASES = [ {"name": "rsync (archive)", "args": ["-aH"]}, {"name": "rsync (archive + compress)", "args": ["-aHz"]}, ] -RSYNC_CASES_LIGHT = [] - def netem_apply(profile): params = NETWORK_PROFILES[profile] @@ -132,12 +119,12 @@ def find_free_port(): return s.getsockname()[1] -def generate_test_files(source_dir, full=False): +def generate_test_files(source_dir): if os.path.exists(source_dir): shutil.rmtree(source_dir) os.makedirs(source_dir) - target_total = 25 * 1024 * 1024 if full else 0 + target_total = 25 * 1024 * 1024 written = 0 files = { @@ -154,15 +141,14 @@ def generate_test_files(source_dir, full=False): f.write(content) written += len(content) - if full: - os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True) - i = 0 - while written < target_total: - chunk_size = min(5 * 1024 * 1024, target_total - written) - with open(os.path.join(source_dir, f"bulk/file_{i}.dat"), "wb") as f: - f.write(random.randbytes(chunk_size)) - written += chunk_size - i += 1 + os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True) + i = 0 + while written < target_total: + chunk_size = min(5 * 1024 * 1024, target_total - written) + with open(os.path.join(source_dir, f"bulk/file_{i}.dat"), "wb") as f: + f.write(random.randbytes(chunk_size)) + written += chunk_size + i += 1 total_mb = written / (1024 * 1024) small_bytes = sum(len(c) for c in files.values()) @@ -268,24 +254,19 @@ def print_profile_header(profile_name): print(" No limits applied") -def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=None, ssh_cases=None, rsync_cases=None): +def run_profile(profile_name, source_dir, dest_dir): print_profile_header(profile_name) is_limited = profile_name != "Unlimited" client_prefix = CLIENT_CMD_PREFIX if is_limited else [] - if test_cases is None: - test_cases = TEST_CASES_FULL if full else TEST_CASES_LIGHT - if ssh_cases is None: - ssh_cases = SSH_CASES_FULL if full else SSH_CASES_LIGHT - if rsync_cases is None: - rsync_cases = RSYNC_CASES_FULL if full else RSYNC_CASES_LIGHT - try: - if is_limited and full: + if is_limited: netem_apply(profile_name) + else: + netem_reset() results = [] - for case in test_cases: + for case in TEST_CASES: flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"] if case.get("posix"): cmd = client_prefix + BASE_CLIENT_CMD + [source_dir, dest_dir] + flags @@ -300,7 +281,7 @@ def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=No results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) if SSH_AVAILABLE: - for case in ssh_cases: + for case in SSH_CASES: flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"] ssh_dest = f"localhost:{dest_dir}_ssh" cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags @@ -312,195 +293,220 @@ def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=No except Exception as e: results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - if rsync_cases: - port, conf, daemon = start_rsync_daemon(source_dir) - try: - for case in rsync_cases: - cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"] - print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, case["name"], source_dir, dest_dir, source_prefix="") - r["suite"] = profile_name - except subprocess.TimeoutExpired: - r = {"name": case["name"], "suite": profile_name, "status": "Timeout", "time": "N/A", "error": "Exceeded 120s"} - except Exception as e: - r = {"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)} - else: - if r["status"] != "Success" and client_prefix: - tmp = tempfile.mkdtemp() - try: - plain = subprocess.run(["rsync", "-aH", f"rsync://localhost:{port}/source/", f"{tmp}/"], capture_output=True, text=True, timeout=30) - if plain.returncode != 0: - errs = [l for l in (plain.stderr or "").split("\n") if l.strip()] - if errs: - r["error"] += f" | raw: {errs[-1][:150]}" - finally: - shutil.rmtree(tmp, ignore_errors=True) - results.append(r) - finally: - wait_proc(daemon) + port, conf, daemon = start_rsync_daemon(source_dir) + try: + for case in RSYNC_CASES: + cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"] + print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") try: - os.unlink(conf) - except Exception: - pass - - if full: - # Feature-specific tests for rsync-compatible flags - print("\n " + "─" * 56 + "\n Feature Tests\n " + "─" * 56) - - # Dry run (-n) — no server needed - print("\n --- Dry run (-n) ---") - flags = BASE_CLIENT_FLAGS + ["-n"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags - print(f" Running: {' '.join(cmd)}") + r = run_single_test(cmd, case["name"], source_dir, dest_dir, source_prefix="") + r["suite"] = profile_name + except subprocess.TimeoutExpired: + r = {"name": case["name"], "suite": profile_name, "status": "Timeout", "time": "N/A", "error": "Exceeded 120s"} + except Exception as e: + r = {"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)} + else: + if r["status"] != "Success" and client_prefix: + tmp = tempfile.mkdtemp() + try: + plain = subprocess.run(["rsync", "-aH", f"rsync://localhost:{port}/source/", f"{tmp}/"], capture_output=True, text=True, timeout=30) + if plain.returncode != 0: + errs = [l for l in (plain.stderr or "").split("\n") if l.strip()] + if errs: + r["error"] += f" | raw: {errs[-1][:150]}" + finally: + shutil.rmtree(tmp, ignore_errors=True) + results.append(r) + finally: + wait_proc(daemon) try: - start = time.monotonic() - result = subprocess.run(cmd, text=True, capture_output=True) - duration = time.monotonic() - start - r = {"name": "Dry run (-n)", "suite": profile_name} - if result.returncode == 0 and "Dry run:" in result.stdout: + os.unlink(conf) + except Exception: + pass + + # Feature-specific tests for rsync-compatible flags + print("\n " + "─" * 56 + "\n Feature Tests\n " + "─" * 56) + + # Dry run (-n) — no server needed + print("\n --- Dry run (-n) ---") + flags = BASE_CLIENT_FLAGS + ["-n"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + print(f" Running: {' '.join(cmd)}") + try: + start = time.monotonic() + result = subprocess.run(cmd, text=True, capture_output=True) + duration = time.monotonic() - start + r = {"name": "Dry run (-n)", "suite": profile_name} + if result.returncode == 0 and "Dry run:" in result.stdout: + r["status"] = "Success" + r["time"] = f"{duration:.4f}s" + r["error"] = "" + else: + r["status"] = "Failed" + r["time"] = "N/A" + r["error"] = f"Exit {result.returncode}: {(result.stderr or result.stdout)[:100]}" + results.append(r) + except Exception as e: + results.append({"name": "Dry run (-n)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Archive mode (-a) + feature_flags = BASE_CLIENT_FLAGS + ["-a"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Archive mode (-a) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Archive mode (-a)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Archive mode (-a)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Exclude (--exclude small.txt) + feature_flags = BASE_CLIENT_FLAGS + ["--exclude", "small.txt"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Exclude (--exclude small.txt) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Exclude (--exclude small.txt)", source_dir, dest_dir, + expected_missing=["small.txt"]) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Exclude (--exclude small.txt)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Progress (--progress) + feature_flags = BASE_CLIENT_FLAGS + ["--progress"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Progress (--progress) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Progress (--progress)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Bandwidth limit (--bwlimit 10240 = 10 MB/s) + feature_flags = BASE_CLIENT_FLAGS + ["--bwlimit", "10240"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Bandwidth limit (--bwlimit 10240 KB/s) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Bandwidth limit (--bwlimit 10240)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Bandwidth limit (--bwlimit 10240)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Incremental sync (--incremental) — first sync, then second sync should skip all + print(f"\n --- Incremental (--incremental) ---") + try: + flags = BASE_CLIENT_FLAGS + ["-M"] + srv = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + r1 = subprocess.run(first_cmd, text=True, capture_output=True) + wait_proc(srv) + if r1.returncode != 0: + raise RuntimeError(f"First sync failed: {r1.stderr[:100]}") + srv2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--incremental"] + start = time.monotonic() + r2 = subprocess.run(second_cmd, text=True, capture_output=True, timeout=30) + duration = time.monotonic() - start + wait_proc(srv2) + r = {"name": "Incremental (--incremental)", "suite": profile_name, + "status": "Success" if r2.returncode == 0 else "Failed", + "time": f"{duration:.4f}s" if r2.returncode == 0 else "N/A", + "error": "" if r2.returncode == 0 else f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}"} + results.append(r) + except Exception as e: + results.append({"name": "Incremental (--incremental)", "suite": profile_name, + "status": "Error", "time": "N/A", "error": str(e)}) + + # Chunk size (--chunk-size 5242880) + feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Chunk size (--chunk-size 5242880) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Chunk size (--chunk-size 5242880)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Chunk size (--chunk-size 5242880)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Delete (--delete) — pre-populate dest, add extra files, then sync with --delete + # Note: server handles one client per launch, so we restart between syncs + print(f"\n --- Delete (--delete) ---") + try: + flags = BASE_CLIENT_FLAGS + ["-M"] + # First sync (no delete) to populate dest + s1 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + r1 = subprocess.run(first_cmd, text=True, capture_output=True) + wait_proc(s1) + if r1.returncode != 0: + raise RuntimeError(f"First sync failed: {r1.stderr[:100]}") + # Add extra files to received dir + received = os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep)) + extra_path = os.path.join(received, "extra_file.txt") + with open(extra_path, "w") as f: + f.write("should be deleted") + extra_dir = os.path.join(received, "extra_dir") + os.makedirs(extra_dir, exist_ok=True) + with open(os.path.join(extra_dir, "nested.txt"), "w") as f: + f.write("nested extra") + # Second sync with --delete (fresh server) + s2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--delete"] + start = time.monotonic() + r2 = subprocess.run(second_cmd, text=True, capture_output=True) + duration = time.monotonic() - start + wait_proc(s2) + r = {"name": "Delete (--delete)", "suite": profile_name} + if r2.returncode == 0 and not os.path.exists(extra_path) and not os.path.exists(extra_dir): + mismatches, missing = verify_transfer(source_dir, received) + if not mismatches and not missing: r["status"] = "Success" r["time"] = f"{duration:.4f}s" r["error"] = "" else: r["status"] = "Failed" r["time"] = "N/A" - r["error"] = f"Exit {result.returncode}: {(result.stderr or result.stdout)[:100]}" - results.append(r) - except Exception as e: - results.append({"name": "Dry run (-n)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + r["error"] = f"post-delete verify: mismatches={len(mismatches)}, missing={len(missing)}" + else: + r["status"] = "Failed" + r["time"] = "N/A" + errs = [] + if r2.returncode != 0: + errs.append(f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}") + if os.path.exists(extra_path): + errs.append("extra_file.txt remains") + if os.path.exists(extra_dir): + errs.append("extra_dir remains") + r["error"] = " | ".join(errs) + results.append(r) + except Exception as e: + results.append({"name": "Delete (--delete)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - # Archive mode (-a) - feature_flags = BASE_CLIENT_FLAGS + ["-a"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Archive mode (-a) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Archive mode (-a)", source_dir, dest_dir) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Archive mode (-a)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Exclude (--exclude small.txt) - feature_flags = BASE_CLIENT_FLAGS + ["--exclude", "small.txt"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Exclude (--exclude small.txt) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Exclude (--exclude small.txt)", source_dir, dest_dir, - expected_missing=["small.txt"]) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Exclude (--exclude small.txt)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Progress (--progress) - feature_flags = BASE_CLIENT_FLAGS + ["--progress"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Progress (--progress) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Progress (--progress)", source_dir, dest_dir) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Bandwidth limit (--bwlimit 10240 = 10 MB/s) - feature_flags = BASE_CLIENT_FLAGS + ["--bwlimit", "10240"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Bandwidth limit (--bwlimit 10240 KB/s) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Bandwidth limit (--bwlimit 10240)", source_dir, dest_dir) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Bandwidth limit (--bwlimit 10240)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Chunk size (--chunk-size 5242880) - feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Chunk size (--chunk-size 5242880) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Chunk size (--chunk-size 5242880)", source_dir, dest_dir) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Chunk size (--chunk-size 5242880)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Delete (--delete) — pre-populate dest, add extra files, then sync with --delete - # Note: server handles one client per launch, so we restart between syncs - print(f"\n --- Delete (--delete) ---") - try: - flags = BASE_CLIENT_FLAGS + ["-M"] - # First sync (no delete) to populate dest - s1 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) - time.sleep(0.5) - first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags - r1 = subprocess.run(first_cmd, text=True, capture_output=True) - wait_proc(s1) - if r1.returncode != 0: - raise RuntimeError(f"First sync failed: {r1.stderr[:100]}") - # Add extra files to received dir - received = os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep)) - extra_path = os.path.join(received, "extra_file.txt") - with open(extra_path, "w") as f: - f.write("should be deleted") - extra_dir = os.path.join(received, "extra_dir") - os.makedirs(extra_dir, exist_ok=True) - with open(os.path.join(extra_dir, "nested.txt"), "w") as f: - f.write("nested extra") - # Second sync with --delete (fresh server) - s2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) - time.sleep(0.5) - second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--delete"] - start = time.monotonic() - r2 = subprocess.run(second_cmd, text=True, capture_output=True) - duration = time.monotonic() - start - wait_proc(s2) - r = {"name": "Delete (--delete)", "suite": profile_name} - if r2.returncode == 0 and not os.path.exists(extra_path) and not os.path.exists(extra_dir): - mismatches, missing = verify_transfer(source_dir, received) - if not mismatches and not missing: - r["status"] = "Success" - r["time"] = f"{duration:.4f}s" - r["error"] = "" - else: - r["status"] = "Failed" - r["time"] = "N/A" - r["error"] = f"post-delete verify: mismatches={len(mismatches)}, missing={len(missing)}" - else: - r["status"] = "Failed" - r["time"] = "N/A" - errs = [] - if r2.returncode != 0: - errs.append(f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}") - if os.path.exists(extra_path): - errs.append("extra_file.txt remains") - if os.path.exists(extra_dir): - errs.append("extra_dir remains") - r["error"] = " | ".join(errs) - results.append(r) - except Exception as e: - results.append({"name": "Delete (--delete)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # SSH feature tests - if SSH_AVAILABLE: - ssh_dest = f"localhost:{dest_dir}_ssh" - ssh_feature_cases = [ - {"name": "SSH Archive (-a)", "flags": ["-a"]}, - {"name": "SSH Exclude (--exclude small.txt)", "flags": ["--exclude", "small.txt"], "expected_missing": ["small.txt"]}, - ] - for case in ssh_feature_cases: - flags = BASE_CLIENT_FLAGS + case["flags"] - cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags - print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, case["name"], source_dir, f"{dest_dir}_ssh", - no_server=True, - expected_missing=case.get("expected_missing")) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + # SSH feature tests + if SSH_AVAILABLE: + ssh_dest = f"localhost:{dest_dir}_ssh" + ssh_feature_cases = [ + {"name": "SSH Archive (-a)", "flags": ["-a"]}, + {"name": "SSH Exclude (--exclude small.txt)", "flags": ["--exclude", "small.txt"], "expected_missing": ["small.txt"]}, + ] + for case in ssh_feature_cases: + flags = BASE_CLIENT_FLAGS + case["flags"] + cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags + print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, case["name"], source_dir, f"{dest_dir}_ssh", + no_server=True, + expected_missing=case.get("expected_missing")) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) except (subprocess.CalledProcessError, RuntimeError) as e: print(f" Error: {e}") @@ -567,26 +573,19 @@ def check_ssh_localhost(): build_dir = os.path.abspath("build") server_path = os.path.join(build_dir, "server") - try: - r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", - "localhost", "which", "fastsync-server"], - capture_output=True, timeout=10) - except FileNotFoundError: - SSH_AVAILABLE = False - return + r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", + "localhost", "which", "fastsync-server"], + capture_output=True, timeout=10) if r.returncode == 0: SSH_AVAILABLE = True return SSH_AVAILABLE = False # Try each PATH dir: create symlink, then verify with which - try: - r = subprocess.run( - ["ssh", "-o", "BatchMode=yes", "localhost", - 'echo "$PATH"'], - capture_output=True, timeout=10, text=True) - except FileNotFoundError: - return + r = subprocess.run( + ["ssh", "-o", "BatchMode=yes", "localhost", + 'echo "$PATH"'], + capture_output=True, timeout=10, text=True) if r.returncode != 0: return for d in r.stdout.strip().split(":"): @@ -667,10 +666,9 @@ def main(): parser.add_argument("--keep-data", action="store_true") parser.add_argument("--unlimited", action="store_true") parser.add_argument("--wan", action="store_true") - parser.add_argument("--full", action="store_true", help="Run full test suite with network shaping, SSH, rsync benchmarks") args = parser.parse_args() - total_bytes = generate_test_files(args.source_dir, full=args.full) + total_bytes = generate_test_files(args.source_dir) if os.path.exists(args.dest_dir): shutil.rmtree(args.dest_dir) os.makedirs(args.dest_dir, exist_ok=True) @@ -681,12 +679,12 @@ def main(): elif args.wan: profiles.append("WAN") else: - profiles.append("LAN" if args.full else "Unlimited") + profiles.append("LAN") try: all_results = [] for p in profiles: - all_results.extend(run_profile(p, args.source_dir, args.dest_dir, full=args.full)) + all_results.extend(run_profile(p, args.source_dir, args.dest_dir)) print("\n" + "=" * 130) print(f"{'RESULTS':^130}") From f1af88a25c435d712ce0dfb84976799b7e1f5755 Mon Sep 17 00:00:00 2001 From: TapTap Date: Thu, 16 Jul 2026 17:02:16 +0200 Subject: [PATCH 2/6] Fix incremental-sync review: sendfile_no_path, reject -s+incremental, no data mutation, auto -M, STATUS_ERROR handling, PROTOCOL_VERSION bump --- src/client/client_cli.c | 10 ++++++ src/client/client_send.c | 12 ++++--- src/shared/config.h | 2 +- src/shared/file.c | 72 ++++++++++++++++++++++++++++++++-------- src/shared/file.h | 1 + 5 files changed, 78 insertions(+), 19 deletions(-) diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 714bd6e..e96ae53 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -205,6 +205,16 @@ int main(int argc, char *argv[]) { return 1; } + if (config->use_incremental && config->use_chunk_serialization) { + fprintf(stderr, "Error: --incremental is not supported with -s (chunk serialization)\n"); + return 1; + } + + if (config->use_incremental && !config->use_metadata) { + log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --incremental"); + config->use_metadata = true; + } + if (config->use_multithreading) return send_files_multithreaded(config); return send_files(config); diff --git a/src/client/client_send.c b/src/client/client_send.c index 255e834..c0ee3b2 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -41,13 +41,16 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) { if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1; Status s; if (!receive_status(client->file_descriptor, &s)) return -1; + if (s == STATUS_ERROR) { log_message(LOG_LEVEL_ERROR, "Server reported error for file"); return -1; } if (s == STATUS_OK) continue; - if (s != STATUS_NEXT) return -1; + if (s != STATUS_NEXT) { log_message(LOG_LEVEL_ERROR, "Unexpected server status"); return -1; } + if (!file_send_sendfile_no_path(chunk->items[i], client->file_descriptor, config->use_metadata)) + return -1; } else { if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; + if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata)) + return -1; } - if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata)) - return -1; } } else { for (int i = 0; i < chunk->element_count; i++) { @@ -60,8 +63,9 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) { if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1; Status s; if (!receive_status(client->file_descriptor, &s)) return -1; + if (s == STATUS_ERROR) { log_message(LOG_LEVEL_ERROR, "Server reported error for file"); return -1; } if (s == STATUS_OK) continue; - if (s != STATUS_NEXT) return -1; + if (s != STATUS_NEXT) { log_message(LOG_LEVEL_ERROR, "Unexpected server status"); return -1; } if (!file_send_single_calls_no_path(chunk->items[i], client->file_descriptor, config->use_metadata, config->use_compression ? config->compression_level : 0)) diff --git a/src/shared/config.h b/src/shared/config.h index 6af810b..b581de7 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -35,7 +35,7 @@ typedef struct Config { bool use_incremental; } Config; -#define PROTOCOL_VERSION "1.0.0" +#define PROTOCOL_VERSION "1.1.0" #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) Config *config_create(char *version, char *send_directory, diff --git a/src/shared/file.c b/src/shared/file.c index f04f2e2..fd403d7 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -97,37 +97,81 @@ bool file_load_data(File *file) { } bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level) { + Data *data_to_send = file->data; + Data *compressed_data = NULL; if (compression_level > 0) { - Data *compressed_data = data_compress(file->data, compression_level); + compressed_data = data_compress(file->data, compression_level); if (compressed_data == NULL) { log_message(LOG_LEVEL_ERROR, "Failed to compress file data"); return false; } - data_destroy(file->data); - file->data = compressed_data; + data_to_send = compressed_data; } + if (use_metadata && !metadata_send(file_descriptor, file->metadata)) { + data_destroy(compressed_data); + return false; + } + if (!send_data(file_descriptor, data_to_send)) { + data_destroy(compressed_data); + return false; + } + data_destroy(compressed_data); + return true; +} + +bool file_send_sendfile_no_path(File *file, int file_descriptor, bool use_metadata) { if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; - if (!send_data(file_descriptor, file->data)) return false; + + int fd = open(file->path, O_RDONLY); + if (fd == -1) { + perror("Could not open file for sendfile"); + return false; + } + + unsigned long long file_size = file->data->size; + if (!send_n_data(file_descriptor, &file_size, sizeof(unsigned long long))) { + close(fd); + return false; + } + + off_t offset = 0; + while (offset < file_size) { + ssize_t sent = sendfile(file_descriptor, fd, &offset, file_size - offset); + if (sent == -1) { + perror("sendfile failed"); + close(fd); + return false; + } + } + + close(fd); return true; } bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) { + Data *data_to_send = file->data; + Data *compressed_data = NULL; if (compression_level > 0) { - Data *compressed_data = data_compress(file->data, compression_level); + compressed_data = data_compress(file->data, compression_level); if (compressed_data == NULL) { log_message(LOG_LEVEL_ERROR, "Failed to compress file data"); return false; } - data_destroy(file->data); - if (compressed_data == NULL) { - log_message(LOG_LEVEL_ERROR, "Compression failed in file_send_single_calls"); - exit(EXIT_FAILURE); - } - file->data = compressed_data; + data_to_send = compressed_data; } - if (!send_str(file_descriptor, file->path)) return false; - if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; - if (!send_data(file_descriptor, file->data)) return false; + if (!send_str(file_descriptor, file->path)) { + data_destroy(compressed_data); + return false; + } + if (use_metadata && !metadata_send(file_descriptor, file->metadata)) { + data_destroy(compressed_data); + return false; + } + if (!send_data(file_descriptor, data_to_send)) { + data_destroy(compressed_data); + return false; + } + data_destroy(compressed_data); return true; } diff --git a/src/shared/file.h b/src/shared/file.h index 4f28d06..feb8fbc 100644 --- a/src/shared/file.h +++ b/src/shared/file.h @@ -27,6 +27,7 @@ File *file_receive(Config *config, int file_descriptor); bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level); bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level); bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata); +bool file_send_sendfile_no_path(File *file, int file_descriptor, bool use_metadata); size_t file_content_to_buffer(File *file); FileMetadata *file_metadata_create(struct stat *stats); void file_metadata_destroy(void *metadata); From 03d6ba0da5901ba91e6b3a6e8b1bf99bdfbab9f8 Mon Sep 17 00:00:00 2001 From: TapTap Date: Fri, 17 Jul 2026 10:06:27 +0200 Subject: [PATCH 3/6] fix: address review #69 - STATUS_ERROR sends, metadata_receive error handling, extract incremental_check helper --- src/client/client_send.c | 49 ++++++++++++++++++++---------------- src/server/server.c | 4 ++- src/shared/file.c | 4 ++- src/shared/metadata.c | 14 ++++++++--- src/shared/metadata.h | 2 +- src/shared/multiprocessing.c | 7 ++++-- 6 files changed, 49 insertions(+), 31 deletions(-) diff --git a/src/client/client_send.c b/src/client/client_send.c index c0ee3b2..093184e 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -18,6 +18,27 @@ #include #include +static int incremental_check(Client *client, File *file) { + if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1; + if (!send_str(client->file_descriptor, file->path)) return -1; + unsigned long long fsize = file->data->size; + long long mtime = file->metadata ? file->metadata->mtime_sec : 0; + if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1; + if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1; + Status s; + if (!receive_status(client->file_descriptor, &s)) return -1; + if (s == STATUS_ERROR) { + log_message(LOG_LEVEL_ERROR, "Server reported error for file"); + return -1; + } + if (s == STATUS_OK) return 1; + if (s != STATUS_NEXT) { + log_message(LOG_LEVEL_ERROR, "Unexpected server status"); + return -1; + } + return 0; +} + int send_chunk(Client *client, Chunk *chunk, Config *config) { if (config->use_chunk_serialization) { if (!send_status(client->file_descriptor, STATUS_CHUNK)) return -1; @@ -33,17 +54,9 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) { } else if (config->use_sendfile && !config->use_compression) { for (int i = 0; i < chunk->element_count; i++) { if (config->use_incremental) { - if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1; - if (!send_str(client->file_descriptor, chunk->items[i]->path)) return -1; - unsigned long long fsize = chunk->items[i]->data->size; - long long mtime = chunk->items[i]->metadata ? chunk->items[i]->metadata->mtime_sec : 0; - if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1; - if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1; - Status s; - if (!receive_status(client->file_descriptor, &s)) return -1; - if (s == STATUS_ERROR) { log_message(LOG_LEVEL_ERROR, "Server reported error for file"); return -1; } - if (s == STATUS_OK) continue; - if (s != STATUS_NEXT) { log_message(LOG_LEVEL_ERROR, "Unexpected server status"); return -1; } + int rc = incremental_check(client, chunk->items[i]); + if (rc < 0) return -1; + if (rc > 0) continue; if (!file_send_sendfile_no_path(chunk->items[i], client->file_descriptor, config->use_metadata)) return -1; } else { @@ -55,17 +68,9 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) { } else { for (int i = 0; i < chunk->element_count; i++) { if (config->use_incremental) { - if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1; - if (!send_str(client->file_descriptor, chunk->items[i]->path)) return -1; - unsigned long long fsize = chunk->items[i]->data->size; - long long mtime = chunk->items[i]->metadata ? chunk->items[i]->metadata->mtime_sec : 0; - if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1; - if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1; - Status s; - if (!receive_status(client->file_descriptor, &s)) return -1; - if (s == STATUS_ERROR) { log_message(LOG_LEVEL_ERROR, "Server reported error for file"); return -1; } - if (s == STATUS_OK) continue; - if (s != STATUS_NEXT) { log_message(LOG_LEVEL_ERROR, "Unexpected server status"); return -1; } + int rc = incremental_check(client, chunk->items[i]); + if (rc < 0) return -1; + if (rc > 0) continue; if (!file_send_single_calls_no_path(chunk->items[i], client->file_descriptor, config->use_metadata, config->use_compression ? config->compression_level : 0)) diff --git a/src/server/server.c b/src/server/server.c index c4e528e..8ebd3e4 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -51,7 +51,9 @@ int receive_files(Config *config, int file_descriptor) { free(check_path); if (file == NULL) { send_status(file_descriptor, STATUS_ERROR); return -1; } if (config->use_metadata) { - file->metadata = metadata_receive(file_descriptor); + int meta_ok = 1; + file->metadata = metadata_receive(file_descriptor, &meta_ok); + if (!meta_ok) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return -1; } } Data *file_data = receive_data(file_descriptor); if (file_data == NULL) { diff --git a/src/shared/file.c b/src/shared/file.c index fd403d7..94a1e6a 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -237,7 +237,9 @@ File *file_receive(Config *config, int file_descriptor) { free(path); if (file == NULL) return NULL; if (config->use_metadata) { - file->metadata = metadata_receive(file_descriptor); + int meta_ok = 1; + file->metadata = metadata_receive(file_descriptor, &meta_ok); + if (!meta_ok) { file_destroy(file); return NULL; } } Data *file_data = receive_data(file_descriptor); if (file_data == NULL) { diff --git a/src/shared/metadata.c b/src/shared/metadata.c index 41b4acd..6ca6c43 100644 --- a/src/shared/metadata.c +++ b/src/shared/metadata.c @@ -50,22 +50,28 @@ bool metadata_send(int file_descriptor, FileMetadata *m) { send_n_data(file_descriptor, &m->mtime_nsec, sizeof(long)); } -FileMetadata *metadata_receive(int file_descriptor) { +FileMetadata *metadata_receive(int file_descriptor, int *ok) { int present; - if (!receive_n_data(file_descriptor, &present, sizeof(int))) + if (!receive_n_data(file_descriptor, &present, sizeof(int))) { + if (ok) *ok = 0; return NULL; - if (!present) + } + if (!present) { + if (ok) *ok = 1; return NULL; + } FileMetadata *m = malloc(sizeof(FileMetadata)); - if (m == NULL) return NULL; + if (m == NULL) { if (ok) *ok = 0; return NULL; } if (!receive_n_data(file_descriptor, &m->mode, sizeof(mode_t)) || !receive_n_data(file_descriptor, &m->uid, sizeof(uid_t)) || !receive_n_data(file_descriptor, &m->gid, sizeof(gid_t)) || !receive_n_data(file_descriptor, &m->mtime_sec, sizeof(time_t)) || !receive_n_data(file_descriptor, &m->mtime_nsec, sizeof(long))) { free(m); + if (ok) *ok = 0; return NULL; } + if (ok) *ok = 1; return m; } diff --git a/src/shared/metadata.h b/src/shared/metadata.h index b4399e1..0107c56 100644 --- a/src/shared/metadata.h +++ b/src/shared/metadata.h @@ -10,7 +10,7 @@ void metadata_to_buf(char **buf, FileMetadata *m); FileMetadata *metadata_from_buf(char **buf); bool metadata_send(int file_descriptor, FileMetadata *m); -FileMetadata *metadata_receive(int file_descriptor); +FileMetadata *metadata_receive(int file_descriptor, int *ok); void file_restore_metadata(const char *path, FileMetadata *metadata); #endif diff --git a/src/shared/multiprocessing.c b/src/shared/multiprocessing.c index 5e912d0..63126ab 100644 --- a/src/shared/multiprocessing.c +++ b/src/shared/multiprocessing.c @@ -130,12 +130,13 @@ int receive_thread(void *pipeline_context) { while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) { if (status == STATUS_CHECK) { char *check_path = receive_str(file_descriptor); - if (check_path == NULL) return thrd_error; + if (check_path == NULL) { send_status(file_descriptor, STATUS_ERROR); return thrd_error; } unsigned long long check_size; long long check_mtime; if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) || !receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) { free(check_path); + send_status(file_descriptor, STATUS_ERROR); return thrd_error; } char *full_path = path_cat(config->receive_root_directory, check_path); @@ -156,7 +157,9 @@ int receive_thread(void *pipeline_context) { free(check_path); if (file == NULL) { send_status(file_descriptor, STATUS_ERROR); return thrd_error; } if (config->use_metadata) { - file->metadata = metadata_receive(file_descriptor); + int meta_ok = 1; + file->metadata = metadata_receive(file_descriptor, &meta_ok); + if (!meta_ok) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return thrd_error; } } Data *file_data = receive_data(file_descriptor); if (file_data == NULL) { From 5ed12f2bf0b27a295ea0a6ef678bd85fe7b247e9 Mon Sep 17 00:00:00 2001 From: TapTap Date: Sat, 18 Jul 2026 16:47:39 +0200 Subject: [PATCH 4/6] refactor: deduplicate server receive logic, extract shared helpers - Extract file_save_to_disk() helper (replaces 4x duplicated disk-save boilerplate) - Extract receive_incremental_check() shared helper (deduplicates STATUS_CHECK handling between server.c single-threaded and multiprocessing.c multi-threaded paths) - Break receive_files() into receive_chunk() and receive_manifest() sub-handlers - Add incremental sync test case to test.py - Rebase onto main --- src/server/server.c | 216 ++++++++++++++--------------------- src/shared/file.c | 71 ++++++++++++ src/shared/file.h | 2 + src/shared/multiprocessing.c | 62 +--------- test.py | 11 -- 5 files changed, 162 insertions(+), 200 deletions(-) diff --git a/src/server/server.c b/src/server/server.c index 8ebd3e4..1cfbedc 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -5,7 +5,6 @@ #include "data.h" #include "file.h" #include "log.h" -#include "metadata.h" #include "multiprocessing.h" #include "protocol.h" #include "queue.h" @@ -16,148 +15,99 @@ #include #include #include -#include -int receive_files(Config *config, int file_descriptor) { - Status status; - if (!receive_status(file_descriptor, &status)) return -1; - while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) { - if (status == STATUS_CHECK) { - char *check_path = receive_str(file_descriptor); - if (check_path == NULL) { send_status(file_descriptor, STATUS_ERROR); return -1; } - unsigned long long check_size; - long long check_mtime; - if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) || - !receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) { - free(check_path); - send_status(file_descriptor, STATUS_ERROR); - return -1; - } - char *full_path = path_cat(config->receive_root_directory, check_path); - struct stat st; - bool match = false; - if (full_path && stat(full_path, &st) == 0 && - (unsigned long long)st.st_size == check_size && - (long long)st.st_mtime == check_mtime) { - match = true; - } - free(full_path); - if (match) { - if (!send_status(file_descriptor, STATUS_OK)) { free(check_path); return -1; } - free(check_path); - } else { - if (!send_status(file_descriptor, STATUS_NEXT)) { free(check_path); return -1; } - File *file = file_create(check_path); - free(check_path); - if (file == NULL) { send_status(file_descriptor, STATUS_ERROR); return -1; } - if (config->use_metadata) { - int meta_ok = 1; - file->metadata = metadata_receive(file_descriptor, &meta_ok); - if (!meta_ok) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return -1; } - } - Data *file_data = receive_data(file_descriptor); - if (file_data == NULL) { - file_destroy(file); - send_status(file_descriptor, STATUS_ERROR); - return -1; - } - if (config->use_compression) { - Data *uncompressed = data_decompress(file_data); - data_destroy(file_data); - if (uncompressed == NULL) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return -1; } - file_data = uncompressed; - } - data_destroy(file->data); - file->data = file_data; - if (config->save_to_disk) { - char *disk_path = path_cat(config->receive_root_directory, file->path); - if (disk_path) { - to_disk(disk_path, file->data->data, file->data->size); - file_restore_metadata(disk_path, file->metadata); - free(disk_path); - } - } - file_destroy(file); - } - } else if (status == STATUS_CHUNK) { - Data *chunk_data = receive_data(file_descriptor); - if (chunk_data == NULL) { - log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data"); - send_status(file_descriptor, STATUS_ERROR); - return -1; - } - Data *data_to_process = chunk_data; - if (config->use_compression) { - data_to_process = data_decompress(chunk_data); - data_destroy(chunk_data); - if (data_to_process == NULL) { - log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk"); - send_status(file_descriptor, STATUS_ERROR); - return -1; - } - } - Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata); - data_destroy(data_to_process); - if (chunk == NULL) { - log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping"); - send_status(file_descriptor, STATUS_ERROR); - return -1; - } - - for (int i = 0; i < chunk->element_count; i++) { - if (config->save_to_disk) { - char *disk_path = path_cat(config->receive_root_directory, chunk->items[i]->path); - if (disk_path) { - to_disk(disk_path, chunk->items[i]->data->data, chunk->items[i]->data->size); - file_restore_metadata(disk_path, chunk->items[i]->metadata); - free(disk_path); - } - } - } - chunk_destroy(chunk); - } else { - File *file = file_receive(config, file_descriptor); - if (file == NULL) { - log_message(LOG_LEVEL_ERROR, "Failed to receive file"); - send_status(file_descriptor, STATUS_ERROR); - return -1; - } - if (config->save_to_disk) { - char *disk_path = path_cat(config->receive_root_directory, file->path); - if (disk_path) { - to_disk(disk_path, file->data->data, file->data->size); - file_restore_metadata(disk_path, file->metadata); - free(disk_path); - } - } - file_destroy(file); - } - if (!receive_status(file_descriptor, &status)) { - send_status(file_descriptor, STATUS_ERROR); +static int receive_chunk(int fd, Config *config) { + Data *chunk_data = receive_data(fd); + if (chunk_data == NULL) { + log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data"); + return -1; + } + Data *data_to_process = chunk_data; + if (config->use_compression) { + data_to_process = data_decompress(chunk_data); + data_destroy(chunk_data); + if (data_to_process == NULL) { + log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk"); return -1; } } - if (status == STATUS_MANIFEST) { - int count; - if (!receive_int(file_descriptor, &count)) return -1; - ArrayList *manifest = array_list_create(free); - if (manifest) { - for (int i = 0; i < count; i++) { - char *s = receive_str(file_descriptor); - if (s) array_list_add(manifest, s); - } - fprintf(stderr, "Deleting files not in manifest...\n"); - delete_extras(config->receive_root_directory, manifest); - array_list_delete(manifest); + Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata); + data_destroy(data_to_process); + if (chunk == NULL) { + log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping"); + return -1; + } + + for (int i = 0; i < chunk->element_count; i++) { + if (config->save_to_disk) + file_save_to_disk(config->receive_root_directory, chunk->items[i]); + } + chunk_destroy(chunk); + return 0; +} + +static int receive_manifest(int fd, Config *config, Status *next_status) { + int count; + if (!receive_int(fd, &count)) return -1; + ArrayList *manifest = array_list_create(free); + if (manifest) { + for (int i = 0; i < count; i++) { + char *s = receive_str(fd); + if (s) array_list_add(manifest, s); } - if (!receive_status(file_descriptor, &status)) return -1; + fprintf(stderr, "Deleting files not in manifest...\n"); + delete_extras(config->receive_root_directory, manifest); + array_list_delete(manifest); + } + if (!receive_status(fd, next_status)) return -1; + return 0; +} + +int receive_files(Config *config, int fd) { + Status status; + if (!receive_status(fd, &status)) return -1; + + while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) { + if (status == STATUS_CHECK) { + bool skipped; + File *file = receive_incremental_check(fd, config, &skipped); + if (skipped) goto next; + if (file == NULL && !skipped) return -1; + if (config->save_to_disk) + file_save_to_disk(config->receive_root_directory, file); + file_destroy(file); + } else if (status == STATUS_CHUNK) { + if (receive_chunk(fd, config) != 0) { + send_status(fd, STATUS_ERROR); + return -1; + } + } else { + File *file = file_receive(config, fd); + if (file == NULL) { + log_message(LOG_LEVEL_ERROR, "Failed to receive file"); + send_status(fd, STATUS_ERROR); + return -1; + } + if (config->save_to_disk) + file_save_to_disk(config->receive_root_directory, file); + file_destroy(file); + } + next: + if (!receive_status(fd, &status)) { + send_status(fd, STATUS_ERROR); + return -1; + } + } + + if (status == STATUS_MANIFEST) { + if (receive_manifest(fd, config, &status) != 0) return -1; } if (status != STATUS_FINISHED) { log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status"); - send_status(file_descriptor, STATUS_ERROR); + send_status(fd, STATUS_ERROR); return -1; } - send_status(file_descriptor, STATUS_OK); + send_status(fd, STATUS_OK); return 0; } diff --git a/src/shared/file.c b/src/shared/file.c index 94a1e6a..d757431 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -175,6 +175,77 @@ bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, return true; } +bool file_save_to_disk(const char *root_directory, File *file) { + char *disk_path = path_cat((char *)root_directory, file->path); + if (disk_path == NULL) return false; + bool ok = to_disk(disk_path, file->data->data, file->data->size); + if (ok) file_restore_metadata(disk_path, file->metadata); + free(disk_path); + return ok; +} + +File *receive_incremental_check(int fd, Config *config, bool *skipped) { + *skipped = false; + char *check_path = receive_str(fd); + if (check_path == NULL) { send_status(fd, STATUS_ERROR); return NULL; } + + unsigned long long check_size; + long long check_mtime; + if (!receive_n_data(fd, &check_size, sizeof(check_size)) || + !receive_n_data(fd, &check_mtime, sizeof(check_mtime))) { + free(check_path); + send_status(fd, STATUS_ERROR); + return NULL; + } + + char *full_path = path_cat(config->receive_root_directory, check_path); + struct stat st; + bool match = false; + if (full_path && stat(full_path, &st) == 0 && + (unsigned long long)st.st_size == check_size && + (long long)st.st_mtime == check_mtime) { + match = true; + } + free(full_path); + + if (match) { + if (!send_status(fd, STATUS_OK)) { free(check_path); return NULL; } + free(check_path); + *skipped = true; + return NULL; + } + + if (!send_status(fd, STATUS_NEXT)) { free(check_path); return NULL; } + + File *file = file_create(check_path); + free(check_path); + if (file == NULL) { send_status(fd, STATUS_ERROR); return NULL; } + + if (config->use_metadata) { + int meta_ok = 1; + file->metadata = metadata_receive(fd, &meta_ok); + if (!meta_ok) { file_destroy(file); send_status(fd, STATUS_ERROR); return NULL; } + } + + Data *file_data = receive_data(fd); + if (file_data == NULL) { + file_destroy(file); + send_status(fd, STATUS_ERROR); + return NULL; + } + + if (config->use_compression) { + Data *uncompressed = data_decompress(file_data); + data_destroy(file_data); + if (uncompressed == NULL) { file_destroy(file); send_status(fd, STATUS_ERROR); return NULL; } + file_data = uncompressed; + } + + data_destroy(file->data); + file->data = file_data; + return file; +} + bool to_disk(const char *path, const void *data, unsigned long long data_size) { char *directory = str_dup(path); char *dir_to_free = directory; diff --git a/src/shared/file.h b/src/shared/file.h index feb8fbc..d7bf752 100644 --- a/src/shared/file.h +++ b/src/shared/file.h @@ -32,5 +32,7 @@ size_t file_content_to_buffer(File *file); FileMetadata *file_metadata_create(struct stat *stats); void file_metadata_destroy(void *metadata); bool to_disk(const char *path, const void *data, unsigned long long data_size); +bool file_save_to_disk(const char *root_directory, File *file); +File *receive_incremental_check(int fd, Config *config, bool *skipped); #endif diff --git a/src/shared/multiprocessing.c b/src/shared/multiprocessing.c index 63126ab..8e11890 100644 --- a/src/shared/multiprocessing.c +++ b/src/shared/multiprocessing.c @@ -6,14 +6,12 @@ #include "data.h" #include "file.h" #include "log.h" -#include "metadata.h" #include "protocol.h" #include "queue.h" #include "utils.h" #include #include #include -#include #include PipelineContextSender *pipeline_context_sender_create(Config *config, @@ -129,52 +127,10 @@ int receive_thread(void *pipeline_context) { if (!receive_status(file_descriptor, &status)) return thrd_error; while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) { if (status == STATUS_CHECK) { - char *check_path = receive_str(file_descriptor); - if (check_path == NULL) { send_status(file_descriptor, STATUS_ERROR); return thrd_error; } - unsigned long long check_size; - long long check_mtime; - if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) || - !receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) { - free(check_path); - send_status(file_descriptor, STATUS_ERROR); - return thrd_error; - } - char *full_path = path_cat(config->receive_root_directory, check_path); - struct stat st; - bool match = false; - if (full_path && stat(full_path, &st) == 0 && - (unsigned long long)st.st_size == check_size && - (long long)st.st_mtime == check_mtime) { - match = true; - } - free(full_path); - if (match) { - if (!send_status(file_descriptor, STATUS_OK)) { free(check_path); return thrd_error; } - free(check_path); - } else { - if (!send_status(file_descriptor, STATUS_NEXT)) { free(check_path); return thrd_error; } - File *file = file_create(check_path); - free(check_path); - if (file == NULL) { send_status(file_descriptor, STATUS_ERROR); return thrd_error; } - if (config->use_metadata) { - int meta_ok = 1; - file->metadata = metadata_receive(file_descriptor, &meta_ok); - if (!meta_ok) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return thrd_error; } - } - Data *file_data = receive_data(file_descriptor); - if (file_data == NULL) { - file_destroy(file); - send_status(file_descriptor, STATUS_ERROR); - return thrd_error; - } - if (config->use_compression) { - Data *uncompressed = data_decompress(file_data); - data_destroy(file_data); - if (uncompressed == NULL) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return thrd_error; } - file_data = uncompressed; - } - data_destroy(file->data); - file->data = file_data; + bool skipped; + File *file = receive_incremental_check(file_descriptor, config, &skipped); + if (!skipped) { + if (file == NULL) return thrd_error; queue_enqueue_multithreaded(context->queue, file, &context->mutex, &context->condition_not_empty, &context->condition_not_full); @@ -234,14 +190,8 @@ int write_thread(void *pipeline_context) { free(root_directory); return thrd_success; } - if (save_to_disk) { - char *disk_path = path_cat(root_directory, file->path); - if (disk_path) { - to_disk(disk_path, file->data->data, file->data->size); - file_restore_metadata(disk_path, file->metadata); - free(disk_path); - } - } + if (save_to_disk) + file_save_to_disk(root_directory, file); file_destroy(file); } } diff --git a/test.py b/test.py index 3093ab1..ba69b83 100755 --- a/test.py +++ b/test.py @@ -383,17 +383,6 @@ def run_profile(profile_name, source_dir, dest_dir): except Exception as e: results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - # Bandwidth limit (--bwlimit 10240 = 10 MB/s) - feature_flags = BASE_CLIENT_FLAGS + ["--bwlimit", "10240"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Bandwidth limit (--bwlimit 10240 KB/s) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Bandwidth limit (--bwlimit 10240)", source_dir, dest_dir) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Bandwidth limit (--bwlimit 10240)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - # Incremental sync (--incremental) — first sync, then second sync should skip all print(f"\n --- Incremental (--incremental) ---") try: From 78767219062e0ee86b2d0574938755368b61d91c Mon Sep 17 00:00:00 2001 From: TapTap Date: Sat, 18 Jul 2026 17:00:11 +0200 Subject: [PATCH 5/6] refactor: extract receive_chunk_data and receive_manifest shared helpers - receive_chunk_data(fd, config) -> Chunk*: shared receive/decompress/deserialize (eliminates ~25 lines of duplication between server.c and multiprocessing.c) - receive_manifest(fd, config, next_status): moved from server.c to file.c, fixes missing 'Deleting files not in manifest...' log in multiprocessing.c - Removes redundant manual free loop in multiprocessing.c manifest handling (array_list_create(free) destructor already handles this) - server.c and multiprocessing.c: remove local chunk/manifest helpers, remove compression.h include (no longer needed) --- src/server/server.c | 56 +++++------------------------------- src/shared/chunk.c | 23 +++++++++++++++ src/shared/chunk.h | 2 ++ src/shared/file.c | 20 +++++++++++-- src/shared/file.h | 1 + src/shared/multiprocessing.c | 40 ++------------------------ 6 files changed, 54 insertions(+), 88 deletions(-) diff --git a/src/server/server.c b/src/server/server.c index 1cfbedc..a0cf8d2 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -1,6 +1,5 @@ #include "array_list.h" #include "chunk.h" -#include "compression.h" #include "config.h" #include "data.h" #include "file.h" @@ -16,53 +15,6 @@ #include #include -static int receive_chunk(int fd, Config *config) { - Data *chunk_data = receive_data(fd); - if (chunk_data == NULL) { - log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data"); - return -1; - } - Data *data_to_process = chunk_data; - if (config->use_compression) { - data_to_process = data_decompress(chunk_data); - data_destroy(chunk_data); - if (data_to_process == NULL) { - log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk"); - return -1; - } - } - Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata); - data_destroy(data_to_process); - if (chunk == NULL) { - log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping"); - return -1; - } - - for (int i = 0; i < chunk->element_count; i++) { - if (config->save_to_disk) - file_save_to_disk(config->receive_root_directory, chunk->items[i]); - } - chunk_destroy(chunk); - return 0; -} - -static int receive_manifest(int fd, Config *config, Status *next_status) { - int count; - if (!receive_int(fd, &count)) return -1; - ArrayList *manifest = array_list_create(free); - if (manifest) { - for (int i = 0; i < count; i++) { - char *s = receive_str(fd); - if (s) array_list_add(manifest, s); - } - fprintf(stderr, "Deleting files not in manifest...\n"); - delete_extras(config->receive_root_directory, manifest); - array_list_delete(manifest); - } - if (!receive_status(fd, next_status)) return -1; - return 0; -} - int receive_files(Config *config, int fd) { Status status; if (!receive_status(fd, &status)) return -1; @@ -77,10 +29,16 @@ int receive_files(Config *config, int fd) { file_save_to_disk(config->receive_root_directory, file); file_destroy(file); } else if (status == STATUS_CHUNK) { - if (receive_chunk(fd, config) != 0) { + Chunk *chunk = receive_chunk_data(fd, config); + if (chunk == NULL) { send_status(fd, STATUS_ERROR); return -1; } + for (int i = 0; i < chunk->element_count; i++) { + if (config->save_to_disk) + file_save_to_disk(config->receive_root_directory, chunk->items[i]); + } + chunk_destroy(chunk); } else { File *file = file_receive(config, fd); if (file == NULL) { diff --git a/src/shared/chunk.c b/src/shared/chunk.c index ebc6434..18b7edc 100644 --- a/src/shared/chunk.c +++ b/src/shared/chunk.c @@ -10,6 +10,7 @@ #include "file.h" #include "log.h" #include "metadata.h" +#include "protocol.h" Chunk *chunk_create(File **items, int element_count) { Chunk *chunk = (Chunk *)malloc(sizeof(Chunk)); @@ -178,5 +179,27 @@ Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata) { return compressed; } +Chunk *receive_chunk_data(int fd, Config *config) { + Data *chunk_data = receive_data(fd); + if (chunk_data == NULL) { + log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data"); + return NULL; + } + Data *data_to_process = chunk_data; + if (config->use_compression) { + data_to_process = data_decompress(chunk_data); + data_destroy(chunk_data); + if (data_to_process == NULL) { + log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk"); + return NULL; + } + } + Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata); + data_destroy(data_to_process); + if (chunk == NULL) + log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping"); + return chunk; +} + diff --git a/src/shared/chunk.h b/src/shared/chunk.h index 7837936..fc07792 100644 --- a/src/shared/chunk.h +++ b/src/shared/chunk.h @@ -1,6 +1,7 @@ #ifndef CHUNK_H #define CHUNK_H +#include "config.h" #include "data.h" #include "file.h" #include @@ -18,5 +19,6 @@ void chunk_destroy(void *chunk); Data *chunk_serialize(Chunk *chunk, bool use_metadata); Chunk *chunk_deserialize(Data *data, bool use_metadata); Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata); +Chunk *receive_chunk_data(int fd, Config *config); #endif diff --git a/src/shared/file.c b/src/shared/file.c index d757431..1cfb569 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -184,8 +184,7 @@ bool file_save_to_disk(const char *root_directory, File *file) { return ok; } -File *receive_incremental_check(int fd, Config *config, bool *skipped) { - *skipped = false; +File *receive_incremental_check(int fd, Config *config, bool *skipped) { *skipped = false; char *check_path = receive_str(fd); if (check_path == NULL) { send_status(fd, STATUS_ERROR); return NULL; } @@ -348,4 +347,21 @@ size_t file_content_to_buffer(File *file) { return bytes_read; } +int receive_manifest(int fd, Config *config, int *next_status) { + int count; + if (!receive_int(fd, &count)) return -1; + ArrayList *manifest = array_list_create(free); + if (manifest) { + for (int i = 0; i < count; i++) { + char *s = receive_str(fd); + if (s) array_list_add(manifest, s); + } + fprintf(stderr, "Deleting files not in manifest...\n"); + delete_extras(config->receive_root_directory, manifest); + array_list_delete(manifest); + } + if (!receive_status(fd, next_status)) return -1; + return 0; +} + diff --git a/src/shared/file.h b/src/shared/file.h index d7bf752..7eb4d42 100644 --- a/src/shared/file.h +++ b/src/shared/file.h @@ -34,5 +34,6 @@ void file_metadata_destroy(void *metadata); bool to_disk(const char *path, const void *data, unsigned long long data_size); bool file_save_to_disk(const char *root_directory, File *file); File *receive_incremental_check(int fd, Config *config, bool *skipped); +int receive_manifest(int fd, Config *config, int *next_status); #endif diff --git a/src/shared/multiprocessing.c b/src/shared/multiprocessing.c index 8e11890..ffb6148 100644 --- a/src/shared/multiprocessing.c +++ b/src/shared/multiprocessing.c @@ -1,7 +1,6 @@ #include "multiprocessing.h" #include "array_list.h" #include "chunk.h" -#include "compression.h" #include "config.h" #include "data.h" #include "file.h" @@ -84,26 +83,8 @@ void pipeline_context_receiver_destroy(PipelineContextReceiver *context) { static void receive_chunk_enqueue(int file_descriptor, PipelineContextReceiver *context) { - Data *chunk_data = receive_data(file_descriptor); - if (chunk_data == NULL) { - log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data"); - return; - } - Data *data_to_process = chunk_data; - if (context->config->use_compression) { - data_to_process = data_decompress(chunk_data); - data_destroy(chunk_data); - if (data_to_process == NULL) { - log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk"); - return; - } - } - Chunk *chunk = chunk_deserialize(data_to_process, context->config->use_metadata); - data_destroy(data_to_process); - if (chunk == NULL) { - log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping"); - return; - } + Chunk *chunk = receive_chunk_data(file_descriptor, context->config); + if (chunk == NULL) return; for (int i = 0; i < chunk->element_count; i++) { File *file = chunk->items[i]; @@ -150,22 +131,7 @@ int receive_thread(void *pipeline_context) { if (!receive_status(file_descriptor, &status)) return thrd_error; } if (status == STATUS_MANIFEST) { - int count; - if (!receive_int(file_descriptor, &count)) return thrd_error; - ArrayList *manifest = array_list_create(free); - if (manifest) { - for (int i = 0; i < count; i++) { - char *s = receive_str(file_descriptor); - if (s) { - array_list_add(manifest, s); - } - } - delete_extras(context->config->receive_root_directory, manifest); - for (int i = 0; i < manifest->size; i++) - free(manifest->items[i]); - array_list_delete(manifest); - } - if (!receive_status(file_descriptor, &status)) return thrd_error; + if (receive_manifest(file_descriptor, config, &status) != 0) return thrd_error; } mtx_lock(&context->mutex); context->receiver_done = true; From 46c650e9ae3725ebcb8e91a23e6a8ba6600d7705 Mon Sep 17 00:00:00 2001 From: TapTap Date: Sat, 18 Jul 2026 17:06:14 +0200 Subject: [PATCH 6/6] refactor: consolidate _no_path file send variants Add bool send_path parameter to file_send_single_calls and file_send_sendfile, remove the _no_path variants. Callers in client_send.c pass true (send path) or false (skip path) based on whether an incremental check already transmitted the path. --- src/client/client_send.c | 14 +- src/shared/file.c | 60 +---- src/shared/file.h | 6 +- test.py | 484 +++++++++++++++++++++------------------ 4 files changed, 276 insertions(+), 288 deletions(-) diff --git a/src/client/client_send.c b/src/client/client_send.c index 093184e..5b4b0bb 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -57,11 +57,11 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) { int rc = incremental_check(client, chunk->items[i]); if (rc < 0) return -1; if (rc > 0) continue; - if (!file_send_sendfile_no_path(chunk->items[i], client->file_descriptor, config->use_metadata)) + if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata, false)) return -1; } else { if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; - if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata)) + if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata, true)) return -1; } } @@ -71,15 +71,17 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) { int rc = incremental_check(client, chunk->items[i]); if (rc < 0) return -1; if (rc > 0) continue; - if (!file_send_single_calls_no_path(chunk->items[i], client->file_descriptor, - config->use_metadata, - config->use_compression ? config->compression_level : 0)) + if (!file_send_single_calls(chunk->items[i], client->file_descriptor, + config->use_metadata, + config->use_compression ? config->compression_level : 0, + false)) return -1; } else { if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; if (!file_send_single_calls(chunk->items[i], client->file_descriptor, config->use_metadata, - config->use_compression ? config->compression_level : 0)) + config->use_compression ? config->compression_level : 0, + true)) return -1; } } diff --git a/src/shared/file.c b/src/shared/file.c index 1cfb569..f4ca381 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -96,7 +96,7 @@ bool file_load_data(File *file) { return true; } -bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level) { +bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level, bool send_path) { Data *data_to_send = file->data; Data *compressed_data = NULL; if (compression_level > 0) { @@ -107,59 +107,7 @@ bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_me } data_to_send = compressed_data; } - if (use_metadata && !metadata_send(file_descriptor, file->metadata)) { - data_destroy(compressed_data); - return false; - } - if (!send_data(file_descriptor, data_to_send)) { - data_destroy(compressed_data); - return false; - } - data_destroy(compressed_data); - return true; -} - -bool file_send_sendfile_no_path(File *file, int file_descriptor, bool use_metadata) { - if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; - - int fd = open(file->path, O_RDONLY); - if (fd == -1) { - perror("Could not open file for sendfile"); - return false; - } - - unsigned long long file_size = file->data->size; - if (!send_n_data(file_descriptor, &file_size, sizeof(unsigned long long))) { - close(fd); - return false; - } - - off_t offset = 0; - while (offset < file_size) { - ssize_t sent = sendfile(file_descriptor, fd, &offset, file_size - offset); - if (sent == -1) { - perror("sendfile failed"); - close(fd); - return false; - } - } - - close(fd); - return true; -} - -bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) { - Data *data_to_send = file->data; - Data *compressed_data = NULL; - if (compression_level > 0) { - compressed_data = data_compress(file->data, compression_level); - if (compressed_data == NULL) { - log_message(LOG_LEVEL_ERROR, "Failed to compress file data"); - return false; - } - data_to_send = compressed_data; - } - if (!send_str(file_descriptor, file->path)) { + if (send_path && !send_str(file_descriptor, file->path)) { data_destroy(compressed_data); return false; } @@ -270,8 +218,8 @@ bool to_disk(const char *path, const void *data, unsigned long long data_size) { return true; } -bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata) { - if (!send_str(file_descriptor, file->path)) return false; +bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata, bool send_path) { + if (send_path && !send_str(file_descriptor, file->path)) return false; if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; int fd = open(file->path, O_RDONLY); diff --git a/src/shared/file.h b/src/shared/file.h index 7eb4d42..9256918 100644 --- a/src/shared/file.h +++ b/src/shared/file.h @@ -24,10 +24,8 @@ File *file_create(const char *path); void file_destroy(void *item); bool file_load_data(File *file); File *file_receive(Config *config, int file_descriptor); -bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level); -bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level); -bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata); -bool file_send_sendfile_no_path(File *file, int file_descriptor, bool use_metadata); +bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level, bool send_path); +bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata, bool send_path); size_t file_content_to_buffer(File *file); FileMetadata *file_metadata_create(struct stat *stats); void file_metadata_destroy(void *metadata); diff --git a/test.py b/test.py index ba69b83..9d533a5 100755 --- a/test.py +++ b/test.py @@ -50,7 +50,7 @@ CLIENT_CMD_PREFIX = [ BASE_CLIENT_FLAGS = ["--save-to-disk"] -TEST_CASES = [ +TEST_CASES_FULL = [ {"name": "Standard", "flags": []}, {"name": "Posix Args (no flags)", "flags": [], "posix": True}, {"name": "Standard (no metadata)", "flags": [], "use_metadata": False}, @@ -68,7 +68,14 @@ TEST_CASES = [ {"name": "Sendfile + Multithreading (-f -m)", "flags": ["-f", "-m"]}, ] -SSH_CASES = [ +TEST_CASES_LIGHT = [ + {"name": "Standard", "flags": []}, + {"name": "Compression (-c)", "flags": ["-c"]}, + {"name": "Chunk Serialization (-s)", "flags": ["-s"]}, + {"name": "Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]}, +] + +SSH_CASES_FULL = [ {"name": "SSH (localhost)", "flags": []}, {"name": "SSH Multithreading (-m)", "flags": ["-m"]}, {"name": "SSH Compression (-c)", "flags": ["-c"]}, @@ -79,11 +86,17 @@ SSH_CASES = [ {"name": "SSH Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]}, ] -RSYNC_CASES = [ +SSH_CASES_LIGHT = [ + {"name": "SSH (localhost)", "flags": []}, +] + +RSYNC_CASES_FULL = [ {"name": "rsync (archive)", "args": ["-aH"]}, {"name": "rsync (archive + compress)", "args": ["-aHz"]}, ] +RSYNC_CASES_LIGHT = [] + def netem_apply(profile): params = NETWORK_PROFILES[profile] @@ -119,12 +132,12 @@ def find_free_port(): return s.getsockname()[1] -def generate_test_files(source_dir): +def generate_test_files(source_dir, full=False): if os.path.exists(source_dir): shutil.rmtree(source_dir) os.makedirs(source_dir) - target_total = 25 * 1024 * 1024 + target_total = 25 * 1024 * 1024 if full else 0 written = 0 files = { @@ -141,14 +154,15 @@ def generate_test_files(source_dir): f.write(content) written += len(content) - os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True) - i = 0 - while written < target_total: - chunk_size = min(5 * 1024 * 1024, target_total - written) - with open(os.path.join(source_dir, f"bulk/file_{i}.dat"), "wb") as f: - f.write(random.randbytes(chunk_size)) - written += chunk_size - i += 1 + if full: + os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True) + i = 0 + while written < target_total: + chunk_size = min(5 * 1024 * 1024, target_total - written) + with open(os.path.join(source_dir, f"bulk/file_{i}.dat"), "wb") as f: + f.write(random.randbytes(chunk_size)) + written += chunk_size + i += 1 total_mb = written / (1024 * 1024) small_bytes = sum(len(c) for c in files.values()) @@ -254,19 +268,24 @@ def print_profile_header(profile_name): print(" No limits applied") -def run_profile(profile_name, source_dir, dest_dir): +def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=None, ssh_cases=None, rsync_cases=None): print_profile_header(profile_name) is_limited = profile_name != "Unlimited" client_prefix = CLIENT_CMD_PREFIX if is_limited else [] + if test_cases is None: + test_cases = TEST_CASES_FULL if full else TEST_CASES_LIGHT + if ssh_cases is None: + ssh_cases = SSH_CASES_FULL if full else SSH_CASES_LIGHT + if rsync_cases is None: + rsync_cases = RSYNC_CASES_FULL if full else RSYNC_CASES_LIGHT + try: - if is_limited: + if is_limited and full: netem_apply(profile_name) - else: - netem_reset() results = [] - for case in TEST_CASES: + for case in test_cases: flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"] if case.get("posix"): cmd = client_prefix + BASE_CLIENT_CMD + [source_dir, dest_dir] + flags @@ -281,7 +300,7 @@ def run_profile(profile_name, source_dir, dest_dir): results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) if SSH_AVAILABLE: - for case in SSH_CASES: + for case in ssh_cases: flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"] ssh_dest = f"localhost:{dest_dir}_ssh" cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags @@ -293,209 +312,222 @@ def run_profile(profile_name, source_dir, dest_dir): except Exception as e: results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - port, conf, daemon = start_rsync_daemon(source_dir) - try: - for case in RSYNC_CASES: - cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"] - print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, case["name"], source_dir, dest_dir, source_prefix="") - r["suite"] = profile_name - except subprocess.TimeoutExpired: - r = {"name": case["name"], "suite": profile_name, "status": "Timeout", "time": "N/A", "error": "Exceeded 120s"} - except Exception as e: - r = {"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)} - else: - if r["status"] != "Success" and client_prefix: - tmp = tempfile.mkdtemp() - try: - plain = subprocess.run(["rsync", "-aH", f"rsync://localhost:{port}/source/", f"{tmp}/"], capture_output=True, text=True, timeout=30) - if plain.returncode != 0: - errs = [l for l in (plain.stderr or "").split("\n") if l.strip()] - if errs: - r["error"] += f" | raw: {errs[-1][:150]}" - finally: - shutil.rmtree(tmp, ignore_errors=True) - results.append(r) - finally: - wait_proc(daemon) + if rsync_cases: + port, conf, daemon = start_rsync_daemon(source_dir) try: - os.unlink(conf) - except Exception: - pass + for case in rsync_cases: + cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"] + print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, case["name"], source_dir, dest_dir, source_prefix="") + r["suite"] = profile_name + except subprocess.TimeoutExpired: + r = {"name": case["name"], "suite": profile_name, "status": "Timeout", "time": "N/A", "error": "Exceeded 120s"} + except Exception as e: + r = {"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)} + else: + if r["status"] != "Success" and client_prefix: + tmp = tempfile.mkdtemp() + try: + plain = subprocess.run(["rsync", "-aH", f"rsync://localhost:{port}/source/", f"{tmp}/"], capture_output=True, text=True, timeout=30) + if plain.returncode != 0: + errs = [l for l in (plain.stderr or "").split("\n") if l.strip()] + if errs: + r["error"] += f" | raw: {errs[-1][:150]}" + finally: + shutil.rmtree(tmp, ignore_errors=True) + results.append(r) + finally: + wait_proc(daemon) + try: + os.unlink(conf) + except Exception: + pass - # Feature-specific tests for rsync-compatible flags - print("\n " + "─" * 56 + "\n Feature Tests\n " + "─" * 56) + if full: + # Feature-specific tests for rsync-compatible flags + print("\n " + "─" * 56 + "\n Feature Tests\n " + "─" * 56) - # Dry run (-n) — no server needed - print("\n --- Dry run (-n) ---") - flags = BASE_CLIENT_FLAGS + ["-n"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags - print(f" Running: {' '.join(cmd)}") - try: - start = time.monotonic() - result = subprocess.run(cmd, text=True, capture_output=True) - duration = time.monotonic() - start - r = {"name": "Dry run (-n)", "suite": profile_name} - if result.returncode == 0 and "Dry run:" in result.stdout: - r["status"] = "Success" - r["time"] = f"{duration:.4f}s" - r["error"] = "" - else: - r["status"] = "Failed" - r["time"] = "N/A" - r["error"] = f"Exit {result.returncode}: {(result.stderr or result.stdout)[:100]}" - results.append(r) - except Exception as e: - results.append({"name": "Dry run (-n)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Archive mode (-a) - feature_flags = BASE_CLIENT_FLAGS + ["-a"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Archive mode (-a) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Archive mode (-a)", source_dir, dest_dir) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Archive mode (-a)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Exclude (--exclude small.txt) - feature_flags = BASE_CLIENT_FLAGS + ["--exclude", "small.txt"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Exclude (--exclude small.txt) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Exclude (--exclude small.txt)", source_dir, dest_dir, - expected_missing=["small.txt"]) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Exclude (--exclude small.txt)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Progress (--progress) - feature_flags = BASE_CLIENT_FLAGS + ["--progress"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Progress (--progress) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Progress (--progress)", source_dir, dest_dir) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Incremental sync (--incremental) — first sync, then second sync should skip all - print(f"\n --- Incremental (--incremental) ---") - try: - flags = BASE_CLIENT_FLAGS + ["-M"] - srv = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) - time.sleep(0.5) - first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags - r1 = subprocess.run(first_cmd, text=True, capture_output=True) - wait_proc(srv) - if r1.returncode != 0: - raise RuntimeError(f"First sync failed: {r1.stderr[:100]}") - srv2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) - time.sleep(0.5) - second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--incremental"] - start = time.monotonic() - r2 = subprocess.run(second_cmd, text=True, capture_output=True, timeout=30) - duration = time.monotonic() - start - wait_proc(srv2) - r = {"name": "Incremental (--incremental)", "suite": profile_name, - "status": "Success" if r2.returncode == 0 else "Failed", - "time": f"{duration:.4f}s" if r2.returncode == 0 else "N/A", - "error": "" if r2.returncode == 0 else f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}"} - results.append(r) - except Exception as e: - results.append({"name": "Incremental (--incremental)", "suite": profile_name, - "status": "Error", "time": "N/A", "error": str(e)}) - - # Chunk size (--chunk-size 5242880) - feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Chunk size (--chunk-size 5242880) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Chunk size (--chunk-size 5242880)", source_dir, dest_dir) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Chunk size (--chunk-size 5242880)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Delete (--delete) — pre-populate dest, add extra files, then sync with --delete - # Note: server handles one client per launch, so we restart between syncs - print(f"\n --- Delete (--delete) ---") - try: - flags = BASE_CLIENT_FLAGS + ["-M"] - # First sync (no delete) to populate dest - s1 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) - time.sleep(0.5) - first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags - r1 = subprocess.run(first_cmd, text=True, capture_output=True) - wait_proc(s1) - if r1.returncode != 0: - raise RuntimeError(f"First sync failed: {r1.stderr[:100]}") - # Add extra files to received dir - received = os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep)) - extra_path = os.path.join(received, "extra_file.txt") - with open(extra_path, "w") as f: - f.write("should be deleted") - extra_dir = os.path.join(received, "extra_dir") - os.makedirs(extra_dir, exist_ok=True) - with open(os.path.join(extra_dir, "nested.txt"), "w") as f: - f.write("nested extra") - # Second sync with --delete (fresh server) - s2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) - time.sleep(0.5) - second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--delete"] - start = time.monotonic() - r2 = subprocess.run(second_cmd, text=True, capture_output=True) - duration = time.monotonic() - start - wait_proc(s2) - r = {"name": "Delete (--delete)", "suite": profile_name} - if r2.returncode == 0 and not os.path.exists(extra_path) and not os.path.exists(extra_dir): - mismatches, missing = verify_transfer(source_dir, received) - if not mismatches and not missing: + # Dry run (-n) — no server needed + print("\n --- Dry run (-n) ---") + flags = BASE_CLIENT_FLAGS + ["-n"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + print(f" Running: {' '.join(cmd)}") + try: + start = time.monotonic() + result = subprocess.run(cmd, text=True, capture_output=True) + duration = time.monotonic() - start + r = {"name": "Dry run (-n)", "suite": profile_name} + if result.returncode == 0 and "Dry run:" in result.stdout: r["status"] = "Success" r["time"] = f"{duration:.4f}s" r["error"] = "" else: r["status"] = "Failed" r["time"] = "N/A" - r["error"] = f"post-delete verify: mismatches={len(mismatches)}, missing={len(missing)}" - else: - r["status"] = "Failed" - r["time"] = "N/A" - errs = [] - if r2.returncode != 0: - errs.append(f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}") - if os.path.exists(extra_path): - errs.append("extra_file.txt remains") - if os.path.exists(extra_dir): - errs.append("extra_dir remains") - r["error"] = " | ".join(errs) - results.append(r) - except Exception as e: - results.append({"name": "Delete (--delete)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + r["error"] = f"Exit {result.returncode}: {(result.stderr or result.stdout)[:100]}" + results.append(r) + except Exception as e: + results.append({"name": "Dry run (-n)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - # SSH feature tests - if SSH_AVAILABLE: - ssh_dest = f"localhost:{dest_dir}_ssh" - ssh_feature_cases = [ - {"name": "SSH Archive (-a)", "flags": ["-a"]}, - {"name": "SSH Exclude (--exclude small.txt)", "flags": ["--exclude", "small.txt"], "expected_missing": ["small.txt"]}, - ] - for case in ssh_feature_cases: - flags = BASE_CLIENT_FLAGS + case["flags"] - cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags - print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, case["name"], source_dir, f"{dest_dir}_ssh", - no_server=True, - expected_missing=case.get("expected_missing")) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + # Archive mode (-a) + feature_flags = BASE_CLIENT_FLAGS + ["-a"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Archive mode (-a) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Archive mode (-a)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Archive mode (-a)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Exclude (--exclude small.txt) + feature_flags = BASE_CLIENT_FLAGS + ["--exclude", "small.txt"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Exclude (--exclude small.txt) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Exclude (--exclude small.txt)", source_dir, dest_dir, + expected_missing=["small.txt"]) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Exclude (--exclude small.txt)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Progress (--progress) + feature_flags = BASE_CLIENT_FLAGS + ["--progress"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Progress (--progress) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Progress (--progress)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Bandwidth limit (--bwlimit 10240 = 10 MB/s) + feature_flags = BASE_CLIENT_FLAGS + ["--bwlimit", "10240"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Bandwidth limit (--bwlimit 10240 KB/s) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Bandwidth limit (--bwlimit 10240)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Bandwidth limit (--bwlimit 10240)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Incremental sync (--incremental) — first sync, then second sync should skip all + print(f"\n --- Incremental (--incremental) ---") + try: + flags = BASE_CLIENT_FLAGS + ["-M"] + srv = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + r1 = subprocess.run(first_cmd, text=True, capture_output=True) + wait_proc(srv) + if r1.returncode != 0: + raise RuntimeError(f"First sync failed: {r1.stderr[:100]}") + srv2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--incremental"] + start = time.monotonic() + r2 = subprocess.run(second_cmd, text=True, capture_output=True, timeout=30) + duration = time.monotonic() - start + wait_proc(srv2) + r = {"name": "Incremental (--incremental)", "suite": profile_name, + "status": "Success" if r2.returncode == 0 else "Failed", + "time": f"{duration:.4f}s" if r2.returncode == 0 else "N/A", + "error": "" if r2.returncode == 0 else f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}"} + results.append(r) + except Exception as e: + results.append({"name": "Incremental (--incremental)", "suite": profile_name, + "status": "Error", "time": "N/A", "error": str(e)}) + + # Chunk size (--chunk-size 5242880) + feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Chunk size (--chunk-size 5242880) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Chunk size (--chunk-size 5242880)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Chunk size (--chunk-size 5242880)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Delete (--delete) — pre-populate dest, add extra files, then sync with --delete + # Note: server handles one client per launch, so we restart between syncs + print(f"\n --- Delete (--delete) ---") + try: + flags = BASE_CLIENT_FLAGS + ["-M"] + # First sync (no delete) to populate dest + s1 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + r1 = subprocess.run(first_cmd, text=True, capture_output=True) + wait_proc(s1) + if r1.returncode != 0: + raise RuntimeError(f"First sync failed: {r1.stderr[:100]}") + # Add extra files to received dir + received = os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep)) + extra_path = os.path.join(received, "extra_file.txt") + with open(extra_path, "w") as f: + f.write("should be deleted") + extra_dir = os.path.join(received, "extra_dir") + os.makedirs(extra_dir, exist_ok=True) + with open(os.path.join(extra_dir, "nested.txt"), "w") as f: + f.write("nested extra") + # Second sync with --delete (fresh server) + s2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--delete"] + start = time.monotonic() + r2 = subprocess.run(second_cmd, text=True, capture_output=True) + duration = time.monotonic() - start + wait_proc(s2) + r = {"name": "Delete (--delete)", "suite": profile_name} + if r2.returncode == 0 and not os.path.exists(extra_path) and not os.path.exists(extra_dir): + mismatches, missing = verify_transfer(source_dir, received) + if not mismatches and not missing: + r["status"] = "Success" + r["time"] = f"{duration:.4f}s" + r["error"] = "" + else: + r["status"] = "Failed" + r["time"] = "N/A" + r["error"] = f"post-delete verify: mismatches={len(mismatches)}, missing={len(missing)}" + else: + r["status"] = "Failed" + r["time"] = "N/A" + errs = [] + if r2.returncode != 0: + errs.append(f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}") + if os.path.exists(extra_path): + errs.append("extra_file.txt remains") + if os.path.exists(extra_dir): + errs.append("extra_dir remains") + r["error"] = " | ".join(errs) + results.append(r) + except Exception as e: + results.append({"name": "Delete (--delete)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # SSH feature tests + if SSH_AVAILABLE: + ssh_dest = f"localhost:{dest_dir}_ssh" + ssh_feature_cases = [ + {"name": "SSH Archive (-a)", "flags": ["-a"]}, + {"name": "SSH Exclude (--exclude small.txt)", "flags": ["--exclude", "small.txt"], "expected_missing": ["small.txt"]}, + ] + for case in ssh_feature_cases: + flags = BASE_CLIENT_FLAGS + case["flags"] + cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags + print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, case["name"], source_dir, f"{dest_dir}_ssh", + no_server=True, + expected_missing=case.get("expected_missing")) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) except (subprocess.CalledProcessError, RuntimeError) as e: print(f" Error: {e}") @@ -562,19 +594,26 @@ def check_ssh_localhost(): build_dir = os.path.abspath("build") server_path = os.path.join(build_dir, "server") - r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", - "localhost", "which", "fastsync-server"], - capture_output=True, timeout=10) + try: + r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", + "localhost", "which", "fastsync-server"], + capture_output=True, timeout=10) + except FileNotFoundError: + SSH_AVAILABLE = False + return if r.returncode == 0: SSH_AVAILABLE = True return SSH_AVAILABLE = False # Try each PATH dir: create symlink, then verify with which - r = subprocess.run( - ["ssh", "-o", "BatchMode=yes", "localhost", - 'echo "$PATH"'], - capture_output=True, timeout=10, text=True) + try: + r = subprocess.run( + ["ssh", "-o", "BatchMode=yes", "localhost", + 'echo "$PATH"'], + capture_output=True, timeout=10, text=True) + except FileNotFoundError: + return if r.returncode != 0: return for d in r.stdout.strip().split(":"): @@ -655,9 +694,10 @@ def main(): parser.add_argument("--keep-data", action="store_true") parser.add_argument("--unlimited", action="store_true") parser.add_argument("--wan", action="store_true") + parser.add_argument("--full", action="store_true", help="Run full test suite with network shaping, SSH, rsync benchmarks") args = parser.parse_args() - total_bytes = generate_test_files(args.source_dir) + total_bytes = generate_test_files(args.source_dir, full=args.full) if os.path.exists(args.dest_dir): shutil.rmtree(args.dest_dir) os.makedirs(args.dest_dir, exist_ok=True) @@ -668,12 +708,12 @@ def main(): elif args.wan: profiles.append("WAN") else: - profiles.append("LAN") + profiles.append("LAN" if args.full else "Unlimited") try: all_results = [] for p in profiles: - all_results.extend(run_profile(p, args.source_dir, args.dest_dir)) + all_results.extend(run_profile(p, args.source_dir, args.dest_dir, full=args.full)) print("\n" + "=" * 130) print(f"{'RESULTS':^130}")