diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 44fca4b..714bd6e 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -34,6 +34,7 @@ static void print_usage(void) { printf(" --include Only include files matching pattern\n"); printf(" --max-size Skip files larger than n bytes\n"); printf(" --min-size Skip files smaller than n bytes\n"); + printf(" --incremental Skip files unchanged since last transfer\n"); printf(" -m Enable multithreading\n"); printf(" -s Enable chunk serialization\n"); printf(" -f Enable sendfile (TCP only, not with -c or -s)\n"); @@ -93,6 +94,8 @@ int main(int argc, char *argv[]) { config->max_size = strtoull(argv[++i], NULL, 10); } else if (strcmp(argv[i], "--min-size") == 0 && i + 1 < argc) { config->min_size = strtoull(argv[++i], NULL, 10); + } else if (strcmp(argv[i], "--incremental") == 0) { + config->use_incremental = true; } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) { config->use_compression = true; log_message(LOG_LEVEL_INFO, "Enabled Compression"); diff --git a/src/client/client_send.c b/src/client/client_send.c index 4c70204..255e834 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -32,17 +32,47 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) { data_destroy(data); } else if (config->use_sendfile && !config->use_compression) { for (int i = 0; i < chunk->element_count; i++) { - if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; + if (config->use_incremental) { + if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1; + if (!send_str(client->file_descriptor, chunk->items[i]->path)) return -1; + unsigned long long fsize = chunk->items[i]->data->size; + long long mtime = chunk->items[i]->metadata ? chunk->items[i]->metadata->mtime_sec : 0; + if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1; + if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1; + Status s; + if (!receive_status(client->file_descriptor, &s)) return -1; + if (s == STATUS_OK) continue; + if (s != STATUS_NEXT) return -1; + } else { + if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; + } if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata)) return -1; } } else { for (int i = 0; i < chunk->element_count; i++) { - if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; - if (!file_send_single_calls(chunk->items[i], client->file_descriptor, - config->use_metadata, - config->use_compression ? config->compression_level : 0)) - return -1; + if (config->use_incremental) { + if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1; + if (!send_str(client->file_descriptor, chunk->items[i]->path)) return -1; + unsigned long long fsize = chunk->items[i]->data->size; + long long mtime = chunk->items[i]->metadata ? chunk->items[i]->metadata->mtime_sec : 0; + if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1; + if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1; + Status s; + if (!receive_status(client->file_descriptor, &s)) return -1; + if (s == STATUS_OK) continue; + if (s != STATUS_NEXT) return -1; + if (!file_send_single_calls_no_path(chunk->items[i], client->file_descriptor, + config->use_metadata, + config->use_compression ? config->compression_level : 0)) + return -1; + } else { + if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; + if (!file_send_single_calls(chunk->items[i], client->file_descriptor, + config->use_metadata, + config->use_compression ? config->compression_level : 0)) + return -1; + } } } return 0; diff --git a/src/server/server.c b/src/server/server.c index a7edce5..c4e528e 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -16,12 +16,68 @@ #include #include #include +#include int receive_files(Config *config, int file_descriptor) { Status status; if (!receive_status(file_descriptor, &status)) return -1; - while (status == STATUS_NEXT || status == STATUS_CHUNK) { - if (status == STATUS_CHUNK) { + while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) { + if (status == STATUS_CHECK) { + char *check_path = receive_str(file_descriptor); + if (check_path == NULL) { send_status(file_descriptor, STATUS_ERROR); return -1; } + unsigned long long check_size; + long long check_mtime; + if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) || + !receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) { + free(check_path); + send_status(file_descriptor, STATUS_ERROR); + return -1; + } + char *full_path = path_cat(config->receive_root_directory, check_path); + struct stat st; + bool match = false; + if (full_path && stat(full_path, &st) == 0 && + (unsigned long long)st.st_size == check_size && + (long long)st.st_mtime == check_mtime) { + match = true; + } + free(full_path); + if (match) { + if (!send_status(file_descriptor, STATUS_OK)) { free(check_path); return -1; } + free(check_path); + } else { + if (!send_status(file_descriptor, STATUS_NEXT)) { free(check_path); return -1; } + File *file = file_create(check_path); + free(check_path); + if (file == NULL) { send_status(file_descriptor, STATUS_ERROR); return -1; } + if (config->use_metadata) { + file->metadata = metadata_receive(file_descriptor); + } + Data *file_data = receive_data(file_descriptor); + if (file_data == NULL) { + file_destroy(file); + send_status(file_descriptor, STATUS_ERROR); + return -1; + } + if (config->use_compression) { + Data *uncompressed = data_decompress(file_data); + data_destroy(file_data); + if (uncompressed == NULL) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return -1; } + file_data = uncompressed; + } + data_destroy(file->data); + file->data = file_data; + if (config->save_to_disk) { + char *disk_path = path_cat(config->receive_root_directory, file->path); + if (disk_path) { + to_disk(disk_path, file->data->data, file->data->size); + file_restore_metadata(disk_path, file->metadata); + free(disk_path); + } + } + file_destroy(file); + } + } else if (status == STATUS_CHUNK) { Data *chunk_data = receive_data(file_descriptor); if (chunk_data == NULL) { log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data"); diff --git a/src/shared/config.c b/src/shared/config.c index a22e489..1838d4d 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -38,6 +38,7 @@ Config *config_create(char *version, char *send_directory, config->include_count = 0; config->max_size = 0; config->min_size = 0; + config->use_incremental = false; return config; } @@ -89,6 +90,7 @@ bool config_send(int file_descriptor, Config *config) { if (!send_int(file_descriptor, (int)config->chunk_size)) return false; if (!send_int(file_descriptor, config->use_sendfile)) return false; if (!send_int(file_descriptor, config->use_delete)) return false; + if (!send_int(file_descriptor, config->use_incremental)) return false; Status status; if (!receive_status(file_descriptor, &status)) return false; if (status != STATUS_OK) { @@ -134,6 +136,8 @@ Config *config_receive(int file_descriptor) { config->use_sendfile = tmp; if (!receive_int(file_descriptor, &tmp)) goto error; config->use_delete = tmp; + if (!receive_int(file_descriptor, &tmp)) goto error; + config->use_incremental = tmp; config->show_progress = false; config->dry_run = false; config->ssh_port = 22; diff --git a/src/shared/config.h b/src/shared/config.h index b08f733..6af810b 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -32,6 +32,7 @@ typedef struct Config { int include_count; unsigned long long max_size; unsigned long long min_size; + bool use_incremental; } Config; #define PROTOCOL_VERSION "1.0.0" diff --git a/src/shared/file.c b/src/shared/file.c index 061343a..f04f2e2 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -96,6 +96,21 @@ bool file_load_data(File *file) { return true; } +bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level) { + if (compression_level > 0) { + Data *compressed_data = data_compress(file->data, compression_level); + if (compressed_data == NULL) { + log_message(LOG_LEVEL_ERROR, "Failed to compress file data"); + return false; + } + data_destroy(file->data); + file->data = compressed_data; + } + if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; + if (!send_data(file_descriptor, file->data)) return false; + return true; +} + bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) { if (compression_level > 0) { Data *compressed_data = data_compress(file->data, compression_level); diff --git a/src/shared/file.h b/src/shared/file.h index 11ec9fe..4f28d06 100644 --- a/src/shared/file.h +++ b/src/shared/file.h @@ -25,6 +25,7 @@ void file_destroy(void *item); bool file_load_data(File *file); File *file_receive(Config *config, int file_descriptor); bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level); +bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level); bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata); size_t file_content_to_buffer(File *file); FileMetadata *file_metadata_create(struct stat *stats); diff --git a/src/shared/multiprocessing.c b/src/shared/multiprocessing.c index ef99386..5e912d0 100644 --- a/src/shared/multiprocessing.c +++ b/src/shared/multiprocessing.c @@ -13,6 +13,7 @@ #include #include #include +#include #include PipelineContextSender *pipeline_context_sender_create(Config *config, @@ -126,8 +127,56 @@ int receive_thread(void *pipeline_context) { Status status; if (!receive_status(file_descriptor, &status)) return thrd_error; - while (status == STATUS_NEXT || status == STATUS_CHUNK) { - if (status == STATUS_CHUNK) { + while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) { + if (status == STATUS_CHECK) { + char *check_path = receive_str(file_descriptor); + if (check_path == NULL) return thrd_error; + unsigned long long check_size; + long long check_mtime; + if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) || + !receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) { + free(check_path); + return thrd_error; + } + char *full_path = path_cat(config->receive_root_directory, check_path); + struct stat st; + bool match = false; + if (full_path && stat(full_path, &st) == 0 && + (unsigned long long)st.st_size == check_size && + (long long)st.st_mtime == check_mtime) { + match = true; + } + free(full_path); + if (match) { + if (!send_status(file_descriptor, STATUS_OK)) { free(check_path); return thrd_error; } + free(check_path); + } else { + if (!send_status(file_descriptor, STATUS_NEXT)) { free(check_path); return thrd_error; } + File *file = file_create(check_path); + free(check_path); + if (file == NULL) { send_status(file_descriptor, STATUS_ERROR); return thrd_error; } + if (config->use_metadata) { + file->metadata = metadata_receive(file_descriptor); + } + Data *file_data = receive_data(file_descriptor); + if (file_data == NULL) { + file_destroy(file); + send_status(file_descriptor, STATUS_ERROR); + return thrd_error; + } + if (config->use_compression) { + Data *uncompressed = data_decompress(file_data); + data_destroy(file_data); + if (uncompressed == NULL) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return thrd_error; } + file_data = uncompressed; + } + data_destroy(file->data); + file->data = file_data; + queue_enqueue_multithreaded(context->queue, file, &context->mutex, + &context->condition_not_empty, + &context->condition_not_full); + } + } else if (status == STATUS_CHUNK) { receive_chunk_enqueue(file_descriptor, context); } else { File *file = file_receive(config, file_descriptor); diff --git a/src/shared/protocol.c b/src/shared/protocol.c index 4f2b82a..a95386e 100644 --- a/src/shared/protocol.c +++ b/src/shared/protocol.c @@ -111,6 +111,8 @@ static const char *status_to_string(Status status) { return "NEXT"; case STATUS_CHUNK: return "CHUNK"; + case STATUS_CHECK: + return "CHECK"; default: return "UNKNOWN"; } diff --git a/src/shared/protocol.h b/src/shared/protocol.h index 1880d24..83b19c4 100644 --- a/src/shared/protocol.h +++ b/src/shared/protocol.h @@ -6,7 +6,7 @@ #include typedef int Status; -enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST }; +enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST, STATUS_CHECK }; void io_set_fds(int read_fd, int write_fd); void io_set_bwlimit(unsigned long long bytes_per_sec); diff --git a/test.py b/test.py index e2efe31..3093ab1 100755 --- a/test.py +++ b/test.py @@ -50,7 +50,7 @@ CLIENT_CMD_PREFIX = [ BASE_CLIENT_FLAGS = ["--save-to-disk"] -TEST_CASES_FULL = [ +TEST_CASES = [ {"name": "Standard", "flags": []}, {"name": "Posix Args (no flags)", "flags": [], "posix": True}, {"name": "Standard (no metadata)", "flags": [], "use_metadata": False}, @@ -68,14 +68,7 @@ TEST_CASES_FULL = [ {"name": "Sendfile + Multithreading (-f -m)", "flags": ["-f", "-m"]}, ] -TEST_CASES_LIGHT = [ - {"name": "Standard", "flags": []}, - {"name": "Compression (-c)", "flags": ["-c"]}, - {"name": "Chunk Serialization (-s)", "flags": ["-s"]}, - {"name": "Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]}, -] - -SSH_CASES_FULL = [ +SSH_CASES = [ {"name": "SSH (localhost)", "flags": []}, {"name": "SSH Multithreading (-m)", "flags": ["-m"]}, {"name": "SSH Compression (-c)", "flags": ["-c"]}, @@ -86,17 +79,11 @@ SSH_CASES_FULL = [ {"name": "SSH Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]}, ] -SSH_CASES_LIGHT = [ - {"name": "SSH (localhost)", "flags": []}, -] - -RSYNC_CASES_FULL = [ +RSYNC_CASES = [ {"name": "rsync (archive)", "args": ["-aH"]}, {"name": "rsync (archive + compress)", "args": ["-aHz"]}, ] -RSYNC_CASES_LIGHT = [] - def netem_apply(profile): params = NETWORK_PROFILES[profile] @@ -132,12 +119,12 @@ def find_free_port(): return s.getsockname()[1] -def generate_test_files(source_dir, full=False): +def generate_test_files(source_dir): if os.path.exists(source_dir): shutil.rmtree(source_dir) os.makedirs(source_dir) - target_total = 25 * 1024 * 1024 if full else 0 + target_total = 25 * 1024 * 1024 written = 0 files = { @@ -154,15 +141,14 @@ def generate_test_files(source_dir, full=False): f.write(content) written += len(content) - if full: - os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True) - i = 0 - while written < target_total: - chunk_size = min(5 * 1024 * 1024, target_total - written) - with open(os.path.join(source_dir, f"bulk/file_{i}.dat"), "wb") as f: - f.write(random.randbytes(chunk_size)) - written += chunk_size - i += 1 + os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True) + i = 0 + while written < target_total: + chunk_size = min(5 * 1024 * 1024, target_total - written) + with open(os.path.join(source_dir, f"bulk/file_{i}.dat"), "wb") as f: + f.write(random.randbytes(chunk_size)) + written += chunk_size + i += 1 total_mb = written / (1024 * 1024) small_bytes = sum(len(c) for c in files.values()) @@ -268,24 +254,19 @@ def print_profile_header(profile_name): print(" No limits applied") -def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=None, ssh_cases=None, rsync_cases=None): +def run_profile(profile_name, source_dir, dest_dir): print_profile_header(profile_name) is_limited = profile_name != "Unlimited" client_prefix = CLIENT_CMD_PREFIX if is_limited else [] - if test_cases is None: - test_cases = TEST_CASES_FULL if full else TEST_CASES_LIGHT - if ssh_cases is None: - ssh_cases = SSH_CASES_FULL if full else SSH_CASES_LIGHT - if rsync_cases is None: - rsync_cases = RSYNC_CASES_FULL if full else RSYNC_CASES_LIGHT - try: - if is_limited and full: + if is_limited: netem_apply(profile_name) + else: + netem_reset() results = [] - for case in test_cases: + for case in TEST_CASES: flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"] if case.get("posix"): cmd = client_prefix + BASE_CLIENT_CMD + [source_dir, dest_dir] + flags @@ -300,7 +281,7 @@ def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=No results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) if SSH_AVAILABLE: - for case in ssh_cases: + for case in SSH_CASES: flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"] ssh_dest = f"localhost:{dest_dir}_ssh" cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags @@ -312,195 +293,220 @@ def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=No except Exception as e: results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - if rsync_cases: - port, conf, daemon = start_rsync_daemon(source_dir) - try: - for case in rsync_cases: - cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"] - print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, case["name"], source_dir, dest_dir, source_prefix="") - r["suite"] = profile_name - except subprocess.TimeoutExpired: - r = {"name": case["name"], "suite": profile_name, "status": "Timeout", "time": "N/A", "error": "Exceeded 120s"} - except Exception as e: - r = {"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)} - else: - if r["status"] != "Success" and client_prefix: - tmp = tempfile.mkdtemp() - try: - plain = subprocess.run(["rsync", "-aH", f"rsync://localhost:{port}/source/", f"{tmp}/"], capture_output=True, text=True, timeout=30) - if plain.returncode != 0: - errs = [l for l in (plain.stderr or "").split("\n") if l.strip()] - if errs: - r["error"] += f" | raw: {errs[-1][:150]}" - finally: - shutil.rmtree(tmp, ignore_errors=True) - results.append(r) - finally: - wait_proc(daemon) + port, conf, daemon = start_rsync_daemon(source_dir) + try: + for case in RSYNC_CASES: + cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"] + print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") try: - os.unlink(conf) - except Exception: - pass - - if full: - # Feature-specific tests for rsync-compatible flags - print("\n " + "─" * 56 + "\n Feature Tests\n " + "─" * 56) - - # Dry run (-n) — no server needed - print("\n --- Dry run (-n) ---") - flags = BASE_CLIENT_FLAGS + ["-n"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags - print(f" Running: {' '.join(cmd)}") + r = run_single_test(cmd, case["name"], source_dir, dest_dir, source_prefix="") + r["suite"] = profile_name + except subprocess.TimeoutExpired: + r = {"name": case["name"], "suite": profile_name, "status": "Timeout", "time": "N/A", "error": "Exceeded 120s"} + except Exception as e: + r = {"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)} + else: + if r["status"] != "Success" and client_prefix: + tmp = tempfile.mkdtemp() + try: + plain = subprocess.run(["rsync", "-aH", f"rsync://localhost:{port}/source/", f"{tmp}/"], capture_output=True, text=True, timeout=30) + if plain.returncode != 0: + errs = [l for l in (plain.stderr or "").split("\n") if l.strip()] + if errs: + r["error"] += f" | raw: {errs[-1][:150]}" + finally: + shutil.rmtree(tmp, ignore_errors=True) + results.append(r) + finally: + wait_proc(daemon) try: - start = time.monotonic() - result = subprocess.run(cmd, text=True, capture_output=True) - duration = time.monotonic() - start - r = {"name": "Dry run (-n)", "suite": profile_name} - if result.returncode == 0 and "Dry run:" in result.stdout: + os.unlink(conf) + except Exception: + pass + + # Feature-specific tests for rsync-compatible flags + print("\n " + "─" * 56 + "\n Feature Tests\n " + "─" * 56) + + # Dry run (-n) — no server needed + print("\n --- Dry run (-n) ---") + flags = BASE_CLIENT_FLAGS + ["-n"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + print(f" Running: {' '.join(cmd)}") + try: + start = time.monotonic() + result = subprocess.run(cmd, text=True, capture_output=True) + duration = time.monotonic() - start + r = {"name": "Dry run (-n)", "suite": profile_name} + if result.returncode == 0 and "Dry run:" in result.stdout: + r["status"] = "Success" + r["time"] = f"{duration:.4f}s" + r["error"] = "" + else: + r["status"] = "Failed" + r["time"] = "N/A" + r["error"] = f"Exit {result.returncode}: {(result.stderr or result.stdout)[:100]}" + results.append(r) + except Exception as e: + results.append({"name": "Dry run (-n)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Archive mode (-a) + feature_flags = BASE_CLIENT_FLAGS + ["-a"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Archive mode (-a) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Archive mode (-a)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Archive mode (-a)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Exclude (--exclude small.txt) + feature_flags = BASE_CLIENT_FLAGS + ["--exclude", "small.txt"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Exclude (--exclude small.txt) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Exclude (--exclude small.txt)", source_dir, dest_dir, + expected_missing=["small.txt"]) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Exclude (--exclude small.txt)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Progress (--progress) + feature_flags = BASE_CLIENT_FLAGS + ["--progress"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Progress (--progress) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Progress (--progress)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Bandwidth limit (--bwlimit 10240 = 10 MB/s) + feature_flags = BASE_CLIENT_FLAGS + ["--bwlimit", "10240"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Bandwidth limit (--bwlimit 10240 KB/s) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Bandwidth limit (--bwlimit 10240)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Bandwidth limit (--bwlimit 10240)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Incremental sync (--incremental) — first sync, then second sync should skip all + print(f"\n --- Incremental (--incremental) ---") + try: + flags = BASE_CLIENT_FLAGS + ["-M"] + srv = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + r1 = subprocess.run(first_cmd, text=True, capture_output=True) + wait_proc(srv) + if r1.returncode != 0: + raise RuntimeError(f"First sync failed: {r1.stderr[:100]}") + srv2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--incremental"] + start = time.monotonic() + r2 = subprocess.run(second_cmd, text=True, capture_output=True, timeout=30) + duration = time.monotonic() - start + wait_proc(srv2) + r = {"name": "Incremental (--incremental)", "suite": profile_name, + "status": "Success" if r2.returncode == 0 else "Failed", + "time": f"{duration:.4f}s" if r2.returncode == 0 else "N/A", + "error": "" if r2.returncode == 0 else f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}"} + results.append(r) + except Exception as e: + results.append({"name": "Incremental (--incremental)", "suite": profile_name, + "status": "Error", "time": "N/A", "error": str(e)}) + + # Chunk size (--chunk-size 5242880) + feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Chunk size (--chunk-size 5242880) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Chunk size (--chunk-size 5242880)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Chunk size (--chunk-size 5242880)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Delete (--delete) — pre-populate dest, add extra files, then sync with --delete + # Note: server handles one client per launch, so we restart between syncs + print(f"\n --- Delete (--delete) ---") + try: + flags = BASE_CLIENT_FLAGS + ["-M"] + # First sync (no delete) to populate dest + s1 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + r1 = subprocess.run(first_cmd, text=True, capture_output=True) + wait_proc(s1) + if r1.returncode != 0: + raise RuntimeError(f"First sync failed: {r1.stderr[:100]}") + # Add extra files to received dir + received = os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep)) + extra_path = os.path.join(received, "extra_file.txt") + with open(extra_path, "w") as f: + f.write("should be deleted") + extra_dir = os.path.join(received, "extra_dir") + os.makedirs(extra_dir, exist_ok=True) + with open(os.path.join(extra_dir, "nested.txt"), "w") as f: + f.write("nested extra") + # Second sync with --delete (fresh server) + s2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--delete"] + start = time.monotonic() + r2 = subprocess.run(second_cmd, text=True, capture_output=True) + duration = time.monotonic() - start + wait_proc(s2) + r = {"name": "Delete (--delete)", "suite": profile_name} + if r2.returncode == 0 and not os.path.exists(extra_path) and not os.path.exists(extra_dir): + mismatches, missing = verify_transfer(source_dir, received) + if not mismatches and not missing: r["status"] = "Success" r["time"] = f"{duration:.4f}s" r["error"] = "" else: r["status"] = "Failed" r["time"] = "N/A" - r["error"] = f"Exit {result.returncode}: {(result.stderr or result.stdout)[:100]}" - results.append(r) - except Exception as e: - results.append({"name": "Dry run (-n)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + r["error"] = f"post-delete verify: mismatches={len(mismatches)}, missing={len(missing)}" + else: + r["status"] = "Failed" + r["time"] = "N/A" + errs = [] + if r2.returncode != 0: + errs.append(f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}") + if os.path.exists(extra_path): + errs.append("extra_file.txt remains") + if os.path.exists(extra_dir): + errs.append("extra_dir remains") + r["error"] = " | ".join(errs) + results.append(r) + except Exception as e: + results.append({"name": "Delete (--delete)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - # Archive mode (-a) - feature_flags = BASE_CLIENT_FLAGS + ["-a"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Archive mode (-a) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Archive mode (-a)", source_dir, dest_dir) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Archive mode (-a)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Exclude (--exclude small.txt) - feature_flags = BASE_CLIENT_FLAGS + ["--exclude", "small.txt"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Exclude (--exclude small.txt) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Exclude (--exclude small.txt)", source_dir, dest_dir, - expected_missing=["small.txt"]) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Exclude (--exclude small.txt)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Progress (--progress) - feature_flags = BASE_CLIENT_FLAGS + ["--progress"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Progress (--progress) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Progress (--progress)", source_dir, dest_dir) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Bandwidth limit (--bwlimit 10240 = 10 MB/s) - feature_flags = BASE_CLIENT_FLAGS + ["--bwlimit", "10240"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Bandwidth limit (--bwlimit 10240 KB/s) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Bandwidth limit (--bwlimit 10240)", source_dir, dest_dir) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Bandwidth limit (--bwlimit 10240)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Chunk size (--chunk-size 5242880) - feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"] - cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags - print(f"\n --- Chunk size (--chunk-size 5242880) ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, "Chunk size (--chunk-size 5242880)", source_dir, dest_dir) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": "Chunk size (--chunk-size 5242880)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # Delete (--delete) — pre-populate dest, add extra files, then sync with --delete - # Note: server handles one client per launch, so we restart between syncs - print(f"\n --- Delete (--delete) ---") - try: - flags = BASE_CLIENT_FLAGS + ["-M"] - # First sync (no delete) to populate dest - s1 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) - time.sleep(0.5) - first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags - r1 = subprocess.run(first_cmd, text=True, capture_output=True) - wait_proc(s1) - if r1.returncode != 0: - raise RuntimeError(f"First sync failed: {r1.stderr[:100]}") - # Add extra files to received dir - received = os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep)) - extra_path = os.path.join(received, "extra_file.txt") - with open(extra_path, "w") as f: - f.write("should be deleted") - extra_dir = os.path.join(received, "extra_dir") - os.makedirs(extra_dir, exist_ok=True) - with open(os.path.join(extra_dir, "nested.txt"), "w") as f: - f.write("nested extra") - # Second sync with --delete (fresh server) - s2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) - time.sleep(0.5) - second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--delete"] - start = time.monotonic() - r2 = subprocess.run(second_cmd, text=True, capture_output=True) - duration = time.monotonic() - start - wait_proc(s2) - r = {"name": "Delete (--delete)", "suite": profile_name} - if r2.returncode == 0 and not os.path.exists(extra_path) and not os.path.exists(extra_dir): - mismatches, missing = verify_transfer(source_dir, received) - if not mismatches and not missing: - r["status"] = "Success" - r["time"] = f"{duration:.4f}s" - r["error"] = "" - else: - r["status"] = "Failed" - r["time"] = "N/A" - r["error"] = f"post-delete verify: mismatches={len(mismatches)}, missing={len(missing)}" - else: - r["status"] = "Failed" - r["time"] = "N/A" - errs = [] - if r2.returncode != 0: - errs.append(f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}") - if os.path.exists(extra_path): - errs.append("extra_file.txt remains") - if os.path.exists(extra_dir): - errs.append("extra_dir remains") - r["error"] = " | ".join(errs) - results.append(r) - except Exception as e: - results.append({"name": "Delete (--delete)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) - - # SSH feature tests - if SSH_AVAILABLE: - ssh_dest = f"localhost:{dest_dir}_ssh" - ssh_feature_cases = [ - {"name": "SSH Archive (-a)", "flags": ["-a"]}, - {"name": "SSH Exclude (--exclude small.txt)", "flags": ["--exclude", "small.txt"], "expected_missing": ["small.txt"]}, - ] - for case in ssh_feature_cases: - flags = BASE_CLIENT_FLAGS + case["flags"] - cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags - print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") - try: - r = run_single_test(cmd, case["name"], source_dir, f"{dest_dir}_ssh", - no_server=True, - expected_missing=case.get("expected_missing")) - r["suite"] = profile_name - results.append(r) - except Exception as e: - results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + # SSH feature tests + if SSH_AVAILABLE: + ssh_dest = f"localhost:{dest_dir}_ssh" + ssh_feature_cases = [ + {"name": "SSH Archive (-a)", "flags": ["-a"]}, + {"name": "SSH Exclude (--exclude small.txt)", "flags": ["--exclude", "small.txt"], "expected_missing": ["small.txt"]}, + ] + for case in ssh_feature_cases: + flags = BASE_CLIENT_FLAGS + case["flags"] + cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags + print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, case["name"], source_dir, f"{dest_dir}_ssh", + no_server=True, + expected_missing=case.get("expected_missing")) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) except (subprocess.CalledProcessError, RuntimeError) as e: print(f" Error: {e}") @@ -567,26 +573,19 @@ def check_ssh_localhost(): build_dir = os.path.abspath("build") server_path = os.path.join(build_dir, "server") - try: - r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", - "localhost", "which", "fastsync-server"], - capture_output=True, timeout=10) - except FileNotFoundError: - SSH_AVAILABLE = False - return + r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", + "localhost", "which", "fastsync-server"], + capture_output=True, timeout=10) if r.returncode == 0: SSH_AVAILABLE = True return SSH_AVAILABLE = False # Try each PATH dir: create symlink, then verify with which - try: - r = subprocess.run( - ["ssh", "-o", "BatchMode=yes", "localhost", - 'echo "$PATH"'], - capture_output=True, timeout=10, text=True) - except FileNotFoundError: - return + r = subprocess.run( + ["ssh", "-o", "BatchMode=yes", "localhost", + 'echo "$PATH"'], + capture_output=True, timeout=10, text=True) if r.returncode != 0: return for d in r.stdout.strip().split(":"): @@ -667,10 +666,9 @@ def main(): parser.add_argument("--keep-data", action="store_true") parser.add_argument("--unlimited", action="store_true") parser.add_argument("--wan", action="store_true") - parser.add_argument("--full", action="store_true", help="Run full test suite with network shaping, SSH, rsync benchmarks") args = parser.parse_args() - total_bytes = generate_test_files(args.source_dir, full=args.full) + total_bytes = generate_test_files(args.source_dir) if os.path.exists(args.dest_dir): shutil.rmtree(args.dest_dir) os.makedirs(args.dest_dir, exist_ok=True) @@ -681,12 +679,12 @@ def main(): elif args.wan: profiles.append("WAN") else: - profiles.append("LAN" if args.full else "Unlimited") + profiles.append("LAN") try: all_results = [] for p in profiles: - all_results.extend(run_profile(p, args.source_dir, args.dest_dir, full=args.full)) + all_results.extend(run_profile(p, args.source_dir, args.dest_dir)) print("\n" + "=" * 130) print(f"{'RESULTS':^130}")