Incremental sync: --incremental flag to skip unchanged files

- New STATUS_CHECK protocol status (value 6)
- Client sends path + size + mtime; server replies OK (skip) or NEXT (send)
- config.h/c: use_incremental field sent/received over wire
- file.c/h: file_send_single_calls_no_path helper for incremental path
- client_cli.c: --incremental flag
- client_send.c: per-file check before send in both single and chunk paths
- server.c: STATUS_CHECK handling in receive_files()
- multiprocessing.c: STATUS_CHECK handling in receive_thread()
- test.py: incremental sync test case
This commit is contained in:
2026-07-16 15:38:48 +02:00
parent a3acf4b8b6
commit 1905369c9f
11 changed files with 406 additions and 247 deletions
+3
View File
@@ -34,6 +34,7 @@ static void print_usage(void) {
printf(" --include <pattern> Only include files matching pattern\n"); printf(" --include <pattern> Only include files matching pattern\n");
printf(" --max-size <n> Skip files larger than n bytes\n"); printf(" --max-size <n> Skip files larger than n bytes\n");
printf(" --min-size <n> Skip files smaller than n bytes\n"); printf(" --min-size <n> Skip files smaller than n bytes\n");
printf(" --incremental Skip files unchanged since last transfer\n");
printf(" -m Enable multithreading\n"); printf(" -m Enable multithreading\n");
printf(" -s Enable chunk serialization\n"); printf(" -s Enable chunk serialization\n");
printf(" -f Enable sendfile (TCP only, not with -c or -s)\n"); printf(" -f Enable sendfile (TCP only, not with -c or -s)\n");
@@ -93,6 +94,8 @@ int main(int argc, char *argv[]) {
config->max_size = strtoull(argv[++i], NULL, 10); config->max_size = strtoull(argv[++i], NULL, 10);
} else if (strcmp(argv[i], "--min-size") == 0 && i + 1 < argc) { } else if (strcmp(argv[i], "--min-size") == 0 && i + 1 < argc) {
config->min_size = strtoull(argv[++i], NULL, 10); config->min_size = strtoull(argv[++i], NULL, 10);
} else if (strcmp(argv[i], "--incremental") == 0) {
config->use_incremental = true;
} else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) { } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) {
config->use_compression = true; config->use_compression = true;
log_message(LOG_LEVEL_INFO, "Enabled Compression"); log_message(LOG_LEVEL_INFO, "Enabled Compression");
+36 -6
View File
@@ -32,17 +32,47 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) {
data_destroy(data); data_destroy(data);
} else if (config->use_sendfile && !config->use_compression) { } else if (config->use_sendfile && !config->use_compression) {
for (int i = 0; i < chunk->element_count; i++) { for (int i = 0; i < chunk->element_count; i++) {
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; if (config->use_incremental) {
if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1;
if (!send_str(client->file_descriptor, chunk->items[i]->path)) return -1;
unsigned long long fsize = chunk->items[i]->data->size;
long long mtime = chunk->items[i]->metadata ? chunk->items[i]->metadata->mtime_sec : 0;
if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1;
if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1;
Status s;
if (!receive_status(client->file_descriptor, &s)) return -1;
if (s == STATUS_OK) continue;
if (s != STATUS_NEXT) return -1;
} else {
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
}
if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata)) if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata))
return -1; return -1;
} }
} else { } else {
for (int i = 0; i < chunk->element_count; i++) { for (int i = 0; i < chunk->element_count; i++) {
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; if (config->use_incremental) {
if (!file_send_single_calls(chunk->items[i], client->file_descriptor, if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1;
config->use_metadata, if (!send_str(client->file_descriptor, chunk->items[i]->path)) return -1;
config->use_compression ? config->compression_level : 0)) unsigned long long fsize = chunk->items[i]->data->size;
return -1; long long mtime = chunk->items[i]->metadata ? chunk->items[i]->metadata->mtime_sec : 0;
if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1;
if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1;
Status s;
if (!receive_status(client->file_descriptor, &s)) return -1;
if (s == STATUS_OK) continue;
if (s != STATUS_NEXT) return -1;
if (!file_send_single_calls_no_path(chunk->items[i], client->file_descriptor,
config->use_metadata,
config->use_compression ? config->compression_level : 0))
return -1;
} else {
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
if (!file_send_single_calls(chunk->items[i], client->file_descriptor,
config->use_metadata,
config->use_compression ? config->compression_level : 0))
return -1;
}
} }
} }
return 0; return 0;
+58 -2
View File
@@ -16,12 +16,68 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <sys/stat.h>
int receive_files(Config *config, int file_descriptor) { int receive_files(Config *config, int file_descriptor) {
Status status; Status status;
if (!receive_status(file_descriptor, &status)) return -1; if (!receive_status(file_descriptor, &status)) return -1;
while (status == STATUS_NEXT || status == STATUS_CHUNK) { while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) {
if (status == STATUS_CHUNK) { if (status == STATUS_CHECK) {
char *check_path = receive_str(file_descriptor);
if (check_path == NULL) { send_status(file_descriptor, STATUS_ERROR); return -1; }
unsigned long long check_size;
long long check_mtime;
if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) ||
!receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) {
free(check_path);
send_status(file_descriptor, STATUS_ERROR);
return -1;
}
char *full_path = path_cat(config->receive_root_directory, check_path);
struct stat st;
bool match = false;
if (full_path && stat(full_path, &st) == 0 &&
(unsigned long long)st.st_size == check_size &&
(long long)st.st_mtime == check_mtime) {
match = true;
}
free(full_path);
if (match) {
if (!send_status(file_descriptor, STATUS_OK)) { free(check_path); return -1; }
free(check_path);
} else {
if (!send_status(file_descriptor, STATUS_NEXT)) { free(check_path); return -1; }
File *file = file_create(check_path);
free(check_path);
if (file == NULL) { send_status(file_descriptor, STATUS_ERROR); return -1; }
if (config->use_metadata) {
file->metadata = metadata_receive(file_descriptor);
}
Data *file_data = receive_data(file_descriptor);
if (file_data == NULL) {
file_destroy(file);
send_status(file_descriptor, STATUS_ERROR);
return -1;
}
if (config->use_compression) {
Data *uncompressed = data_decompress(file_data);
data_destroy(file_data);
if (uncompressed == NULL) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return -1; }
file_data = uncompressed;
}
data_destroy(file->data);
file->data = file_data;
if (config->save_to_disk) {
char *disk_path = path_cat(config->receive_root_directory, file->path);
if (disk_path) {
to_disk(disk_path, file->data->data, file->data->size);
file_restore_metadata(disk_path, file->metadata);
free(disk_path);
}
}
file_destroy(file);
}
} else if (status == STATUS_CHUNK) {
Data *chunk_data = receive_data(file_descriptor); Data *chunk_data = receive_data(file_descriptor);
if (chunk_data == NULL) { if (chunk_data == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data"); log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data");
+4
View File
@@ -38,6 +38,7 @@ Config *config_create(char *version, char *send_directory,
config->include_count = 0; config->include_count = 0;
config->max_size = 0; config->max_size = 0;
config->min_size = 0; config->min_size = 0;
config->use_incremental = false;
return config; return config;
} }
@@ -89,6 +90,7 @@ bool config_send(int file_descriptor, Config *config) {
if (!send_int(file_descriptor, (int)config->chunk_size)) return false; if (!send_int(file_descriptor, (int)config->chunk_size)) return false;
if (!send_int(file_descriptor, config->use_sendfile)) return false; if (!send_int(file_descriptor, config->use_sendfile)) return false;
if (!send_int(file_descriptor, config->use_delete)) return false; if (!send_int(file_descriptor, config->use_delete)) return false;
if (!send_int(file_descriptor, config->use_incremental)) return false;
Status status; Status status;
if (!receive_status(file_descriptor, &status)) return false; if (!receive_status(file_descriptor, &status)) return false;
if (status != STATUS_OK) { if (status != STATUS_OK) {
@@ -134,6 +136,8 @@ Config *config_receive(int file_descriptor) {
config->use_sendfile = tmp; config->use_sendfile = tmp;
if (!receive_int(file_descriptor, &tmp)) goto error; if (!receive_int(file_descriptor, &tmp)) goto error;
config->use_delete = tmp; config->use_delete = tmp;
if (!receive_int(file_descriptor, &tmp)) goto error;
config->use_incremental = tmp;
config->show_progress = false; config->show_progress = false;
config->dry_run = false; config->dry_run = false;
config->ssh_port = 22; config->ssh_port = 22;
+1
View File
@@ -32,6 +32,7 @@ typedef struct Config {
int include_count; int include_count;
unsigned long long max_size; unsigned long long max_size;
unsigned long long min_size; unsigned long long min_size;
bool use_incremental;
} Config; } Config;
#define PROTOCOL_VERSION "1.0.0" #define PROTOCOL_VERSION "1.0.0"
+15
View File
@@ -96,6 +96,21 @@ bool file_load_data(File *file) {
return true; return true;
} }
bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level) {
if (compression_level > 0) {
Data *compressed_data = data_compress(file->data, compression_level);
if (compressed_data == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
return false;
}
data_destroy(file->data);
file->data = compressed_data;
}
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false;
if (!send_data(file_descriptor, file->data)) return false;
return true;
}
bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) { bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) {
if (compression_level > 0) { if (compression_level > 0) {
Data *compressed_data = data_compress(file->data, compression_level); Data *compressed_data = data_compress(file->data, compression_level);
+1
View File
@@ -25,6 +25,7 @@ void file_destroy(void *item);
bool file_load_data(File *file); bool file_load_data(File *file);
File *file_receive(Config *config, int file_descriptor); File *file_receive(Config *config, int file_descriptor);
bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level); bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level);
bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level);
bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata); bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata);
size_t file_content_to_buffer(File *file); size_t file_content_to_buffer(File *file);
FileMetadata *file_metadata_create(struct stat *stats); FileMetadata *file_metadata_create(struct stat *stats);
+51 -2
View File
@@ -13,6 +13,7 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <sys/stat.h>
#include <threads.h> #include <threads.h>
PipelineContextSender *pipeline_context_sender_create(Config *config, PipelineContextSender *pipeline_context_sender_create(Config *config,
@@ -126,8 +127,56 @@ int receive_thread(void *pipeline_context) {
Status status; Status status;
if (!receive_status(file_descriptor, &status)) return thrd_error; if (!receive_status(file_descriptor, &status)) return thrd_error;
while (status == STATUS_NEXT || status == STATUS_CHUNK) { while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) {
if (status == STATUS_CHUNK) { if (status == STATUS_CHECK) {
char *check_path = receive_str(file_descriptor);
if (check_path == NULL) return thrd_error;
unsigned long long check_size;
long long check_mtime;
if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) ||
!receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) {
free(check_path);
return thrd_error;
}
char *full_path = path_cat(config->receive_root_directory, check_path);
struct stat st;
bool match = false;
if (full_path && stat(full_path, &st) == 0 &&
(unsigned long long)st.st_size == check_size &&
(long long)st.st_mtime == check_mtime) {
match = true;
}
free(full_path);
if (match) {
if (!send_status(file_descriptor, STATUS_OK)) { free(check_path); return thrd_error; }
free(check_path);
} else {
if (!send_status(file_descriptor, STATUS_NEXT)) { free(check_path); return thrd_error; }
File *file = file_create(check_path);
free(check_path);
if (file == NULL) { send_status(file_descriptor, STATUS_ERROR); return thrd_error; }
if (config->use_metadata) {
file->metadata = metadata_receive(file_descriptor);
}
Data *file_data = receive_data(file_descriptor);
if (file_data == NULL) {
file_destroy(file);
send_status(file_descriptor, STATUS_ERROR);
return thrd_error;
}
if (config->use_compression) {
Data *uncompressed = data_decompress(file_data);
data_destroy(file_data);
if (uncompressed == NULL) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return thrd_error; }
file_data = uncompressed;
}
data_destroy(file->data);
file->data = file_data;
queue_enqueue_multithreaded(context->queue, file, &context->mutex,
&context->condition_not_empty,
&context->condition_not_full);
}
} else if (status == STATUS_CHUNK) {
receive_chunk_enqueue(file_descriptor, context); receive_chunk_enqueue(file_descriptor, context);
} else { } else {
File *file = file_receive(config, file_descriptor); File *file = file_receive(config, file_descriptor);
+2
View File
@@ -111,6 +111,8 @@ static const char *status_to_string(Status status) {
return "NEXT"; return "NEXT";
case STATUS_CHUNK: case STATUS_CHUNK:
return "CHUNK"; return "CHUNK";
case STATUS_CHECK:
return "CHECK";
default: default:
return "UNKNOWN"; return "UNKNOWN";
} }
+1 -1
View File
@@ -6,7 +6,7 @@
#include <stddef.h> #include <stddef.h>
typedef int Status; typedef int Status;
enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST }; enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST, STATUS_CHECK };
void io_set_fds(int read_fd, int write_fd); void io_set_fds(int read_fd, int write_fd);
void io_set_bwlimit(unsigned long long bytes_per_sec); void io_set_bwlimit(unsigned long long bytes_per_sec);
+234 -236
View File
@@ -50,7 +50,7 @@ CLIENT_CMD_PREFIX = [
BASE_CLIENT_FLAGS = ["--save-to-disk"] BASE_CLIENT_FLAGS = ["--save-to-disk"]
TEST_CASES_FULL = [ TEST_CASES = [
{"name": "Standard", "flags": []}, {"name": "Standard", "flags": []},
{"name": "Posix Args (no flags)", "flags": [], "posix": True}, {"name": "Posix Args (no flags)", "flags": [], "posix": True},
{"name": "Standard (no metadata)", "flags": [], "use_metadata": False}, {"name": "Standard (no metadata)", "flags": [], "use_metadata": False},
@@ -68,14 +68,7 @@ TEST_CASES_FULL = [
{"name": "Sendfile + Multithreading (-f -m)", "flags": ["-f", "-m"]}, {"name": "Sendfile + Multithreading (-f -m)", "flags": ["-f", "-m"]},
] ]
TEST_CASES_LIGHT = [ SSH_CASES = [
{"name": "Standard", "flags": []},
{"name": "Compression (-c)", "flags": ["-c"]},
{"name": "Chunk Serialization (-s)", "flags": ["-s"]},
{"name": "Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]},
]
SSH_CASES_FULL = [
{"name": "SSH (localhost)", "flags": []}, {"name": "SSH (localhost)", "flags": []},
{"name": "SSH Multithreading (-m)", "flags": ["-m"]}, {"name": "SSH Multithreading (-m)", "flags": ["-m"]},
{"name": "SSH Compression (-c)", "flags": ["-c"]}, {"name": "SSH Compression (-c)", "flags": ["-c"]},
@@ -86,17 +79,11 @@ SSH_CASES_FULL = [
{"name": "SSH Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]}, {"name": "SSH Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]},
] ]
SSH_CASES_LIGHT = [ RSYNC_CASES = [
{"name": "SSH (localhost)", "flags": []},
]
RSYNC_CASES_FULL = [
{"name": "rsync (archive)", "args": ["-aH"]}, {"name": "rsync (archive)", "args": ["-aH"]},
{"name": "rsync (archive + compress)", "args": ["-aHz"]}, {"name": "rsync (archive + compress)", "args": ["-aHz"]},
] ]
RSYNC_CASES_LIGHT = []
def netem_apply(profile): def netem_apply(profile):
params = NETWORK_PROFILES[profile] params = NETWORK_PROFILES[profile]
@@ -132,12 +119,12 @@ def find_free_port():
return s.getsockname()[1] return s.getsockname()[1]
def generate_test_files(source_dir, full=False): def generate_test_files(source_dir):
if os.path.exists(source_dir): if os.path.exists(source_dir):
shutil.rmtree(source_dir) shutil.rmtree(source_dir)
os.makedirs(source_dir) os.makedirs(source_dir)
target_total = 25 * 1024 * 1024 if full else 0 target_total = 25 * 1024 * 1024
written = 0 written = 0
files = { files = {
@@ -154,15 +141,14 @@ def generate_test_files(source_dir, full=False):
f.write(content) f.write(content)
written += len(content) written += len(content)
if full: os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True)
os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True) i = 0
i = 0 while written < target_total:
while written < target_total: chunk_size = min(5 * 1024 * 1024, target_total - written)
chunk_size = min(5 * 1024 * 1024, target_total - written) with open(os.path.join(source_dir, f"bulk/file_{i}.dat"), "wb") as f:
with open(os.path.join(source_dir, f"bulk/file_{i}.dat"), "wb") as f: f.write(random.randbytes(chunk_size))
f.write(random.randbytes(chunk_size)) written += chunk_size
written += chunk_size i += 1
i += 1
total_mb = written / (1024 * 1024) total_mb = written / (1024 * 1024)
small_bytes = sum(len(c) for c in files.values()) small_bytes = sum(len(c) for c in files.values())
@@ -268,24 +254,19 @@ def print_profile_header(profile_name):
print(" No limits applied") print(" No limits applied")
def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=None, ssh_cases=None, rsync_cases=None): def run_profile(profile_name, source_dir, dest_dir):
print_profile_header(profile_name) print_profile_header(profile_name)
is_limited = profile_name != "Unlimited" is_limited = profile_name != "Unlimited"
client_prefix = CLIENT_CMD_PREFIX if is_limited else [] client_prefix = CLIENT_CMD_PREFIX if is_limited else []
if test_cases is None:
test_cases = TEST_CASES_FULL if full else TEST_CASES_LIGHT
if ssh_cases is None:
ssh_cases = SSH_CASES_FULL if full else SSH_CASES_LIGHT
if rsync_cases is None:
rsync_cases = RSYNC_CASES_FULL if full else RSYNC_CASES_LIGHT
try: try:
if is_limited and full: if is_limited:
netem_apply(profile_name) netem_apply(profile_name)
else:
netem_reset()
results = [] results = []
for case in test_cases: for case in TEST_CASES:
flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"] flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"]
if case.get("posix"): if case.get("posix"):
cmd = client_prefix + BASE_CLIENT_CMD + [source_dir, dest_dir] + flags cmd = client_prefix + BASE_CLIENT_CMD + [source_dir, dest_dir] + flags
@@ -300,7 +281,7 @@ def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=No
results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
if SSH_AVAILABLE: if SSH_AVAILABLE:
for case in ssh_cases: for case in SSH_CASES:
flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"] flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"]
ssh_dest = f"localhost:{dest_dir}_ssh" ssh_dest = f"localhost:{dest_dir}_ssh"
cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags
@@ -312,195 +293,220 @@ def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=No
except Exception as e: except Exception as e:
results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
if rsync_cases: port, conf, daemon = start_rsync_daemon(source_dir)
port, conf, daemon = start_rsync_daemon(source_dir) try:
try: for case in RSYNC_CASES:
for case in rsync_cases: cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"]
cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"] print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}")
print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, case["name"], source_dir, dest_dir, source_prefix="")
r["suite"] = profile_name
except subprocess.TimeoutExpired:
r = {"name": case["name"], "suite": profile_name, "status": "Timeout", "time": "N/A", "error": "Exceeded 120s"}
except Exception as e:
r = {"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}
else:
if r["status"] != "Success" and client_prefix:
tmp = tempfile.mkdtemp()
try:
plain = subprocess.run(["rsync", "-aH", f"rsync://localhost:{port}/source/", f"{tmp}/"], capture_output=True, text=True, timeout=30)
if plain.returncode != 0:
errs = [l for l in (plain.stderr or "").split("\n") if l.strip()]
if errs:
r["error"] += f" | raw: {errs[-1][:150]}"
finally:
shutil.rmtree(tmp, ignore_errors=True)
results.append(r)
finally:
wait_proc(daemon)
try: try:
os.unlink(conf) r = run_single_test(cmd, case["name"], source_dir, dest_dir, source_prefix="")
except Exception: r["suite"] = profile_name
pass except subprocess.TimeoutExpired:
r = {"name": case["name"], "suite": profile_name, "status": "Timeout", "time": "N/A", "error": "Exceeded 120s"}
if full: except Exception as e:
# Feature-specific tests for rsync-compatible flags r = {"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}
print("\n " + "" * 56 + "\n Feature Tests\n " + "" * 56) else:
if r["status"] != "Success" and client_prefix:
# Dry run (-n) — no server needed tmp = tempfile.mkdtemp()
print("\n --- Dry run (-n) ---") try:
flags = BASE_CLIENT_FLAGS + ["-n"] plain = subprocess.run(["rsync", "-aH", f"rsync://localhost:{port}/source/", f"{tmp}/"], capture_output=True, text=True, timeout=30)
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags if plain.returncode != 0:
print(f" Running: {' '.join(cmd)}") errs = [l for l in (plain.stderr or "").split("\n") if l.strip()]
if errs:
r["error"] += f" | raw: {errs[-1][:150]}"
finally:
shutil.rmtree(tmp, ignore_errors=True)
results.append(r)
finally:
wait_proc(daemon)
try: try:
start = time.monotonic() os.unlink(conf)
result = subprocess.run(cmd, text=True, capture_output=True) except Exception:
duration = time.monotonic() - start pass
r = {"name": "Dry run (-n)", "suite": profile_name}
if result.returncode == 0 and "Dry run:" in result.stdout: # Feature-specific tests for rsync-compatible flags
print("\n " + "" * 56 + "\n Feature Tests\n " + "" * 56)
# Dry run (-n) — no server needed
print("\n --- Dry run (-n) ---")
flags = BASE_CLIENT_FLAGS + ["-n"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
print(f" Running: {' '.join(cmd)}")
try:
start = time.monotonic()
result = subprocess.run(cmd, text=True, capture_output=True)
duration = time.monotonic() - start
r = {"name": "Dry run (-n)", "suite": profile_name}
if result.returncode == 0 and "Dry run:" in result.stdout:
r["status"] = "Success"
r["time"] = f"{duration:.4f}s"
r["error"] = ""
else:
r["status"] = "Failed"
r["time"] = "N/A"
r["error"] = f"Exit {result.returncode}: {(result.stderr or result.stdout)[:100]}"
results.append(r)
except Exception as e:
results.append({"name": "Dry run (-n)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Archive mode (-a)
feature_flags = BASE_CLIENT_FLAGS + ["-a"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Archive mode (-a) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Archive mode (-a)", source_dir, dest_dir)
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Archive mode (-a)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Exclude (--exclude small.txt)
feature_flags = BASE_CLIENT_FLAGS + ["--exclude", "small.txt"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Exclude (--exclude small.txt) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Exclude (--exclude small.txt)", source_dir, dest_dir,
expected_missing=["small.txt"])
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Exclude (--exclude small.txt)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Progress (--progress)
feature_flags = BASE_CLIENT_FLAGS + ["--progress"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Progress (--progress) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Progress (--progress)", source_dir, dest_dir)
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Bandwidth limit (--bwlimit 10240 = 10 MB/s)
feature_flags = BASE_CLIENT_FLAGS + ["--bwlimit", "10240"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Bandwidth limit (--bwlimit 10240 KB/s) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Bandwidth limit (--bwlimit 10240)", source_dir, dest_dir)
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Bandwidth limit (--bwlimit 10240)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Incremental sync (--incremental) — first sync, then second sync should skip all
print(f"\n --- Incremental (--incremental) ---")
try:
flags = BASE_CLIENT_FLAGS + ["-M"]
srv = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
time.sleep(0.5)
first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
r1 = subprocess.run(first_cmd, text=True, capture_output=True)
wait_proc(srv)
if r1.returncode != 0:
raise RuntimeError(f"First sync failed: {r1.stderr[:100]}")
srv2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
time.sleep(0.5)
second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--incremental"]
start = time.monotonic()
r2 = subprocess.run(second_cmd, text=True, capture_output=True, timeout=30)
duration = time.monotonic() - start
wait_proc(srv2)
r = {"name": "Incremental (--incremental)", "suite": profile_name,
"status": "Success" if r2.returncode == 0 else "Failed",
"time": f"{duration:.4f}s" if r2.returncode == 0 else "N/A",
"error": "" if r2.returncode == 0 else f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}"}
results.append(r)
except Exception as e:
results.append({"name": "Incremental (--incremental)", "suite": profile_name,
"status": "Error", "time": "N/A", "error": str(e)})
# Chunk size (--chunk-size 5242880)
feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Chunk size (--chunk-size 5242880) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Chunk size (--chunk-size 5242880)", source_dir, dest_dir)
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Chunk size (--chunk-size 5242880)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Delete (--delete) — pre-populate dest, add extra files, then sync with --delete
# Note: server handles one client per launch, so we restart between syncs
print(f"\n --- Delete (--delete) ---")
try:
flags = BASE_CLIENT_FLAGS + ["-M"]
# First sync (no delete) to populate dest
s1 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
time.sleep(0.5)
first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
r1 = subprocess.run(first_cmd, text=True, capture_output=True)
wait_proc(s1)
if r1.returncode != 0:
raise RuntimeError(f"First sync failed: {r1.stderr[:100]}")
# Add extra files to received dir
received = os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep))
extra_path = os.path.join(received, "extra_file.txt")
with open(extra_path, "w") as f:
f.write("should be deleted")
extra_dir = os.path.join(received, "extra_dir")
os.makedirs(extra_dir, exist_ok=True)
with open(os.path.join(extra_dir, "nested.txt"), "w") as f:
f.write("nested extra")
# Second sync with --delete (fresh server)
s2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
time.sleep(0.5)
second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--delete"]
start = time.monotonic()
r2 = subprocess.run(second_cmd, text=True, capture_output=True)
duration = time.monotonic() - start
wait_proc(s2)
r = {"name": "Delete (--delete)", "suite": profile_name}
if r2.returncode == 0 and not os.path.exists(extra_path) and not os.path.exists(extra_dir):
mismatches, missing = verify_transfer(source_dir, received)
if not mismatches and not missing:
r["status"] = "Success" r["status"] = "Success"
r["time"] = f"{duration:.4f}s" r["time"] = f"{duration:.4f}s"
r["error"] = "" r["error"] = ""
else: else:
r["status"] = "Failed" r["status"] = "Failed"
r["time"] = "N/A" r["time"] = "N/A"
r["error"] = f"Exit {result.returncode}: {(result.stderr or result.stdout)[:100]}" r["error"] = f"post-delete verify: mismatches={len(mismatches)}, missing={len(missing)}"
results.append(r) else:
except Exception as e: r["status"] = "Failed"
results.append({"name": "Dry run (-n)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) r["time"] = "N/A"
errs = []
if r2.returncode != 0:
errs.append(f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}")
if os.path.exists(extra_path):
errs.append("extra_file.txt remains")
if os.path.exists(extra_dir):
errs.append("extra_dir remains")
r["error"] = " | ".join(errs)
results.append(r)
except Exception as e:
results.append({"name": "Delete (--delete)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Archive mode (-a) # SSH feature tests
feature_flags = BASE_CLIENT_FLAGS + ["-a"] if SSH_AVAILABLE:
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags ssh_dest = f"localhost:{dest_dir}_ssh"
print(f"\n --- Archive mode (-a) ---\n Running: {' '.join(cmd)}") ssh_feature_cases = [
try: {"name": "SSH Archive (-a)", "flags": ["-a"]},
r = run_single_test(cmd, "Archive mode (-a)", source_dir, dest_dir) {"name": "SSH Exclude (--exclude small.txt)", "flags": ["--exclude", "small.txt"], "expected_missing": ["small.txt"]},
r["suite"] = profile_name ]
results.append(r) for case in ssh_feature_cases:
except Exception as e: flags = BASE_CLIENT_FLAGS + case["flags"]
results.append({"name": "Archive mode (-a)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags
print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}")
# Exclude (--exclude small.txt) try:
feature_flags = BASE_CLIENT_FLAGS + ["--exclude", "small.txt"] r = run_single_test(cmd, case["name"], source_dir, f"{dest_dir}_ssh",
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags no_server=True,
print(f"\n --- Exclude (--exclude small.txt) ---\n Running: {' '.join(cmd)}") expected_missing=case.get("expected_missing"))
try: r["suite"] = profile_name
r = run_single_test(cmd, "Exclude (--exclude small.txt)", source_dir, dest_dir, results.append(r)
expected_missing=["small.txt"]) except Exception as e:
r["suite"] = profile_name results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
results.append(r)
except Exception as e:
results.append({"name": "Exclude (--exclude small.txt)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Progress (--progress)
feature_flags = BASE_CLIENT_FLAGS + ["--progress"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Progress (--progress) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Progress (--progress)", source_dir, dest_dir)
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Bandwidth limit (--bwlimit 10240 = 10 MB/s)
feature_flags = BASE_CLIENT_FLAGS + ["--bwlimit", "10240"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Bandwidth limit (--bwlimit 10240 KB/s) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Bandwidth limit (--bwlimit 10240)", source_dir, dest_dir)
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Bandwidth limit (--bwlimit 10240)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Chunk size (--chunk-size 5242880)
feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Chunk size (--chunk-size 5242880) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Chunk size (--chunk-size 5242880)", source_dir, dest_dir)
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Chunk size (--chunk-size 5242880)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Delete (--delete) — pre-populate dest, add extra files, then sync with --delete
# Note: server handles one client per launch, so we restart between syncs
print(f"\n --- Delete (--delete) ---")
try:
flags = BASE_CLIENT_FLAGS + ["-M"]
# First sync (no delete) to populate dest
s1 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
time.sleep(0.5)
first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
r1 = subprocess.run(first_cmd, text=True, capture_output=True)
wait_proc(s1)
if r1.returncode != 0:
raise RuntimeError(f"First sync failed: {r1.stderr[:100]}")
# Add extra files to received dir
received = os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep))
extra_path = os.path.join(received, "extra_file.txt")
with open(extra_path, "w") as f:
f.write("should be deleted")
extra_dir = os.path.join(received, "extra_dir")
os.makedirs(extra_dir, exist_ok=True)
with open(os.path.join(extra_dir, "nested.txt"), "w") as f:
f.write("nested extra")
# Second sync with --delete (fresh server)
s2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
time.sleep(0.5)
second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--delete"]
start = time.monotonic()
r2 = subprocess.run(second_cmd, text=True, capture_output=True)
duration = time.monotonic() - start
wait_proc(s2)
r = {"name": "Delete (--delete)", "suite": profile_name}
if r2.returncode == 0 and not os.path.exists(extra_path) and not os.path.exists(extra_dir):
mismatches, missing = verify_transfer(source_dir, received)
if not mismatches and not missing:
r["status"] = "Success"
r["time"] = f"{duration:.4f}s"
r["error"] = ""
else:
r["status"] = "Failed"
r["time"] = "N/A"
r["error"] = f"post-delete verify: mismatches={len(mismatches)}, missing={len(missing)}"
else:
r["status"] = "Failed"
r["time"] = "N/A"
errs = []
if r2.returncode != 0:
errs.append(f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}")
if os.path.exists(extra_path):
errs.append("extra_file.txt remains")
if os.path.exists(extra_dir):
errs.append("extra_dir remains")
r["error"] = " | ".join(errs)
results.append(r)
except Exception as e:
results.append({"name": "Delete (--delete)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# SSH feature tests
if SSH_AVAILABLE:
ssh_dest = f"localhost:{dest_dir}_ssh"
ssh_feature_cases = [
{"name": "SSH Archive (-a)", "flags": ["-a"]},
{"name": "SSH Exclude (--exclude small.txt)", "flags": ["--exclude", "small.txt"], "expected_missing": ["small.txt"]},
]
for case in ssh_feature_cases:
flags = BASE_CLIENT_FLAGS + case["flags"]
cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags
print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, case["name"], source_dir, f"{dest_dir}_ssh",
no_server=True,
expected_missing=case.get("expected_missing"))
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
except (subprocess.CalledProcessError, RuntimeError) as e: except (subprocess.CalledProcessError, RuntimeError) as e:
print(f" Error: {e}") print(f" Error: {e}")
@@ -567,26 +573,19 @@ def check_ssh_localhost():
build_dir = os.path.abspath("build") build_dir = os.path.abspath("build")
server_path = os.path.join(build_dir, "server") server_path = os.path.join(build_dir, "server")
try: r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5",
r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", "localhost", "which", "fastsync-server"],
"localhost", "which", "fastsync-server"], capture_output=True, timeout=10)
capture_output=True, timeout=10)
except FileNotFoundError:
SSH_AVAILABLE = False
return
if r.returncode == 0: if r.returncode == 0:
SSH_AVAILABLE = True SSH_AVAILABLE = True
return return
SSH_AVAILABLE = False SSH_AVAILABLE = False
# Try each PATH dir: create symlink, then verify with which # Try each PATH dir: create symlink, then verify with which
try: r = subprocess.run(
r = subprocess.run( ["ssh", "-o", "BatchMode=yes", "localhost",
["ssh", "-o", "BatchMode=yes", "localhost", 'echo "$PATH"'],
'echo "$PATH"'], capture_output=True, timeout=10, text=True)
capture_output=True, timeout=10, text=True)
except FileNotFoundError:
return
if r.returncode != 0: if r.returncode != 0:
return return
for d in r.stdout.strip().split(":"): for d in r.stdout.strip().split(":"):
@@ -667,10 +666,9 @@ def main():
parser.add_argument("--keep-data", action="store_true") parser.add_argument("--keep-data", action="store_true")
parser.add_argument("--unlimited", action="store_true") parser.add_argument("--unlimited", action="store_true")
parser.add_argument("--wan", action="store_true") parser.add_argument("--wan", action="store_true")
parser.add_argument("--full", action="store_true", help="Run full test suite with network shaping, SSH, rsync benchmarks")
args = parser.parse_args() args = parser.parse_args()
total_bytes = generate_test_files(args.source_dir, full=args.full) total_bytes = generate_test_files(args.source_dir)
if os.path.exists(args.dest_dir): if os.path.exists(args.dest_dir):
shutil.rmtree(args.dest_dir) shutil.rmtree(args.dest_dir)
os.makedirs(args.dest_dir, exist_ok=True) os.makedirs(args.dest_dir, exist_ok=True)
@@ -681,12 +679,12 @@ def main():
elif args.wan: elif args.wan:
profiles.append("WAN") profiles.append("WAN")
else: else:
profiles.append("LAN" if args.full else "Unlimited") profiles.append("LAN")
try: try:
all_results = [] all_results = []
for p in profiles: for p in profiles:
all_results.extend(run_profile(p, args.source_dir, args.dest_dir, full=args.full)) all_results.extend(run_profile(p, args.source_dir, args.dest_dir))
print("\n" + "=" * 130) print("\n" + "=" * 130)
print(f"{'RESULTS':^130}") print(f"{'RESULTS':^130}")