Incremental sync: --incremental to skip unchanged files #14

Merged
TapTap merged 6 commits from incremental-sync into main 2026-07-18 17:36:26 +02:00
4 changed files with 276 additions and 288 deletions
Showing only changes of commit 46c650e9ae - Show all commits
+7 -5
View File
@@ -57,11 +57,11 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) {
int rc = incremental_check(client, chunk->items[i]); int rc = incremental_check(client, chunk->items[i]);
if (rc < 0) return -1; if (rc < 0) return -1;
if (rc > 0) continue; if (rc > 0) continue;
if (!file_send_sendfile_no_path(chunk->items[i], client->file_descriptor, config->use_metadata)) if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata, false))
return -1; return -1;
} else { } else {
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata)) if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata, true))
return -1; return -1;
} }
} }
@@ -71,15 +71,17 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) {
int rc = incremental_check(client, chunk->items[i]); int rc = incremental_check(client, chunk->items[i]);
if (rc < 0) return -1; if (rc < 0) return -1;
if (rc > 0) continue; if (rc > 0) continue;
if (!file_send_single_calls_no_path(chunk->items[i], client->file_descriptor, if (!file_send_single_calls(chunk->items[i], client->file_descriptor,
config->use_metadata, config->use_metadata,
config->use_compression ? config->compression_level : 0)) config->use_compression ? config->compression_level : 0,
false))
return -1; return -1;
} else { } else {
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
if (!file_send_single_calls(chunk->items[i], client->file_descriptor, if (!file_send_single_calls(chunk->items[i], client->file_descriptor,
config->use_metadata, config->use_metadata,
config->use_compression ? config->compression_level : 0)) config->use_compression ? config->compression_level : 0,
true))
return -1; return -1;
} }
} }
+4 -56
View File
@@ -96,7 +96,7 @@ bool file_load_data(File *file) {
return true; return true;
} }
bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level) { bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level, bool send_path) {
Data *data_to_send = file->data; Data *data_to_send = file->data;
Data *compressed_data = NULL; Data *compressed_data = NULL;
if (compression_level > 0) { if (compression_level > 0) {
@@ -107,59 +107,7 @@ bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_me
} }
data_to_send = compressed_data; data_to_send = compressed_data;
} }
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) { if (send_path && !send_str(file_descriptor, file->path)) {
data_destroy(compressed_data);
return false;
}
if (!send_data(file_descriptor, data_to_send)) {
data_destroy(compressed_data);
return false;
}
data_destroy(compressed_data);
return true;
}
bool file_send_sendfile_no_path(File *file, int file_descriptor, bool use_metadata) {
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false;
int fd = open(file->path, O_RDONLY);
if (fd == -1) {
perror("Could not open file for sendfile");
return false;
}
unsigned long long file_size = file->data->size;
if (!send_n_data(file_descriptor, &file_size, sizeof(unsigned long long))) {
close(fd);
return false;
}
off_t offset = 0;
while (offset < file_size) {
ssize_t sent = sendfile(file_descriptor, fd, &offset, file_size - offset);
if (sent == -1) {
perror("sendfile failed");
close(fd);
return false;
}
}
close(fd);
return true;
}
bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) {
Data *data_to_send = file->data;
Data *compressed_data = NULL;
if (compression_level > 0) {
compressed_data = data_compress(file->data, compression_level);
if (compressed_data == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
return false;
}
data_to_send = compressed_data;
}
if (!send_str(file_descriptor, file->path)) {
data_destroy(compressed_data); data_destroy(compressed_data);
return false; return false;
} }
@@ -270,8 +218,8 @@ bool to_disk(const char *path, const void *data, unsigned long long data_size) {
return true; return true;
} }
bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata) { bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata, bool send_path) {
if (!send_str(file_descriptor, file->path)) return false; if (send_path && !send_str(file_descriptor, file->path)) return false;
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false;
int fd = open(file->path, O_RDONLY); int fd = open(file->path, O_RDONLY);
+2 -4
View File
@@ -24,10 +24,8 @@ File *file_create(const char *path);
void file_destroy(void *item); void file_destroy(void *item);
bool file_load_data(File *file); bool file_load_data(File *file);
File *file_receive(Config *config, int file_descriptor); File *file_receive(Config *config, int file_descriptor);
bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level); bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level, bool send_path);
bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level); bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata, bool send_path);
bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata);
bool file_send_sendfile_no_path(File *file, int file_descriptor, bool use_metadata);
size_t file_content_to_buffer(File *file); size_t file_content_to_buffer(File *file);
FileMetadata *file_metadata_create(struct stat *stats); FileMetadata *file_metadata_create(struct stat *stats);
void file_metadata_destroy(void *metadata); void file_metadata_destroy(void *metadata);
+55 -15
View File
@@ -50,7 +50,7 @@ CLIENT_CMD_PREFIX = [
BASE_CLIENT_FLAGS = ["--save-to-disk"] BASE_CLIENT_FLAGS = ["--save-to-disk"]
TEST_CASES = [ TEST_CASES_FULL = [
{"name": "Standard", "flags": []}, {"name": "Standard", "flags": []},
{"name": "Posix Args (no flags)", "flags": [], "posix": True}, {"name": "Posix Args (no flags)", "flags": [], "posix": True},
{"name": "Standard (no metadata)", "flags": [], "use_metadata": False}, {"name": "Standard (no metadata)", "flags": [], "use_metadata": False},
@@ -68,7 +68,14 @@ TEST_CASES = [
{"name": "Sendfile + Multithreading (-f -m)", "flags": ["-f", "-m"]}, {"name": "Sendfile + Multithreading (-f -m)", "flags": ["-f", "-m"]},
] ]
SSH_CASES = [ TEST_CASES_LIGHT = [
{"name": "Standard", "flags": []},
{"name": "Compression (-c)", "flags": ["-c"]},
{"name": "Chunk Serialization (-s)", "flags": ["-s"]},
{"name": "Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]},
]
SSH_CASES_FULL = [
{"name": "SSH (localhost)", "flags": []}, {"name": "SSH (localhost)", "flags": []},
{"name": "SSH Multithreading (-m)", "flags": ["-m"]}, {"name": "SSH Multithreading (-m)", "flags": ["-m"]},
{"name": "SSH Compression (-c)", "flags": ["-c"]}, {"name": "SSH Compression (-c)", "flags": ["-c"]},
@@ -79,11 +86,17 @@ SSH_CASES = [
{"name": "SSH Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]}, {"name": "SSH Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]},
] ]
RSYNC_CASES = [ SSH_CASES_LIGHT = [
{"name": "SSH (localhost)", "flags": []},
]
RSYNC_CASES_FULL = [
{"name": "rsync (archive)", "args": ["-aH"]}, {"name": "rsync (archive)", "args": ["-aH"]},
{"name": "rsync (archive + compress)", "args": ["-aHz"]}, {"name": "rsync (archive + compress)", "args": ["-aHz"]},
] ]
RSYNC_CASES_LIGHT = []
def netem_apply(profile): def netem_apply(profile):
params = NETWORK_PROFILES[profile] params = NETWORK_PROFILES[profile]
@@ -119,12 +132,12 @@ def find_free_port():
return s.getsockname()[1] return s.getsockname()[1]
def generate_test_files(source_dir): def generate_test_files(source_dir, full=False):
if os.path.exists(source_dir): if os.path.exists(source_dir):
shutil.rmtree(source_dir) shutil.rmtree(source_dir)
os.makedirs(source_dir) os.makedirs(source_dir)
target_total = 25 * 1024 * 1024 target_total = 25 * 1024 * 1024 if full else 0
written = 0 written = 0
files = { files = {
@@ -141,6 +154,7 @@ def generate_test_files(source_dir):
f.write(content) f.write(content)
written += len(content) written += len(content)
if full:
os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True) os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True)
i = 0 i = 0
while written < target_total: while written < target_total:
@@ -254,19 +268,24 @@ def print_profile_header(profile_name):
print(" No limits applied") print(" No limits applied")
def run_profile(profile_name, source_dir, dest_dir): def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=None, ssh_cases=None, rsync_cases=None):
print_profile_header(profile_name) print_profile_header(profile_name)
is_limited = profile_name != "Unlimited" is_limited = profile_name != "Unlimited"
client_prefix = CLIENT_CMD_PREFIX if is_limited else [] client_prefix = CLIENT_CMD_PREFIX if is_limited else []
if test_cases is None:
test_cases = TEST_CASES_FULL if full else TEST_CASES_LIGHT
if ssh_cases is None:
ssh_cases = SSH_CASES_FULL if full else SSH_CASES_LIGHT
if rsync_cases is None:
rsync_cases = RSYNC_CASES_FULL if full else RSYNC_CASES_LIGHT
try: try:
if is_limited: if is_limited and full:
netem_apply(profile_name) netem_apply(profile_name)
else:
netem_reset()
results = [] results = []
for case in TEST_CASES: for case in test_cases:
flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"] flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"]
if case.get("posix"): if case.get("posix"):
cmd = client_prefix + BASE_CLIENT_CMD + [source_dir, dest_dir] + flags cmd = client_prefix + BASE_CLIENT_CMD + [source_dir, dest_dir] + flags
@@ -281,7 +300,7 @@ def run_profile(profile_name, source_dir, dest_dir):
results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
if SSH_AVAILABLE: if SSH_AVAILABLE:
for case in SSH_CASES: for case in ssh_cases:
flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"] flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"]
ssh_dest = f"localhost:{dest_dir}_ssh" ssh_dest = f"localhost:{dest_dir}_ssh"
cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags
@@ -293,9 +312,10 @@ def run_profile(profile_name, source_dir, dest_dir):
except Exception as e: except Exception as e:
results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
if rsync_cases:
port, conf, daemon = start_rsync_daemon(source_dir) port, conf, daemon = start_rsync_daemon(source_dir)
try: try:
for case in RSYNC_CASES: for case in rsync_cases:
cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"] cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"]
print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}")
try: try:
@@ -324,6 +344,7 @@ def run_profile(profile_name, source_dir, dest_dir):
except Exception: except Exception:
pass pass
if full:
# Feature-specific tests for rsync-compatible flags # Feature-specific tests for rsync-compatible flags
print("\n " + "" * 56 + "\n Feature Tests\n " + "" * 56) print("\n " + "" * 56 + "\n Feature Tests\n " + "" * 56)
@@ -383,6 +404,17 @@ def run_profile(profile_name, source_dir, dest_dir):
except Exception as e: except Exception as e:
results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Bandwidth limit (--bwlimit 10240 = 10 MB/s)
feature_flags = BASE_CLIENT_FLAGS + ["--bwlimit", "10240"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Bandwidth limit (--bwlimit 10240 KB/s) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Bandwidth limit (--bwlimit 10240)", source_dir, dest_dir)
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Bandwidth limit (--bwlimit 10240)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Incremental sync (--incremental) — first sync, then second sync should skip all # Incremental sync (--incremental) — first sync, then second sync should skip all
print(f"\n --- Incremental (--incremental) ---") print(f"\n --- Incremental (--incremental) ---")
try: try:
@@ -562,19 +594,26 @@ def check_ssh_localhost():
build_dir = os.path.abspath("build") build_dir = os.path.abspath("build")
server_path = os.path.join(build_dir, "server") server_path = os.path.join(build_dir, "server")
try:
r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5",
"localhost", "which", "fastsync-server"], "localhost", "which", "fastsync-server"],
capture_output=True, timeout=10) capture_output=True, timeout=10)
except FileNotFoundError:
SSH_AVAILABLE = False
return
if r.returncode == 0: if r.returncode == 0:
SSH_AVAILABLE = True SSH_AVAILABLE = True
return return
SSH_AVAILABLE = False SSH_AVAILABLE = False
# Try each PATH dir: create symlink, then verify with which # Try each PATH dir: create symlink, then verify with which
try:
r = subprocess.run( r = subprocess.run(
["ssh", "-o", "BatchMode=yes", "localhost", ["ssh", "-o", "BatchMode=yes", "localhost",
'echo "$PATH"'], 'echo "$PATH"'],
capture_output=True, timeout=10, text=True) capture_output=True, timeout=10, text=True)
except FileNotFoundError:
return
if r.returncode != 0: if r.returncode != 0:
return return
for d in r.stdout.strip().split(":"): for d in r.stdout.strip().split(":"):
@@ -655,9 +694,10 @@ def main():
parser.add_argument("--keep-data", action="store_true") parser.add_argument("--keep-data", action="store_true")
parser.add_argument("--unlimited", action="store_true") parser.add_argument("--unlimited", action="store_true")
parser.add_argument("--wan", action="store_true") parser.add_argument("--wan", action="store_true")
parser.add_argument("--full", action="store_true", help="Run full test suite with network shaping, SSH, rsync benchmarks")
args = parser.parse_args() args = parser.parse_args()
total_bytes = generate_test_files(args.source_dir) total_bytes = generate_test_files(args.source_dir, full=args.full)
if os.path.exists(args.dest_dir): if os.path.exists(args.dest_dir):
shutil.rmtree(args.dest_dir) shutil.rmtree(args.dest_dir)
os.makedirs(args.dest_dir, exist_ok=True) os.makedirs(args.dest_dir, exist_ok=True)
@@ -668,12 +708,12 @@ def main():
elif args.wan: elif args.wan:
profiles.append("WAN") profiles.append("WAN")
else: else:
profiles.append("LAN") profiles.append("LAN" if args.full else "Unlimited")
try: try:
all_results = [] all_results = []
for p in profiles: for p in profiles:
all_results.extend(run_profile(p, args.source_dir, args.dest_dir)) all_results.extend(run_profile(p, args.source_dir, args.dest_dir, full=args.full))
print("\n" + "=" * 130) print("\n" + "=" * 130)
print(f"{'RESULTS':^130}") print(f"{'RESULTS':^130}")