From 0bfa68fee7179dc237323bd21529ced729651cc2 Mon Sep 17 00:00:00 2001 From: TapTap Date: Thu, 16 Jul 2026 15:38:48 +0200 Subject: [PATCH] Incremental sync: --incremental flag to skip unchanged files - New STATUS_CHECK protocol status (value 6) - Client sends path + size + mtime; server replies OK (skip) or NEXT (send) - config.h/c: use_incremental field sent/received over wire - file.c/h: file_send_single_calls_no_path helper for incremental path - client_cli.c: --incremental flag - client_send.c: per-file check before send in both single and chunk paths - server.c: STATUS_CHECK handling in receive_files() - multiprocessing.c: STATUS_CHECK handling in receive_thread() - test.py: incremental sync test case --- src/client/client_cli.c | 3 ++ src/client/client_send.c | 42 +++++++++++++++++++++---- src/server/server.c | 60 ++++++++++++++++++++++++++++++++++-- src/shared/config.c | 4 +++ src/shared/config.h | 1 + src/shared/file.c | 15 +++++++++ src/shared/file.h | 1 + src/shared/multiprocessing.c | 53 +++++++++++++++++++++++++++++-- src/shared/protocol.c | 2 ++ src/shared/protocol.h | 2 +- test.py | 27 ++++++++++++++++ 11 files changed, 199 insertions(+), 11 deletions(-) diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 44fca4b..714bd6e 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -34,6 +34,7 @@ static void print_usage(void) { printf(" --include Only include files matching pattern\n"); printf(" --max-size Skip files larger than n bytes\n"); printf(" --min-size Skip files smaller than n bytes\n"); + printf(" --incremental Skip files unchanged since last transfer\n"); printf(" -m Enable multithreading\n"); printf(" -s Enable chunk serialization\n"); printf(" -f Enable sendfile (TCP only, not with -c or -s)\n"); @@ -93,6 +94,8 @@ int main(int argc, char *argv[]) { config->max_size = strtoull(argv[++i], NULL, 10); } else if (strcmp(argv[i], "--min-size") == 0 && i + 1 < argc) { config->min_size = strtoull(argv[++i], NULL, 10); + } else if (strcmp(argv[i], "--incremental") == 0) { + config->use_incremental = true; } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) { config->use_compression = true; log_message(LOG_LEVEL_INFO, "Enabled Compression"); diff --git a/src/client/client_send.c b/src/client/client_send.c index 4c70204..255e834 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -32,17 +32,47 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) { data_destroy(data); } else if (config->use_sendfile && !config->use_compression) { for (int i = 0; i < chunk->element_count; i++) { - if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; + if (config->use_incremental) { + if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1; + if (!send_str(client->file_descriptor, chunk->items[i]->path)) return -1; + unsigned long long fsize = chunk->items[i]->data->size; + long long mtime = chunk->items[i]->metadata ? chunk->items[i]->metadata->mtime_sec : 0; + if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1; + if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1; + Status s; + if (!receive_status(client->file_descriptor, &s)) return -1; + if (s == STATUS_OK) continue; + if (s != STATUS_NEXT) return -1; + } else { + if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; + } if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata)) return -1; } } else { for (int i = 0; i < chunk->element_count; i++) { - if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; - if (!file_send_single_calls(chunk->items[i], client->file_descriptor, - config->use_metadata, - config->use_compression ? config->compression_level : 0)) - return -1; + if (config->use_incremental) { + if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1; + if (!send_str(client->file_descriptor, chunk->items[i]->path)) return -1; + unsigned long long fsize = chunk->items[i]->data->size; + long long mtime = chunk->items[i]->metadata ? chunk->items[i]->metadata->mtime_sec : 0; + if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1; + if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1; + Status s; + if (!receive_status(client->file_descriptor, &s)) return -1; + if (s == STATUS_OK) continue; + if (s != STATUS_NEXT) return -1; + if (!file_send_single_calls_no_path(chunk->items[i], client->file_descriptor, + config->use_metadata, + config->use_compression ? config->compression_level : 0)) + return -1; + } else { + if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; + if (!file_send_single_calls(chunk->items[i], client->file_descriptor, + config->use_metadata, + config->use_compression ? config->compression_level : 0)) + return -1; + } } } return 0; diff --git a/src/server/server.c b/src/server/server.c index a7edce5..c4e528e 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -16,12 +16,68 @@ #include #include #include +#include int receive_files(Config *config, int file_descriptor) { Status status; if (!receive_status(file_descriptor, &status)) return -1; - while (status == STATUS_NEXT || status == STATUS_CHUNK) { - if (status == STATUS_CHUNK) { + while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) { + if (status == STATUS_CHECK) { + char *check_path = receive_str(file_descriptor); + if (check_path == NULL) { send_status(file_descriptor, STATUS_ERROR); return -1; } + unsigned long long check_size; + long long check_mtime; + if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) || + !receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) { + free(check_path); + send_status(file_descriptor, STATUS_ERROR); + return -1; + } + char *full_path = path_cat(config->receive_root_directory, check_path); + struct stat st; + bool match = false; + if (full_path && stat(full_path, &st) == 0 && + (unsigned long long)st.st_size == check_size && + (long long)st.st_mtime == check_mtime) { + match = true; + } + free(full_path); + if (match) { + if (!send_status(file_descriptor, STATUS_OK)) { free(check_path); return -1; } + free(check_path); + } else { + if (!send_status(file_descriptor, STATUS_NEXT)) { free(check_path); return -1; } + File *file = file_create(check_path); + free(check_path); + if (file == NULL) { send_status(file_descriptor, STATUS_ERROR); return -1; } + if (config->use_metadata) { + file->metadata = metadata_receive(file_descriptor); + } + Data *file_data = receive_data(file_descriptor); + if (file_data == NULL) { + file_destroy(file); + send_status(file_descriptor, STATUS_ERROR); + return -1; + } + if (config->use_compression) { + Data *uncompressed = data_decompress(file_data); + data_destroy(file_data); + if (uncompressed == NULL) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return -1; } + file_data = uncompressed; + } + data_destroy(file->data); + file->data = file_data; + if (config->save_to_disk) { + char *disk_path = path_cat(config->receive_root_directory, file->path); + if (disk_path) { + to_disk(disk_path, file->data->data, file->data->size); + file_restore_metadata(disk_path, file->metadata); + free(disk_path); + } + } + file_destroy(file); + } + } else if (status == STATUS_CHUNK) { Data *chunk_data = receive_data(file_descriptor); if (chunk_data == NULL) { log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data"); diff --git a/src/shared/config.c b/src/shared/config.c index a22e489..1838d4d 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -38,6 +38,7 @@ Config *config_create(char *version, char *send_directory, config->include_count = 0; config->max_size = 0; config->min_size = 0; + config->use_incremental = false; return config; } @@ -89,6 +90,7 @@ bool config_send(int file_descriptor, Config *config) { if (!send_int(file_descriptor, (int)config->chunk_size)) return false; if (!send_int(file_descriptor, config->use_sendfile)) return false; if (!send_int(file_descriptor, config->use_delete)) return false; + if (!send_int(file_descriptor, config->use_incremental)) return false; Status status; if (!receive_status(file_descriptor, &status)) return false; if (status != STATUS_OK) { @@ -134,6 +136,8 @@ Config *config_receive(int file_descriptor) { config->use_sendfile = tmp; if (!receive_int(file_descriptor, &tmp)) goto error; config->use_delete = tmp; + if (!receive_int(file_descriptor, &tmp)) goto error; + config->use_incremental = tmp; config->show_progress = false; config->dry_run = false; config->ssh_port = 22; diff --git a/src/shared/config.h b/src/shared/config.h index b08f733..6af810b 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -32,6 +32,7 @@ typedef struct Config { int include_count; unsigned long long max_size; unsigned long long min_size; + bool use_incremental; } Config; #define PROTOCOL_VERSION "1.0.0" diff --git a/src/shared/file.c b/src/shared/file.c index 061343a..f04f2e2 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -96,6 +96,21 @@ bool file_load_data(File *file) { return true; } +bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level) { + if (compression_level > 0) { + Data *compressed_data = data_compress(file->data, compression_level); + if (compressed_data == NULL) { + log_message(LOG_LEVEL_ERROR, "Failed to compress file data"); + return false; + } + data_destroy(file->data); + file->data = compressed_data; + } + if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; + if (!send_data(file_descriptor, file->data)) return false; + return true; +} + bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) { if (compression_level > 0) { Data *compressed_data = data_compress(file->data, compression_level); diff --git a/src/shared/file.h b/src/shared/file.h index 11ec9fe..4f28d06 100644 --- a/src/shared/file.h +++ b/src/shared/file.h @@ -25,6 +25,7 @@ void file_destroy(void *item); bool file_load_data(File *file); File *file_receive(Config *config, int file_descriptor); bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level); +bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level); bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata); size_t file_content_to_buffer(File *file); FileMetadata *file_metadata_create(struct stat *stats); diff --git a/src/shared/multiprocessing.c b/src/shared/multiprocessing.c index ef99386..5e912d0 100644 --- a/src/shared/multiprocessing.c +++ b/src/shared/multiprocessing.c @@ -13,6 +13,7 @@ #include #include #include +#include #include PipelineContextSender *pipeline_context_sender_create(Config *config, @@ -126,8 +127,56 @@ int receive_thread(void *pipeline_context) { Status status; if (!receive_status(file_descriptor, &status)) return thrd_error; - while (status == STATUS_NEXT || status == STATUS_CHUNK) { - if (status == STATUS_CHUNK) { + while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) { + if (status == STATUS_CHECK) { + char *check_path = receive_str(file_descriptor); + if (check_path == NULL) return thrd_error; + unsigned long long check_size; + long long check_mtime; + if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) || + !receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) { + free(check_path); + return thrd_error; + } + char *full_path = path_cat(config->receive_root_directory, check_path); + struct stat st; + bool match = false; + if (full_path && stat(full_path, &st) == 0 && + (unsigned long long)st.st_size == check_size && + (long long)st.st_mtime == check_mtime) { + match = true; + } + free(full_path); + if (match) { + if (!send_status(file_descriptor, STATUS_OK)) { free(check_path); return thrd_error; } + free(check_path); + } else { + if (!send_status(file_descriptor, STATUS_NEXT)) { free(check_path); return thrd_error; } + File *file = file_create(check_path); + free(check_path); + if (file == NULL) { send_status(file_descriptor, STATUS_ERROR); return thrd_error; } + if (config->use_metadata) { + file->metadata = metadata_receive(file_descriptor); + } + Data *file_data = receive_data(file_descriptor); + if (file_data == NULL) { + file_destroy(file); + send_status(file_descriptor, STATUS_ERROR); + return thrd_error; + } + if (config->use_compression) { + Data *uncompressed = data_decompress(file_data); + data_destroy(file_data); + if (uncompressed == NULL) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return thrd_error; } + file_data = uncompressed; + } + data_destroy(file->data); + file->data = file_data; + queue_enqueue_multithreaded(context->queue, file, &context->mutex, + &context->condition_not_empty, + &context->condition_not_full); + } + } else if (status == STATUS_CHUNK) { receive_chunk_enqueue(file_descriptor, context); } else { File *file = file_receive(config, file_descriptor); diff --git a/src/shared/protocol.c b/src/shared/protocol.c index 4f2b82a..a95386e 100644 --- a/src/shared/protocol.c +++ b/src/shared/protocol.c @@ -111,6 +111,8 @@ static const char *status_to_string(Status status) { return "NEXT"; case STATUS_CHUNK: return "CHUNK"; + case STATUS_CHECK: + return "CHECK"; default: return "UNKNOWN"; } diff --git a/src/shared/protocol.h b/src/shared/protocol.h index 1880d24..83b19c4 100644 --- a/src/shared/protocol.h +++ b/src/shared/protocol.h @@ -6,7 +6,7 @@ #include typedef int Status; -enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST }; +enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST, STATUS_CHECK }; void io_set_fds(int read_fd, int write_fd); void io_set_bwlimit(unsigned long long bytes_per_sec); diff --git a/test.py b/test.py index d708258..3093ab1 100755 --- a/test.py +++ b/test.py @@ -394,6 +394,33 @@ def run_profile(profile_name, source_dir, dest_dir): except Exception as e: results.append({"name": "Bandwidth limit (--bwlimit 10240)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + # Incremental sync (--incremental) — first sync, then second sync should skip all + print(f"\n --- Incremental (--incremental) ---") + try: + flags = BASE_CLIENT_FLAGS + ["-M"] + srv = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + r1 = subprocess.run(first_cmd, text=True, capture_output=True) + wait_proc(srv) + if r1.returncode != 0: + raise RuntimeError(f"First sync failed: {r1.stderr[:100]}") + srv2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--incremental"] + start = time.monotonic() + r2 = subprocess.run(second_cmd, text=True, capture_output=True, timeout=30) + duration = time.monotonic() - start + wait_proc(srv2) + r = {"name": "Incremental (--incremental)", "suite": profile_name, + "status": "Success" if r2.returncode == 0 else "Failed", + "time": f"{duration:.4f}s" if r2.returncode == 0 else "N/A", + "error": "" if r2.returncode == 0 else f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}"} + results.append(r) + except Exception as e: + results.append({"name": "Incremental (--incremental)", "suite": profile_name, + "status": "Error", "time": "N/A", "error": str(e)}) + # Chunk size (--chunk-size 5242880) feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"] cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags