feat: add delta transfer for incremental sync
Implement rsync-style delta transfer using rolling checksums (Adler-32 + xxHash32). When a file exists on both sides but has changed, only the changed blocks are transmitted instead of the entire file. - New status codes: STATUS_DELTA_SIGNATURE, STATUS_DELTA_DATA - Protocol version bumped to 1.2.0 - Server generates block signature, client computes delta - Auto-fallback to whole-file when delta >= 70% of file size - Works with zstd compression on delta stream - Configurable block size (default 8KB, --delta-block flag) - 13 unit tests covering hashing, signature roundtrip, delta compute/apply, file growth/shrink, and decision logic
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
#include "client_send.h"
|
||||
#include "config.h"
|
||||
#include "delta.h"
|
||||
#include "log.h"
|
||||
#include "protocol.h"
|
||||
#include "transport_tls.h"
|
||||
@@ -36,6 +37,8 @@ static void print_usage(void) {
|
||||
printf(" --max-size <n> Skip files larger than n bytes\n");
|
||||
printf(" --min-size <n> Skip files smaller than n bytes\n");
|
||||
printf(" --incremental Skip files unchanged since last transfer\n");
|
||||
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
|
||||
printf(" --delta-block <n> Delta block size in bytes (default: %d)\n", DELTA_BLOCK_SIZE_DEFAULT);
|
||||
printf(" -m Enable multithreading\n");
|
||||
printf(" -s Enable chunk serialization\n");
|
||||
printf(" -f Enable sendfile (TCP only, not with -c or -s)\n");
|
||||
@@ -101,6 +104,14 @@ int main(int argc, char *argv[]) {
|
||||
config->min_size = strtoull(argv[++i], NULL, 10);
|
||||
} else if (strcmp(argv[i], "--incremental") == 0) {
|
||||
config->use_incremental = true;
|
||||
} else if (strcmp(argv[i], "--delta") == 0) {
|
||||
config->use_delta = true;
|
||||
} else if (strcmp(argv[i], "--delta-block") == 0 && i + 1 < argc) {
|
||||
unsigned long long val = strtoull(argv[++i], NULL, 10);
|
||||
if (val >= DELTA_BLOCK_SIZE_MIN && val <= DELTA_BLOCK_SIZE_MAX)
|
||||
config->delta_block_size = (uint32_t)val;
|
||||
else
|
||||
fprintf(stderr, "Warning: --delta-block value %llu out of range, using default\n", val);
|
||||
} else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) {
|
||||
config->use_compression = true;
|
||||
log_message(LOG_LEVEL_INFO, "Enabled Compression");
|
||||
@@ -231,6 +242,23 @@ int main(int argc, char *argv[]) {
|
||||
config->use_metadata = true;
|
||||
}
|
||||
|
||||
if (config->use_delta && !config->use_incremental) {
|
||||
fprintf(stderr, "Error: --delta requires --incremental\n");
|
||||
return 1;
|
||||
}
|
||||
if (config->use_delta && config->use_chunk_serialization) {
|
||||
fprintf(stderr, "Error: --delta cannot be combined with -s (chunk serialization)\n");
|
||||
return 1;
|
||||
}
|
||||
if (config->use_delta && config->use_sendfile) {
|
||||
fprintf(stderr, "Error: --delta cannot be combined with -f (sendfile)\n");
|
||||
return 1;
|
||||
}
|
||||
if (config->use_delta && !config->use_metadata) {
|
||||
log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --delta");
|
||||
config->use_metadata = true;
|
||||
}
|
||||
|
||||
if (config->use_tls) {
|
||||
if (!config->tls_cert || !config->tls_key) {
|
||||
fprintf(stderr, "Error: --tls requires --cert and --key\n");
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
#include "client_send.h"
|
||||
#include "array_list.h"
|
||||
#include "chunk.h"
|
||||
#include "compression.h"
|
||||
#include "config.h"
|
||||
#include "data.h"
|
||||
#include "delta.h"
|
||||
#include "file.h"
|
||||
#include "metadata.h"
|
||||
#include "log.h"
|
||||
#include "multiprocessing.h"
|
||||
#include "protocol.h"
|
||||
@@ -19,7 +22,9 @@
|
||||
#include <threads.h>
|
||||
#include <time.h>
|
||||
|
||||
static int incremental_check(Client *client, File *file) {
|
||||
static int incremental_check(Client *client, File *file,
|
||||
DeltaSignature **out_sig) {
|
||||
*out_sig = NULL;
|
||||
if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1;
|
||||
if (!send_str(client->file_descriptor, file->path)) return -1;
|
||||
unsigned long long fsize = file->data->size;
|
||||
@@ -33,6 +38,15 @@ static int incremental_check(Client *client, File *file) {
|
||||
return -1;
|
||||
}
|
||||
if (s == STATUS_OK) return 1;
|
||||
if (s == STATUS_DELTA_SIGNATURE) {
|
||||
Data *sig_data = receive_data(client->file_descriptor);
|
||||
if (!sig_data) return -1;
|
||||
DeltaSignature *sig = delta_signature_deserialize(sig_data);
|
||||
data_destroy(sig_data);
|
||||
if (!sig) return -1;
|
||||
*out_sig = sig;
|
||||
return 2;
|
||||
}
|
||||
if (s != STATUS_NEXT) {
|
||||
log_message(LOG_LEVEL_ERROR, "Unexpected server status");
|
||||
return -1;
|
||||
@@ -40,6 +54,39 @@ static int incremental_check(Client *client, File *file) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int send_delta(Client *client, File *file, DeltaSignature *sig,
|
||||
Config *config) {
|
||||
Delta *delta = delta_compute(file->data->data, file->data->size,
|
||||
sig, config->delta_block_size);
|
||||
if (!delta) return 1;
|
||||
|
||||
if (!delta_is_worthwhile(delta, file->data->size)) {
|
||||
delta_destroy(delta);
|
||||
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
|
||||
return 1;
|
||||
}
|
||||
|
||||
Data *delta_data = delta_serialize(delta);
|
||||
delta_destroy(delta);
|
||||
if (!delta_data) return -1;
|
||||
|
||||
Data *to_send = delta_data;
|
||||
if (config->use_compression) {
|
||||
to_send = data_compress(delta_data, config->compression_level);
|
||||
data_destroy(delta_data);
|
||||
if (!to_send) return -1;
|
||||
}
|
||||
|
||||
bool ok = send_status(client->file_descriptor, STATUS_DELTA_DATA) &&
|
||||
send_data(client->file_descriptor, to_send);
|
||||
|
||||
if (ok && config->use_metadata)
|
||||
ok = metadata_send(client->file_descriptor, file->metadata);
|
||||
|
||||
data_destroy(to_send);
|
||||
return ok ? 0 : -1;
|
||||
}
|
||||
|
||||
int send_chunk(Client *client, Chunk *chunk, Config *config) {
|
||||
if (config->use_chunk_serialization) {
|
||||
if (!send_status(client->file_descriptor, STATUS_CHUNK)) return -1;
|
||||
@@ -55,9 +102,18 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) {
|
||||
} else if (config->use_sendfile && !config->use_compression) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
if (config->use_incremental) {
|
||||
int rc = incremental_check(client, chunk->items[i]);
|
||||
if (rc < 0) return -1;
|
||||
if (rc > 0) continue;
|
||||
DeltaSignature *sig = NULL;
|
||||
int rc = incremental_check(client, chunk->items[i], &sig);
|
||||
if (rc < 0) { delta_signature_destroy(sig); return -1; }
|
||||
if (rc == 1) { delta_signature_destroy(sig); continue; }
|
||||
if (rc == 2 && config->use_delta) {
|
||||
int drc = send_delta(client, chunk->items[i], sig, config);
|
||||
delta_signature_destroy(sig);
|
||||
if (drc == 0) continue;
|
||||
if (drc < 0) return -1;
|
||||
} else {
|
||||
delta_signature_destroy(sig);
|
||||
}
|
||||
if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata, false))
|
||||
return -1;
|
||||
} else {
|
||||
@@ -69,9 +125,18 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) {
|
||||
} else {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
if (config->use_incremental) {
|
||||
int rc = incremental_check(client, chunk->items[i]);
|
||||
if (rc < 0) return -1;
|
||||
if (rc > 0) continue;
|
||||
DeltaSignature *sig = NULL;
|
||||
int rc = incremental_check(client, chunk->items[i], &sig);
|
||||
if (rc < 0) { delta_signature_destroy(sig); return -1; }
|
||||
if (rc == 1) { delta_signature_destroy(sig); continue; }
|
||||
if (rc == 2 && config->use_delta) {
|
||||
int drc = send_delta(client, chunk->items[i], sig, config);
|
||||
delta_signature_destroy(sig);
|
||||
if (drc == 0) continue;
|
||||
if (drc < 0) return -1;
|
||||
} else {
|
||||
delta_signature_destroy(sig);
|
||||
}
|
||||
if (!file_send_single_calls(chunk->items[i], client->file_descriptor,
|
||||
config->use_metadata,
|
||||
config->use_compression ? config->compression_level : 0,
|
||||
|
||||
Reference in New Issue
Block a user