feat: add delta transfer for incremental sync
Implement rsync-style delta transfer using rolling checksums (Adler-32 + xxHash32). When a file exists on both sides but has changed, only the changed blocks are transmitted instead of the entire file. - New status codes: STATUS_DELTA_SIGNATURE, STATUS_DELTA_DATA - Protocol version bumped to 1.2.0 - Server generates block signature, client computes delta - Auto-fallback to whole-file when delta >= 70% of file size - Works with zstd compression on delta stream - Configurable block size (default 8KB, --delta-block flag) - 13 unit tests covering hashing, signature roundtrip, delta compute/apply, file growth/shrink, and decision logic
This commit is contained in:
+198
-10
@@ -10,6 +10,7 @@
|
||||
#include <unistd.h>
|
||||
|
||||
#include "compression.h"
|
||||
#include "delta.h"
|
||||
#include "log.h"
|
||||
#include "config.h"
|
||||
#include "data.h"
|
||||
@@ -132,7 +133,8 @@ bool file_save_to_disk(const char *root_directory, File *file) {
|
||||
return ok;
|
||||
}
|
||||
|
||||
File *receive_incremental_check(int fd, Config *config, bool *skipped) { *skipped = false;
|
||||
File *receive_incremental_check(int fd, Config *config, bool *skipped) {
|
||||
*skipped = false;
|
||||
char *check_path = receive_str(fd);
|
||||
if (check_path == NULL) { send_status(fd, STATUS_ERROR); return NULL; }
|
||||
|
||||
@@ -147,25 +149,211 @@ File *receive_incremental_check(int fd, Config *config, bool *skipped) { *skipp
|
||||
|
||||
char *full_path = path_cat(config->receive_root_directory, check_path);
|
||||
struct stat st;
|
||||
bool match = false;
|
||||
if (full_path && stat(full_path, &st) == 0 &&
|
||||
(unsigned long long)st.st_size == check_size &&
|
||||
(long long)st.st_mtime == check_mtime) {
|
||||
match = true;
|
||||
}
|
||||
free(full_path);
|
||||
bool has_old_file = (full_path && stat(full_path, &st) == 0);
|
||||
unsigned long long old_size = has_old_file ? (unsigned long long)st.st_size : 0;
|
||||
|
||||
bool match = has_old_file &&
|
||||
(unsigned long long)st.st_size == check_size &&
|
||||
(long long)st.st_mtime == check_mtime;
|
||||
|
||||
if (match) {
|
||||
if (!send_status(fd, STATUS_OK)) { free(check_path); return NULL; }
|
||||
if (!send_status(fd, STATUS_OK)) { free(full_path); free(check_path); return NULL; }
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
*skipped = true;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (!send_status(fd, STATUS_NEXT)) { free(check_path); return NULL; }
|
||||
bool try_delta = config->use_delta && has_old_file &&
|
||||
delta_should_attempt(old_size, check_size);
|
||||
|
||||
if (try_delta) {
|
||||
void *old_data = malloc((size_t)old_size);
|
||||
if (!old_data) {
|
||||
try_delta = false;
|
||||
} else {
|
||||
FILE *fp = fopen(full_path, "rb");
|
||||
if (!fp) {
|
||||
free(old_data);
|
||||
try_delta = false;
|
||||
} else {
|
||||
size_t nread = fread(old_data, 1, (size_t)old_size, fp);
|
||||
fclose(fp);
|
||||
if (nread != (size_t)old_size) {
|
||||
free(old_data);
|
||||
try_delta = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (try_delta) {
|
||||
DeltaSignature *sig = delta_signature_create(old_data, old_size,
|
||||
config->delta_block_size);
|
||||
if (!sig) {
|
||||
free(old_data);
|
||||
try_delta = false;
|
||||
} else {
|
||||
Data *sig_data = delta_signature_serialize(sig);
|
||||
if (!sig_data) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
try_delta = false;
|
||||
} else {
|
||||
bool sig_sent = send_status(fd, STATUS_DELTA_SIGNATURE) &&
|
||||
send_data(fd, sig_data);
|
||||
data_destroy(sig_data);
|
||||
|
||||
if (!sig_sent) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
try_delta = false;
|
||||
} else {
|
||||
Status resp;
|
||||
if (!receive_status(fd, &resp)) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (resp == STATUS_DELTA_DATA) {
|
||||
Data *delta_data = receive_data(fd);
|
||||
if (!delta_data) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
Data *raw_delta = delta_data;
|
||||
if (config->use_compression) {
|
||||
raw_delta = data_decompress(delta_data);
|
||||
data_destroy(delta_data);
|
||||
if (!raw_delta) {
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
Delta *delta = delta_deserialize(raw_delta);
|
||||
data_destroy(raw_delta);
|
||||
if (!delta) {
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void *new_data = delta_apply(old_data, old_size, delta,
|
||||
config->delta_block_size);
|
||||
uint64_t new_size = delta->new_file_size;
|
||||
delta_destroy(delta);
|
||||
|
||||
if (!new_data) {
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
File *file = file_create(check_path);
|
||||
free(check_path);
|
||||
free(full_path);
|
||||
|
||||
if (!file) {
|
||||
free(new_data);
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (config->use_metadata) {
|
||||
int meta_ok = 1;
|
||||
file->metadata = metadata_receive(fd, &meta_ok);
|
||||
if (!meta_ok) {
|
||||
file_destroy(file);
|
||||
free(new_data);
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
data_destroy(file->data);
|
||||
file->data = data_create(new_data, (size_t)new_size);
|
||||
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
return file;
|
||||
}
|
||||
|
||||
if (resp == STATUS_NEXT) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
|
||||
File *file = file_create(check_path);
|
||||
free(check_path);
|
||||
free(full_path);
|
||||
if (!file) { send_status(fd, STATUS_ERROR); return NULL; }
|
||||
|
||||
if (config->use_metadata) {
|
||||
int meta_ok = 1;
|
||||
file->metadata = metadata_receive(fd, &meta_ok);
|
||||
if (!meta_ok) { file_destroy(file); send_status(fd, STATUS_ERROR); return NULL; }
|
||||
}
|
||||
|
||||
Data *file_data = receive_data(fd);
|
||||
if (file_data == NULL) {
|
||||
file_destroy(file);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (config->use_compression) {
|
||||
Data *uncompressed = data_decompress(file_data);
|
||||
data_destroy(file_data);
|
||||
if (uncompressed == NULL) { file_destroy(file); send_status(fd, STATUS_ERROR); return NULL; }
|
||||
file_data = uncompressed;
|
||||
}
|
||||
|
||||
data_destroy(file->data);
|
||||
file->data = file_data;
|
||||
return file;
|
||||
}
|
||||
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
try_delta = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!try_delta) {
|
||||
if (!send_status(fd, STATUS_NEXT)) {
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
File *file = file_create(check_path);
|
||||
free(check_path);
|
||||
free(full_path);
|
||||
if (file == NULL) { send_status(fd, STATUS_ERROR); return NULL; }
|
||||
|
||||
if (config->use_metadata) {
|
||||
|
||||
Reference in New Issue
Block a user