diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d1cf9b..e97d020 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,15 @@ if(STRICT_WARNINGS) add_compile_options(-Wextra -Wpedantic -Werror) endif() +include(FetchContent) +FetchContent_Declare( + xxhash + GIT_REPOSITORY https://github.com/Cyan4973/xxHash + GIT_TAG v0.8.3 + SOURCE_SUBDIR cmake_unofficial +) +FetchContent_MakeAvailable(xxhash) + set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) @@ -45,13 +54,13 @@ file(GLOB TEST_SRCS "tests/*.c") add_executable(server ${SERVER_SRCS} ${SHARED_SRCS}) target_include_directories(server PRIVATE src/shared src/server src/client) -target_link_libraries(server PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto) +target_link_libraries(server PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash) add_executable(client ${CLIENT_SRCS} ${SHARED_SRCS}) target_include_directories(client PRIVATE src/shared src/server src/client) -target_link_libraries(client PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto) +target_link_libraries(client PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash) add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} src/client/scanner.c) target_include_directories(tests PRIVATE tests src/shared src/server src/client) -target_link_libraries(tests PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto) +target_link_libraries(tests PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash) diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 55b510e..ecf152b 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -1,5 +1,6 @@ #include "client_send.h" #include "config.h" +#include "delta.h" #include "log.h" #include "protocol.h" #include "transport_tls.h" @@ -36,6 +37,9 @@ static void print_usage(void) { printf(" --max-size Skip files larger than n bytes\n"); printf(" --min-size Skip files smaller than n bytes\n"); printf(" --incremental Skip files unchanged since last transfer\n"); + printf(" --delta Delta transfer for changed files (requires --incremental)\n"); + printf(" --delta-block Delta block size in bytes (default: %d)\n", DELTA_BLOCK_SIZE_DEFAULT); + printf(" --delta-max Max file size for delta transfer (default: %llu)\n", DELTA_MAX_FILE_SIZE); printf(" -m Enable multithreading\n"); printf(" -s Enable chunk serialization\n"); printf(" -f Enable sendfile (TCP only, not with -c or -s)\n"); @@ -102,6 +106,20 @@ int main(int argc, char* argv[]) { config->min_size = strtoull(argv[++i], NULL, 10); } else if (strcmp(argv[i], "--incremental") == 0) { config->use_incremental = true; + } else if (strcmp(argv[i], "--delta") == 0) { + config->use_delta = true; + } else if (strcmp(argv[i], "--delta-block") == 0 && i + 1 < argc) { + unsigned long long val = strtoull(argv[++i], NULL, 10); + if (val >= DELTA_BLOCK_SIZE_MIN && val <= DELTA_BLOCK_SIZE_MAX) + config->delta_block_size = (uint32_t)val; + else + fprintf(stderr, "Warning: --delta-block value %llu out of range, using default\n", val); + } else if (strcmp(argv[i], "--delta-max") == 0 && i + 1 < argc) { + unsigned long long val = strtoull(argv[++i], NULL, 10); + if (val >= DELTA_MIN_FILE_SIZE) + config->delta_max_file_size = val; + else + fprintf(stderr, "Warning: --delta-max value %llu too small, using default\n", val); } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) { config->use_compression = true; log_message(LOG_LEVEL_INFO, "Enabled Compression"); @@ -232,6 +250,23 @@ int main(int argc, char* argv[]) { config->use_metadata = true; } + if (config->use_delta && !config->use_incremental) { + fprintf(stderr, "Error: --delta requires --incremental\n"); + return 1; + } + if (config->use_delta && config->use_chunk_serialization) { + fprintf(stderr, "Error: --delta cannot be combined with -s (chunk serialization)\n"); + return 1; + } + if (config->use_delta && config->use_sendfile) { + fprintf(stderr, "Error: --delta cannot be combined with -f (sendfile)\n"); + return 1; + } + if (config->use_delta && !config->use_metadata) { + log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --delta"); + config->use_metadata = true; + } + if (config->use_tls) { if (!config->tls_cert || !config->tls_key) { fprintf(stderr, "Error: --tls requires --cert and --key\n"); diff --git a/src/client/client_send.c b/src/client/client_send.c index a832f51..b682828 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -1,9 +1,12 @@ #include "client_send.h" #include "array_list.h" #include "chunk.h" +#include "compression.h" #include "config.h" #include "data.h" +#include "delta.h" #include "file.h" +#include "metadata.h" #include "log.h" #include "multiprocessing.h" #include "protocol.h" @@ -19,11 +22,11 @@ #include #include -static int incremental_check(Client* client, File* file) { - if (!send_status(client->file_descriptor, STATUS_CHECK)) - return -1; - if (!send_str(client->file_descriptor, file->path)) - return -1; +static int incremental_check(Client *client, File *file, + DeltaSignature **out_sig) { + *out_sig = NULL; + if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1; + if (!send_str(client->file_descriptor, file->path)) return -1; unsigned long long fsize = file->data->size; long long mtime = file->metadata ? file->metadata->mtime_sec : 0; if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) @@ -37,8 +40,16 @@ static int incremental_check(Client* client, File* file) { log_message(LOG_LEVEL_ERROR, "Server reported error for file"); return -1; } - if (s == STATUS_OK) - return 1; + if (s == STATUS_OK) return 1; + if (s == STATUS_DELTA_SIGNATURE) { + Data *sig_data = receive_data(client->file_descriptor); + if (!sig_data) return -1; + DeltaSignature *sig = delta_signature_deserialize(sig_data); + data_destroy(sig_data); + if (!sig) return -1; + *out_sig = sig; + return 2; + } if (s != STATUS_NEXT) { log_message(LOG_LEVEL_ERROR, "Unexpected server status"); return -1; @@ -46,7 +57,62 @@ static int incremental_check(Client* client, File* file) { return 0; } -int send_chunk(Client* client, Chunk* chunk, Config* config) { +static int send_delta(Client *client, File *file, DeltaSignature *sig, + Config *config) { + Delta *delta = delta_compute(file->data->data, file->data->size, + sig, config->delta_block_size); + if (!delta) return 1; + + if (!delta_is_worthwhile(delta, file->data->size)) { + delta_destroy(delta); + if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; + return 1; + } + + Data *delta_data = delta_serialize(delta); + delta_destroy(delta); + if (!delta_data) return -1; + + Data *to_send = delta_data; + if (config->use_compression) { + to_send = data_compress(delta_data, config->compression_level); + data_destroy(delta_data); + if (!to_send) return -1; + } + + bool ok = send_status(client->file_descriptor, STATUS_DELTA_DATA) && + send_data(client->file_descriptor, to_send); + + if (ok && config->use_metadata) + ok = metadata_send(client->file_descriptor, file->metadata); + + data_destroy(to_send); + return ok ? 0 : -1; +} + +typedef bool (*file_send_fn)(File *, int, bool, int, bool); + +static int send_file_incremental(Client *client, File *file, Config *config, + file_send_fn send_fn) { + DeltaSignature *sig = NULL; + int rc = incremental_check(client, file, &sig); + if (rc < 0) { delta_signature_destroy(sig); return -1; } + if (rc == 1) { delta_signature_destroy(sig); return 1; } + if (rc == 2 && config->use_delta) { + int drc = send_delta(client, file, sig, config); + delta_signature_destroy(sig); + if (drc == 0) return 0; + if (drc < 0) return -1; + } else { + delta_signature_destroy(sig); + } + if (!send_fn(file, client->file_descriptor, config->use_metadata, + config->use_compression ? config->compression_level : 0, false)) + return -1; + return 0; +} + +int send_chunk(Client *client, Chunk *chunk, Config *config) { if (config->use_chunk_serialization) { if (!send_status(client->file_descriptor, STATUS_CHUNK)) return -1; @@ -66,33 +132,23 @@ int send_chunk(Client* client, Chunk* chunk, Config* config) { } else if (config->use_sendfile && !config->use_compression) { for (int i = 0; i < chunk->element_count; i++) { if (config->use_incremental) { - int rc = incremental_check(client, chunk->items[i]); - if (rc < 0) - return -1; - if (rc > 0) - continue; - if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata, - false)) - return -1; + int rc = send_file_incremental(client, chunk->items[i], config, + (file_send_fn)file_send_sendfile); + if (rc == 1) continue; + if (rc < 0) return -1; } else { - if (!send_status(client->file_descriptor, STATUS_NEXT)) - return -1; - if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata, - true)) + if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; + if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata, 0, true)) return -1; } } } else { for (int i = 0; i < chunk->element_count; i++) { if (config->use_incremental) { - int rc = incremental_check(client, chunk->items[i]); - if (rc < 0) - return -1; - if (rc > 0) - continue; - if (!file_send_single_calls(chunk->items[i], client->file_descriptor, config->use_metadata, - config->use_compression ? config->compression_level : 0, false)) - return -1; + int rc = send_file_incremental(client, chunk->items[i], config, + (file_send_fn)file_send_single_calls); + if (rc == 1) continue; + if (rc < 0) return -1; } else { if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; diff --git a/src/shared/config.c b/src/shared/config.c index b7b143c..42aaa41 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -1,4 +1,5 @@ #include "config.h" +#include "delta.h" #include "log.h" #include "protocol.h" #include "utils.h" @@ -37,6 +38,9 @@ Config* config_create(char* version, char* send_directory, char* receive_directo config->max_size = 0; config->min_size = 0; config->use_incremental = false; + config->use_delta = false; + config->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT; + config->delta_max_file_size = DELTA_MAX_FILE_SIZE; config->use_tls = false; config->tls_cert = NULL; config->tls_key = NULL; @@ -88,32 +92,22 @@ void config_delete(Config* config) { } bool config_send(int file_descriptor, const Config* config) { - if (!send_str(file_descriptor, config->version)) - return false; - if (!send_str(file_descriptor, config->send_directory)) - return false; - if (!send_str(file_descriptor, config->receive_root_directory)) - return false; - if (!send_int(file_descriptor, config->save_to_disk)) - return false; - if (!send_int(file_descriptor, config->use_multithreading)) - return false; - if (!send_int(file_descriptor, config->use_chunk_serialization)) - return false; - if (!send_int(file_descriptor, config->use_compression)) - return false; - if (!send_int(file_descriptor, config->use_metadata)) - return false; - if (!send_int(file_descriptor, config->compression_level)) - return false; - if (!send_int(file_descriptor, (int)config->chunk_size)) - return false; - if (!send_int(file_descriptor, config->use_sendfile)) - return false; - if (!send_int(file_descriptor, config->use_delete)) - return false; - if (!send_int(file_descriptor, config->use_incremental)) - return false; + if (!send_str(file_descriptor, config->version)) return false; + if (!send_str(file_descriptor, config->send_directory)) return false; + if (!send_str(file_descriptor, config->receive_root_directory)) return false; + if (!send_int(file_descriptor, config->save_to_disk)) return false; + if (!send_int(file_descriptor, config->use_multithreading)) return false; + if (!send_int(file_descriptor, config->use_chunk_serialization)) return false; + if (!send_int(file_descriptor, config->use_compression)) return false; + if (!send_int(file_descriptor, config->use_metadata)) return false; + if (!send_int(file_descriptor, config->compression_level)) return false; + if (!send_int(file_descriptor, (int)config->chunk_size)) return false; + if (!send_int(file_descriptor, config->use_sendfile)) return false; + if (!send_int(file_descriptor, config->use_delete)) return false; + if (!send_int(file_descriptor, config->use_incremental)) return false; + if (!send_int(file_descriptor, config->use_delta)) return false; + if (!send_int(file_descriptor, (int)config->delta_block_size)) return false; + if (!send_n_data(file_descriptor, &config->delta_max_file_size, sizeof(unsigned long long))) return false; Status status; if (!receive_status(file_descriptor, &status)) return false; @@ -185,6 +179,11 @@ Config* config_receive(int file_descriptor) { if (!receive_int(file_descriptor, &tmp)) goto error; config->use_incremental = tmp; + if (!receive_int(file_descriptor, &tmp)) goto error; + config->use_delta = tmp; + if (!receive_int(file_descriptor, &tmp)) goto error; + config->delta_block_size = (uint32_t)tmp; + if (!receive_n_data(file_descriptor, &config->delta_max_file_size, sizeof(unsigned long long))) goto error; config->show_progress = false; config->dry_run = false; config->ssh_port = 22; diff --git a/src/shared/config.h b/src/shared/config.h index eb646cf..c1f025b 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -2,6 +2,7 @@ #define CONFIG_H #include +#include typedef enum { TRANSPORT_TCP, TRANSPORT_SSH } TransportType; @@ -30,13 +31,16 @@ typedef struct Config { unsigned long long max_size; unsigned long long min_size; bool use_incremental; + bool use_delta; + uint32_t delta_block_size; + unsigned long long delta_max_file_size; bool use_tls; char* tls_cert; char* tls_key; char* tls_ca; } Config; -#define PROTOCOL_VERSION "1.1.0" +#define PROTOCOL_VERSION "1.2.0" #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) Config* config_create(char* version, char* send_directory, char* receive_directory, diff --git a/src/shared/delta.c b/src/shared/delta.c new file mode 100644 index 0000000..fc13a1e --- /dev/null +++ b/src/shared/delta.c @@ -0,0 +1,486 @@ +#include "delta.h" +#include "log.h" +#include +#include + +#define XXH_STATIC_LINKING_ONLY +#define XXH_IMPLEMENTATION +#include + +uint32_t delta_adler32(const void *data, uint32_t len) { + const uint8_t *p = (const uint8_t *)data; + uint32_t s1 = 1; + uint32_t s2 = 0; + for (uint32_t i = 0; i < len; i++) { + s1 = (s1 + p[i]) % DELTA_ADLER32_MODULUS; + s2 = (s2 + s1) % DELTA_ADLER32_MODULUS; + } + return (s2 << 16) | s1; +} + +uint32_t delta_xxhash32(const void *data, uint32_t len) { + return XXH32(data, len, 0); +} + +DeltaSignature *delta_signature_create(const void *old_file_data, + uint64_t old_file_size, + uint32_t block_size) { + if (old_file_data == NULL || old_file_size == 0 || block_size == 0) + return NULL; + + uint32_t block_count = (uint32_t)((old_file_size + block_size - 1) / block_size); + + DeltaSignature *sig = malloc(sizeof(DeltaSignature)); + if (!sig) return NULL; + + sig->file_size = old_file_size; + sig->block_size = block_size; + sig->block_count = block_count; + sig->blocks = malloc(block_count * sizeof(DeltaBlockSig)); + if (!sig->blocks) { + free(sig); + return NULL; + } + + const uint8_t *data = (const uint8_t *)old_file_data; + for (uint32_t i = 0; i < block_count; i++) { + uint64_t offset = (uint64_t)i * block_size; + uint32_t len = (uint32_t)((old_file_size - offset < block_size) + ? (old_file_size - offset) + : block_size); + sig->blocks[i].adler32 = delta_adler32(data + offset, len); + sig->blocks[i].xxhash = delta_xxhash32(data + offset, len); + } + + return sig; +} + +Data *delta_signature_serialize(const DeltaSignature *sig) { + if (!sig) return NULL; + + uint64_t total = sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint32_t) + + (uint64_t)sig->block_count * (sizeof(uint32_t) + sizeof(uint32_t)); + + uint8_t *buf = malloc((size_t)total); + if (!buf) return NULL; + + size_t pos = 0; + memcpy(buf + pos, &sig->file_size, sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(buf + pos, &sig->block_size, sizeof(uint32_t)); + pos += sizeof(uint32_t); + memcpy(buf + pos, &sig->block_count, sizeof(uint32_t)); + pos += sizeof(uint32_t); + + for (uint32_t i = 0; i < sig->block_count; i++) { + memcpy(buf + pos, &sig->blocks[i].adler32, sizeof(uint32_t)); + pos += sizeof(uint32_t); + memcpy(buf + pos, &sig->blocks[i].xxhash, sizeof(uint32_t)); + pos += sizeof(uint32_t); + } + + return data_create(buf, (size_t)total); +} + +DeltaSignature *delta_signature_deserialize(const Data *data) { + if (!data || data->size < sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint32_t)) + return NULL; + + const uint8_t *buf = (const uint8_t *)data->data; + size_t pos = 0; + + DeltaSignature *sig = malloc(sizeof(DeltaSignature)); + if (!sig) return NULL; + + memcpy(&sig->file_size, buf + pos, sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(&sig->block_size, buf + pos, sizeof(uint32_t)); + pos += sizeof(uint32_t); + memcpy(&sig->block_count, buf + pos, sizeof(uint32_t)); + pos += sizeof(uint32_t); + + uint64_t expected = sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint32_t) + + (uint64_t)sig->block_count * (sizeof(uint32_t) + sizeof(uint32_t)); + if (data->size < expected) { + free(sig); + return NULL; + } + + sig->blocks = malloc(sig->block_count * sizeof(DeltaBlockSig)); + if (!sig->blocks) { + free(sig); + return NULL; + } + + for (uint32_t i = 0; i < sig->block_count; i++) { + memcpy(&sig->blocks[i].adler32, buf + pos, sizeof(uint32_t)); + pos += sizeof(uint32_t); + memcpy(&sig->blocks[i].xxhash, buf + pos, sizeof(uint32_t)); + pos += sizeof(uint32_t); + } + + return sig; +} + +void delta_signature_destroy(DeltaSignature *sig) { + if (!sig) return; + free(sig->blocks); + free(sig); +} + +static bool ensure_capacity(DeltaInstruction **instrs, uint32_t *capacity, + uint32_t count) { + if (count < *capacity) return true; + uint32_t new_cap = *capacity * 2; + DeltaInstruction *tmp = realloc(*instrs, new_cap * sizeof(DeltaInstruction)); + if (!tmp) return false; + *instrs = tmp; + *capacity = new_cap; + return true; +} + +static bool flush_literal(DeltaInstruction **instrs, uint32_t *capacity, + uint32_t *count, const uint8_t *data, + uint64_t start, uint64_t end) { + if (start >= end) return true; + uint32_t lit_len = (uint32_t)(end - start); + if (!ensure_capacity(instrs, capacity, *count)) return false; + uint8_t *lit_data = malloc(lit_len); + if (!lit_data) return false; + memcpy(lit_data, data + start, lit_len); + (*instrs)[*count].type = DELTA_INSTR_LITERAL; + (*instrs)[*count].literal.data = lit_data; + (*instrs)[*count].literal.length = lit_len; + (*count)++; + return true; +} + +Delta *delta_compute(const void *new_file_data, uint64_t new_file_size, + const DeltaSignature *sig, uint32_t block_size) { + if (!new_file_data || !sig || new_file_size == 0 || block_size == 0) + return NULL; + + const uint8_t *new_data = (const uint8_t *)new_file_data; + + uint32_t capacity = 64; + uint32_t count = 0; + DeltaInstruction *instrs = malloc(capacity * sizeof(DeltaInstruction)); + if (!instrs) return NULL; + + uint64_t literal_start = 0; + bool has_literal = false; + + uint64_t i = 0; + + uint32_t s1 = 1, s2 = 0; + bool rolling_valid = false; + + while (i < new_file_size) { + uint32_t window_len = (uint32_t)((new_file_size - i < block_size) + ? (new_file_size - i) + : block_size); + bool full_window = (window_len == block_size); + + uint32_t adler; + if (rolling_valid && full_window) { + uint8_t old_byte = new_data[i - 1]; + uint8_t new_byte = new_data[i + block_size - 1]; + s1 = (s1 + DELTA_ADLER32_MODULUS - old_byte + new_byte) % + DELTA_ADLER32_MODULUS; + s2 = (s2 + DELTA_ADLER32_MODULUS - + (uint32_t)((uint64_t)block_size * old_byte % DELTA_ADLER32_MODULUS) + + s1 - 1) % + DELTA_ADLER32_MODULUS; + adler = (s2 << 16) | s1; + } else { + s1 = 1; + s2 = 0; + for (uint32_t k = 0; k < window_len; k++) { + s1 = (s1 + new_data[i + k]) % DELTA_ADLER32_MODULUS; + s2 = (s2 + s1) % DELTA_ADLER32_MODULUS; + } + adler = (s2 << 16) | s1; + rolling_valid = full_window; + } + + bool matched = false; + for (uint32_t j = 0; j < sig->block_count; j++) { + if (adler == sig->blocks[j].adler32 && full_window) { + uint32_t xxh = delta_xxhash32(new_data + i, window_len); + if (xxh == sig->blocks[j].xxhash) { + if (has_literal) { + if (!flush_literal(&instrs, &capacity, &count, new_data, + literal_start, i)) { + free(instrs); + return NULL; + } + has_literal = false; + } + + if (!ensure_capacity(&instrs, &capacity, count)) { + free(instrs); + return NULL; + } + instrs[count].type = DELTA_INSTR_BLOCK_MATCH; + instrs[count].match.block_index = j; + instrs[count].match.block_offset = 0; + instrs[count].match.length = window_len; + count++; + + i += window_len; + rolling_valid = false; + matched = true; + break; + } + } + } + + if (!matched) { + if (!has_literal) { + literal_start = i; + has_literal = true; + } + i++; + } + } + + if (has_literal) { + if (!flush_literal(&instrs, &capacity, &count, new_data, + literal_start, new_file_size)) { + free(instrs); + return NULL; + } + } + + Delta *delta = malloc(sizeof(Delta)); + if (!delta) { + for (uint32_t k = 0; k < count; k++) { + if (instrs[k].type == DELTA_INSTR_LITERAL) + free(instrs[k].literal.data); + } + free(instrs); + return NULL; + } + + delta->new_file_size = new_file_size; + delta->instruction_count = count; + delta->instructions = instrs; + delta->delta_size = 0; + + for (uint32_t k = 0; k < count; k++) { + delta->delta_size += 1; + if (instrs[k].type == DELTA_INSTR_BLOCK_MATCH) { + delta->delta_size += sizeof(uint32_t) * 3; + } else { + delta->delta_size += sizeof(uint32_t) + instrs[k].literal.length; + } + } + + return delta; +} + +Data *delta_serialize(const Delta *delta) { + if (!delta) return NULL; + + uint64_t total = sizeof(uint64_t) + sizeof(uint32_t) + delta->delta_size; + uint8_t *buf = malloc((size_t)total); + if (!buf) return NULL; + + size_t pos = 0; + memcpy(buf + pos, &delta->new_file_size, sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(buf + pos, &delta->instruction_count, sizeof(uint32_t)); + pos += sizeof(uint32_t); + + for (uint32_t i = 0; i < delta->instruction_count; i++) { + uint8_t type = (uint8_t)delta->instructions[i].type; + memcpy(buf + pos, &type, sizeof(uint8_t)); + pos += sizeof(uint8_t); + + if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH) { + memcpy(buf + pos, &delta->instructions[i].match.block_index, sizeof(uint32_t)); + pos += sizeof(uint32_t); + memcpy(buf + pos, &delta->instructions[i].match.block_offset, sizeof(uint32_t)); + pos += sizeof(uint32_t); + memcpy(buf + pos, &delta->instructions[i].match.length, sizeof(uint32_t)); + pos += sizeof(uint32_t); + } else { + memcpy(buf + pos, &delta->instructions[i].literal.length, sizeof(uint32_t)); + pos += sizeof(uint32_t); + memcpy(buf + pos, delta->instructions[i].literal.data, + delta->instructions[i].literal.length); + pos += delta->instructions[i].literal.length; + } + } + + return data_create(buf, (size_t)total); +} + +Delta *delta_deserialize(const Data *data) { + if (!data || data->size < sizeof(uint64_t) + sizeof(uint32_t)) + return NULL; + + const uint8_t *buf = (const uint8_t *)data->data; + size_t pos = 0; + + Delta *delta = malloc(sizeof(Delta)); + if (!delta) return NULL; + + memcpy(&delta->new_file_size, buf + pos, sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(&delta->instruction_count, buf + pos, sizeof(uint32_t)); + pos += sizeof(uint32_t); + + delta->instructions = malloc(delta->instruction_count * sizeof(DeltaInstruction)); + if (!delta->instructions) { + free(delta); + return NULL; + } + + delta->delta_size = 0; + + for (uint32_t i = 0; i < delta->instruction_count; i++) { + if (pos >= data->size) { + for (uint32_t k = 0; k < i; k++) { + if (delta->instructions[k].type == DELTA_INSTR_LITERAL) + free(delta->instructions[k].literal.data); + } + free(delta->instructions); + free(delta); + return NULL; + } + + uint8_t type; + memcpy(&type, buf + pos, sizeof(uint8_t)); + pos += sizeof(uint8_t); + + delta->delta_size += 1; + + if (type == DELTA_OP_BLOCK_MATCH) { + if (pos + sizeof(uint32_t) * 3 > data->size) { + free(delta->instructions); + free(delta); + return NULL; + } + delta->instructions[i].type = DELTA_INSTR_BLOCK_MATCH; + memcpy(&delta->instructions[i].match.block_index, buf + pos, sizeof(uint32_t)); + pos += sizeof(uint32_t); + memcpy(&delta->instructions[i].match.block_offset, buf + pos, sizeof(uint32_t)); + pos += sizeof(uint32_t); + memcpy(&delta->instructions[i].match.length, buf + pos, sizeof(uint32_t)); + pos += sizeof(uint32_t); + delta->delta_size += sizeof(uint32_t) * 3; + } else if (type == DELTA_OP_LITERAL) { + if (pos + sizeof(uint32_t) > data->size) { + free(delta->instructions); + free(delta); + return NULL; + } + memcpy(&delta->instructions[i].literal.length, buf + pos, sizeof(uint32_t)); + pos += sizeof(uint32_t); + + uint32_t lit_len = delta->instructions[i].literal.length; + if (pos + lit_len > data->size) { + free(delta->instructions); + free(delta); + return NULL; + } + delta->instructions[i].literal.data = malloc(lit_len); + if (!delta->instructions[i].literal.data) { + free(delta->instructions); + free(delta); + return NULL; + } + memcpy(delta->instructions[i].literal.data, buf + pos, lit_len); + pos += lit_len; + delta->delta_size += sizeof(uint32_t) + lit_len; + } else { + for (uint32_t k = 0; k < i; k++) { + if (delta->instructions[k].type == DELTA_INSTR_LITERAL) + free(delta->instructions[k].literal.data); + } + free(delta->instructions); + free(delta); + return NULL; + } + } + + return delta; +} + +void *delta_apply(const void *old_data, uint64_t old_size, const Delta *delta, + uint32_t block_size) { + if (!old_data || !delta) return NULL; + + void *output = malloc((size_t)delta->new_file_size); + if (!output) return NULL; + + uint8_t *out = (uint8_t *)output; + uint8_t *old = (uint8_t *)old_data; + uint64_t out_pos = 0; + + for (uint32_t i = 0; i < delta->instruction_count; i++) { + if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH) { + uint64_t src_offset = (uint64_t)delta->instructions[i].match.block_index * + block_size; + src_offset += delta->instructions[i].match.block_offset; + uint32_t len = delta->instructions[i].match.length; + + if (src_offset + len > old_size) { + free(output); + return NULL; + } + memcpy(out + out_pos, old + src_offset, len); + out_pos += len; + } else { + uint32_t len = delta->instructions[i].literal.length; + memcpy(out + out_pos, delta->instructions[i].literal.data, len); + out_pos += len; + } + } + + if (out_pos != delta->new_file_size) { + free(output); + return NULL; + } + + return output; +} + +void delta_destroy(Delta *delta) { + if (!delta) return; + for (uint32_t i = 0; i < delta->instruction_count; i++) { + if (delta->instructions[i].type == DELTA_INSTR_LITERAL) + free(delta->instructions[i].literal.data); + } + free(delta->instructions); + free(delta); +} + +bool delta_should_attempt(uint64_t old_size, uint64_t new_size, uint64_t max_file_size) { + if (old_size < DELTA_MIN_FILE_SIZE || new_size < DELTA_MIN_FILE_SIZE) + return false; + if (old_size > max_file_size || new_size > max_file_size) + return false; + double large = (old_size > new_size) ? (double)old_size : (double)new_size; + double small = (old_size > new_size) ? (double)new_size : (double)old_size; + if (small == 0 || large / small > DELTA_MAX_SIZE_RATIO) + return false; + return true; +} + +bool delta_is_worthwhile(const Delta *delta, uint64_t new_file_size) { + if (!delta || delta->instruction_count == 0) return false; + + bool has_match = false; + for (uint32_t i = 0; i < delta->instruction_count; i++) { + if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH) { + has_match = true; + break; + } + } + if (!has_match) return false; + + double ratio = (double)delta->delta_size / (double)new_file_size; + return ratio < DELTA_FALLBACK_RATIO; +} diff --git a/src/shared/delta.h b/src/shared/delta.h new file mode 100644 index 0000000..209162d --- /dev/null +++ b/src/shared/delta.h @@ -0,0 +1,81 @@ +#ifndef DELTA_H +#define DELTA_H + +#include "data.h" +#include +#include +#include + +#define DELTA_BLOCK_SIZE_DEFAULT 8192U +#define DELTA_BLOCK_SIZE_MIN 1024U +#define DELTA_BLOCK_SIZE_MAX 65536U +#define DELTA_MIN_FILE_SIZE 16384ULL +#define DELTA_MAX_FILE_SIZE (256ULL * 1024 * 1024) +#define DELTA_MAX_SIZE_RATIO 10.0 +#define DELTA_FALLBACK_RATIO 0.7 +#define DELTA_ADLER32_MODULUS 65521U + +#define DELTA_OP_BLOCK_MATCH 0x01 +#define DELTA_OP_LITERAL 0x02 + +typedef struct { + uint32_t adler32; + uint32_t xxhash; +} DeltaBlockSig; + +typedef struct { + uint64_t file_size; + uint32_t block_size; + uint32_t block_count; + DeltaBlockSig *blocks; +} DeltaSignature; + +typedef enum { + DELTA_INSTR_BLOCK_MATCH = 0x01, + DELTA_INSTR_LITERAL = 0x02 +} DeltaInstrType; + +typedef struct { + DeltaInstrType type; + union { + struct { + uint32_t block_index; + uint32_t block_offset; + uint32_t length; + } match; + struct { + uint8_t *data; + uint32_t length; + } literal; + }; +} DeltaInstruction; + +typedef struct { + uint64_t new_file_size; + uint32_t instruction_count; + DeltaInstruction *instructions; + uint64_t delta_size; +} Delta; + +DeltaSignature *delta_signature_create(const void *old_file_data, + uint64_t old_file_size, + uint32_t block_size); +Data *delta_signature_serialize(const DeltaSignature *sig); +DeltaSignature *delta_signature_deserialize(const Data *data); +void delta_signature_destroy(DeltaSignature *sig); + +Delta *delta_compute(const void *new_file_data, uint64_t new_file_size, + const DeltaSignature *sig, uint32_t block_size); +Data *delta_serialize(const Delta *delta); +Delta *delta_deserialize(const Data *data); +void *delta_apply(const void *old_data, uint64_t old_size, const Delta *delta, + uint32_t block_size); +void delta_destroy(Delta *delta); + +bool delta_should_attempt(uint64_t old_size, uint64_t new_size, uint64_t max_file_size); +bool delta_is_worthwhile(const Delta *delta, uint64_t new_file_size); + +uint32_t delta_adler32(const void *data, uint32_t len); +uint32_t delta_xxhash32(const void *data, uint32_t len); + +#endif diff --git a/src/shared/file.c b/src/shared/file.c index b73c12d..6eeabf5 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -10,6 +10,7 @@ #include #include "compression.h" +#include "delta.h" #include "log.h" #include "config.h" #include "data.h" @@ -136,13 +137,154 @@ bool file_save_to_disk(const char* root_directory, File* file) { return ok; } +static void *old_data_from_path(const char *full_path, unsigned long long old_size) { + void *data = malloc((size_t)old_size); + if (!data) return NULL; + FILE *fp = fopen(full_path, "rb"); + if (!fp) { free(data); return NULL; } + size_t nread = fread(data, 1, (size_t)old_size, fp); + fclose(fp); + if (nread != (size_t)old_size) { free(data); return NULL; } + return data; +} + +static File *receive_delta_file(int fd, const Config *config, const char *check_path, + void *old_data, unsigned long long old_size) { + if (!old_data) return NULL; + + DeltaSignature *sig = delta_signature_create(old_data, old_size, + config->delta_block_size); + if (!sig) { free(old_data); return NULL; } + + Data *sig_data = delta_signature_serialize(sig); + if (!sig_data) { delta_signature_destroy(sig); free(old_data); return NULL; } + + bool sig_sent = send_status(fd, STATUS_DELTA_SIGNATURE) && + send_data(fd, sig_data); + data_destroy(sig_data); + + if (!sig_sent) { delta_signature_destroy(sig); free(old_data); return NULL; } + + Status resp; + if (!receive_status(fd, &resp)) { + delta_signature_destroy(sig); + free(old_data); + return NULL; + } + + if (resp == STATUS_DELTA_DATA) { + Data *delta_data = receive_data(fd); + if (!delta_data) { + delta_signature_destroy(sig); + free(old_data); + send_status(fd, STATUS_ERROR); + return NULL; + } + + Data *raw_delta = delta_data; + if (config->use_compression) { + raw_delta = data_decompress(delta_data); + data_destroy(delta_data); + if (!raw_delta) { + free(old_data); + delta_signature_destroy(sig); + send_status(fd, STATUS_ERROR); + return NULL; + } + } + + Delta *delta = delta_deserialize(raw_delta); + data_destroy(raw_delta); + if (!delta) { + free(old_data); + delta_signature_destroy(sig); + send_status(fd, STATUS_ERROR); + return NULL; + } + + void *new_data = delta_apply(old_data, old_size, delta, + config->delta_block_size); + uint64_t new_size = delta->new_file_size; + delta_destroy(delta); + + if (!new_data) { + free(old_data); + delta_signature_destroy(sig); + send_status(fd, STATUS_ERROR); + return NULL; + } + + File *file = file_create(check_path); + if (!file) { + free(new_data); + free(old_data); + delta_signature_destroy(sig); + send_status(fd, STATUS_ERROR); + return NULL; + } + + if (config->use_metadata) { + int meta_ok = 1; + file->metadata = metadata_receive(fd, &meta_ok); + if (!meta_ok) { + file_destroy(file); + free(new_data); + free(old_data); + delta_signature_destroy(sig); + send_status(fd, STATUS_ERROR); + return NULL; + } + } + + data_destroy(file->data); + file->data = data_create(new_data, (size_t)new_size); + + free(old_data); + delta_signature_destroy(sig); + return file; + } + + if (resp == STATUS_NEXT) { + delta_signature_destroy(sig); + free(old_data); + + File *file = file_create(check_path); + if (!file) { send_status(fd, STATUS_ERROR); return NULL; } + + if (config->use_metadata) { + int meta_ok = 1; + file->metadata = metadata_receive(fd, &meta_ok); + if (!meta_ok) { file_destroy(file); send_status(fd, STATUS_ERROR); return NULL; } + } + + Data *file_data = receive_data(fd); + if (file_data == NULL) { + file_destroy(file); + send_status(fd, STATUS_ERROR); + return NULL; + } + + if (config->use_compression) { + Data *uncompressed = data_decompress(file_data); + data_destroy(file_data); + if (uncompressed == NULL) { file_destroy(file); send_status(fd, STATUS_ERROR); return NULL; } + file_data = uncompressed; + } + + data_destroy(file->data); + file->data = file_data; + return file; + } + + delta_signature_destroy(sig); + free(old_data); + return NULL; +} + File* receive_incremental_check(int fd, const Config* config, bool* skipped) { *skipped = false; char* check_path = receive_str(fd); - if (check_path == NULL) { - send_status(fd, STATUS_ERROR); - return NULL; - } + if (check_path == NULL) { send_status(fd, STATUS_ERROR); return NULL; } unsigned long long check_size; long long check_mtime; @@ -155,34 +297,48 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) { char* full_path = path_cat(config->receive_root_directory, check_path); struct stat st; - bool match = false; - if (full_path && stat(full_path, &st) == 0 && (unsigned long long)st.st_size == check_size && - (long long)st.st_mtime == check_mtime) { - match = true; - } - free(full_path); + bool has_old_file = (full_path && stat(full_path, &st) == 0); + unsigned long long old_size = has_old_file ? (unsigned long long)st.st_size : 0; + + bool match = has_old_file && + (unsigned long long)st.st_size == check_size && + (long long)st.st_mtime == check_mtime; if (match) { - if (!send_status(fd, STATUS_OK)) { - free(check_path); - return NULL; - } + if (!send_status(fd, STATUS_OK)) { free(full_path); free(check_path); return NULL; } + free(full_path); free(check_path); *skipped = true; return NULL; } - if (!send_status(fd, STATUS_NEXT)) { - free(check_path); - return NULL; + bool try_delta = config->use_delta && has_old_file && + delta_should_attempt(old_size, check_size, config->delta_max_file_size); + + if (try_delta) { + void *old_data = old_data_from_path(full_path, old_size); + File *delta_file = receive_delta_file(fd, config, check_path, + old_data, old_size); + if (delta_file) { + free(full_path); + free(check_path); + return delta_file; + } + try_delta = false; + } + + if (!try_delta) { + if (!send_status(fd, STATUS_NEXT)) { + free(full_path); + free(check_path); + return NULL; + } } File* file = file_create(check_path); free(check_path); - if (file == NULL) { - send_status(fd, STATUS_ERROR); - return NULL; - } + free(full_path); + if (file == NULL) { send_status(fd, STATUS_ERROR); return NULL; } if (config->use_metadata) { int meta_ok = 1; @@ -242,11 +398,10 @@ bool to_disk(const char* path, const void* data, unsigned long long data_size) { return true; } -bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, bool send_path) { - if (send_path && !send_str(file_descriptor, file->path)) - return false; - if (use_metadata && !metadata_send(file_descriptor, file->metadata)) - return false; +bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata, int compression_level, bool send_path) { + (void)compression_level; + if (send_path && !send_str(file_descriptor, file->path)) return false; + if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; int fd = open(file->path, O_RDONLY); if (fd == -1) { diff --git a/src/shared/file.h b/src/shared/file.h index f2d692e..60e799c 100644 --- a/src/shared/file.h +++ b/src/shared/file.h @@ -26,7 +26,8 @@ bool file_load_data(File* file); File* file_receive(const Config* config, int file_descriptor); bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata, int compression_level, bool send_path); -bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, bool send_path); +bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, + int compression_level, bool send_path); size_t file_content_to_buffer(File* file); FileMetadata* file_metadata_create(const struct stat* stats); void file_metadata_destroy(void* metadata); diff --git a/src/shared/protocol.c b/src/shared/protocol.c index d050869..4d91ce6 100644 --- a/src/shared/protocol.c +++ b/src/shared/protocol.c @@ -128,6 +128,10 @@ static const char* status_to_string(Status status) { return "CHUNK"; case STATUS_CHECK: return "CHECK"; + case STATUS_DELTA_SIGNATURE: + return "DELTA_SIGNATURE"; + case STATUS_DELTA_DATA: + return "DELTA_DATA"; default: return "UNKNOWN"; } diff --git a/src/shared/protocol.h b/src/shared/protocol.h index 0289ab4..a7854f1 100644 --- a/src/shared/protocol.h +++ b/src/shared/protocol.h @@ -15,7 +15,9 @@ enum NET_STATUS { STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST, - STATUS_CHECK + STATUS_CHECK, + STATUS_DELTA_SIGNATURE, + STATUS_DELTA_DATA }; void io_set_fds(int read_fd, int write_fd); diff --git a/tests/runner.c b/tests/runner.c index b05f007..775bc9c 100644 --- a/tests/runner.c +++ b/tests/runner.c @@ -2,6 +2,7 @@ #include "test_chunk.h" #include "test_compression.h" #include "test_config.h" +#include "test_delta.h" #include "test_queue.h" #include "test_scanner.h" #include "test_shared_utils.h" @@ -23,6 +24,7 @@ int main() { RUN_TEST(test_config); RUN_TEST(test_compression); RUN_TEST(test_scanner); + RUN_TEST(test_delta); printf("\n\033[1;36m=== TEST SUMMARY ===\033[0m\n"); printf("Total Tests Run: %d\n", tests_run); diff --git a/tests/test_delta.c b/tests/test_delta.c new file mode 100644 index 0000000..4fbbe26 --- /dev/null +++ b/tests/test_delta.c @@ -0,0 +1,334 @@ +#include "test_utils.h" +#include "delta.h" +#include +#include + +static void test_adler32_basic() { + const char *data = "Hello"; + uint32_t h = delta_adler32(data, 5); + EXPECT_TRUE(h != 0); + uint32_t h2 = delta_adler32(data, 5); + EXPECT_EQ_INT((int)h, (int)h2); +} + +static void test_adler32_different_data() { + const char *a = "AAAA"; + const char *b = "BBBB"; + uint32_t ha = delta_adler32(a, 4); + uint32_t hb = delta_adler32(b, 4); + EXPECT_TRUE(ha != hb); +} + +static void test_xxhash32_basic() { + const char *data = "Hello"; + uint32_t h = delta_xxhash32(data, 5); + EXPECT_TRUE(h != 0); + uint32_t h2 = delta_xxhash32(data, 5); + EXPECT_EQ_INT((int)h, (int)h2); +} + +static void test_xxhash32_different_data() { + const char *a = "AAAA"; + const char *b = "BBBB"; + uint32_t ha = delta_xxhash32(a, 4); + uint32_t hb = delta_xxhash32(b, 4); + EXPECT_TRUE(ha != hb); +} + +static void test_signature_roundtrip() { + char old_data[4096]; + for (int i = 0; i < 4096; i++) old_data[i] = (char)(i % 256); + + DeltaSignature *sig = delta_signature_create(old_data, 4096, 1024); + EXPECT_NOT_NULL(sig); + EXPECT_EQ_INT((int)sig->block_count, 4); + EXPECT_EQ_INT((int)sig->block_size, 1024); + + Data *serialized = delta_signature_serialize(sig); + EXPECT_NOT_NULL(serialized); + + DeltaSignature *deserialized = delta_signature_deserialize(serialized); + EXPECT_NOT_NULL(deserialized); + EXPECT_EQ_INT((int)deserialized->block_count, (int)sig->block_count); + EXPECT_EQ_INT((int)deserialized->block_size, (int)sig->block_size); + + for (uint32_t i = 0; i < sig->block_count; i++) { + EXPECT_EQ_INT((int)deserialized->blocks[i].adler32, (int)sig->blocks[i].adler32); + EXPECT_EQ_INT((int)deserialized->blocks[i].xxhash, (int)sig->blocks[i].xxhash); + } + + delta_signature_destroy(sig); + data_destroy(serialized); + delta_signature_destroy(deserialized); +} + +static void test_delta_identical_files() { + char data[2048]; + for (int i = 0; i < 2048; i++) data[i] = (char)(i % 128); + + DeltaSignature *sig = delta_signature_create(data, 2048, 512); + EXPECT_NOT_NULL(sig); + + Delta *delta = delta_compute(data, 2048, sig, 512); + EXPECT_NOT_NULL(delta); + + bool has_match = false; + for (uint32_t i = 0; i < delta->instruction_count; i++) { + if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH) { + has_match = true; + break; + } + } + EXPECT_TRUE(has_match); + + bool all_match = true; + for (uint32_t i = 0; i < delta->instruction_count; i++) { + if (delta->instructions[i].type != DELTA_INSTR_BLOCK_MATCH) { + all_match = false; + break; + } + } + EXPECT_TRUE(all_match); + + delta_signature_destroy(sig); + delta_destroy(delta); +} + +static void test_delta_small_edit() { + char old_data[4096]; + char new_data[4096]; + for (int i = 0; i < 4096; i++) { + old_data[i] = (char)(i % 256); + new_data[i] = old_data[i]; + } + new_data[100] = 'X'; + new_data[101] = 'Y'; + new_data[102] = 'Z'; + + DeltaSignature *sig = delta_signature_create(old_data, 4096, 1024); + EXPECT_NOT_NULL(sig); + + Delta *delta = delta_compute(new_data, 4096, sig, 1024); + EXPECT_NOT_NULL(delta); + + uint64_t total_literal = 0; + uint32_t match_count = 0; + for (uint32_t i = 0; i < delta->instruction_count; i++) { + if (delta->instructions[i].type == DELTA_INSTR_LITERAL) + total_literal += delta->instructions[i].literal.length; + else + match_count++; + } + EXPECT_TRUE(match_count > 0); + EXPECT_TRUE(total_literal < 4096); + + void *reconstructed = delta_apply(old_data, 4096, delta, 1024); + EXPECT_NOT_NULL(reconstructed); + EXPECT_EQ_INT(memcmp(reconstructed, new_data, 4096), 0); + + free(reconstructed); + delta_signature_destroy(sig); + delta_destroy(delta); +} + +static void test_delta_completely_different() { + char old_data[4096]; + char new_data[4096]; + for (int i = 0; i < 4096; i++) { + old_data[i] = (char)(i * 7 + 3); + new_data[i] = (char)(i * 13 + 97); + } + + DeltaSignature *sig = delta_signature_create(old_data, 4096, 1024); + EXPECT_NOT_NULL(sig); + + Delta *delta = delta_compute(new_data, 4096, sig, 1024); + EXPECT_NOT_NULL(delta); + + bool has_match = false; + for (uint32_t i = 0; i < delta->instruction_count; i++) { + if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH) { + has_match = true; + break; + } + } + EXPECT_TRUE(!has_match); + EXPECT_TRUE(!delta_is_worthwhile(delta, 4096)); + + delta_signature_destroy(sig); + delta_destroy(delta); +} + +static void test_delta_serialize_roundtrip() { + char old_data[4096]; + char new_data[4096]; + for (int i = 0; i < 4096; i++) { + old_data[i] = (char)(i % 256); + new_data[i] = old_data[i]; + } + new_data[500] = 'A'; + new_data[501] = 'B'; + + DeltaSignature *sig = delta_signature_create(old_data, 4096, 1024); + Delta *delta = delta_compute(new_data, 4096, sig, 1024); + EXPECT_NOT_NULL(delta); + + Data *serialized = delta_serialize(delta); + EXPECT_NOT_NULL(serialized); + + Delta *deserialized = delta_deserialize(serialized); + EXPECT_NOT_NULL(deserialized); + EXPECT_EQ_INT((int)deserialized->new_file_size, (int)delta->new_file_size); + EXPECT_EQ_INT((int)deserialized->instruction_count, (int)delta->instruction_count); + + void *reconstructed = delta_apply(old_data, 4096, deserialized, 1024); + EXPECT_NOT_NULL(reconstructed); + EXPECT_EQ_INT(memcmp(reconstructed, new_data, 4096), 0); + + free(reconstructed); + data_destroy(serialized); + delta_destroy(deserialized); + delta_signature_destroy(sig); + delta_destroy(delta); +} + +static void test_delta_file_growth() { + char old_data[2048]; + char new_data[3072]; + for (int i = 0; i < 2048; i++) old_data[i] = (char)(i % 256); + memcpy(new_data, old_data, 2048); + for (int i = 2048; i < 3072; i++) new_data[i] = (char)(i % 256); + + DeltaSignature *sig = delta_signature_create(old_data, 2048, 512); + EXPECT_NOT_NULL(sig); + + Delta *delta = delta_compute(new_data, 3072, sig, 512); + EXPECT_NOT_NULL(delta); + + void *reconstructed = delta_apply(old_data, 2048, delta, 512); + EXPECT_NOT_NULL(reconstructed); + EXPECT_EQ_INT(delta->new_file_size, 3072); + EXPECT_EQ_INT(memcmp(reconstructed, new_data, 3072), 0); + + free(reconstructed); + delta_signature_destroy(sig); + delta_destroy(delta); +} + +static void test_delta_file_shrink() { + char old_data[3072]; + char new_data[2048]; + for (int i = 0; i < 3072; i++) old_data[i] = (char)(i % 256); + for (int i = 0; i < 2048; i++) new_data[i] = old_data[i]; + + DeltaSignature *sig = delta_signature_create(old_data, 3072, 512); + EXPECT_NOT_NULL(sig); + + Delta *delta = delta_compute(new_data, 2048, sig, 512); + EXPECT_NOT_NULL(delta); + + void *reconstructed = delta_apply(old_data, 3072, delta, 512); + EXPECT_NOT_NULL(reconstructed); + EXPECT_EQ_INT(delta->new_file_size, 2048); + EXPECT_EQ_INT(memcmp(reconstructed, new_data, 2048), 0); + + free(reconstructed); + delta_signature_destroy(sig); + delta_destroy(delta); +} + +static void test_should_attempt() { + EXPECT_TRUE(delta_should_attempt(100000, 100000, DELTA_MAX_FILE_SIZE)); + EXPECT_TRUE(!delta_should_attempt(100, 100, DELTA_MAX_FILE_SIZE)); + EXPECT_TRUE(!delta_should_attempt(100000, 10, DELTA_MAX_FILE_SIZE)); + EXPECT_TRUE(!delta_should_attempt(300000000, 300000000, DELTA_MAX_FILE_SIZE)); + EXPECT_TRUE(delta_should_attempt(50000, 60000, DELTA_MAX_FILE_SIZE)); + EXPECT_TRUE(!delta_should_attempt(50000, 600000, DELTA_MAX_FILE_SIZE)); + EXPECT_TRUE(delta_should_attempt(50000, 60000, 500000)); + EXPECT_TRUE(!delta_should_attempt(100000, 100000, 50000)); +} + +static void test_is_worthwhile() { + Delta d; + d.instruction_count = 1; + DeltaInstruction instr; + instr.type = DELTA_INSTR_BLOCK_MATCH; + d.instructions = &instr; + + d.delta_size = 100; + EXPECT_TRUE(delta_is_worthwhile(&d, 1000)); + + d.delta_size = 800; + EXPECT_TRUE(!delta_is_worthwhile(&d, 1000)); + + d.instruction_count = 1; + instr.type = DELTA_INSTR_LITERAL; + EXPECT_TRUE(!delta_is_worthwhile(&d, 1000)); + + EXPECT_TRUE(!delta_is_worthwhile(NULL, 1000)); +} + +static void test_large_file_delta() { + uint32_t block_size = 8192; + uint64_t old_size = 200000; + uint64_t new_size = 200000; + + void *old_data = malloc((size_t)old_size); + void *new_data = malloc((size_t)new_size); + EXPECT_TRUE(old_data != NULL && new_data != NULL); + + for (uint64_t i = 0; i < old_size; i++) + ((uint8_t *)old_data)[i] = (uint8_t)(i % 251); + memcpy(new_data, old_data, (size_t)old_size); + + uint64_t offset = 100000; + uint32_t change_len = 4096; + for (uint32_t i = 0; i < change_len; i++) + ((uint8_t *)new_data)[offset + i] = (uint8_t)((i * 7 + 13) % 256); + + DeltaSignature *sig = delta_signature_create(old_data, old_size, block_size); + EXPECT_TRUE(sig != NULL); + EXPECT_TRUE(sig->block_count == (uint32_t)((old_size + block_size - 1) / block_size)); + + Delta *delta = delta_compute(new_data, new_size, sig, block_size); + EXPECT_TRUE(delta != NULL); + + uint64_t total_literal = 0; + uint32_t match_count = 0; + for (uint32_t i = 0; i < delta->instruction_count; i++) { + if (delta->instructions[i].type == DELTA_INSTR_LITERAL) + total_literal += delta->instructions[i].literal.length; + else + match_count++; + } + EXPECT_TRUE(match_count > 0); + EXPECT_TRUE(delta->delta_size < new_size / 2); + + void *result = delta_apply(old_data, old_size, delta, block_size); + EXPECT_TRUE(result != NULL); + EXPECT_TRUE(delta->new_file_size == new_size); + EXPECT_TRUE(memcmp(result, new_data, (size_t)new_size) == 0); + + free(result); + delta_destroy(delta); + delta_signature_destroy(sig); + free(old_data); + free(new_data); +} + +void test_delta() { + test_adler32_basic(); + test_adler32_different_data(); + test_xxhash32_basic(); + test_xxhash32_different_data(); + test_signature_roundtrip(); + test_delta_identical_files(); + test_delta_small_edit(); + test_delta_completely_different(); + test_delta_serialize_roundtrip(); + test_delta_file_growth(); + test_delta_file_shrink(); + test_should_attempt(); + test_is_worthwhile(); + test_large_file_delta(); +} diff --git a/tests/test_delta.h b/tests/test_delta.h new file mode 100644 index 0000000..2ea7aec --- /dev/null +++ b/tests/test_delta.h @@ -0,0 +1,6 @@ +#ifndef TEST_DELTA_H +#define TEST_DELTA_H + +void test_delta(void); + +#endif