feat: add delta transfer for incremental sync
Implement rsync-style delta transfer using rolling checksums (Adler-32 + xxHash32). When a file exists on both sides but has changed, only the changed blocks are transmitted instead of the entire file. - New status codes: STATUS_DELTA_SIGNATURE, STATUS_DELTA_DATA - Protocol version bumped to 1.2.0 - Server generates block signature, client computes delta - Auto-fallback to whole-file when delta >= 70% of file size - Works with zstd compression on delta stream - Configurable block size (default 8KB, --delta-block flag) - 13 unit tests covering hashing, signature roundtrip, delta compute/apply, file growth/shrink, and decision logic
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
#include "config.h"
|
||||
#include "delta.h"
|
||||
#include "log.h"
|
||||
#include "protocol.h"
|
||||
#include "utils.h"
|
||||
@@ -39,6 +40,8 @@ Config *config_create(char *version, char *send_directory,
|
||||
config->max_size = 0;
|
||||
config->min_size = 0;
|
||||
config->use_incremental = false;
|
||||
config->use_delta = false;
|
||||
config->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT;
|
||||
config->use_tls = false;
|
||||
config->tls_cert = NULL;
|
||||
config->tls_key = NULL;
|
||||
@@ -98,6 +101,8 @@ bool config_send(int file_descriptor, Config *config) {
|
||||
if (!send_int(file_descriptor, config->use_sendfile)) return false;
|
||||
if (!send_int(file_descriptor, config->use_delete)) return false;
|
||||
if (!send_int(file_descriptor, config->use_incremental)) return false;
|
||||
if (!send_int(file_descriptor, config->use_delta)) return false;
|
||||
if (!send_int(file_descriptor, (int)config->delta_block_size)) return false;
|
||||
Status status;
|
||||
if (!receive_status(file_descriptor, &status)) return false;
|
||||
if (status != STATUS_OK) {
|
||||
@@ -145,6 +150,10 @@ Config *config_receive(int file_descriptor) {
|
||||
config->use_delete = tmp;
|
||||
if (!receive_int(file_descriptor, &tmp)) goto error;
|
||||
config->use_incremental = tmp;
|
||||
if (!receive_int(file_descriptor, &tmp)) goto error;
|
||||
config->use_delta = tmp;
|
||||
if (!receive_int(file_descriptor, &tmp)) goto error;
|
||||
config->delta_block_size = (uint32_t)tmp;
|
||||
config->show_progress = false;
|
||||
config->dry_run = false;
|
||||
config->ssh_port = 22;
|
||||
|
||||
+4
-1
@@ -2,6 +2,7 @@
|
||||
#define CONFIG_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
typedef enum {
|
||||
TRANSPORT_TCP,
|
||||
@@ -33,13 +34,15 @@ typedef struct Config {
|
||||
unsigned long long max_size;
|
||||
unsigned long long min_size;
|
||||
bool use_incremental;
|
||||
bool use_delta;
|
||||
uint32_t delta_block_size;
|
||||
bool use_tls;
|
||||
char *tls_cert;
|
||||
char *tls_key;
|
||||
char *tls_ca;
|
||||
} Config;
|
||||
|
||||
#define PROTOCOL_VERSION "1.1.0"
|
||||
#define PROTOCOL_VERSION "1.2.0"
|
||||
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
||||
|
||||
Config *config_create(char *version, char *send_directory,
|
||||
|
||||
@@ -0,0 +1,450 @@
|
||||
#include "delta.h"
|
||||
#include "log.h"
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#define XXH_IMPLEMENTATION
|
||||
#include "xxhash.h"
|
||||
|
||||
uint32_t delta_adler32(const void *data, uint32_t len) {
|
||||
const uint8_t *p = (const uint8_t *)data;
|
||||
uint32_t s1 = 1;
|
||||
uint32_t s2 = 0;
|
||||
for (uint32_t i = 0; i < len; i++) {
|
||||
s1 = (s1 + p[i]) % DELTA_ADLER32_MODULUS;
|
||||
s2 = (s2 + s1) % DELTA_ADLER32_MODULUS;
|
||||
}
|
||||
return (s2 << 16) | s1;
|
||||
}
|
||||
|
||||
uint32_t delta_xxhash32(const void *data, uint32_t len) {
|
||||
return XXH32(data, len, 0);
|
||||
}
|
||||
|
||||
DeltaSignature *delta_signature_create(const void *old_file_data,
|
||||
uint64_t old_file_size,
|
||||
uint32_t block_size) {
|
||||
if (old_file_data == NULL || old_file_size == 0 || block_size == 0)
|
||||
return NULL;
|
||||
|
||||
uint32_t block_count = (uint32_t)((old_file_size + block_size - 1) / block_size);
|
||||
|
||||
DeltaSignature *sig = malloc(sizeof(DeltaSignature));
|
||||
if (!sig) return NULL;
|
||||
|
||||
sig->file_size = old_file_size;
|
||||
sig->block_size = block_size;
|
||||
sig->block_count = block_count;
|
||||
sig->blocks = malloc(block_count * sizeof(DeltaBlockSig));
|
||||
if (!sig->blocks) {
|
||||
free(sig);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const uint8_t *data = (const uint8_t *)old_file_data;
|
||||
for (uint32_t i = 0; i < block_count; i++) {
|
||||
uint64_t offset = (uint64_t)i * block_size;
|
||||
uint32_t len = (uint32_t)((old_file_size - offset < block_size)
|
||||
? (old_file_size - offset)
|
||||
: block_size);
|
||||
sig->blocks[i].adler32 = delta_adler32(data + offset, len);
|
||||
sig->blocks[i].xxhash = delta_xxhash32(data + offset, len);
|
||||
}
|
||||
|
||||
return sig;
|
||||
}
|
||||
|
||||
Data *delta_signature_serialize(const DeltaSignature *sig) {
|
||||
if (!sig) return NULL;
|
||||
|
||||
uint64_t total = sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint32_t) +
|
||||
(uint64_t)sig->block_count * (sizeof(uint32_t) + sizeof(uint32_t));
|
||||
|
||||
uint8_t *buf = malloc((size_t)total);
|
||||
if (!buf) return NULL;
|
||||
|
||||
size_t pos = 0;
|
||||
memcpy(buf + pos, &sig->file_size, sizeof(uint64_t));
|
||||
pos += sizeof(uint64_t);
|
||||
memcpy(buf + pos, &sig->block_size, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
memcpy(buf + pos, &sig->block_count, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
|
||||
for (uint32_t i = 0; i < sig->block_count; i++) {
|
||||
memcpy(buf + pos, &sig->blocks[i].adler32, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
memcpy(buf + pos, &sig->blocks[i].xxhash, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
}
|
||||
|
||||
return data_create(buf, (size_t)total);
|
||||
}
|
||||
|
||||
DeltaSignature *delta_signature_deserialize(const Data *data) {
|
||||
if (!data || data->size < sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint32_t))
|
||||
return NULL;
|
||||
|
||||
const uint8_t *buf = (const uint8_t *)data->data;
|
||||
size_t pos = 0;
|
||||
|
||||
DeltaSignature *sig = malloc(sizeof(DeltaSignature));
|
||||
if (!sig) return NULL;
|
||||
|
||||
memcpy(&sig->file_size, buf + pos, sizeof(uint64_t));
|
||||
pos += sizeof(uint64_t);
|
||||
memcpy(&sig->block_size, buf + pos, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
memcpy(&sig->block_count, buf + pos, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
|
||||
uint64_t expected = sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint32_t) +
|
||||
(uint64_t)sig->block_count * (sizeof(uint32_t) + sizeof(uint32_t));
|
||||
if (data->size < expected) {
|
||||
free(sig);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
sig->blocks = malloc(sig->block_count * sizeof(DeltaBlockSig));
|
||||
if (!sig->blocks) {
|
||||
free(sig);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for (uint32_t i = 0; i < sig->block_count; i++) {
|
||||
memcpy(&sig->blocks[i].adler32, buf + pos, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
memcpy(&sig->blocks[i].xxhash, buf + pos, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
}
|
||||
|
||||
return sig;
|
||||
}
|
||||
|
||||
void delta_signature_destroy(DeltaSignature *sig) {
|
||||
if (!sig) return;
|
||||
free(sig->blocks);
|
||||
free(sig);
|
||||
}
|
||||
|
||||
Delta *delta_compute(const void *new_file_data, uint64_t new_file_size,
|
||||
const DeltaSignature *sig, uint32_t block_size) {
|
||||
if (!new_file_data || !sig || new_file_size == 0 || block_size == 0)
|
||||
return NULL;
|
||||
|
||||
const uint8_t *new_data = (const uint8_t *)new_file_data;
|
||||
|
||||
uint32_t capacity = 64;
|
||||
uint32_t count = 0;
|
||||
DeltaInstruction *instrs = malloc(capacity * sizeof(DeltaInstruction));
|
||||
if (!instrs) return NULL;
|
||||
|
||||
uint64_t literal_start = 0;
|
||||
bool has_literal = false;
|
||||
|
||||
uint64_t i = 0;
|
||||
while (i < new_file_size) {
|
||||
uint32_t window_len = (uint32_t)((new_file_size - i < block_size)
|
||||
? (new_file_size - i)
|
||||
: block_size);
|
||||
uint32_t adler = delta_adler32(new_data + i, window_len);
|
||||
uint32_t xxh = delta_xxhash32(new_data + i, window_len);
|
||||
|
||||
bool matched = false;
|
||||
for (uint32_t j = 0; j < sig->block_count; j++) {
|
||||
if (adler == sig->blocks[j].adler32 && xxh == sig->blocks[j].xxhash) {
|
||||
if (has_literal) {
|
||||
uint32_t lit_len = (uint32_t)(i - literal_start);
|
||||
if (count == capacity) {
|
||||
capacity *= 2;
|
||||
DeltaInstruction *tmp = realloc(instrs, capacity * sizeof(DeltaInstruction));
|
||||
if (!tmp) { free(instrs); return NULL; }
|
||||
instrs = tmp;
|
||||
}
|
||||
uint8_t *lit_data = malloc(lit_len);
|
||||
if (!lit_data) { free(instrs); return NULL; }
|
||||
memcpy(lit_data, new_data + literal_start, lit_len);
|
||||
instrs[count].type = DELTA_INSTR_LITERAL;
|
||||
instrs[count].literal.data = lit_data;
|
||||
instrs[count].literal.length = lit_len;
|
||||
count++;
|
||||
has_literal = false;
|
||||
}
|
||||
|
||||
if (count == capacity) {
|
||||
capacity *= 2;
|
||||
DeltaInstruction *tmp = realloc(instrs, capacity * sizeof(DeltaInstruction));
|
||||
if (!tmp) { free(instrs); return NULL; }
|
||||
instrs = tmp;
|
||||
}
|
||||
instrs[count].type = DELTA_INSTR_BLOCK_MATCH;
|
||||
instrs[count].match.block_index = j;
|
||||
instrs[count].match.block_offset = 0;
|
||||
instrs[count].match.length = window_len;
|
||||
count++;
|
||||
|
||||
i += window_len;
|
||||
matched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!matched) {
|
||||
if (!has_literal) {
|
||||
literal_start = i;
|
||||
has_literal = true;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
if (has_literal) {
|
||||
uint32_t lit_len = (uint32_t)(new_file_size - literal_start);
|
||||
if (count == capacity) {
|
||||
capacity *= 2;
|
||||
DeltaInstruction *tmp = realloc(instrs, capacity * sizeof(DeltaInstruction));
|
||||
if (!tmp) { free(instrs); return NULL; }
|
||||
instrs = tmp;
|
||||
}
|
||||
uint8_t *lit_data = malloc(lit_len);
|
||||
if (!lit_data) { free(instrs); return NULL; }
|
||||
memcpy(lit_data, new_data + literal_start, lit_len);
|
||||
instrs[count].type = DELTA_INSTR_LITERAL;
|
||||
instrs[count].literal.data = lit_data;
|
||||
instrs[count].literal.length = lit_len;
|
||||
count++;
|
||||
}
|
||||
|
||||
Delta *delta = malloc(sizeof(Delta));
|
||||
if (!delta) {
|
||||
for (uint32_t k = 0; k < count; k++) {
|
||||
if (instrs[k].type == DELTA_INSTR_LITERAL)
|
||||
free(instrs[k].literal.data);
|
||||
}
|
||||
free(instrs);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
delta->new_file_size = new_file_size;
|
||||
delta->instruction_count = count;
|
||||
delta->instructions = instrs;
|
||||
delta->delta_size = 0;
|
||||
|
||||
for (uint32_t k = 0; k < count; k++) {
|
||||
delta->delta_size += 1;
|
||||
if (instrs[k].type == DELTA_INSTR_BLOCK_MATCH) {
|
||||
delta->delta_size += sizeof(uint32_t) * 3;
|
||||
} else {
|
||||
delta->delta_size += sizeof(uint32_t) + instrs[k].literal.length;
|
||||
}
|
||||
}
|
||||
|
||||
return delta;
|
||||
}
|
||||
|
||||
Data *delta_serialize(const Delta *delta) {
|
||||
if (!delta) return NULL;
|
||||
|
||||
uint64_t total = sizeof(uint64_t) + sizeof(uint32_t) + delta->delta_size;
|
||||
uint8_t *buf = malloc((size_t)total);
|
||||
if (!buf) return NULL;
|
||||
|
||||
size_t pos = 0;
|
||||
memcpy(buf + pos, &delta->new_file_size, sizeof(uint64_t));
|
||||
pos += sizeof(uint64_t);
|
||||
memcpy(buf + pos, &delta->instruction_count, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
|
||||
for (uint32_t i = 0; i < delta->instruction_count; i++) {
|
||||
uint8_t type = (uint8_t)delta->instructions[i].type;
|
||||
memcpy(buf + pos, &type, sizeof(uint8_t));
|
||||
pos += sizeof(uint8_t);
|
||||
|
||||
if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH) {
|
||||
memcpy(buf + pos, &delta->instructions[i].match.block_index, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
memcpy(buf + pos, &delta->instructions[i].match.block_offset, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
memcpy(buf + pos, &delta->instructions[i].match.length, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
} else {
|
||||
memcpy(buf + pos, &delta->instructions[i].literal.length, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
memcpy(buf + pos, delta->instructions[i].literal.data,
|
||||
delta->instructions[i].literal.length);
|
||||
pos += delta->instructions[i].literal.length;
|
||||
}
|
||||
}
|
||||
|
||||
return data_create(buf, (size_t)total);
|
||||
}
|
||||
|
||||
Delta *delta_deserialize(const Data *data) {
|
||||
if (!data || data->size < sizeof(uint64_t) + sizeof(uint32_t))
|
||||
return NULL;
|
||||
|
||||
const uint8_t *buf = (const uint8_t *)data->data;
|
||||
size_t pos = 0;
|
||||
|
||||
Delta *delta = malloc(sizeof(Delta));
|
||||
if (!delta) return NULL;
|
||||
|
||||
memcpy(&delta->new_file_size, buf + pos, sizeof(uint64_t));
|
||||
pos += sizeof(uint64_t);
|
||||
memcpy(&delta->instruction_count, buf + pos, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
|
||||
delta->instructions = malloc(delta->instruction_count * sizeof(DeltaInstruction));
|
||||
if (!delta->instructions) {
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
delta->delta_size = 0;
|
||||
|
||||
for (uint32_t i = 0; i < delta->instruction_count; i++) {
|
||||
if (pos >= data->size) {
|
||||
for (uint32_t k = 0; k < i; k++) {
|
||||
if (delta->instructions[k].type == DELTA_INSTR_LITERAL)
|
||||
free(delta->instructions[k].literal.data);
|
||||
}
|
||||
free(delta->instructions);
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
uint8_t type;
|
||||
memcpy(&type, buf + pos, sizeof(uint8_t));
|
||||
pos += sizeof(uint8_t);
|
||||
|
||||
delta->delta_size += 1;
|
||||
|
||||
if (type == DELTA_OP_BLOCK_MATCH) {
|
||||
if (pos + sizeof(uint32_t) * 3 > data->size) {
|
||||
free(delta->instructions);
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
delta->instructions[i].type = DELTA_INSTR_BLOCK_MATCH;
|
||||
memcpy(&delta->instructions[i].match.block_index, buf + pos, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
memcpy(&delta->instructions[i].match.block_offset, buf + pos, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
memcpy(&delta->instructions[i].match.length, buf + pos, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
delta->delta_size += sizeof(uint32_t) * 3;
|
||||
} else if (type == DELTA_OP_LITERAL) {
|
||||
if (pos + sizeof(uint32_t) > data->size) {
|
||||
free(delta->instructions);
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
memcpy(&delta->instructions[i].literal.length, buf + pos, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
|
||||
uint32_t lit_len = delta->instructions[i].literal.length;
|
||||
if (pos + lit_len > data->size) {
|
||||
free(delta->instructions);
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
delta->instructions[i].literal.data = malloc(lit_len);
|
||||
if (!delta->instructions[i].literal.data) {
|
||||
free(delta->instructions);
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
memcpy(delta->instructions[i].literal.data, buf + pos, lit_len);
|
||||
pos += lit_len;
|
||||
delta->delta_size += sizeof(uint32_t) + lit_len;
|
||||
} else {
|
||||
for (uint32_t k = 0; k < i; k++) {
|
||||
if (delta->instructions[k].type == DELTA_INSTR_LITERAL)
|
||||
free(delta->instructions[k].literal.data);
|
||||
}
|
||||
free(delta->instructions);
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
return delta;
|
||||
}
|
||||
|
||||
void *delta_apply(const void *old_data, uint64_t old_size, const Delta *delta,
|
||||
uint32_t block_size) {
|
||||
if (!old_data || !delta) return NULL;
|
||||
|
||||
void *output = malloc((size_t)delta->new_file_size);
|
||||
if (!output) return NULL;
|
||||
|
||||
uint8_t *out = (uint8_t *)output;
|
||||
uint8_t *old = (uint8_t *)old_data;
|
||||
uint64_t out_pos = 0;
|
||||
|
||||
for (uint32_t i = 0; i < delta->instruction_count; i++) {
|
||||
if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH) {
|
||||
uint64_t src_offset = (uint64_t)delta->instructions[i].match.block_index *
|
||||
block_size;
|
||||
src_offset += delta->instructions[i].match.block_offset;
|
||||
uint32_t len = delta->instructions[i].match.length;
|
||||
|
||||
if (src_offset + len > old_size) {
|
||||
free(output);
|
||||
return NULL;
|
||||
}
|
||||
memcpy(out + out_pos, old + src_offset, len);
|
||||
out_pos += len;
|
||||
} else {
|
||||
uint32_t len = delta->instructions[i].literal.length;
|
||||
memcpy(out + out_pos, delta->instructions[i].literal.data, len);
|
||||
out_pos += len;
|
||||
}
|
||||
}
|
||||
|
||||
if (out_pos != delta->new_file_size) {
|
||||
free(output);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
void delta_destroy(Delta *delta) {
|
||||
if (!delta) return;
|
||||
for (uint32_t i = 0; i < delta->instruction_count; i++) {
|
||||
if (delta->instructions[i].type == DELTA_INSTR_LITERAL)
|
||||
free(delta->instructions[i].literal.data);
|
||||
}
|
||||
free(delta->instructions);
|
||||
free(delta);
|
||||
}
|
||||
|
||||
bool delta_should_attempt(uint64_t old_size, uint64_t new_size) {
|
||||
if (old_size < DELTA_MIN_FILE_SIZE || new_size < DELTA_MIN_FILE_SIZE)
|
||||
return false;
|
||||
if (old_size > DELTA_MAX_FILE_SIZE || new_size > DELTA_MAX_FILE_SIZE)
|
||||
return false;
|
||||
double large = (old_size > new_size) ? (double)old_size : (double)new_size;
|
||||
double small = (old_size > new_size) ? (double)new_size : (double)old_size;
|
||||
if (small == 0 || large / small > DELTA_MAX_SIZE_RATIO)
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool delta_is_worthwhile(const Delta *delta, uint64_t new_file_size) {
|
||||
if (!delta || delta->instruction_count == 0) return false;
|
||||
|
||||
bool has_match = false;
|
||||
for (uint32_t i = 0; i < delta->instruction_count; i++) {
|
||||
if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH) {
|
||||
has_match = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!has_match) return false;
|
||||
|
||||
double ratio = (double)delta->delta_size / (double)new_file_size;
|
||||
return ratio < DELTA_FALLBACK_RATIO;
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
#ifndef DELTA_H
|
||||
#define DELTA_H
|
||||
|
||||
#include "data.h"
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
|
||||
#define DELTA_BLOCK_SIZE_DEFAULT 8192U
|
||||
#define DELTA_BLOCK_SIZE_MIN 1024U
|
||||
#define DELTA_BLOCK_SIZE_MAX 65536U
|
||||
#define DELTA_MIN_FILE_SIZE 16384ULL
|
||||
#define DELTA_MAX_FILE_SIZE (256ULL * 1024 * 1024)
|
||||
#define DELTA_MAX_SIZE_RATIO 10.0
|
||||
#define DELTA_FALLBACK_RATIO 0.7
|
||||
#define DELTA_ADLER32_MODULUS 65521U
|
||||
|
||||
#define DELTA_OP_BLOCK_MATCH 0x01
|
||||
#define DELTA_OP_LITERAL 0x02
|
||||
|
||||
typedef struct {
|
||||
uint32_t adler32;
|
||||
uint32_t xxhash;
|
||||
} DeltaBlockSig;
|
||||
|
||||
typedef struct {
|
||||
uint64_t file_size;
|
||||
uint32_t block_size;
|
||||
uint32_t block_count;
|
||||
DeltaBlockSig *blocks;
|
||||
} DeltaSignature;
|
||||
|
||||
typedef enum {
|
||||
DELTA_INSTR_BLOCK_MATCH = 0x01,
|
||||
DELTA_INSTR_LITERAL = 0x02
|
||||
} DeltaInstrType;
|
||||
|
||||
typedef struct {
|
||||
DeltaInstrType type;
|
||||
union {
|
||||
struct {
|
||||
uint32_t block_index;
|
||||
uint32_t block_offset;
|
||||
uint32_t length;
|
||||
} match;
|
||||
struct {
|
||||
uint8_t *data;
|
||||
uint32_t length;
|
||||
} literal;
|
||||
};
|
||||
} DeltaInstruction;
|
||||
|
||||
typedef struct {
|
||||
uint64_t new_file_size;
|
||||
uint32_t instruction_count;
|
||||
DeltaInstruction *instructions;
|
||||
uint64_t delta_size;
|
||||
} Delta;
|
||||
|
||||
DeltaSignature *delta_signature_create(const void *old_file_data,
|
||||
uint64_t old_file_size,
|
||||
uint32_t block_size);
|
||||
Data *delta_signature_serialize(const DeltaSignature *sig);
|
||||
DeltaSignature *delta_signature_deserialize(const Data *data);
|
||||
void delta_signature_destroy(DeltaSignature *sig);
|
||||
|
||||
Delta *delta_compute(const void *new_file_data, uint64_t new_file_size,
|
||||
const DeltaSignature *sig, uint32_t block_size);
|
||||
Data *delta_serialize(const Delta *delta);
|
||||
Delta *delta_deserialize(const Data *data);
|
||||
void *delta_apply(const void *old_data, uint64_t old_size, const Delta *delta,
|
||||
uint32_t block_size);
|
||||
void delta_destroy(Delta *delta);
|
||||
|
||||
bool delta_should_attempt(uint64_t old_size, uint64_t new_size);
|
||||
bool delta_is_worthwhile(const Delta *delta, uint64_t new_file_size);
|
||||
|
||||
uint32_t delta_adler32(const void *data, uint32_t len);
|
||||
uint32_t delta_xxhash32(const void *data, uint32_t len);
|
||||
|
||||
#endif
|
||||
+198
-10
@@ -10,6 +10,7 @@
|
||||
#include <unistd.h>
|
||||
|
||||
#include "compression.h"
|
||||
#include "delta.h"
|
||||
#include "log.h"
|
||||
#include "config.h"
|
||||
#include "data.h"
|
||||
@@ -132,7 +133,8 @@ bool file_save_to_disk(const char *root_directory, File *file) {
|
||||
return ok;
|
||||
}
|
||||
|
||||
File *receive_incremental_check(int fd, Config *config, bool *skipped) { *skipped = false;
|
||||
File *receive_incremental_check(int fd, Config *config, bool *skipped) {
|
||||
*skipped = false;
|
||||
char *check_path = receive_str(fd);
|
||||
if (check_path == NULL) { send_status(fd, STATUS_ERROR); return NULL; }
|
||||
|
||||
@@ -147,25 +149,211 @@ File *receive_incremental_check(int fd, Config *config, bool *skipped) { *skipp
|
||||
|
||||
char *full_path = path_cat(config->receive_root_directory, check_path);
|
||||
struct stat st;
|
||||
bool match = false;
|
||||
if (full_path && stat(full_path, &st) == 0 &&
|
||||
(unsigned long long)st.st_size == check_size &&
|
||||
(long long)st.st_mtime == check_mtime) {
|
||||
match = true;
|
||||
}
|
||||
free(full_path);
|
||||
bool has_old_file = (full_path && stat(full_path, &st) == 0);
|
||||
unsigned long long old_size = has_old_file ? (unsigned long long)st.st_size : 0;
|
||||
|
||||
bool match = has_old_file &&
|
||||
(unsigned long long)st.st_size == check_size &&
|
||||
(long long)st.st_mtime == check_mtime;
|
||||
|
||||
if (match) {
|
||||
if (!send_status(fd, STATUS_OK)) { free(check_path); return NULL; }
|
||||
if (!send_status(fd, STATUS_OK)) { free(full_path); free(check_path); return NULL; }
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
*skipped = true;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (!send_status(fd, STATUS_NEXT)) { free(check_path); return NULL; }
|
||||
bool try_delta = config->use_delta && has_old_file &&
|
||||
delta_should_attempt(old_size, check_size);
|
||||
|
||||
if (try_delta) {
|
||||
void *old_data = malloc((size_t)old_size);
|
||||
if (!old_data) {
|
||||
try_delta = false;
|
||||
} else {
|
||||
FILE *fp = fopen(full_path, "rb");
|
||||
if (!fp) {
|
||||
free(old_data);
|
||||
try_delta = false;
|
||||
} else {
|
||||
size_t nread = fread(old_data, 1, (size_t)old_size, fp);
|
||||
fclose(fp);
|
||||
if (nread != (size_t)old_size) {
|
||||
free(old_data);
|
||||
try_delta = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (try_delta) {
|
||||
DeltaSignature *sig = delta_signature_create(old_data, old_size,
|
||||
config->delta_block_size);
|
||||
if (!sig) {
|
||||
free(old_data);
|
||||
try_delta = false;
|
||||
} else {
|
||||
Data *sig_data = delta_signature_serialize(sig);
|
||||
if (!sig_data) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
try_delta = false;
|
||||
} else {
|
||||
bool sig_sent = send_status(fd, STATUS_DELTA_SIGNATURE) &&
|
||||
send_data(fd, sig_data);
|
||||
data_destroy(sig_data);
|
||||
|
||||
if (!sig_sent) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
try_delta = false;
|
||||
} else {
|
||||
Status resp;
|
||||
if (!receive_status(fd, &resp)) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (resp == STATUS_DELTA_DATA) {
|
||||
Data *delta_data = receive_data(fd);
|
||||
if (!delta_data) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
Data *raw_delta = delta_data;
|
||||
if (config->use_compression) {
|
||||
raw_delta = data_decompress(delta_data);
|
||||
data_destroy(delta_data);
|
||||
if (!raw_delta) {
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
Delta *delta = delta_deserialize(raw_delta);
|
||||
data_destroy(raw_delta);
|
||||
if (!delta) {
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void *new_data = delta_apply(old_data, old_size, delta,
|
||||
config->delta_block_size);
|
||||
uint64_t new_size = delta->new_file_size;
|
||||
delta_destroy(delta);
|
||||
|
||||
if (!new_data) {
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
File *file = file_create(check_path);
|
||||
free(check_path);
|
||||
free(full_path);
|
||||
|
||||
if (!file) {
|
||||
free(new_data);
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (config->use_metadata) {
|
||||
int meta_ok = 1;
|
||||
file->metadata = metadata_receive(fd, &meta_ok);
|
||||
if (!meta_ok) {
|
||||
file_destroy(file);
|
||||
free(new_data);
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
data_destroy(file->data);
|
||||
file->data = data_create(new_data, (size_t)new_size);
|
||||
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
return file;
|
||||
}
|
||||
|
||||
if (resp == STATUS_NEXT) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
|
||||
File *file = file_create(check_path);
|
||||
free(check_path);
|
||||
free(full_path);
|
||||
if (!file) { send_status(fd, STATUS_ERROR); return NULL; }
|
||||
|
||||
if (config->use_metadata) {
|
||||
int meta_ok = 1;
|
||||
file->metadata = metadata_receive(fd, &meta_ok);
|
||||
if (!meta_ok) { file_destroy(file); send_status(fd, STATUS_ERROR); return NULL; }
|
||||
}
|
||||
|
||||
Data *file_data = receive_data(fd);
|
||||
if (file_data == NULL) {
|
||||
file_destroy(file);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (config->use_compression) {
|
||||
Data *uncompressed = data_decompress(file_data);
|
||||
data_destroy(file_data);
|
||||
if (uncompressed == NULL) { file_destroy(file); send_status(fd, STATUS_ERROR); return NULL; }
|
||||
file_data = uncompressed;
|
||||
}
|
||||
|
||||
data_destroy(file->data);
|
||||
file->data = file_data;
|
||||
return file;
|
||||
}
|
||||
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
try_delta = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!try_delta) {
|
||||
if (!send_status(fd, STATUS_NEXT)) {
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
File *file = file_create(check_path);
|
||||
free(check_path);
|
||||
free(full_path);
|
||||
if (file == NULL) { send_status(fd, STATUS_ERROR); return NULL; }
|
||||
|
||||
if (config->use_metadata) {
|
||||
|
||||
@@ -127,6 +127,10 @@ static const char *status_to_string(Status status) {
|
||||
return "CHUNK";
|
||||
case STATUS_CHECK:
|
||||
return "CHECK";
|
||||
case STATUS_DELTA_SIGNATURE:
|
||||
return "DELTA_SIGNATURE";
|
||||
case STATUS_DELTA_DATA:
|
||||
return "DELTA_DATA";
|
||||
default:
|
||||
return "UNKNOWN";
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
typedef struct ssl_st SSL;
|
||||
|
||||
typedef int Status;
|
||||
enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST, STATUS_CHECK };
|
||||
enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST, STATUS_CHECK, STATUS_DELTA_SIGNATURE, STATUS_DELTA_DATA };
|
||||
|
||||
void io_set_fds(int read_fd, int write_fd);
|
||||
void io_set_bwlimit(unsigned long long bytes_per_sec);
|
||||
|
||||
+7490
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user