feat(checksum): --checksum-choice/--cc and --checksum-seed for whole-file digest

Adds real algorithm selection (xxh64 default, plus md5 via OpenSSL EVP) and a
64-bit seed for the per-file whole-file digest used by the --incremental/
--checksum handshake and basis-dir content verification. The seed also feeds
the delta path's per-block xxHash32 strong checksum (low 32 bits) so an
explicit seed deterministically changes those digests too. Sender and receiver
hash identically: the algorithm id and seed cross the config wire frame and the
STATUS_CHECK handshake now carries a length-prefixed, bounded digest instead of
a fixed 64-bit value. Unsupported algorithm names are rejected at parse time
(never a silent no-op). PROTOCOL_VERSION bumped 2.9.0 -> 2.10.0; defaults
(xxh64, seed 0) preserve prior byte-for-byte behavior.
This commit is contained in:
2026-09-07 17:42:47 +02:00
parent 6701c103cb
commit 0afda6b094
12 changed files with 316 additions and 43 deletions
+58 -1
View File
@@ -94,6 +94,47 @@ static int set_compression_choice(Config* config, const char* value) {
return 0; return 0;
} }
/* Validate and store the --checksum-choice/--cc algorithm. Only the algorithms
* the engine genuinely supports are accepted (xxHash64 and md5); anything else
* is a clear error, never a silent no-op. "xxhash" is accepted as rsync's
* spelling of xxHash64. */
static int set_checksum_choice(Config* config, const char* value) {
int algo = checksum_algo_from_name(value);
if (algo < 0) {
log_message(LOG_LEVEL_ERROR, "--checksum-choice must be xxh64 (or xxhash) or md5 (got '%s')",
value);
return -1;
}
config->checksum_algo = algo;
return 0;
}
/* parse_ull_arg is defined later in this file; declared here for the seed
parser below. */
static int parse_ull_arg(const char* val, unsigned long long* out, const char* optname);
/* Parse --checksum-seed=NUM as a strict decimal 0..UINT64_MAX. A blank value,
* a sign, or any non-digit (which parse_ull_arg's strtoull would silently
* coerce) is rejected: an explicit seed must be an exact unsigned integer or
* the run fails with a clear error rather than quietly ignoring the value. */
static int set_checksum_seed(Config* config, const char* value) {
if (!value || *value == '\0') {
log_message(LOG_LEVEL_ERROR, "--checksum-seed must be a non-negative integer");
return -1;
}
for (const char* p = value; *p; p++) {
if (*p < '0' || *p > '9') {
log_message(LOG_LEVEL_ERROR, "--checksum-seed must be a non-negative integer");
return -1;
}
}
unsigned long long seed;
if (parse_ull_arg(value, &seed, "--checksum-seed") != 0)
return -1;
config->checksum_seed = seed;
return 0;
}
static int set_compression_threads_option(int* dest, const char* value) { static int set_compression_threads_option(int* dest, const char* value) {
if (set_positive_int_option(dest, value, "--compress-threads") != 0) if (set_positive_int_option(dest, value, "--compress-threads") != 0)
return -1; return -1;
@@ -1009,7 +1050,23 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]); log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
return -1; return -1;
} }
log_message(LOG_LEVEL_ERROR, "%s is not supported yet (xxHash64 is used)", argv[i]); if (set_checksum_choice(config, argv[++i]) != 0)
return -1;
} else if (strncmp(argv[i], "--checksum-choice=", 18) == 0) {
if (set_checksum_choice(config, argv[i] + 18) != 0)
return -1;
} else if (strncmp(argv[i], "--cc=", 5) == 0) {
if (set_checksum_choice(config, argv[i] + 5) != 0)
return -1;
} else if (strncmp(argv[i], "--checksum-seed=", 16) == 0) {
if (set_checksum_seed(config, argv[i] + 16) != 0)
return -1;
} else if (opt_is(argv[i], "--checksum-seed", NULL)) {
if (i + 1 >= argc) {
log_message(LOG_LEVEL_ERROR, "missing argument for --checksum-seed");
return -1;
}
if (set_checksum_seed(config, argv[++i]) != 0)
return -1; return -1;
} else if (strncmp(argv[i], "--compare-dest=", 15) == 0) { } else if (strncmp(argv[i], "--compare-dest=", 15) == 0) {
if (set_basis_dest_option(config, BASIS_DEST_COMPARE, argv[i] + 15, "--compare-dest") != 0) if (set_basis_dest_option(config, BASIS_DEST_COMPARE, argv[i] + 15, "--compare-dest") != 0)
+13 -6
View File
@@ -733,12 +733,18 @@ static int incremental_check(Client* client, File* file, const Config* config,
if (!send_n_data(client->file_descriptor, &mtime_nsec, sizeof(mtime_nsec))) if (!send_n_data(client->file_descriptor, &mtime_nsec, sizeof(mtime_nsec)))
return -1; return -1;
/* With alternate basis directories the receiver must be able to verify the /* With alternate basis directories the receiver must be able to verify the
* content of every candidate basis file, so the sender supplies its xxHash64 * content of every candidate basis file, so the sender supplies its whole-file
* for every file even when --checksum was not requested. */ * digest (computed with the negotiated --checksum-choice algorithm and
* --checksum-seed) for every file even when --checksum was not requested. */
if (config->checksum || config_has_basis(config)) { if (config->checksum || config_has_basis(config)) {
uint64_t checksum; uint8_t digest[CHECKSUM_MAX_DIGEST_LEN];
if (!file_checksum(file, &checksum) || size_t digest_len = 0;
!send_n_data(client->file_descriptor, &checksum, sizeof(checksum))) if (!file_checksum(file, (ChecksumAlgo)config->checksum_algo, config->checksum_seed, digest,
sizeof(digest), &digest_len))
return -1;
uint8_t wire_len = (uint8_t)digest_len;
if (!send_n_data(client->file_descriptor, &wire_len, sizeof(wire_len)) ||
!send_n_data(client->file_descriptor, digest, wire_len))
return -1; return -1;
} }
Status s; Status s;
@@ -774,7 +780,8 @@ static int incremental_check(Client* client, File* file, const Config* config,
} }
static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* config) { static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* config) {
Delta* delta = delta_compute(file->data->data, file->data->size, sig, config->delta_block_size); Delta* delta = delta_compute_seeded(file->data->data, file->data->size, sig,
config->delta_block_size, (uint32_t)config->checksum_seed);
/* The receiver is blocked after sending the signature. Every local /* The receiver is blocked after sending the signature. Every local
fallback therefore needs the explicit NEXT response before full data. */ fallback therefore needs the explicit NEXT response before full data. */
if (!delta) if (!delta)
+6 -2
View File
@@ -91,8 +91,12 @@ void print_usage(void) {
printf(" into the destination instead of transferring its data\n"); printf(" into the destination instead of transferring its data\n");
printf(" --link-dest <dir> Like --copy-dest, but hard-links the unchanged file from DIR\n"); printf(" --link-dest <dir> Like --copy-dest, but hard-links the unchanged file from DIR\n");
printf(" into the destination (repeatable; earlier DIRs win)\n"); printf(" into the destination (repeatable; earlier DIRs win)\n");
printf(" --checksum-choice, --cc <alg> Checksum algorithm (not supported yet; xxHash64 is " printf(" --checksum-choice, --cc <alg> Whole-file checksum algorithm for --incremental/\n");
"used)\n"); printf(" --checksum compares (xxh64/xxhash or md5; default xxh64 with\n");
printf(" seed 0). The seed comes from --checksum-seed\n");
printf(" --checksum-seed <num> Seed for the whole-file xxHash64 digest (and the delta\n");
printf(" block strong hash, low 32 bits); md5 ignores the seed. The\n");
printf(" digest algorithm and seed must match on sender and receiver\n");
printf(" --delta Delta transfer for changed files (requires --incremental)\n"); printf(" --delta Delta transfer for changed files (requires --incremental)\n");
printf(" -W, --whole-file Transfer changed files without delta processing\n"); printf(" -W, --whole-file Transfer changed files without delta processing\n");
printf(" -y, --fuzzy Use a similar-named file already in the destination\n"); printf(" -y, --fuzzy Use a similar-named file already in the destination\n");
+78
View File
@@ -0,0 +1,78 @@
#include "checksum.h"
#include <openssl/evp.h>
#include <string.h>
#include <strings.h>
/* delta.c owns the single XXH_IMPLEMENTATION that provides the xxHash symbols
* for the whole binary; this TU only needs the declarations. */
#include <xxhash.h>
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
size_t out_capacity, size_t* out_len) {
if (!out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
return false;
if (data == NULL && size != 0)
return false;
if (algo == CHECKSUM_ALGO_XXH64) {
uint64_t digest = XXH64(data, size, seed);
uint8_t buf[CHECKSUM_MAX_DIGEST_LEN];
memcpy(buf, &digest, sizeof(digest));
memcpy(out, buf, sizeof(digest));
*out_len = sizeof(digest);
return true;
}
if (algo == CHECKSUM_ALGO_MD5) {
/* md5 takes no seed; the caller's seed is deliberately ignored (documented
* in RSYNC_COMPAT.md). OpenSSL's one-shot EVP_Digest needs a non-NULL
* buffer even for an empty input, so map a NULL data + size==0 to an empty
* buffer. */
static const uint8_t empty = 0;
const void* input = data ? data : &empty;
unsigned int digest_len = 0;
if (EVP_Digest(input, size, out, &digest_len, EVP_md5(), NULL) != 1)
return false;
if (digest_len > out_capacity)
return false;
*out_len = digest_len;
return true;
}
return false;
}
int checksum_algo_from_name(const char* name) {
if (!name)
return -1;
if (strcasecmp(name, "xxh64") == 0 || strcasecmp(name, "xxhash") == 0 ||
strcasecmp(name, "xxh3") == 0)
return (int)CHECKSUM_ALGO_XXH64;
if (strcasecmp(name, "md5") == 0)
return (int)CHECKSUM_ALGO_MD5;
return -1;
}
const char* checksum_algo_name(ChecksumAlgo algo) {
switch (algo) {
case CHECKSUM_ALGO_XXH64:
return "xxh64";
case CHECKSUM_ALGO_MD5:
return "md5";
}
return "<unknown>";
}
bool checksum_algo_valid(int algo) {
return algo == (int)CHECKSUM_ALGO_XXH64 || algo == (int)CHECKSUM_ALGO_MD5;
}
uint8_t checksum_digest_len(ChecksumAlgo algo) {
switch (algo) {
case CHECKSUM_ALGO_XXH64:
return 8;
case CHECKSUM_ALGO_MD5:
return 16;
}
return 0;
}
+45
View File
@@ -0,0 +1,45 @@
#ifndef CHECKSUM_H
#define CHECKSUM_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
/* Whole-file content-digest algorithms selectable with --checksum-choice and
* seeded with --checksum-seed. The ids are the values actually placed on the
* wire (config frame), so they must be kept stable and validated on receive.
* CHECKSUM_ALGO_XXH64 == 0 is the default and is byte-for-byte what FastSync
* computed before these options existed (xxHash64 with seed 0). */
typedef enum { CHECKSUM_ALGO_XXH64 = 0, CHECKSUM_ALGO_MD5 = 1 } ChecksumAlgo;
/* md5 digest is 16 bytes, the longest supported. */
#define CHECKSUM_MAX_DIGEST_LEN 16
/* Compute the whole-file digest of the first `size` bytes of `data`.
*
* - CHECKSUM_ALGO_XXH64: xxHash64(data, size, seed) (full 64-bit seed).
* - CHECKSUM_ALGO_MD5: md5(data, size) via OpenSSL EVP.
* md5 has no seed, so `seed` is ignored (documented).
* - `size == 0` hashes the empty input (plus its seed), not a NULL input.
*
* Writes up to `out_capacity` bytes into `out`, storing the digest length in
* *out_len. Returns false on NULL out* or when the digest would not fit.
* Never writes more than CHECKSUM_MAX_DIGEST_LEN bytes. */
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
size_t out_capacity, size_t* out_len);
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id.
* Accepts "xxh64" and "xxhash" (both map to CHECKSUM_ALGO_XXH64, rsync's
* xxhash spelling) and "md5". Returns -1 for any unsupported name. */
int checksum_algo_from_name(const char* name);
/* Canonical name of an algorithm (used in CLI error messages). */
const char* checksum_algo_name(ChecksumAlgo algo);
/* True when `algo` is a supported id (used by config receive validation). */
bool checksum_algo_valid(int algo);
/* Digest length in bytes for an algorithm (xxx64 = 8, md5 = 16). */
uint8_t checksum_digest_len(ChecksumAlgo algo);
#endif /* CHECKSUM_H */
+24 -3
View File
@@ -127,6 +127,8 @@ static void config_set_defaults(Config* config) {
config->daemon_config = NULL; config->daemon_config = NULL;
config->server_mode = false; config->server_mode = false;
config->checksum = false; config->checksum = false;
config->checksum_algo = CHECKSUM_ALGO_XXH64;
config->checksum_seed = 0;
config->compress_choice = NULL; config->compress_choice = NULL;
config->chmod_spec = NULL; config->chmod_spec = NULL;
config->skip_compress_suffixes = NULL; config->skip_compress_suffixes = NULL;
@@ -172,7 +174,7 @@ static bool validate_received_config(const Config* config) {
!(config->delay_updates && delay_updates_staging_name_conflict(config->backup_dir)) && !(config->delay_updates && delay_updates_staging_name_conflict(config->backup_dir)) &&
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) && valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) && valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
config_has_valid_delete_timing(config) && checksum_algo_valid(config->checksum_algo) && config_has_valid_delete_timing(config) &&
!(config->skip_compress_set && config->use_chunk_serialization) && !(config->skip_compress_set && config->use_chunk_serialization) &&
(!config->use_compression || (!config->use_compression ||
(config->compression_level >= 1 && config->compression_level <= 22)) && (config->compression_level >= 1 && config->compression_level <= 22)) &&
@@ -462,6 +464,15 @@ static bool send_fuzzy_option(int fd, const Config* c) {
return send_int(fd, c->fuzzy); return send_int(fd, c->fuzzy);
} }
/* --checksum-choice/--cc + --checksum-seed. The algorithm id and seed travel
* with the config so the receiver hashes the on-disk old file with the same
* parameters the sender used for its digest (see checksum.h). Trailing fields
* on the config frame; protocol 2.10.0. */
static bool send_checksum_options(int fd, const Config* c) {
return send_int(fd, c->checksum_algo) &&
send_n_data(fd, &c->checksum_seed, sizeof(c->checksum_seed));
}
static bool receive_core_fields(int fd, Config* c) { static bool receive_core_fields(int fd, Config* c) {
int value; int value;
if (!receive_wire_bool(fd, &c->eight_bit_output)) if (!receive_wire_bool(fd, &c->eight_bit_output))
@@ -637,13 +648,22 @@ static bool receive_fuzzy_option(int fd, Config* c) {
return receive_wire_bool(fd, &c->fuzzy); return receive_wire_bool(fd, &c->fuzzy);
} }
static bool receive_checksum_options(int fd, Config* c) {
int algo;
if (!receive_int(fd, &algo) || !checksum_algo_valid(algo))
return false;
c->checksum_algo = algo;
return receive_n_data(fd, &c->checksum_seed, sizeof(c->checksum_seed));
}
bool config_send(int file_descriptor, const Config* config) { bool config_send(int file_descriptor, const Config* config) {
protocol_session_set_max_alloc(NULL, config->max_alloc); protocol_session_set_max_alloc(NULL, config->max_alloc);
if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) || if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) ||
!send_file_options(file_descriptor, config) || !send_file_options(file_descriptor, config) ||
!send_selection_options(file_descriptor, config) || !send_selection_options(file_descriptor, config) ||
!send_resume_options(file_descriptor, config) || !send_resume_options(file_descriptor, config) ||
!send_basis_options(file_descriptor, config) || !send_fuzzy_option(file_descriptor, config)) !send_basis_options(file_descriptor, config) || !send_fuzzy_option(file_descriptor, config) ||
!send_checksum_options(file_descriptor, config))
return false; return false;
Status status; Status status;
if (!receive_status(file_descriptor, &status)) if (!receive_status(file_descriptor, &status))
@@ -677,7 +697,8 @@ Config* config_receive(int file_descriptor) {
!receive_selection_options(file_descriptor, config) || !receive_selection_options(file_descriptor, config) ||
!receive_resume_options(file_descriptor, config) || !receive_resume_options(file_descriptor, config) ||
!receive_basis_options(file_descriptor, config) || !receive_basis_options(file_descriptor, config) ||
!receive_fuzzy_option(file_descriptor, config)) !receive_fuzzy_option(file_descriptor, config) ||
!receive_checksum_options(file_descriptor, config))
goto error; goto error;
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 && if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
strcmp(config->compress_choice, "none") != 0) { strcmp(config->compress_choice, "none") != 0) {
+14 -1
View File
@@ -2,6 +2,7 @@
#define CONFIG_H #define CONFIG_H
#include "array_list.h" #include "array_list.h"
#include "checksum.h"
#include <stdbool.h> #include <stdbool.h>
#include <stdint.h> #include <stdint.h>
#include <stdio.h> #include <stdio.h>
@@ -222,6 +223,18 @@ typedef struct Config {
char* compress_choice; char* compress_choice;
char* chmod_spec; char* chmod_spec;
/* --checksum-choice / --cc and --checksum-seed. checksum_algo is the id of
* the whole-file content-digest algorithm used by the per-file --incremental
* handshake (sender computes it, receiver compares it to skip unchanged
* files) and by the basis-dir content verification. checksum_seed is passed
* to xxHash64 (and to the delta block strong hash, low 32 bits); md5 has no
* seed so it is ignored there. Both cross the wire: the receiver MUST hash
* the on-disk old file with the same algorithm and seed to reach a matching
* digest. Defaults (XXH64 / seed 0) reproduce the pre-existing behavior
* byte-for-byte. */
int checksum_algo; /* ChecksumAlgo, default CHECKSUM_ALGO_XXH64 */
uint64_t checksum_seed; /* default 0 */
char** skip_compress_suffixes; char** skip_compress_suffixes;
int skip_compress_count; int skip_compress_count;
bool skip_compress_set; bool skip_compress_set;
@@ -231,7 +244,7 @@ typedef struct Config {
DelayUpdatesContext* delay_context; DelayUpdatesContext* delay_context;
} Config; } Config;
#define PROTOCOL_VERSION "2.9.0" #define PROTOCOL_VERSION "2.10.0"
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */ /* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
#define MAX_BASIS_DIRS 64 #define MAX_BASIS_DIRS 64
+19 -5
View File
@@ -29,12 +29,21 @@ uint32_t delta_xxhash32(const void* data, uint32_t len) {
return XXH32(data, len, 0); return XXH32(data, len, 0);
} }
uint32_t delta_xxhash32_seeded(const void* data, uint32_t len, uint32_t seed) {
return XXH32(data, len, seed);
}
uint64_t delta_xxhash64(const void* data, size_t len) { uint64_t delta_xxhash64(const void* data, size_t len) {
return XXH64(data, len, 0); return XXH64(data, len, 0);
} }
DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_file_size, DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_file_size,
uint32_t block_size) { uint32_t block_size) {
return delta_signature_create_seeded(old_file_data, old_file_size, block_size, 0);
}
DeltaSignature* delta_signature_create_seeded(const void* old_file_data, uint64_t old_file_size,
uint32_t block_size, uint32_t seed) {
if (old_file_data == NULL || old_file_size == 0 || block_size == 0) if (old_file_data == NULL || old_file_size == 0 || block_size == 0)
return NULL; return NULL;
@@ -67,7 +76,7 @@ DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_f
uint32_t len = uint32_t len =
(uint32_t)((old_file_size - offset < block_size) ? (old_file_size - offset) : block_size); (uint32_t)((old_file_size - offset < block_size) ? (old_file_size - offset) : block_size);
sig->blocks[i].adler32 = delta_adler32(data + offset, len); sig->blocks[i].adler32 = delta_adler32(data + offset, len);
sig->blocks[i].xxhash = delta_xxhash32(data + offset, len); sig->blocks[i].xxhash = delta_xxhash32_seeded(data + offset, len, seed);
} }
return sig; return sig;
@@ -297,7 +306,7 @@ static uint32_t* delta_build_index(const DeltaSignature* sig, uint32_t bucket_co
* O(1) per window); otherwise an exact linear scan is used. */ * O(1) per window); otherwise an exact linear scan is used. */
static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uint32_t adler, static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uint32_t adler,
bool full_window, const DeltaSignature* sig, const uint32_t* heads, bool full_window, const DeltaSignature* sig, const uint32_t* heads,
const uint32_t* next, uint32_t mask) { const uint32_t* next, uint32_t mask, uint32_t seed) {
if (!full_window || sig->block_count == 0) if (!full_window || sig->block_count == 0)
return DELTA_NO_BLOCK; return DELTA_NO_BLOCK;
@@ -309,7 +318,7 @@ static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uin
if (sig->blocks[j].adler32 != adler) if (sig->blocks[j].adler32 != adler)
continue; continue;
if (!have_xxh) { if (!have_xxh) {
window_xxh = delta_xxhash32(window, window_len); window_xxh = delta_xxhash32_seeded(window, window_len, seed);
have_xxh = true; have_xxh = true;
} }
if (window_xxh == sig->blocks[j].xxhash) if (window_xxh == sig->blocks[j].xxhash)
@@ -321,7 +330,7 @@ static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uin
/* Fallback used when the index could not be allocated. */ /* Fallback used when the index could not be allocated. */
for (uint32_t j = 0; j < sig->block_count; j++) { for (uint32_t j = 0; j < sig->block_count; j++) {
if (sig->blocks[j].adler32 == adler) { if (sig->blocks[j].adler32 == adler) {
uint32_t window_xxh = delta_xxhash32(window, window_len); uint32_t window_xxh = delta_xxhash32_seeded(window, window_len, seed);
if (window_xxh == sig->blocks[j].xxhash) if (window_xxh == sig->blocks[j].xxhash)
return j; return j;
} }
@@ -331,6 +340,11 @@ static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uin
Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const DeltaSignature* sig, Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const DeltaSignature* sig,
uint32_t block_size) { uint32_t block_size) {
return delta_compute_seeded(new_file_data, new_file_size, sig, block_size, 0);
}
Delta* delta_compute_seeded(const void* new_file_data, uint64_t new_file_size,
const DeltaSignature* sig, uint32_t block_size, uint32_t seed) {
if (!new_file_data || !sig || !sig->blocks || new_file_size == 0 || block_size == 0 || if (!new_file_data || !sig || !sig->blocks || new_file_size == 0 || block_size == 0 ||
block_size > DELTA_BLOCK_SIZE_MAX || sig->block_size != block_size) block_size > DELTA_BLOCK_SIZE_MAX || sig->block_size != block_size)
return NULL; return NULL;
@@ -397,7 +411,7 @@ Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const De
bool matched = false; bool matched = false;
uint32_t match_block = delta_find_match(new_data + i, window_len, adler, full_window, sig, uint32_t match_block = delta_find_match(new_data + i, window_len, adler, full_window, sig,
index, chain_next, mask); index, chain_next, mask, seed);
if (match_block != DELTA_NO_BLOCK) { if (match_block != DELTA_NO_BLOCK) {
if (has_literal) { if (has_literal) {
if (!flush_literal(&instrs, &capacity, &count, new_data, literal_start, i)) { if (!flush_literal(&instrs, &capacity, &count, new_data, literal_start, i)) {
+11
View File
@@ -56,12 +56,22 @@ typedef struct {
DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_file_size, DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_file_size,
uint32_t block_size); uint32_t block_size);
/* Seeded equivalent of delta_signature_create: the per-block strong (xxHash32)
* checksum uses `seed` (the low 32 bits of --checksum-seed). Passing seed 0 is
* identical to the unseeded function. */
DeltaSignature* delta_signature_create_seeded(const void* old_file_data, uint64_t old_file_size,
uint32_t block_size, uint32_t seed);
Data* delta_signature_serialize(const DeltaSignature* sig); Data* delta_signature_serialize(const DeltaSignature* sig);
DeltaSignature* delta_signature_deserialize(const Data* data); DeltaSignature* delta_signature_deserialize(const Data* data);
void delta_signature_destroy(DeltaSignature* sig); void delta_signature_destroy(DeltaSignature* sig);
Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const DeltaSignature* sig, Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const DeltaSignature* sig,
uint32_t block_size); uint32_t block_size);
/* Seeded equivalent of delta_compute: the per-window strong (xxHash32) check
* uses `seed` (the low 32 bits of --checksum-seed). The receiver's signature
* must have been built with the same seed for matching. */
Delta* delta_compute_seeded(const void* new_file_data, uint64_t new_file_size,
const DeltaSignature* sig, uint32_t block_size, uint32_t seed);
Data* delta_serialize(const Delta* delta); Data* delta_serialize(const Delta* delta);
Delta* delta_deserialize(const Data* data); Delta* delta_deserialize(const Data* data);
void* delta_apply(const void* old_data, uint64_t old_size, const Delta* delta, uint32_t block_size); void* delta_apply(const void* old_data, uint64_t old_size, const Delta* delta, uint32_t block_size);
@@ -72,6 +82,7 @@ bool delta_is_worthwhile(const Delta* delta, uint64_t new_file_size);
uint32_t delta_adler32(const void* data, uint32_t len); uint32_t delta_adler32(const void* data, uint32_t len);
uint32_t delta_xxhash32(const void* data, uint32_t len); uint32_t delta_xxhash32(const void* data, uint32_t len);
uint32_t delta_xxhash32_seeded(const void* data, uint32_t len, uint32_t seed);
uint64_t delta_xxhash64(const void* data, size_t len); uint64_t delta_xxhash64(const void* data, size_t len);
#endif #endif
+6 -6
View File
@@ -43,17 +43,17 @@ static unsigned long long next_temp_sequence(void) {
return atomic_fetch_add_explicit(&sequence, 1, memory_order_relaxed); return atomic_fetch_add_explicit(&sequence, 1, memory_order_relaxed);
} }
bool file_checksum(File* file, uint64_t* checksum) { bool file_checksum(File* file, ChecksumAlgo algo, uint64_t seed, uint8_t* out, size_t out_capacity,
if (!file || !checksum || !file->data) size_t* out_len) {
if (!file || !out || !out_len || !file->data)
return false; return false;
if (file->data->size == 0) { if (file->data->size == 0) {
*checksum = delta_xxhash64("", 0); return checksum_digest(algo, seed, "", 0, out, out_capacity, out_len);
return true;
} }
if (!file->data->data && !file_load_data(file)) if (!file->data->data && !file_load_data(file))
return false; return false;
*checksum = delta_xxhash64(file->data->data, file->data->size); return checksum_digest(algo, seed, file->data->data, file->data->size, out, out_capacity,
return true; out_len);
} }
File* file_create(const char* path) { File* file_create(const char* path) {
+7 -1
View File
@@ -4,6 +4,7 @@
#include "file_send.h" #include "file_send.h"
#include "file_receive.h" #include "file_receive.h"
#include "file_types.h" #include "file_types.h"
#include "checksum.h"
#include <stdbool.h> #include <stdbool.h>
#include <stdint.h> #include <stdint.h>
#include <sys/stat.h> #include <sys/stat.h>
@@ -14,7 +15,12 @@
File* file_create(const char* path); File* file_create(const char* path);
void file_destroy(void* item); void file_destroy(void* item);
bool file_load_data(File* file); bool file_load_data(File* file);
bool file_checksum(File* file, uint64_t* checksum); /* Compute the whole-file content digest of `file` with the negotiated
* --checksum-choice algorithm and --checksum-seed. Writes the digest into
* `out` (capacity `out_capacity`) and its length into `*out_len`. Returns
* false on read/allocation failure or when the digest would not fit. */
bool file_checksum(File* file, ChecksumAlgo algo, uint64_t seed, uint8_t* out, size_t out_capacity,
size_t* out_len);
size_t file_content_to_buffer(File* file); size_t file_content_to_buffer(File* file);
FileMetadata* file_metadata_create(const struct stat* stats); FileMetadata* file_metadata_create(const struct stat* stats);
void file_metadata_destroy(void* metadata); void file_metadata_destroy(void* metadata);
+32 -15
View File
@@ -355,7 +355,8 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
return NULL; return NULL;
} }
DeltaSignature* sig = delta_signature_create(old_data, old_size, config->delta_block_size); DeltaSignature* sig = delta_signature_create_seeded(old_data, old_size, config->delta_block_size,
(uint32_t)config->checksum_seed);
if (!sig) { if (!sig) {
free(old_data); free(old_data);
*failed = true; *failed = true;
@@ -630,8 +631,8 @@ static bool basis_quick_matches(const Config* config, const struct stat* st, tim
so the caller can materialize the file without re-reading it. */ so the caller can materialize the file without re-reading it. */
static bool basis_match_find(const Config* config, const char* check_path, static bool basis_match_find(const Config* config, const char* check_path,
unsigned long long check_size, time_t check_mtime, unsigned long long check_size, time_t check_mtime,
long check_mtime_nsec, uint64_t check_checksum, bool load_content, long check_mtime_nsec, const uint8_t* check_digest,
BasisMatch* out) { size_t check_digest_len, bool load_content, BasisMatch* out) {
memset(out, 0, sizeof(*out)); memset(out, 0, sizeof(*out));
if (!config || !config_has_basis(config) || config->ignore_times) if (!config || !config_has_basis(config) || config->ignore_times)
return false; return false;
@@ -651,10 +652,13 @@ static bool basis_match_find(const Config* config, const char* check_path,
if (basis_quick_matches(config, &st, check_mtime, check_mtime_nsec)) { if (basis_quick_matches(config, &st, check_mtime, check_mtime_nsec)) {
Data* content = basis_read_content(fd, check_size); Data* content = basis_read_content(fd, check_size);
if (content) { if (content) {
uint64_t basis_hash = check_size == 0 ? delta_xxhash64("", 0) uint8_t basis_digest[CHECKSUM_MAX_DIGEST_LEN];
: content->data ? delta_xxhash64(content->data, content->size) size_t basis_len = 0;
: 0; bool hashed = checksum_digest((ChecksumAlgo)config->checksum_algo, config->checksum_seed,
if (basis_hash == check_checksum) { content->data, content->size, basis_digest,
sizeof(basis_digest), &basis_len);
if (hashed && basis_len == check_digest_len && check_digest_len > 0 &&
memcmp(basis_digest, check_digest, check_digest_len) == 0) {
out->hit = true; out->hit = true;
out->type = entry->type; out->type = entry->type;
out->basis_path = candidate; out->basis_path = candidate;
@@ -997,7 +1001,8 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
unsigned long long check_size; unsigned long long check_size;
long long check_mtime; long long check_mtime;
long long check_mtime_nsec; long long check_mtime_nsec;
uint64_t check_checksum = 0; uint8_t check_digest[CHECKSUM_MAX_DIGEST_LEN];
size_t check_digest_len = 0;
if (!receive_n_data(fd, &check_size, sizeof(check_size)) || if (!receive_n_data(fd, &check_size, sizeof(check_size)) ||
!receive_n_data(fd, &check_mtime, sizeof(check_mtime))) { !receive_n_data(fd, &check_mtime, sizeof(check_mtime))) {
free(check_path); free(check_path);
@@ -1009,11 +1014,20 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
send_status(fd, STATUS_ERROR); send_status(fd, STATUS_ERROR);
return NULL; return NULL;
} }
if ((config->checksum || config_has_basis(config)) && if ((config->checksum || config_has_basis(config))) {
!receive_n_data(fd, &check_checksum, sizeof(check_checksum))) { uint8_t wire_len;
if (!receive_n_data(fd, &wire_len, sizeof(wire_len)) || wire_len == 0 ||
wire_len > CHECKSUM_MAX_DIGEST_LEN) {
free(check_path);
send_status(fd, STATUS_ERROR);
return NULL;
}
check_digest_len = wire_len;
if (!receive_n_data(fd, check_digest, check_digest_len)) {
free(check_path); free(check_path);
return NULL; return NULL;
} }
}
if (check_size > MAX_RECEIVE_WHOLE_FILE_SIZE) { if (check_size > MAX_RECEIVE_WHOLE_FILE_SIZE) {
free(check_path); free(check_path);
@@ -1098,10 +1112,13 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
skipped and the transfer proceeds with the full new contents. */ skipped and the transfer proceeds with the full new contents. */
bool match = false; bool match = false;
if (checksum_needs_read) { if (checksum_needs_read) {
if (old_size == 0) uint8_t old_digest[CHECKSUM_MAX_DIGEST_LEN];
match = delta_xxhash64("", 0) == check_checksum; size_t old_len = 0;
else bool hashed = checksum_digest((ChecksumAlgo)config->checksum_algo, config->checksum_seed,
match = old_data != NULL && delta_xxhash64(old_data, (size_t)old_size) == check_checksum; old_size == 0 ? "" : old_data, (size_t)old_size, old_digest,
sizeof(old_digest), &old_len);
match = hashed && old_len == check_digest_len && check_digest_len > 0 &&
memcmp(old_digest, check_digest, check_digest_len) == 0;
} else if (size_equal && !config->ignore_times) { } else if (size_equal && !config->ignore_times) {
match = config->size_only || match_by_metadata; match = config->size_only || match_by_metadata;
} }
@@ -1125,7 +1142,7 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
if (config_has_basis(config)) { if (config_has_basis(config)) {
BasisMatch basis; BasisMatch basis;
basis_match_find(config, check_path, check_size, (time_t)check_mtime, (long)check_mtime_nsec, basis_match_find(config, check_path, check_size, (time_t)check_mtime, (long)check_mtime_nsec,
check_checksum, true, &basis); check_digest, check_digest_len, true, &basis);
if (basis.hit) { if (basis.hit) {
if (basis.type == BASIS_DEST_COMPARE) { if (basis.type == BASIS_DEST_COMPARE) {
/* compare-dest never copies: an exact match only suppresses the data /* compare-dest never copies: an exact match only suppresses the data