Merge feat/p3-checksum-choice: --checksum-choice/--cc and --checksum-seed
This commit is contained in:
+59
-2
@@ -94,6 +94,47 @@ static int set_compression_choice(Config* config, const char* value) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Validate and store the --checksum-choice/--cc algorithm. Only the algorithms
|
||||
* the engine genuinely supports are accepted (xxHash64 and md5); anything else
|
||||
* is a clear error, never a silent no-op. "xxhash" is accepted as rsync's
|
||||
* spelling of xxHash64. */
|
||||
static int set_checksum_choice(Config* config, const char* value) {
|
||||
int algo = checksum_algo_from_name(value);
|
||||
if (algo < 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "--checksum-choice must be xxh64 (or xxhash) or md5 (got '%s')",
|
||||
value);
|
||||
return -1;
|
||||
}
|
||||
config->checksum_algo = algo;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* parse_ull_arg is defined later in this file; declared here for the seed
|
||||
parser below. */
|
||||
static int parse_ull_arg(const char* val, unsigned long long* out, const char* optname);
|
||||
|
||||
/* Parse --checksum-seed=NUM as a strict decimal 0..UINT64_MAX. A blank value,
|
||||
* a sign, or any non-digit (which parse_ull_arg's strtoull would silently
|
||||
* coerce) is rejected: an explicit seed must be an exact unsigned integer or
|
||||
* the run fails with a clear error rather than quietly ignoring the value. */
|
||||
static int set_checksum_seed(Config* config, const char* value) {
|
||||
if (!value || *value == '\0') {
|
||||
log_message(LOG_LEVEL_ERROR, "--checksum-seed must be a non-negative integer");
|
||||
return -1;
|
||||
}
|
||||
for (const char* p = value; *p; p++) {
|
||||
if (*p < '0' || *p > '9') {
|
||||
log_message(LOG_LEVEL_ERROR, "--checksum-seed must be a non-negative integer");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
unsigned long long seed;
|
||||
if (parse_ull_arg(value, &seed, "--checksum-seed") != 0)
|
||||
return -1;
|
||||
config->checksum_seed = seed;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int set_compression_threads_option(int* dest, const char* value) {
|
||||
if (set_positive_int_option(dest, value, "--compress-threads") != 0)
|
||||
return -1;
|
||||
@@ -1019,8 +1060,24 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
||||
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||
return -1;
|
||||
}
|
||||
log_message(LOG_LEVEL_ERROR, "%s is not supported yet (xxHash64 is used)", argv[i]);
|
||||
return -1;
|
||||
if (set_checksum_choice(config, argv[++i]) != 0)
|
||||
return -1;
|
||||
} else if (strncmp(argv[i], "--checksum-choice=", 18) == 0) {
|
||||
if (set_checksum_choice(config, argv[i] + 18) != 0)
|
||||
return -1;
|
||||
} else if (strncmp(argv[i], "--cc=", 5) == 0) {
|
||||
if (set_checksum_choice(config, argv[i] + 5) != 0)
|
||||
return -1;
|
||||
} else if (strncmp(argv[i], "--checksum-seed=", 16) == 0) {
|
||||
if (set_checksum_seed(config, argv[i] + 16) != 0)
|
||||
return -1;
|
||||
} else if (opt_is(argv[i], "--checksum-seed", NULL)) {
|
||||
if (i + 1 >= argc) {
|
||||
log_message(LOG_LEVEL_ERROR, "missing argument for --checksum-seed");
|
||||
return -1;
|
||||
}
|
||||
if (set_checksum_seed(config, argv[++i]) != 0)
|
||||
return -1;
|
||||
} else if (strncmp(argv[i], "--compare-dest=", 15) == 0) {
|
||||
if (set_basis_dest_option(config, BASIS_DEST_COMPARE, argv[i] + 15, "--compare-dest") != 0)
|
||||
return -1;
|
||||
|
||||
@@ -843,12 +843,18 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
||||
if (!send_n_data(client->file_descriptor, &mtime_nsec, sizeof(mtime_nsec)))
|
||||
return -1;
|
||||
/* With alternate basis directories the receiver must be able to verify the
|
||||
* content of every candidate basis file, so the sender supplies its xxHash64
|
||||
* for every file even when --checksum was not requested. */
|
||||
* content of every candidate basis file, so the sender supplies its whole-file
|
||||
* digest (computed with the negotiated --checksum-choice algorithm and
|
||||
* --checksum-seed) for every file even when --checksum was not requested. */
|
||||
if (config->checksum || config_has_basis(config)) {
|
||||
uint64_t checksum;
|
||||
if (!file_checksum(file, &checksum) ||
|
||||
!send_n_data(client->file_descriptor, &checksum, sizeof(checksum)))
|
||||
uint8_t digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||
size_t digest_len = 0;
|
||||
if (!file_checksum(file, (ChecksumAlgo)config->checksum_algo, config->checksum_seed, digest,
|
||||
sizeof(digest), &digest_len))
|
||||
return -1;
|
||||
uint8_t wire_len = (uint8_t)digest_len;
|
||||
if (!send_n_data(client->file_descriptor, &wire_len, sizeof(wire_len)) ||
|
||||
!send_n_data(client->file_descriptor, digest, wire_len))
|
||||
return -1;
|
||||
}
|
||||
Status s;
|
||||
@@ -897,7 +903,8 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
||||
}
|
||||
|
||||
static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* config) {
|
||||
Delta* delta = delta_compute(file->data->data, file->data->size, sig, config->delta_block_size);
|
||||
Delta* delta = delta_compute_seeded(file->data->data, file->data->size, sig,
|
||||
config->delta_block_size, (uint32_t)config->checksum_seed);
|
||||
/* The receiver is blocked after sending the signature. Every local
|
||||
fallback therefore needs the explicit NEXT response before full data. */
|
||||
if (!delta)
|
||||
|
||||
+6
-2
@@ -97,8 +97,12 @@ void print_usage(void) {
|
||||
printf(" into the destination instead of transferring its data\n");
|
||||
printf(" --link-dest <dir> Like --copy-dest, but hard-links the unchanged file from DIR\n");
|
||||
printf(" into the destination (repeatable; earlier DIRs win)\n");
|
||||
printf(" --checksum-choice, --cc <alg> Checksum algorithm (not supported yet; xxHash64 is "
|
||||
"used)\n");
|
||||
printf(" --checksum-choice, --cc <alg> Whole-file checksum algorithm for --incremental/\n");
|
||||
printf(" --checksum compares (xxh64/xxhash or md5; default xxh64 with\n");
|
||||
printf(" seed 0). The seed comes from --checksum-seed\n");
|
||||
printf(" --checksum-seed <num> Seed for the whole-file xxHash64 digest (and the delta\n");
|
||||
printf(" block strong hash, low 32 bits); md5 ignores the seed. The\n");
|
||||
printf(" digest algorithm and seed must match on sender and receiver\n");
|
||||
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
|
||||
printf(" -W, --whole-file Transfer changed files without delta processing\n");
|
||||
printf(" -y, --fuzzy Use a similar-named file already in the destination\n");
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
#include "checksum.h"
|
||||
#include <openssl/evp.h>
|
||||
#include <string.h>
|
||||
#include <strings.h>
|
||||
|
||||
/* delta.c owns the single XXH_IMPLEMENTATION that provides the xxHash symbols
|
||||
* for the whole binary; this TU only needs the declarations. */
|
||||
#include <xxhash.h>
|
||||
|
||||
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
|
||||
size_t out_capacity, size_t* out_len) {
|
||||
if (!out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
|
||||
return false;
|
||||
if (data == NULL && size != 0)
|
||||
return false;
|
||||
|
||||
if (algo == CHECKSUM_ALGO_XXH64) {
|
||||
uint64_t digest = XXH64(data, size, seed);
|
||||
memcpy(out, &digest, sizeof(digest));
|
||||
*out_len = sizeof(digest);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (algo == CHECKSUM_ALGO_MD5) {
|
||||
/* md5 takes no seed; the caller's seed is deliberately ignored (documented
|
||||
* in RSYNC_COMPAT.md). OpenSSL's one-shot EVP_Digest needs a non-NULL
|
||||
* buffer even for an empty input, so map a NULL data + size==0 to an empty
|
||||
* buffer. */
|
||||
static const uint8_t empty = 0;
|
||||
const void* input = data ? data : ∅
|
||||
unsigned int digest_len = 0;
|
||||
if (EVP_Digest(input, size, out, &digest_len, EVP_md5(), NULL) != 1)
|
||||
return false;
|
||||
if (digest_len > out_capacity)
|
||||
return false;
|
||||
*out_len = digest_len;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
int checksum_algo_from_name(const char* name) {
|
||||
if (!name)
|
||||
return -1;
|
||||
if (strcasecmp(name, "xxh64") == 0 || strcasecmp(name, "xxhash") == 0)
|
||||
return (int)CHECKSUM_ALGO_XXH64;
|
||||
if (strcasecmp(name, "md5") == 0)
|
||||
return (int)CHECKSUM_ALGO_MD5;
|
||||
return -1;
|
||||
}
|
||||
|
||||
const char* checksum_algo_name(ChecksumAlgo algo) {
|
||||
switch (algo) {
|
||||
case CHECKSUM_ALGO_XXH64:
|
||||
return "xxh64";
|
||||
case CHECKSUM_ALGO_MD5:
|
||||
return "md5";
|
||||
}
|
||||
return "<unknown>";
|
||||
}
|
||||
|
||||
bool checksum_algo_valid(int algo) {
|
||||
return algo == (int)CHECKSUM_ALGO_XXH64 || algo == (int)CHECKSUM_ALGO_MD5;
|
||||
}
|
||||
|
||||
uint8_t checksum_digest_len(ChecksumAlgo algo) {
|
||||
switch (algo) {
|
||||
case CHECKSUM_ALGO_XXH64:
|
||||
return 8;
|
||||
case CHECKSUM_ALGO_MD5:
|
||||
return 16;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
#ifndef CHECKSUM_H
|
||||
#define CHECKSUM_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
/* Whole-file content-digest algorithms selectable with --checksum-choice and
|
||||
* seeded with --checksum-seed. The ids are the values actually placed on the
|
||||
* wire (config frame), so they must be kept stable and validated on receive.
|
||||
* CHECKSUM_ALGO_XXH64 == 0 is the default and is byte-for-byte what FastSync
|
||||
* computed before these options existed (xxHash64 with seed 0). */
|
||||
typedef enum { CHECKSUM_ALGO_XXH64 = 0, CHECKSUM_ALGO_MD5 = 1 } ChecksumAlgo;
|
||||
|
||||
/* md5 digest is 16 bytes, the longest supported. */
|
||||
#define CHECKSUM_MAX_DIGEST_LEN 16
|
||||
|
||||
/* Compute the whole-file digest of the first `size` bytes of `data`.
|
||||
*
|
||||
* - CHECKSUM_ALGO_XXH64: xxHash64(data, size, seed) (full 64-bit seed).
|
||||
* - CHECKSUM_ALGO_MD5: md5(data, size) via OpenSSL EVP.
|
||||
* md5 has no seed, so `seed` is ignored (documented).
|
||||
* - `size == 0` hashes the empty input (plus its seed), not a NULL input.
|
||||
*
|
||||
* Writes up to `out_capacity` bytes into `out`, storing the digest length in
|
||||
* *out_len. Returns false on NULL out* or when the digest would not fit.
|
||||
* Never writes more than CHECKSUM_MAX_DIGEST_LEN bytes. */
|
||||
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
|
||||
size_t out_capacity, size_t* out_len);
|
||||
|
||||
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id.
|
||||
* Accepts "xxh64" and "xxhash" (both map to CHECKSUM_ALGO_XXH64, rsync's
|
||||
* xxhash spelling) and "md5". Returns -1 for any unsupported name. */
|
||||
int checksum_algo_from_name(const char* name);
|
||||
|
||||
/* Canonical name of an algorithm (used in CLI error messages). */
|
||||
const char* checksum_algo_name(ChecksumAlgo algo);
|
||||
|
||||
/* True when `algo` is a supported id (used by config receive validation). */
|
||||
bool checksum_algo_valid(int algo);
|
||||
|
||||
/* Digest length in bytes for an algorithm (xxx64 = 8, md5 = 16). */
|
||||
uint8_t checksum_digest_len(ChecksumAlgo algo);
|
||||
|
||||
#endif /* CHECKSUM_H */
|
||||
+24
-3
@@ -129,6 +129,8 @@ static void config_set_defaults(Config* config) {
|
||||
config->daemon_config = NULL;
|
||||
config->server_mode = false;
|
||||
config->checksum = false;
|
||||
config->checksum_algo = CHECKSUM_ALGO_XXH64;
|
||||
config->checksum_seed = 0;
|
||||
config->compress_choice = NULL;
|
||||
config->chmod_spec = NULL;
|
||||
config->skip_compress_suffixes = NULL;
|
||||
@@ -175,7 +177,7 @@ static bool validate_received_config(const Config* config) {
|
||||
!(config->delay_updates && delay_updates_staging_name_conflict(config->backup_dir)) &&
|
||||
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
|
||||
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
|
||||
config_has_valid_delete_timing(config) &&
|
||||
checksum_algo_valid(config->checksum_algo) && config_has_valid_delete_timing(config) &&
|
||||
!(config->skip_compress_set && config->use_chunk_serialization) &&
|
||||
/* --append / --append-verify tail resume needs the per-file check,
|
||||
which chunk serialization -s disables: reject on the receiver too
|
||||
@@ -470,6 +472,15 @@ static bool send_fuzzy_option(int fd, const Config* c) {
|
||||
return send_int(fd, c->fuzzy);
|
||||
}
|
||||
|
||||
/* --checksum-choice/--cc + --checksum-seed. The algorithm id and seed travel
|
||||
* with the config so the receiver hashes the on-disk old file with the same
|
||||
* parameters the sender used for its digest (see checksum.h). Trailing fields
|
||||
* on the config frame; protocol 2.10.0. */
|
||||
static bool send_checksum_options(int fd, const Config* c) {
|
||||
return send_int(fd, c->checksum_algo) &&
|
||||
send_n_data(fd, &c->checksum_seed, sizeof(c->checksum_seed));
|
||||
}
|
||||
|
||||
static bool receive_core_fields(int fd, Config* c) {
|
||||
int value;
|
||||
if (!receive_wire_bool(fd, &c->eight_bit_output))
|
||||
@@ -654,13 +665,22 @@ static bool receive_fuzzy_option(int fd, Config* c) {
|
||||
return receive_wire_bool(fd, &c->fuzzy);
|
||||
}
|
||||
|
||||
static bool receive_checksum_options(int fd, Config* c) {
|
||||
int algo;
|
||||
if (!receive_int(fd, &algo) || !checksum_algo_valid(algo))
|
||||
return false;
|
||||
c->checksum_algo = algo;
|
||||
return receive_n_data(fd, &c->checksum_seed, sizeof(c->checksum_seed));
|
||||
}
|
||||
|
||||
bool config_send(int file_descriptor, const Config* config) {
|
||||
protocol_session_set_max_alloc(NULL, config->max_alloc);
|
||||
if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) ||
|
||||
!send_file_options(file_descriptor, config) ||
|
||||
!send_selection_options(file_descriptor, config) ||
|
||||
!send_resume_options(file_descriptor, config) ||
|
||||
!send_basis_options(file_descriptor, config) || !send_fuzzy_option(file_descriptor, config))
|
||||
!send_basis_options(file_descriptor, config) || !send_fuzzy_option(file_descriptor, config) ||
|
||||
!send_checksum_options(file_descriptor, config))
|
||||
return false;
|
||||
Status status;
|
||||
if (!receive_status(file_descriptor, &status))
|
||||
@@ -694,7 +714,8 @@ Config* config_receive(int file_descriptor) {
|
||||
!receive_selection_options(file_descriptor, config) ||
|
||||
!receive_resume_options(file_descriptor, config) ||
|
||||
!receive_basis_options(file_descriptor, config) ||
|
||||
!receive_fuzzy_option(file_descriptor, config))
|
||||
!receive_fuzzy_option(file_descriptor, config) ||
|
||||
!receive_checksum_options(file_descriptor, config))
|
||||
goto error;
|
||||
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
|
||||
strcmp(config->compress_choice, "none") != 0) {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#define CONFIG_H
|
||||
|
||||
#include "array_list.h"
|
||||
#include "checksum.h"
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
@@ -235,6 +236,18 @@ typedef struct Config {
|
||||
char* compress_choice;
|
||||
char* chmod_spec;
|
||||
|
||||
/* --checksum-choice / --cc and --checksum-seed. checksum_algo is the id of
|
||||
* the whole-file content-digest algorithm used by the per-file --incremental
|
||||
* handshake (sender computes it, receiver compares it to skip unchanged
|
||||
* files) and by the basis-dir content verification. checksum_seed is passed
|
||||
* to xxHash64 (and to the delta block strong hash, low 32 bits); md5 has no
|
||||
* seed so it is ignored there. Both cross the wire: the receiver MUST hash
|
||||
* the on-disk old file with the same algorithm and seed to reach a matching
|
||||
* digest. Defaults (XXH64 / seed 0) reproduce the pre-existing behavior
|
||||
* byte-for-byte. */
|
||||
int checksum_algo; /* ChecksumAlgo, default CHECKSUM_ALGO_XXH64 */
|
||||
uint64_t checksum_seed; /* default 0 */
|
||||
|
||||
char** skip_compress_suffixes;
|
||||
int skip_compress_count;
|
||||
bool skip_compress_set;
|
||||
|
||||
+19
-5
@@ -29,12 +29,21 @@ uint32_t delta_xxhash32(const void* data, uint32_t len) {
|
||||
return XXH32(data, len, 0);
|
||||
}
|
||||
|
||||
uint32_t delta_xxhash32_seeded(const void* data, uint32_t len, uint32_t seed) {
|
||||
return XXH32(data, len, seed);
|
||||
}
|
||||
|
||||
uint64_t delta_xxhash64(const void* data, size_t len) {
|
||||
return XXH64(data, len, 0);
|
||||
}
|
||||
|
||||
DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_file_size,
|
||||
uint32_t block_size) {
|
||||
return delta_signature_create_seeded(old_file_data, old_file_size, block_size, 0);
|
||||
}
|
||||
|
||||
DeltaSignature* delta_signature_create_seeded(const void* old_file_data, uint64_t old_file_size,
|
||||
uint32_t block_size, uint32_t seed) {
|
||||
if (old_file_data == NULL || old_file_size == 0 || block_size == 0)
|
||||
return NULL;
|
||||
|
||||
@@ -67,7 +76,7 @@ DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_f
|
||||
uint32_t len =
|
||||
(uint32_t)((old_file_size - offset < block_size) ? (old_file_size - offset) : block_size);
|
||||
sig->blocks[i].adler32 = delta_adler32(data + offset, len);
|
||||
sig->blocks[i].xxhash = delta_xxhash32(data + offset, len);
|
||||
sig->blocks[i].xxhash = delta_xxhash32_seeded(data + offset, len, seed);
|
||||
}
|
||||
|
||||
return sig;
|
||||
@@ -297,7 +306,7 @@ static uint32_t* delta_build_index(const DeltaSignature* sig, uint32_t bucket_co
|
||||
* O(1) per window); otherwise an exact linear scan is used. */
|
||||
static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uint32_t adler,
|
||||
bool full_window, const DeltaSignature* sig, const uint32_t* heads,
|
||||
const uint32_t* next, uint32_t mask) {
|
||||
const uint32_t* next, uint32_t mask, uint32_t seed) {
|
||||
if (!full_window || sig->block_count == 0)
|
||||
return DELTA_NO_BLOCK;
|
||||
|
||||
@@ -309,7 +318,7 @@ static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uin
|
||||
if (sig->blocks[j].adler32 != adler)
|
||||
continue;
|
||||
if (!have_xxh) {
|
||||
window_xxh = delta_xxhash32(window, window_len);
|
||||
window_xxh = delta_xxhash32_seeded(window, window_len, seed);
|
||||
have_xxh = true;
|
||||
}
|
||||
if (window_xxh == sig->blocks[j].xxhash)
|
||||
@@ -321,7 +330,7 @@ static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uin
|
||||
/* Fallback used when the index could not be allocated. */
|
||||
for (uint32_t j = 0; j < sig->block_count; j++) {
|
||||
if (sig->blocks[j].adler32 == adler) {
|
||||
uint32_t window_xxh = delta_xxhash32(window, window_len);
|
||||
uint32_t window_xxh = delta_xxhash32_seeded(window, window_len, seed);
|
||||
if (window_xxh == sig->blocks[j].xxhash)
|
||||
return j;
|
||||
}
|
||||
@@ -331,6 +340,11 @@ static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uin
|
||||
|
||||
Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const DeltaSignature* sig,
|
||||
uint32_t block_size) {
|
||||
return delta_compute_seeded(new_file_data, new_file_size, sig, block_size, 0);
|
||||
}
|
||||
|
||||
Delta* delta_compute_seeded(const void* new_file_data, uint64_t new_file_size,
|
||||
const DeltaSignature* sig, uint32_t block_size, uint32_t seed) {
|
||||
if (!new_file_data || !sig || !sig->blocks || new_file_size == 0 || block_size == 0 ||
|
||||
block_size > DELTA_BLOCK_SIZE_MAX || sig->block_size != block_size)
|
||||
return NULL;
|
||||
@@ -397,7 +411,7 @@ Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const De
|
||||
|
||||
bool matched = false;
|
||||
uint32_t match_block = delta_find_match(new_data + i, window_len, adler, full_window, sig,
|
||||
index, chain_next, mask);
|
||||
index, chain_next, mask, seed);
|
||||
if (match_block != DELTA_NO_BLOCK) {
|
||||
if (has_literal) {
|
||||
if (!flush_literal(&instrs, &capacity, &count, new_data, literal_start, i)) {
|
||||
|
||||
@@ -56,12 +56,22 @@ typedef struct {
|
||||
|
||||
DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_file_size,
|
||||
uint32_t block_size);
|
||||
/* Seeded equivalent of delta_signature_create: the per-block strong (xxHash32)
|
||||
* checksum uses `seed` (the low 32 bits of --checksum-seed). Passing seed 0 is
|
||||
* identical to the unseeded function. */
|
||||
DeltaSignature* delta_signature_create_seeded(const void* old_file_data, uint64_t old_file_size,
|
||||
uint32_t block_size, uint32_t seed);
|
||||
Data* delta_signature_serialize(const DeltaSignature* sig);
|
||||
DeltaSignature* delta_signature_deserialize(const Data* data);
|
||||
void delta_signature_destroy(DeltaSignature* sig);
|
||||
|
||||
Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const DeltaSignature* sig,
|
||||
uint32_t block_size);
|
||||
/* Seeded equivalent of delta_compute: the per-window strong (xxHash32) check
|
||||
* uses `seed` (the low 32 bits of --checksum-seed). The receiver's signature
|
||||
* must have been built with the same seed for matching. */
|
||||
Delta* delta_compute_seeded(const void* new_file_data, uint64_t new_file_size,
|
||||
const DeltaSignature* sig, uint32_t block_size, uint32_t seed);
|
||||
Data* delta_serialize(const Delta* delta);
|
||||
Delta* delta_deserialize(const Data* data);
|
||||
void* delta_apply(const void* old_data, uint64_t old_size, const Delta* delta, uint32_t block_size);
|
||||
@@ -72,6 +82,7 @@ bool delta_is_worthwhile(const Delta* delta, uint64_t new_file_size);
|
||||
|
||||
uint32_t delta_adler32(const void* data, uint32_t len);
|
||||
uint32_t delta_xxhash32(const void* data, uint32_t len);
|
||||
uint32_t delta_xxhash32_seeded(const void* data, uint32_t len, uint32_t seed);
|
||||
uint64_t delta_xxhash64(const void* data, size_t len);
|
||||
|
||||
#endif
|
||||
|
||||
+6
-6
@@ -43,17 +43,17 @@ static unsigned long long next_temp_sequence(void) {
|
||||
return atomic_fetch_add_explicit(&sequence, 1, memory_order_relaxed);
|
||||
}
|
||||
|
||||
bool file_checksum(File* file, uint64_t* checksum) {
|
||||
if (!file || !checksum || !file->data)
|
||||
bool file_checksum(File* file, ChecksumAlgo algo, uint64_t seed, uint8_t* out, size_t out_capacity,
|
||||
size_t* out_len) {
|
||||
if (!file || !out || !out_len || !file->data)
|
||||
return false;
|
||||
if (file->data->size == 0) {
|
||||
*checksum = delta_xxhash64("", 0);
|
||||
return true;
|
||||
return checksum_digest(algo, seed, "", 0, out, out_capacity, out_len);
|
||||
}
|
||||
if (!file->data->data && !file_load_data(file))
|
||||
return false;
|
||||
*checksum = delta_xxhash64(file->data->data, file->data->size);
|
||||
return true;
|
||||
return checksum_digest(algo, seed, file->data->data, file->data->size, out, out_capacity,
|
||||
out_len);
|
||||
}
|
||||
|
||||
File* file_create(const char* path) {
|
||||
|
||||
+7
-1
@@ -4,6 +4,7 @@
|
||||
#include "file_send.h"
|
||||
#include "file_receive.h"
|
||||
#include "file_types.h"
|
||||
#include "checksum.h"
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <sys/stat.h>
|
||||
@@ -14,7 +15,12 @@
|
||||
File* file_create(const char* path);
|
||||
void file_destroy(void* item);
|
||||
bool file_load_data(File* file);
|
||||
bool file_checksum(File* file, uint64_t* checksum);
|
||||
/* Compute the whole-file content digest of `file` with the negotiated
|
||||
* --checksum-choice algorithm and --checksum-seed. Writes the digest into
|
||||
* `out` (capacity `out_capacity`) and its length into `*out_len`. Returns
|
||||
* false on read/allocation failure or when the digest would not fit. */
|
||||
bool file_checksum(File* file, ChecksumAlgo algo, uint64_t seed, uint8_t* out, size_t out_capacity,
|
||||
size_t* out_len);
|
||||
size_t file_content_to_buffer(File* file);
|
||||
FileMetadata* file_metadata_create(const struct stat* stats);
|
||||
void file_metadata_destroy(void* metadata);
|
||||
|
||||
+35
-17
@@ -355,7 +355,8 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
||||
return NULL;
|
||||
}
|
||||
|
||||
DeltaSignature* sig = delta_signature_create(old_data, old_size, config->delta_block_size);
|
||||
DeltaSignature* sig = delta_signature_create_seeded(old_data, old_size, config->delta_block_size,
|
||||
(uint32_t)config->checksum_seed);
|
||||
if (!sig) {
|
||||
free(old_data);
|
||||
*failed = true;
|
||||
@@ -630,8 +631,8 @@ static bool basis_quick_matches(const Config* config, const struct stat* st, tim
|
||||
so the caller can materialize the file without re-reading it. */
|
||||
static bool basis_match_find(const Config* config, const char* check_path,
|
||||
unsigned long long check_size, time_t check_mtime,
|
||||
long check_mtime_nsec, uint64_t check_checksum, bool load_content,
|
||||
BasisMatch* out) {
|
||||
long check_mtime_nsec, const uint8_t* check_digest,
|
||||
size_t check_digest_len, bool load_content, BasisMatch* out) {
|
||||
memset(out, 0, sizeof(*out));
|
||||
if (!config || !config_has_basis(config) || config->ignore_times)
|
||||
return false;
|
||||
@@ -651,10 +652,13 @@ static bool basis_match_find(const Config* config, const char* check_path,
|
||||
if (basis_quick_matches(config, &st, check_mtime, check_mtime_nsec)) {
|
||||
Data* content = basis_read_content(fd, check_size);
|
||||
if (content) {
|
||||
uint64_t basis_hash = check_size == 0 ? delta_xxhash64("", 0)
|
||||
: content->data ? delta_xxhash64(content->data, content->size)
|
||||
: 0;
|
||||
if (basis_hash == check_checksum) {
|
||||
uint8_t basis_digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||
size_t basis_len = 0;
|
||||
bool hashed = checksum_digest((ChecksumAlgo)config->checksum_algo, config->checksum_seed,
|
||||
content->data, content->size, basis_digest,
|
||||
sizeof(basis_digest), &basis_len);
|
||||
if (hashed && basis_len == check_digest_len && check_digest_len > 0 &&
|
||||
memcmp(basis_digest, check_digest, check_digest_len) == 0) {
|
||||
out->hit = true;
|
||||
out->type = entry->type;
|
||||
out->basis_path = candidate;
|
||||
@@ -1041,7 +1045,8 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
||||
unsigned long long check_size;
|
||||
long long check_mtime;
|
||||
long long check_mtime_nsec;
|
||||
uint64_t check_checksum = 0;
|
||||
uint8_t check_digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||
size_t check_digest_len = 0;
|
||||
if (!receive_n_data(fd, &check_size, sizeof(check_size)) ||
|
||||
!receive_n_data(fd, &check_mtime, sizeof(check_mtime))) {
|
||||
free(check_path);
|
||||
@@ -1053,10 +1058,20 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return NULL;
|
||||
}
|
||||
if ((config->checksum || config_has_basis(config)) &&
|
||||
!receive_n_data(fd, &check_checksum, sizeof(check_checksum))) {
|
||||
free(check_path);
|
||||
return NULL;
|
||||
if ((config->checksum || config_has_basis(config))) {
|
||||
uint8_t wire_len;
|
||||
if (!receive_n_data(fd, &wire_len, sizeof(wire_len)) || wire_len == 0 ||
|
||||
wire_len > CHECKSUM_MAX_DIGEST_LEN ||
|
||||
wire_len != checksum_digest_len((ChecksumAlgo)config->checksum_algo)) {
|
||||
free(check_path);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return NULL;
|
||||
}
|
||||
check_digest_len = wire_len;
|
||||
if (!receive_n_data(fd, check_digest, check_digest_len)) {
|
||||
free(check_path);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
if (check_size > MAX_RECEIVE_WHOLE_FILE_SIZE) {
|
||||
@@ -1142,10 +1157,13 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
||||
skipped and the transfer proceeds with the full new contents. */
|
||||
bool match = false;
|
||||
if (checksum_needs_read) {
|
||||
if (old_size == 0)
|
||||
match = delta_xxhash64("", 0) == check_checksum;
|
||||
else
|
||||
match = old_data != NULL && delta_xxhash64(old_data, (size_t)old_size) == check_checksum;
|
||||
uint8_t old_digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||
size_t old_len = 0;
|
||||
bool hashed = checksum_digest((ChecksumAlgo)config->checksum_algo, config->checksum_seed,
|
||||
old_size == 0 ? "" : old_data, (size_t)old_size, old_digest,
|
||||
sizeof(old_digest), &old_len);
|
||||
match = hashed && old_len == check_digest_len && check_digest_len > 0 &&
|
||||
memcmp(old_digest, check_digest, check_digest_len) == 0;
|
||||
} else if (size_equal && !config->ignore_times) {
|
||||
match = config->size_only || match_by_metadata;
|
||||
}
|
||||
@@ -1169,7 +1187,7 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
||||
if (config_has_basis(config)) {
|
||||
BasisMatch basis;
|
||||
basis_match_find(config, check_path, check_size, (time_t)check_mtime, (long)check_mtime_nsec,
|
||||
check_checksum, true, &basis);
|
||||
check_digest, check_digest_len, true, &basis);
|
||||
if (basis.hit) {
|
||||
if (basis.type == BASIS_DEST_COMPARE) {
|
||||
/* compare-dest never copies: an exact match only suppresses the data
|
||||
|
||||
Reference in New Issue
Block a user