TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Merge branch 'feat/parity-codecs' into feat/parity-completion

Browse Source 
# Conflicts:
#	src/shared/checksum.h
#	src/shared/config.h
#	tests/integration/test_fault_injection.py
#	tests/integration/test_preflight.py
#	tests/test_client_cli.c
#	tests/test_config.c
#	tests/test_fuzz_smoke.c
This commit is contained in:
TapTap
2026-09-16 23:49:41 +02:00
parent 9dd5288381 76a81f1684
commit 125921c11b
21 changed files with 1500 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/client_cli.c
+102 -33
View File
@@ -150,47 +150,96 @@ static int set_positive_int_option(int* dest, const char* value, const char* opt
}
/* Set and validate the compression algorithm selected by the client. rsync
* 3.4.1 can be built with zstd, none, lz4, zlibx, zlib and auto; FastSync only
* implements zstd (and no compression). "auto" is accepted as the default
* zstd choice; any other rsync choice is rejected by name instead of being
* silently accepted and ignored. */
* 3.4.1 can be built with zstd, none, lz4, zlibx, zlib and auto; all of those
* names are accepted and mapped to a real codec here. "auto" resolves through
* FastSync's compiled-in preference order (rsync 3.4.1's list). An unknown
* name is a hard error with rsync's exit code 4, never a silent no-op. */
static int set_compression_choice(Config* config, const char* value) {
/* rsync's "auto" is normalized to the canonical "zstd" at parse time (like
--checksum-choice=auto), so the value that crosses the wire is always one
the receiver accepts. */
const char* canonical = strcmp(value, "auto") == 0 ? "zstd" : value;
if (strcmp(canonical, "zstd") != 0 && strcmp(canonical, "none") != 0) {
log_message(LOG_LEVEL_ERROR,
"--compress-choice '%s' is not implemented; FastSync supports zstd, none or auto "
"(rsync's lz4/zlib/zlibx are rejected, never silently ignored)",
value);
if (!value) {
config->cli_exit_code = 4;
return -1;
}
int algo;
if (strcasecmp(value, "auto") == 0)
algo = (int)compression_negotiate_default();
else
algo = compression_algo_from_name(value);
if (algo < 0) {
log_message(LOG_LEVEL_ERROR,
"--compress-choice '%s' is not a supported algorithm; FastSync supports zstd, "
"lz4, zlib, zlibx, none or auto",
value);
config->cli_exit_code = 4;
return -1;
}
const char* canonical = compression_algo_name((CompressionAlgo)algo);
if (set_string_option(&config->compress_choice, canonical, "--compress-choice") != 0)
return -1;
config->use_compression = strcmp(canonical, "none") != 0;
config->compression_algo = algo;
config->use_compression = (algo != (int)COMPRESSION_ALGO_NONE);
return 0;
}
/* Validate and store the --checksum-choice/--cc algorithm. Only the algorithms
* the engine genuinely supports are accepted (xxh64/xxhash, xxh3, xxh128, md5);
* rsync's compiled-in choices that FastSync does not implement (md4, sha1,
* none) and the two-name transfer/pre-transfer syntax are a clear error, never
* a silent no-op. "auto" (rsync's default automatic choice) selects FastSync's
* default algorithm. */
/* Store one algorithm name into *out. Returns 0 for a valid name, 1 for
* "auto" (caller resolves it), -1 for an unknown/too-long name. */
static int resolve_checksum_name(const char* name, size_t len, int* out) {
char buf[64];
if (len == 0 || len >= sizeof(buf))
return -1;
memcpy(buf, name, len);
buf[len] = '\0';
if (strcasecmp(buf, "auto") == 0)
return 1;
int algo = checksum_algo_from_name(buf);
if (algo < 0)
return -1;
*out = algo;
return 0;
}
/* Validate and store the --checksum-choice/--cc algorithm. rsync 3.4.1 accepts
* a single name (used for both the transfer and pre-transfer checksums) or the
* two-name "TRANSFER,PRE-TRANSFER" form (only one comma is significant). The
* pre-transfer half is FastSync's whole-file digest; the transfer half is
* validated for parity and, when "none", forces --whole-file like rsync. An
* unknown name (including an empty half or a second comma) is exit 4. "auto"
* resolves to FastSync's negotiated default (xxh128). */
static int set_checksum_choice(Config* config, const char* value) {
if (strcasecmp(value, "auto") == 0)
return 0;
int algo = checksum_algo_from_name(value);
if (algo < 0) {
log_message(LOG_LEVEL_ERROR,
"--checksum-choice '%s' is not implemented; FastSync supports xxh64 (or xxhash), "
"xxh3, xxh128, md5 or auto (rsync's md4/sha1/none and the two-name "
"transfer,pre-transfer form are rejected, never silently ignored)",
value);
if (!value) {
config->cli_exit_code = 4;
return -1;
}
config->checksum_algo = algo;
const char* comma = strchr(value, ',');
const char* name1 = value;
size_t len1 = comma ? (size_t)(comma - value) : strlen(value);
const char* name2 = comma ? comma + 1 : NULL;
size_t len2 = name2 ? strlen(name2) : 0;
int transfer = -1;
int pre = -1;
int rc1 = resolve_checksum_name(name1, len1, &transfer);
int rc2 = name2 ? resolve_checksum_name(name2, len2, &pre) : 1;
if (rc1 < 0 || rc2 < 0) {
log_message(LOG_LEVEL_ERROR,
"--checksum-choice '%s' is invalid; FastSync supports xxh64 (or xxhash), xxh128, "
"xxh3, md5, md4, sha1, none or auto, optionally as 'transfer,pre-transfer'",
value);
config->cli_exit_code = 4;
return -1;
}
ChecksumAlgo negotiated = checksum_negotiate_default();
if (rc1 == 1)
transfer = (int)negotiated;
if (!name2)
pre = transfer;
else if (rc2 == 1)
pre = (int)negotiated;
config->checksum_algo = pre;
config->checksum_transfer_algo = transfer;
/* rsync: "none" for the transfer checksum forces --whole-file. */
if (transfer == (int)CHECKSUM_ALGO_NONE)
config->whole_file = true;
return 0;
}
@@ -2326,8 +2375,23 @@ static bool cli_handle_outbuf_option(CliParseCtx* ctx) {
* -1 on error. */
static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool no_incremental) {
set_log_level(config->quiet ? LOG_LEVEL_ERROR : (verbose ? LOG_LEVEL_DEBUG : LOG_LEVEL_WARNING));
if (config->compress_choice)
config->use_compression = strcmp(config->compress_choice, "none") != 0;
if (config->compress_choice) {
int algo = compression_algo_from_name(config->compress_choice);
if (algo >= 0) {
config->compression_algo = algo;
config->use_compression = (algo != (int)COMPRESSION_ALGO_NONE);
}
}
if (config->use_compression && config->compression_algo == (int)COMPRESSION_ALGO_NONE)
config->compression_algo = (int)compression_negotiate_default();
/* rsync parity: "none" as the pre-transfer checksum cannot be combined with
* --checksum (exit 4). The check runs here because --checksum may appear on
* either side of --checksum-choice. */
if (config->checksum && config->checksum_algo == (int)CHECKSUM_ALGO_NONE) {
log_message(LOG_LEVEL_ERROR, "Invalid checksum-choice for --checksum: none");
config->cli_exit_code = 4;
return -1;
}
/* rsync randomizes the checksum seed for every transfer when the user did not
* supply one (a seed of 0, including an explicit --checksum-seed=0), using
@@ -2786,7 +2850,7 @@ int main(int argc, char* argv[]) {
int parse_ret = parse_args(config, argc, argv, positional_args, &positional_count);
if (parse_ret != 0) {
if (parse_ret < 0)
exit_code = 1;
exit_code = config->cli_exit_code ? config->cli_exit_code : 1;
goto cleanup;
}
@@ -2876,6 +2940,11 @@ int main(int argc, char* argv[]) {
goto cleanup;
}
/* Install the negotiated codec for this process before any transfer thread
* is spawned; the compressed frames are self-describing, so the receiver's
* decompressor does not need this, but the sender compressor does. */
compression_set_algo((CompressionAlgo)config->compression_algo);
/* --iconv: install the sender-side local->wire conversion before any path is
scanned or serialized (the scanner and the chunk/data path read windows are
all driven from this process, so one global initialization covers every
src/client/usage.c
+6 -5
View File
@@ -138,10 +138,10 @@ void print_usage(void) {
printf(" --link-dest <dir> Like --copy-dest, but hard-links the unchanged file from DIR\n");
printf(" into the destination (repeatable; earlier DIRs win)\n");
printf(" --checksum-choice, --cc <alg> Whole-file checksum algorithm for --incremental/\n");
printf(" --checksum compares. Accepted: xxh64 (aka xxhash), xxh3,\n");
printf(" xxh128, md5, or auto (default xxh64). rsync choices FastSync\n");
printf(" does not implement (md4, sha1, none) and the two-name\n");
printf(" transfer,pre-transfer form are rejected by name\n");
printf(" --checksum compares. Accepted: xxh128 (default), xxh3, xxh64\n");
printf(" (aka xxhash), md5, md4, sha1, or none. A two-name\n");
printf(" 'transfer,pre-transfer' form is accepted like rsync; 'none' as\n");
printf(" the pre-transfer algorithm is rejected with --checksum\n");
printf(" --checksum-seed <num> Seed for the whole-file xxHash digest (and the delta\n");
printf(" block strong hash, low 32 bits); md5 ignores the seed. A seed\n");
printf(" of 0 (the default) is randomized per transfer, exactly like\n");
@@ -168,7 +168,8 @@ void print_usage(void) {
printf(" SSH argv is already built injection-safe)\n");
printf(" --sendfile Enable sendfile zero-copy (TCP only; long form only;\n");
printf(" -f is bound to --filter, not --sendfile)\n");
printf(" --compress-choice <alg> Compression algorithm (default: zstd)\n");
printf(" --compress-choice <alg> Compression algorithm: zstd (default), lz4, zlib,\n");
printf(" zlibx, none, or auto\n");
printf(" --zc <alg> Alias for --compress-choice\n");
printf(" -v, --verbose Enable debug logging\n");
printf(" -q, --quiet Suppress non-error output\n");
src/server/server.c
+4
View File
@@ -741,6 +741,10 @@ void handler(int file_descriptor) {
* received config. */
if (gate_ctx.super_mode_override != -1)
config->super_mode = (SuperMode)gate_ctx.super_mode_override;
/* Install the codec this connection negotiated before the receiver/writer
* threads start (the server forks per connection, so the process-global
* codec is private to this session). */
compression_set_algo((CompressionAlgo)config->compression_algo);
/* If the client requested ownership but the effective super mode forbids it
* (operator --no-super, a privileged standalone receiver's secure default, or
* a daemon module without `client owner = yes`), say so ONCE per connection so
src/shared/checksum.c
+210 -21
View File
@@ -11,6 +11,160 @@
#define XXH_STATIC_LINKING_ONLY
#include <xxhash.h>
/* ---------------------------------------------------------------------------
* Self-contained MD4 (RFC 1320). OpenSSL's MD4 lives in the legacy provider
* and is not guaranteed present, so FastSync carries its own implementation to
* keep --checksum-choice=md4 working on every build.
* ------------------------------------------------------------------------- */
typedef struct {
uint32_t state[4];
uint64_t bit_count;
uint8_t buffer[64];
size_t buffer_len;
} Md4Ctx;
static uint32_t md4_rotl(uint32_t x, int n) {
return (x << n) | (x >> (32 - n));
}
static void md4_transform(uint32_t state[4], const uint8_t block[64]) {
uint32_t x[16];
for (int i = 0; i < 16; i++)
x[i] = (uint32_t)block[i * 4] | ((uint32_t)block[i * 4 + 1] << 8) |
((uint32_t)block[i * 4 + 2] << 16) | ((uint32_t)block[i * 4 + 3] << 24);
uint32_t a = state[0], b = state[1], c = state[2], d = state[3];
#define F(x, y, z) (((x) & (y)) | (~(x) & (z)))
#define G(x, y, z) (((x) & (y)) | ((x) & (z)) | ((y) & (z)))
#define H(x, y, z) ((x) ^ (y) ^ (z))
#define ROUND1(a, b, c, d, k, s) a = md4_rotl(a + F(b, c, d) + x[k], s)
#define ROUND2(a, b, c, d, k, s) a = md4_rotl(a + G(b, c, d) + x[k] + 0x5a827999u, s)
#define ROUND3(a, b, c, d, k, s) a = md4_rotl(a + H(b, c, d) + x[k] + 0x6ed9eba1u, s)
ROUND1(a, b, c, d, 0, 3);
ROUND1(d, a, b, c, 1, 7);
ROUND1(c, d, a, b, 2, 11);
ROUND1(b, c, d, a, 3, 19);
ROUND1(a, b, c, d, 4, 3);
ROUND1(d, a, b, c, 5, 7);
ROUND1(c, d, a, b, 6, 11);
ROUND1(b, c, d, a, 7, 19);
ROUND1(a, b, c, d, 8, 3);
ROUND1(d, a, b, c, 9, 7);
ROUND1(c, d, a, b, 10, 11);
ROUND1(b, c, d, a, 11, 19);
ROUND1(a, b, c, d, 12, 3);
ROUND1(d, a, b, c, 13, 7);
ROUND1(c, d, a, b, 14, 11);
ROUND1(b, c, d, a, 15, 19);
ROUND2(a, b, c, d, 0, 3);
ROUND2(d, a, b, c, 4, 5);
ROUND2(c, d, a, b, 8, 9);
ROUND2(b, c, d, a, 12, 13);
ROUND2(a, b, c, d, 1, 3);
ROUND2(d, a, b, c, 5, 5);
ROUND2(c, d, a, b, 9, 9);
ROUND2(b, c, d, a, 13, 13);
ROUND2(a, b, c, d, 2, 3);
ROUND2(d, a, b, c, 6, 5);
ROUND2(c, d, a, b, 10, 9);
ROUND2(b, c, d, a, 14, 13);
ROUND2(a, b, c, d, 3, 3);
ROUND2(d, a, b, c, 7, 5);
ROUND2(c, d, a, b, 11, 9);
ROUND2(b, c, d, a, 15, 13);
ROUND3(a, b, c, d, 0, 3);
ROUND3(d, a, b, c, 8, 9);
ROUND3(c, d, a, b, 4, 11);
ROUND3(b, c, d, a, 12, 15);
ROUND3(a, b, c, d, 2, 3);
ROUND3(d, a, b, c, 10, 9);
ROUND3(c, d, a, b, 6, 11);
ROUND3(b, c, d, a, 14, 15);
ROUND3(a, b, c, d, 1, 3);
ROUND3(d, a, b, c, 9, 9);
ROUND3(c, d, a, b, 5, 11);
ROUND3(b, c, d, a, 13, 15);
ROUND3(a, b, c, d, 3, 3);
ROUND3(d, a, b, c, 11, 9);
ROUND3(c, d, a, b, 7, 11);
ROUND3(b, c, d, a, 15, 15);
#undef F
#undef G
#undef H
#undef ROUND1
#undef ROUND2
#undef ROUND3
state[0] += a;
state[1] += b;
state[2] += c;
state[3] += d;
}
static void md4_init(Md4Ctx* ctx) {
ctx->state[0] = 0x67452301u;
ctx->state[1] = 0xefcdab89u;
ctx->state[2] = 0x98badcfeu;
ctx->state[3] = 0x10325476u;
ctx->bit_count = 0;
ctx->buffer_len = 0;
}
static void md4_update(Md4Ctx* ctx, const uint8_t* data, size_t len) {
ctx->bit_count += (uint64_t)len * 8;
while (len > 0) {
size_t space = sizeof(ctx->buffer) - ctx->buffer_len;
size_t take = len < space ? len : space;
memcpy(ctx->buffer + ctx->buffer_len, data, take);
ctx->buffer_len += take;
data += take;
len -= take;
if (ctx->buffer_len == sizeof(ctx->buffer)) {
md4_transform(ctx->state, ctx->buffer);
ctx->buffer_len = 0;
}
}
}
static void md4_final(Md4Ctx* ctx, uint8_t out[16]) {
uint64_t bit_count = ctx->bit_count;
uint8_t pad = 0x80;
md4_update(ctx, &pad, 1);
uint8_t zero = 0;
while (ctx->buffer_len != 56)
md4_update(ctx, &zero, 1);
uint8_t length_le[8];
for (int i = 0; i < 8; i++)
length_le[i] = (uint8_t)((bit_count >> (8 * i)) & 0xff);
md4_update(ctx, length_le, sizeof(length_le));
for (int i = 0; i < 4; i++) {
out[i * 4] = (uint8_t)(ctx->state[i] & 0xff);
out[i * 4 + 1] = (uint8_t)((ctx->state[i] >> 8) & 0xff);
out[i * 4 + 2] = (uint8_t)((ctx->state[i] >> 16) & 0xff);
out[i * 4 + 3] = (uint8_t)((ctx->state[i] >> 24) & 0xff);
}
}
/* One-shot EVP digest (md5/sha1). Returns false when OpenSSL refuses. */
static bool evp_digest(const EVP_MD* md, const void* data, size_t size, uint8_t* out,
size_t out_capacity, size_t* out_len) {
static const uint8_t empty = 0;
const void* input = data ? data : &empty;
unsigned int digest_len = 0;
if (EVP_Digest(input, size, out, &digest_len, md, NULL) != 1)
return false;
if (digest_len > out_capacity)
return false;
*out_len = digest_len;
return true;
}
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
size_t out_capacity, size_t* out_len) {
if (!out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
@@ -18,43 +172,45 @@ bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t
if (data == NULL && size != 0)
return false;
if (algo == CHECKSUM_ALGO_XXH64) {
switch (algo) {
case CHECKSUM_ALGO_XXH64: {
uint64_t digest = XXH64(data, size, seed);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
return true;
}
if (algo == CHECKSUM_ALGO_XXH3) {
case CHECKSUM_ALGO_XXH3: {
uint64_t digest = XXH3_64bits_withSeed(data, size, seed);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
return true;
}
if (algo == CHECKSUM_ALGO_XXH128) {
case CHECKSUM_ALGO_XXH128: {
XXH128_hash_t digest = XXH3_128bits_withSeed(data, size, seed);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
return true;
}
if (algo == CHECKSUM_ALGO_MD5) {
case CHECKSUM_ALGO_MD5:
/* md5 takes no seed; the caller's seed is deliberately ignored (documented
* in RSYNC_COMPAT.md). OpenSSL's one-shot EVP_Digest needs a non-NULL
* buffer even for an empty input, so map a NULL data + size==0 to an empty
* buffer. */
static const uint8_t empty = 0;
const void* input = data ? data : &empty;
unsigned int digest_len = 0;
if (EVP_Digest(input, size, out, &digest_len, EVP_md5(), NULL) != 1)
return false;
if (digest_len > out_capacity)
return false;
*out_len = digest_len;
* in RSYNC_COMPAT.md). */
return evp_digest(EVP_md5(), data, size, out, out_capacity, out_len);
case CHECKSUM_ALGO_MD4: {
Md4Ctx ctx;
md4_init(&ctx);
md4_update(&ctx, (const uint8_t*)data, size);
md4_final(&ctx, out);
*out_len = 16;
return true;
}
case CHECKSUM_ALGO_SHA1:
/* sha1 takes no seed; the caller's seed is deliberately ignored. */
return evp_digest(EVP_sha1(), data, size, out, out_capacity, out_len);
case CHECKSUM_ALGO_NONE:
/* No checksum requested: an empty digest is the successful result. */
*out_len = 0;
return true;
}
return false;
}
@@ -162,6 +318,12 @@ int checksum_algo_from_name(const char* name) {
return (int)CHECKSUM_ALGO_XXH128;
if (strcasecmp(name, "md5") == 0)
return (int)CHECKSUM_ALGO_MD5;
if (strcasecmp(name, "md4") == 0)
return (int)CHECKSUM_ALGO_MD4;
if (strcasecmp(name, "sha1") == 0)
return (int)CHECKSUM_ALGO_SHA1;
if (strcasecmp(name, "none") == 0)
return (int)CHECKSUM_ALGO_NONE;
return -1;
}
@@ -175,13 +337,21 @@ const char* checksum_algo_name(ChecksumAlgo algo) {
return "xxh128";
case CHECKSUM_ALGO_MD5:
return "md5";
case CHECKSUM_ALGO_MD4:
return "md4";
case CHECKSUM_ALGO_SHA1:
return "sha1";
case CHECKSUM_ALGO_NONE:
return "none";
}
return "<unknown>";
}
bool checksum_algo_valid(int algo) {
return algo == (int)CHECKSUM_ALGO_XXH64 || algo == (int)CHECKSUM_ALGO_MD5 ||
algo == (int)CHECKSUM_ALGO_XXH3 || algo == (int)CHECKSUM_ALGO_XXH128;
algo == (int)CHECKSUM_ALGO_XXH3 || algo == (int)CHECKSUM_ALGO_XXH128 ||
algo == (int)CHECKSUM_ALGO_MD4 || algo == (int)CHECKSUM_ALGO_SHA1 ||
algo == (int)CHECKSUM_ALGO_NONE;
}
uint8_t checksum_digest_len(ChecksumAlgo algo) {
@@ -191,7 +361,26 @@ uint8_t checksum_digest_len(ChecksumAlgo algo) {
return 8;
case CHECKSUM_ALGO_XXH128:
case CHECKSUM_ALGO_MD5:
case CHECKSUM_ALGO_MD4:
return 16;
case CHECKSUM_ALGO_SHA1:
return 20;
case CHECKSUM_ALGO_NONE:
return 0;
}
return 0;
}
}
ChecksumAlgo checksum_negotiate_default(void) {
/* rsync 3.4.1 default preference order; every entry is compiled in, so this
* resolves to xxh128. */
static const ChecksumAlgo preference[] = {
CHECKSUM_ALGO_XXH128, CHECKSUM_ALGO_XXH3, CHECKSUM_ALGO_XXH64, CHECKSUM_ALGO_MD5,
CHECKSUM_ALGO_MD4, CHECKSUM_ALGO_SHA1, CHECKSUM_ALGO_NONE,
};
for (size_t i = 0; i < sizeof(preference) / sizeof(preference[0]); i++) {
if (checksum_algo_valid((int)preference[i]))
return preference[i];
}
return CHECKSUM_ALGO_XXH64;
}
src/shared/checksum.h
+31 -15
View File
@@ -8,25 +8,35 @@
/* Whole-file content-digest algorithms selectable with --checksum-choice and
* seeded with --checksum-seed. The ids are the values actually placed on the
* wire (config frame), so they must be kept stable and validated on receive.
* CHECKSUM_ALGO_XXH64 == 0 is the default and is byte-for-byte what FastSync
* computed before these options existed (xxHash64 with seed 0). The set mirrors
* the algorithms rsync 3.4.1 can be built with; the ones FastSync does not
* implement (md4, sha1, none) are rejected by name at parse time. */
* CHECKSUM_ALGO_XXH64 == 0 is the historical FastSync default and its numeric
* value is preserved. The full set mirrors the algorithms rsync 3.4.1 can be
* built with; every one of them is implemented here. */
typedef enum {
CHECKSUM_ALGO_XXH64 = 0,
CHECKSUM_ALGO_MD5 = 1,
CHECKSUM_ALGO_XXH3 = 2,
CHECKSUM_ALGO_XXH128 = 3
CHECKSUM_ALGO_XXH128 = 3,
CHECKSUM_ALGO_MD4 = 4,
CHECKSUM_ALGO_SHA1 = 5,
CHECKSUM_ALGO_NONE = 6
} ChecksumAlgo;
/* xxh128 digest is 16 bytes, the longest supported. */
#define CHECKSUM_MAX_DIGEST_LEN 16
/* FastSync's negotiated default (rsync 3.4.1 auto-negotiates xxh128 first).
* The wire default for Config->checksum_algo is this value. */
#define CHECKSUM_ALGO_DEFAULT CHECKSUM_ALGO_XXH128
/* sha1 digest is 20 bytes, the longest supported. */
#define CHECKSUM_MAX_DIGEST_LEN 20
/* Compute the whole-file digest of the first `size` bytes of `data`.
*
* - CHECKSUM_ALGO_XXH64: xxHash64(data, size, seed) (full 64-bit seed).
* - CHECKSUM_ALGO_MD5: md5(data, size) via OpenSSL EVP.
* md5 has no seed, so `seed` is ignored (documented).
* - CHECKSUM_ALGO_XXH3: XXH3_64bits_withSeed(data, size, seed).
* - CHECKSUM_ALGO_XXH128: XXH3_128bits_withSeed(data, size, seed).
* - CHECKSUM_ALGO_MD5: md5(data, size) via OpenSSL EVP (seed ignored).
* - CHECKSUM_ALGO_MD4: md4(data, size), self-contained RFC 1320 (seed ignored).
* - CHECKSUM_ALGO_SHA1: sha1(data, size) via OpenSSL EVP (seed ignored).
* - CHECKSUM_ALGO_NONE: no digest; *out_len is 0 and nothing is written.
* - `size == 0` hashes the empty input (plus its seed), not a NULL input.
*
* Writes up to `out_capacity` bytes into `out`, storing the digest length in
@@ -41,10 +51,10 @@ bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t
bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, uint8_t* out,
size_t out_capacity, size_t* out_len);
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id. * Accepts
* "xxh64"/"xxhash", "xxh3", "xxh128" and "md5". "auto", rsync's default automatic choice, is
* resolved to the default by the caller (it is not a distinct algorithm here). Returns -1 for any
* name FastSync does not implement (md4/sha1/none included). */
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id.
* Accepts "xxh64"/"xxhash", "xxh3", "xxh128", "md5", "md4", "sha1", "none".
* "auto" is not an algorithm here; the caller resolves it to the negotiated
* default. Returns -1 for any unrecognized name. */
int checksum_algo_from_name(const char* name);
/* Canonical name of an algorithm (used in CLI error messages). */
@@ -53,7 +63,13 @@ const char* checksum_algo_name(ChecksumAlgo algo);
/* True when `algo` is a supported id (used by config receive validation). */
bool checksum_algo_valid(int algo);
/* Digest length in bytes for an algorithm (xxh64/xxh3 = 8, md5/xxh128 = 16). */
/* Digest length in bytes for an algorithm (xxh64/xxh3 = 8,
* md5/md4/xxh128 = 16, sha1 = 20, none = 0). */
uint8_t checksum_digest_len(ChecksumAlgo algo);
#endif /* CHECKSUM_H */
/* Pick the first algorithm from FastSync's compiled-in preference list that is
* supported on this build (rsync 3.4.1's `--version` order:
* xxh128 xxh3 xxh64 md5 md4 sha1 none). Used to resolve "auto". */
ChecksumAlgo checksum_negotiate_default(void);
#endif /* CHECKSUM_H */
src/shared/compression.c
+298 -24
View File
@@ -3,12 +3,15 @@
#include "log.h"
#include "protocol.h"
#include <limits.h>
#include <lz4.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <threads.h>
#include <unistd.h>
#include <zlib.h>
#include <zstd.h>
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
@@ -29,6 +32,13 @@
"z " \
"zip zst"
/* Self-describing compressed frames: the first byte is the CompressionAlgo id.
* zlib/lz4 store the uncompressed size as a little-endian uint32 after the
* codec byte so decompression can be exactly pre-sized and bounded. */
#define LZ4_SIZE_PREFIX_LEN 4
static _Atomic int g_compression_algo = COMPRESSION_ALGO_ZSTD;
/* Case-insensitive match of a bare suffix (no leading dot) against a
* space-separated suffix list. */
static bool suffix_in_list(const char* name, const char* list) {
@@ -67,6 +77,75 @@ bool compression_should_skip_with_suffixes(const char* path, char* const* suffix
return false;
}
CompressionAlgo compression_default_algo(void) {
return COMPRESSION_ALGO_ZSTD;
}
int compression_algo_from_name(const char* name) {
if (!name)
return -1;
if (strcasecmp(name, "zstd") == 0)
return (int)COMPRESSION_ALGO_ZSTD;
if (strcasecmp(name, "lz4") == 0)
return (int)COMPRESSION_ALGO_LZ4;
if (strcasecmp(name, "zlib") == 0)
return (int)COMPRESSION_ALGO_ZLIB;
if (strcasecmp(name, "zlibx") == 0)
return (int)COMPRESSION_ALGO_ZLIBX;
if (strcasecmp(name, "none") == 0)
return (int)COMPRESSION_ALGO_NONE;
return -1;
}
const char* compression_algo_name(CompressionAlgo algo) {
switch (algo) {
case COMPRESSION_ALGO_NONE:
return "none";
case COMPRESSION_ALGO_ZSTD:
return "zstd";
case COMPRESSION_ALGO_LZ4:
return "lz4";
case COMPRESSION_ALGO_ZLIB:
return "zlib";
case COMPRESSION_ALGO_ZLIBX:
return "zlibx";
}
return "<unknown>";
}
bool compression_algo_valid(int algo) {
return algo == (int)COMPRESSION_ALGO_NONE || algo == (int)COMPRESSION_ALGO_ZSTD ||
algo == (int)COMPRESSION_ALGO_LZ4 || algo == (int)COMPRESSION_ALGO_ZLIB ||
algo == (int)COMPRESSION_ALGO_ZLIBX;
}
bool compression_algo_enabled(CompressionAlgo algo) {
return algo != COMPRESSION_ALGO_NONE;
}
CompressionAlgo compression_negotiate_default(void) {
/* rsync 3.4.1 default preference order; every entry is compiled in, so this
* resolves to zstd. */
static const CompressionAlgo preference[] = {
COMPRESSION_ALGO_ZSTD, COMPRESSION_ALGO_LZ4, COMPRESSION_ALGO_ZLIBX,
COMPRESSION_ALGO_ZLIB, COMPRESSION_ALGO_NONE,
};
for (size_t i = 0; i < sizeof(preference) / sizeof(preference[0]); i++) {
if (compression_algo_valid((int)preference[i]))
return preference[i];
}
return COMPRESSION_ALGO_ZSTD;
}
void compression_set_algo(CompressionAlgo algo) {
if (compression_algo_valid((int)algo))
atomic_store(&g_compression_algo, (int)algo);
}
CompressionAlgo compression_get_algo(void) {
return (CompressionAlgo)atomic_load(&g_compression_algo);
}
/* Per-thread cache of zstd contexts plus the grow-only compression scratch
* buffer. zstd contexts are stateful and not safe to share between threads,
* so each thread keeps its own (see compression_get_thread_ctx). The cache is
@@ -157,17 +236,25 @@ static void compression_ctx_put(CompressionThreadCtx* ctx) {
compression_ctx_free(ctx);
}
Data* data_compress(Data* data_to_compress, int compression_level) {
return data_compress_with_threads(data_to_compress, compression_level, 0);
/* Build a frame consisting of a copy of `src` prefixed by `codec`. */
static Data* frame_with_codec(const void* src, size_t size, CompressionAlgo codec) {
if (size > SIZE_MAX - 1)
return NULL;
Data* out = data_create_empty(size + 1);
if (!out)
return NULL;
((uint8_t*)out->data)[0] = (uint8_t)codec;
if (size > 0)
memcpy((uint8_t*)out->data + 1, src, size);
out->size = size + 1;
return out;
}
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
int compression_threads) {
if (!data_to_compress || (!data_to_compress->data && data_to_compress->size != 0) ||
compression_threads < 0 || compression_threads > COMPRESSION_MAX_THREADS)
static Data* zstd_compress(Data* in, int compression_level, int compression_threads) {
size_t dst_size = ZSTD_compressBound(in->size);
if (dst_size > SIZE_MAX - 1)
return NULL;
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
size_t dst_size = ZSTD_compressBound(data_to_compress->size);
dst_size += 1; /* codec prefix */
CompressionThreadCtx* ctx = compression_get_thread_ctx();
if (ctx == NULL) {
@@ -218,7 +305,7 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
if (available_threads > 0) {
/* Streaming compression needs the source size before threaded mode can end a frame. */
size_t zret = ZSTD_CCtx_setPledgedSrcSize(ctx->cctx, data_to_compress->size);
size_t zret = ZSTD_CCtx_setPledgedSrcSize(ctx->cctx, in->size);
if (ZSTD_isError(zret)) {
log_message(LOG_LEVEL_ERROR, "Failed to set compression source size: %s",
ZSTD_getErrorName(zret));
@@ -236,8 +323,8 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
ctx->out_cap = dst_size;
}
ZSTD_inBuffer input = {data_to_compress->data, data_to_compress->size, 0};
ZSTD_outBuffer output = {ctx->out_buf, dst_size, 0};
ZSTD_inBuffer input = {in->data, in->size, 0};
ZSTD_outBuffer output = {(uint8_t*)ctx->out_buf + 1, dst_size - 1, 0};
size_t ret;
do {
@@ -250,30 +337,192 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
/* Hand off an exactly-sized copy; the scratch buffer stays cached so the next
* call does not reallocate a ZSTD_compressBound-sized block. */
compressed_data = data_create_empty(output.pos);
compressed_data = data_create_empty(output.pos + 1);
if (compressed_data == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to allocate compressed data");
goto cleanup;
}
((uint8_t*)compressed_data->data)[0] = (uint8_t)COMPRESSION_ALGO_ZSTD;
if (output.pos > 0)
memcpy(compressed_data->data, ctx->out_buf, output.pos);
compressed_data->size = output.pos;
memcpy((uint8_t*)compressed_data->data + 1, (uint8_t*)ctx->out_buf + 1, output.pos);
compressed_data->size = output.pos + 1;
log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu",
data_to_compress->size, compressed_data->size);
log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu", in->size,
compressed_data->size);
cleanup:
compression_ctx_put(ctx);
return compressed_data;
}
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
if (!compressed_data || (!compressed_data->data && compressed_data->size != 0) ||
maximum_size == 0)
static Data* lz4_compress(Data* in) {
int bound = LZ4_compressBound((int)in->size);
if (bound < 0 || in->size > (size_t)INT_MAX)
return NULL;
Data* out = data_create_empty((size_t)bound + 1 + LZ4_SIZE_PREFIX_LEN);
if (!out)
return NULL;
uint32_t raw_size = (uint32_t)in->size;
uint8_t* p = (uint8_t*)out->data;
p[0] = (uint8_t)COMPRESSION_ALGO_LZ4;
for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++)
p[1 + i] = (uint8_t)((raw_size >> (8 * i)) & 0xff);
int written = 0;
if (in->size > 0) {
written = LZ4_compress_default((const char*)in->data, (char*)p + 1 + LZ4_SIZE_PREFIX_LEN,
(int)in->size, bound);
if (written <= 0) {
data_destroy(out);
return NULL;
}
}
out->size = (size_t)written + 1 + LZ4_SIZE_PREFIX_LEN;
return out;
}
static Data* zlib_compress(Data* in, CompressionAlgo algo, int compression_level) {
int level = compression_level;
if (level < 1)
level = Z_DEFAULT_COMPRESSION;
if (level > 9)
level = 9;
uLong bound = compressBound((uLong)in->size);
if (in->size > (size_t)ULONG_MAX)
return NULL;
Data* out = data_create_empty((size_t)bound + 1 + LZ4_SIZE_PREFIX_LEN);
if (!out)
return NULL;
uint32_t raw_size = (uint32_t)in->size;
uint8_t* p = (uint8_t*)out->data;
p[0] = (uint8_t)algo;
for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++)
p[1 + i] = (uint8_t)((raw_size >> (8 * i)) & 0xff);
uLongf dest_len = bound;
int rc = compress2(p + 1 + LZ4_SIZE_PREFIX_LEN, &dest_len, (const Bytef*)in->data,
(uLong)in->size, level);
if (rc != Z_OK) {
data_destroy(out);
return NULL;
}
out->size = (size_t)dest_len + 1 + LZ4_SIZE_PREFIX_LEN;
return out;
}
Data* data_compress_codec(Data* data_to_compress, CompressionAlgo algo, int compression_level,
int compression_threads) {
if (!data_to_compress || (!data_to_compress->data && data_to_compress->size != 0) ||
compression_threads < 0 || compression_threads > COMPRESSION_MAX_THREADS)
return NULL;
if (!compression_algo_valid((int)algo))
return NULL;
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
switch (algo) {
case COMPRESSION_ALGO_NONE:
return frame_with_codec(data_to_compress->data, data_to_compress->size, COMPRESSION_ALGO_NONE);
case COMPRESSION_ALGO_ZSTD:
return zstd_compress(data_to_compress, compression_level, compression_threads);
case COMPRESSION_ALGO_LZ4:
return lz4_compress(data_to_compress);
case COMPRESSION_ALGO_ZLIB:
case COMPRESSION_ALGO_ZLIBX:
return zlib_compress(data_to_compress, algo, compression_level);
}
return NULL;
}
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
int compression_threads) {
return data_compress_codec(data_to_compress, compression_get_algo(), compression_level,
compression_threads);
}
Data* data_compress(Data* data_to_compress, int compression_level) {
return data_compress_codec(data_to_compress, compression_get_algo(), compression_level, 0);
}
static Data* decompress_none(const Data* compressed_data, size_t maximum_size) {
size_t size = compressed_data->size - 1;
if (size > maximum_size)
return NULL;
Data* out = data_create_empty(size);
if (!out)
return NULL;
if (size > 0)
memcpy(out->data, (const uint8_t*)compressed_data->data + 1, size);
out->size = size;
return out;
}
/* Read the 4-byte little-endian raw size stored after the codec byte. */
static bool read_raw_size(const Data* in, uint32_t* raw_size) {
if (in->size < 1 + LZ4_SIZE_PREFIX_LEN)
return false;
const uint8_t* p = (const uint8_t*)in->data;
uint32_t v = 0;
for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++)
v |= (uint32_t)p[1 + i] << (8 * i);
*raw_size = v;
return true;
}
static Data* lz4_decompress(Data* compressed_data, size_t maximum_size, size_t hard_limit) {
uint32_t raw_size = 0;
if (!read_raw_size(compressed_data, &raw_size))
return NULL;
if (raw_size > hard_limit || raw_size > maximum_size)
return NULL;
size_t comp_size = compressed_data->size - 1 - LZ4_SIZE_PREFIX_LEN;
Data* out = data_create_empty(raw_size);
if (!out)
return NULL;
if (raw_size == 0) {
out->size = 0;
return out;
}
int rc = LZ4_decompress_safe((const char*)compressed_data->data + 1 + LZ4_SIZE_PREFIX_LEN,
(char*)out->data, (int)comp_size, (int)raw_size);
if (rc < 0 || (uint32_t)rc != raw_size) {
log_message(LOG_LEVEL_ERROR, "LZ4 decompression failed");
data_destroy(out);
return NULL;
}
out->size = raw_size;
return out;
}
static Data* zlib_decompress(Data* compressed_data, size_t maximum_size, size_t hard_limit) {
uint32_t raw_size = 0;
if (!read_raw_size(compressed_data, &raw_size))
return NULL;
if (raw_size > hard_limit || raw_size > maximum_size)
return NULL;
size_t comp_size = compressed_data->size - 1 - LZ4_SIZE_PREFIX_LEN;
Data* out = data_create_empty(raw_size);
if (!out)
return NULL;
if (raw_size == 0) {
out->size = 0;
return out;
}
uLongf dest_len = raw_size;
int rc =
uncompress((Bytef*)out->data, &dest_len,
(const Bytef*)compressed_data->data + 1 + LZ4_SIZE_PREFIX_LEN, (uLong)comp_size);
if (rc != Z_OK || dest_len != raw_size) {
log_message(LOG_LEVEL_ERROR, "zlib decompression failed");
data_destroy(out);
return NULL;
}
out->size = raw_size;
return out;
}
static Data* zstd_decompress(Data* compressed_data, size_t maximum_size) {
/* The zstd frame starts after the codec byte. */
const void* frame = (const uint8_t*)compressed_data->data + 1;
size_t frame_size = compressed_data->size - 1;
log_debug_message(LOG_DEBUG_UTIL, "Start to decompress data");
unsigned long long dst_size =
ZSTD_getFrameContentSize(compressed_data->data, compressed_data->size);
unsigned long long dst_size = ZSTD_getFrameContentSize(frame, frame_size);
/* ZSTD_isError() is also true for ZSTD_CONTENTSIZE_ERROR and
* ZSTD_CONTENTSIZE_UNKNOWN (both are encoded near (size_t)-1), so test the
* sentinels explicitly instead of blanket-rejecting every error-ish value:
@@ -287,9 +536,9 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
// ZSTD_CONTENTSIZE_UNKNOWN (~2^64) can cause massive allocation;
// fall back to a conservative estimate (3x compressed size) when unknown.
if (dst_size == ZSTD_CONTENTSIZE_UNKNOWN) {
if (compressed_data->size > ULLONG_MAX / 3)
if (frame_size > ULLONG_MAX / 3)
return NULL;
dst_size = compressed_data->size * 3;
dst_size = frame_size * 3;
if (dst_size < INITIAL_DECOMPRESS_BUF_SIZE)
dst_size = INITIAL_DECOMPRESS_BUF_SIZE;
}
@@ -326,7 +575,7 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
goto cleanup;
}
ZSTD_inBuffer input = {compressed_data->data, compressed_data->size, 0};
ZSTD_inBuffer input = {frame, frame_size, 0};
ZSTD_outBuffer output = {uncompressed_data->data, buf_size, 0};
size_t ret;
@@ -385,6 +634,31 @@ cleanup:
return uncompressed_data;
}
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
if (!compressed_data || (!compressed_data->data && compressed_data->size != 0) ||
maximum_size == 0)
return NULL;
if (compressed_data->size < 1)
return NULL;
unsigned long long hard_limit =
maximum_size < MAX_DECOMPRESSED_SIZE ? maximum_size : MAX_DECOMPRESSED_SIZE;
uint8_t codec = ((const uint8_t*)compressed_data->data)[0];
if (!compression_algo_valid(codec))
return NULL;
switch ((CompressionAlgo)codec) {
case COMPRESSION_ALGO_NONE:
return decompress_none(compressed_data, (size_t)hard_limit);
case COMPRESSION_ALGO_ZSTD:
return zstd_decompress(compressed_data, (size_t)hard_limit);
case COMPRESSION_ALGO_LZ4:
return lz4_decompress(compressed_data, maximum_size, (size_t)hard_limit);
case COMPRESSION_ALGO_ZLIB:
case COMPRESSION_ALGO_ZLIBX:
return zlib_decompress(compressed_data, maximum_size, (size_t)hard_limit);
}
return NULL;
}
Data* data_decompress(Data* compressed_data) {
return data_decompress_limited(compressed_data, MAX_DECOMPRESSED_SIZE);
}
src/shared/compression.h
+51 -1
View File
@@ -6,11 +6,61 @@
#define COMPRESSION_MAX_THREADS 64
/* Compression algorithms selectable with --compress-choice / -z. The ids are
* the values placed on the wire (Config->compression_algo), so they must be
* kept stable. NONE is "no compression"; ZSTD is the historical FastSync
* default and the negotiated "auto" choice. ZLIBX is rsync's zlib-without-
* matched-data variant: FastSync compresses only the delta/token bytes (it does
* not put matched file data in the compression stream), so its zlib codec is
* already the "x" form and zlib/zlibx share the same implementation, recorded
* under distinct ids. */
typedef enum {
COMPRESSION_ALGO_NONE = 0,
COMPRESSION_ALGO_ZSTD = 1,
COMPRESSION_ALGO_LZ4 = 2,
COMPRESSION_ALGO_ZLIB = 3,
COMPRESSION_ALGO_ZLIBX = 4
} CompressionAlgo;
CompressionAlgo compression_default_algo(void);
/* Resolve a --compress-choice string (case-insensitive) to an algorithm id.
* Accepts "zstd", "lz4", "zlib", "zlibx", "none". "auto" is not an algorithm
* here; the caller resolves it to the negotiated default. Returns -1 for any
* unrecognized name. */
int compression_algo_from_name(const char* name);
const char* compression_algo_name(CompressionAlgo algo);
bool compression_algo_valid(int algo);
/* Pick the first algorithm from FastSync's compiled-in preference list
* (rsync 3.4.1's `--version` order: zstd lz4 zlibx zlib none). Resolves
* "auto". */
CompressionAlgo compression_negotiate_default(void);
/* True when the algorithm actually compresses (i.e. is not NONE). */
bool compression_algo_enabled(CompressionAlgo algo);
/* Select the process-wide codec used by the legacy wrappers below. Each
* process serves exactly one transfer config (the server forks per connection,
* the client configures itself before spawning transfer threads), so a
* process-global default is sufficient and constant for the lifetime of a
* transfer. Defaults to ZSTD when never set. Thread-safe. */
void compression_set_algo(CompressionAlgo algo);
CompressionAlgo compression_get_algo(void);
/* Codec-aware primitives. The compressed buffer is self-describing: its first
* byte is the CompressionAlgo id, so decompression never needs the codec passed
* separately (this keeps every existing Decompress call site source-compatible).
* `data_compress_codec` returns NULL on invalid input or an unsupported codec. */
Data* data_compress_codec(Data* data_to_compress, CompressionAlgo algo, int compression_level,
int compression_threads);
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
/* Legacy zstd-default wrappers retained for existing callers/tests. */
Data* data_compress(Data* data_to_compress, int compression_level);
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
int compression_threads);
Data* data_decompress(Data* compressed_data);
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
/* Release the calling thread's cached zstd contexts (compressor, decompressor
src/shared/config.c
+75 -27
View File
@@ -14,6 +14,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <limits.h>
#include <errno.h>
@@ -67,6 +68,8 @@ static void config_set_defaults(Config* config) {
config->human_readable = false;
config->ignore_errors = false;
config->ignore_missing_args = false;
config->checksum_transfer_algo = CHECKSUM_ALGO_DEFAULT;
config->cli_exit_code = 0;
config->filters = NULL;
config->files_from = NULL;
config->files_from_set = NULL;
@@ -196,12 +199,13 @@ static bool validate_received_config(const Config* config) {
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
valid_wire_bool(config->dry_run) && checksum_algo_valid(config->checksum_algo) &&
identity_wire_valid(config) && valid_wire_bool(config->preserve_atimes) &&
valid_wire_bool(config->preserve_crtimes) && valid_wire_bool(config->omit_dir_times) &&
valid_wire_bool(config->omit_link_times) && valid_wire_bool(config->preserve_perms) &&
valid_wire_bool(config->preserve_times) && valid_wire_bool(config->preserve_owner) &&
valid_wire_bool(config->preserve_group) && valid_wire_bool(config->munge_links) &&
valid_wire_bool(config->keep_dirlinks) && valid_wire_bool(config->fake_super) &&
compression_algo_valid(config->compression_algo) && identity_wire_valid(config) &&
valid_wire_bool(config->preserve_atimes) && valid_wire_bool(config->preserve_crtimes) &&
valid_wire_bool(config->omit_dir_times) && valid_wire_bool(config->omit_link_times) &&
valid_wire_bool(config->preserve_perms) && valid_wire_bool(config->preserve_times) &&
valid_wire_bool(config->preserve_owner) && valid_wire_bool(config->preserve_group) &&
valid_wire_bool(config->munge_links) && valid_wire_bool(config->keep_dirlinks) &&
valid_wire_bool(config->fake_super) &&
(!config->copy_as_set || (config->copy_as_uid >= 0 && config->copy_as_gid >= 0)) &&
(!config->use_compression ||
(config->compression_level >= 1 && config->compression_level <= 22)) &&
@@ -894,6 +898,14 @@ static bool config_receive_checksum_algo(int fd, int* value) {
return true;
}
static bool config_receive_compression_algo(int fd, int* value) {
int algo;
if (!receive_int(fd, &algo) || !compression_algo_valid(algo))
return false;
*value = algo;
return true;
}
static bool config_receive_super_mode(int fd, SuperMode* value) {
int mode;
if (!receive_int(fd, &mode) || mode < SUPER_MODE_AUTO || mode > SUPER_MODE_OFF)
@@ -1095,6 +1107,9 @@ fail:
#define CONFIG_SEND_INT_CHECKSUM_ALGO(name) send_int(fd, c->name)
#define CONFIG_RECV_INT_CHECKSUM_ALGO(name) config_receive_checksum_algo(fd, &c->name)
#define CONFIG_SEND_INT_COMPRESSION_ALGO(name) send_int(fd, c->name)
#define CONFIG_RECV_INT_COMPRESSION_ALGO(name) config_receive_compression_algo(fd, &c->name)
#define CONFIG_SEND_SUPERMODE(name) send_int(fd, (int)c->name)
#define CONFIG_RECV_SUPERMODE(name) config_receive_super_mode(fd, &c->name)
@@ -1170,6 +1185,7 @@ CONFIG_DEFINE_SEND(send_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
CONFIG_DEFINE_SEND(send_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
CONFIG_DEFINE_SEND(send_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
CONFIG_DEFINE_SEND(send_output_options, CONFIG_WIRE_OUTPUT_FIELDS)
CONFIG_DEFINE_SEND(send_codec_options, CONFIG_WIRE_CODEC_FIELDS)
CONFIG_DEFINE_RECV(receive_core_fields, CONFIG_WIRE_CORE_FIELDS)
CONFIG_DEFINE_RECV(receive_delta_fields, CONFIG_WIRE_DELTA_FIELDS)
@@ -1189,6 +1205,7 @@ CONFIG_DEFINE_RECV(receive_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
CONFIG_DEFINE_RECV(receive_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
CONFIG_DEFINE_RECV(receive_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
CONFIG_DEFINE_RECV(receive_output_options, CONFIG_WIRE_OUTPUT_FIELDS)
CONFIG_DEFINE_RECV(receive_codec_options, CONFIG_WIRE_CODEC_FIELDS)
#undef XSEND
#undef XRECV
@@ -1306,7 +1323,8 @@ bool config_send_wire_block(int file_descriptor, const Config* config) {
send_iconv_spec(file_descriptor, config) &&
send_privilege_options(file_descriptor, config) &&
send_copy_as_options(file_descriptor, config) &&
send_output_options(file_descriptor, config);
send_output_options(file_descriptor, config) &&
send_codec_options(file_descriptor, config);
}
bool config_send(int file_descriptor, const Config* config) {
@@ -1377,29 +1395,59 @@ Config* config_receive_with_validate(int file_descriptor, ConfigValidateFunc val
!receive_iconv_spec(file_descriptor, config, &budget) ||
!receive_privilege_options(file_descriptor, config, &budget) ||
!receive_copy_as_options(file_descriptor, config, &budget) ||
!receive_output_options(file_descriptor, config, &budget))
!receive_output_options(file_descriptor, config, &budget) ||
!receive_codec_options(file_descriptor, config, &budget))
goto error;
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
strcmp(config->compress_choice, "none") != 0 &&
strcmp(config->compress_choice, "auto") != 0) {
char* escaped_choice = output_escape(config->compress_choice, config->eight_bit_output);
log_message(LOG_LEVEL_ERROR, "Unsupported compression choice: %s",
escaped_choice ? escaped_choice : "<allocation failed>");
char detail[128];
snprintf(detail, sizeof(detail), "unsupported compression choice: %s",
escaped_choice ? escaped_choice : "<allocation failed>");
send_error_detail(file_descriptor, detail);
free(escaped_choice);
/* Validate/normalize the negotiated codec. compress_choice is the human
* spelling (NULL or "" when -z was not given); compression_algo is the
* concrete codec id the sender used. They must agree, and "auto" is
* canonicalized to FastSync's negotiated default so the stored spelling is
* always concrete (a hostile/older client may still send "auto"). */
if (config->compress_choice && config->compress_choice[0] != '\0') {
int choice_algo = compression_algo_from_name(config->compress_choice);
if (choice_algo < 0 && strcasecmp(config->compress_choice, "auto") != 0) {
char* escaped_choice = output_escape(config->compress_choice, config->eight_bit_output);
log_message(LOG_LEVEL_ERROR, "Unsupported compression choice: %s",
escaped_choice ? escaped_choice : "<allocation failed>");
char detail[160];
snprintf(detail, sizeof(detail), "unsupported compression choice: %s",
escaped_choice ? escaped_choice : "<allocation failed>");
send_error_detail(file_descriptor, detail);
free(escaped_choice);
goto error;
}
if (choice_algo < 0)
choice_algo = (int)compression_negotiate_default();
if (strcasecmp(config->compress_choice, "auto") == 0 ||
choice_algo == (int)COMPRESSION_ALGO_NONE) {
const char* canonical = compression_algo_name((CompressionAlgo)choice_algo);
char* dup = str_dup(canonical);
if (!dup)
goto error;
free(config->compress_choice);
config->compress_choice = dup;
}
if (config->compression_algo != choice_algo) {
log_message(LOG_LEVEL_ERROR, "Compression choice '%s' does not match codec id %d",
config->compress_choice, config->compression_algo);
send_error_detail(file_descriptor, "compression choice/codec mismatch");
goto error;
}
}
/* The concrete codec must exist only when compression is on. A client that
* left -z off has no codec in effect, but the field keeps whatever id it
* carried (the receiver never dispatches on it without use_compression), so
* the wire value round-trips untouched. */
if (config->use_compression && config->compression_algo == (int)COMPRESSION_ALGO_NONE) {
log_message(LOG_LEVEL_ERROR, "Compression requested with the 'none' codec");
send_error_detail(file_descriptor, "compression requested with the none codec");
goto error;
}
/* Defensive: an older/hostile client may still send "auto"; canonicalize it
to zstd (its effective choice) so the stored value is always concrete. */
if (strcmp(config->compress_choice, "auto") == 0) {
char* canonical = str_dup("zstd");
if (!canonical)
goto error;
free(config->compress_choice);
config->compress_choice = canonical;
/* rsync: "none" as the pre-transfer checksum is invalid with --checksum. */
if (config->checksum && config->checksum_algo == (int)CHECKSUM_ALGO_NONE) {
log_message(LOG_LEVEL_ERROR, "Invalid checksum-choice for --checksum: none");
send_error_detail(file_descriptor, "checksum-choice 'none' cannot be used with --checksum");
goto error;
}
if (!validate_received_config(config)) {
log_message(LOG_LEVEL_ERROR, "Invalid configuration received from client");
src/shared/config.h
+51 -6
View File
@@ -3,6 +3,7 @@
#include "array_list.h"
#include "checksum.h"
#include "compression.h"
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
@@ -81,7 +82,7 @@ typedef struct {
typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode;
/* ===========================================================================
* Config wire-field table (single source of truth for protocol 2.23.0).
* Config wire-field table (single source of truth for protocol 2.26.0).
*
* Every field below crosses the wire. The table is the ONLY place a
* serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare
@@ -203,7 +204,7 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
#define CONFIG_WIRE_FUZZY_FIELDS(X) X(fuzzy, bool, false, BOOL)
#define CONFIG_WIRE_CHECKSUM_FIELDS(X) \
X(checksum_algo, int, CHECKSUM_ALGO_XXH64, INT_CHECKSUM_ALGO) \
X(checksum_algo, int, CHECKSUM_ALGO_DEFAULT, INT_CHECKSUM_ALGO) \
X(checksum_seed, uint64_t, 0, RAW)
#define CONFIG_WIRE_IDENTITY_FIELDS(X) \
@@ -262,6 +263,27 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
#define CONFIG_WIRE_OUTPUT_FIELDS(X) \
X(report_dest_info, bool, false, BOOL) X(report_stats, bool, false, BOOL)
/* Codec-negotiation wave (protocol 2.26.0). compression_algo is the concrete
* codec the client selected for this transfer (a CompressionAlgo id) and is the
* value the receiver validates and installs. It is the resolved result of
* --compress-choice / the "auto" negotiation so both peers agree exactly.
*
* Negotiation model: FastSync enforces a strict same-version handshake, so both
* peers carry the identical compiled-in codec set. The client resolves the
* effective algorithm deterministically and serializes it here; "auto" picks
* the first entry of the rsync 3.4.1 preference order
* (compression: zstd lz4 zlibx zlib none; checksum: xxh128 xxh3 xxh64 md5 md4
* sha1 none), and an explicit request wins. The receiver rejects (before
* STATUS_OK) any algorithm outside its own supported set, which is rsync's
* "no common choice is an error" behavior. The same resolver runs on both
* sides (compression_negotiate_default / checksum_negotiate_default), so the
* fallback is consistent.
*
* The field is appended after the output block so every pre-2.26 field keeps
* its wire position. */
#define CONFIG_WIRE_CODEC_FIELDS(X) \
X(compression_algo, int, COMPRESSION_ALGO_ZSTD, INT_COMPRESSION_ALGO)
/* All serialized fields, in exact wire order. Concatenating the per-segment
* lists here is what keeps the declaration order = the wire order. */
#define CONFIG_WIRE_FIELDS(X) \
@@ -283,7 +305,8 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
CONFIG_WIRE_ICONV_FIELDS(X) \
CONFIG_WIRE_PRIVILEGE_FIELDS(X) \
CONFIG_WIRE_COPY_AS_FIELDS(X) \
CONFIG_WIRE_OUTPUT_FIELDS(X)
CONFIG_WIRE_OUTPUT_FIELDS(X) \
CONFIG_WIRE_CODEC_FIELDS(X)
typedef struct Config {
/* -j/--threads=N: number of parallel scanner worker threads for the -m
@@ -378,6 +401,16 @@ typedef struct Config {
* enters the keep-set. Implied by --delete-missing-args. */
bool ignore_missing_args;
/* Codec-negotiation CLI state (all client-only, never serialized). The
* effective pre-transfer checksum is Config->checksum_algo (serialized);
* checksum_transfer_algo is the rsync "transfer" half of a two-name
* --checksum-choice form (validated and used only to mirror rsync's
* whole-file forcing, since FastSync's per-block strong hash is fixed).
* cli_exit_code carries a parser-requested process exit status (rsync uses 4
* for an unsupported checksum/compress algorithm) so main() can mirror it. */
int checksum_transfer_algo;
int cli_exit_code;
// Issue #129: Advanced file selection. These fields are CLIENT-ONLY: they are
// never serialized to the wire (the receiver must not learn them).
ArrayList* filters; /* --filter=RULE rule strings, in order */
@@ -941,9 +974,21 @@ typedef struct Config {
* would have removed without deleting anything. The config frame gains one
* trailing report_stats bool and the receiver emits a new STATUS_STATS frame
* (carrying matched data, the deleted-file count and the would-delete path
* list) immediately before its terminal success status. Both a config-frame
* layout change and a frame-sequence change, hence the bump. */
#define PROTOCOL_VERSION "2.25.0"
* list) immediately before its terminal success status.
*
* (6) Codec breadth + negotiation (protocol 2.26.0): the config frame gains one
* trailing int, compression_algo (a CompressionAlgo id), appended after the
* output block. It is the negotiated/effective compression codec and is what
* the receiver's self-describing decompressor validates against its own
* supported set. The checksum_algo wire value now also accepts md4/sha1/none,
* and its default changes to the rsync 3.4.1 auto-negotiated xxh128.
*
* Any config-frame layout change must bump the protocol version: a peer that
* does not parse the new trailing bytes would desynchronize on the frame
* boundary, and the strict same-version handshake (config_receive rejects a
* mismatched version before parsing anything else) keeps mixed deployments from
* ever reaching that state. */
#define PROTOCOL_VERSION "2.26.0"
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
#define MAX_BASIS_DIRS 64