rsync 3.4.1 drop-in parity (#285-#297) + parity completion (protocol 2.26.0) #298

Merged
TapTap merged 77 commits from feat/rsync-parity into dev 2026-09-17 20:33:17 +02:00
21 changed files with 1500 additions and 195 deletions
Showing only changes of commit 125921c11b - Show all commits
+15 -5
View File
@@ -1,6 +1,6 @@
cmake_minimum_required(VERSION 3.22)
project(FastFileTransfer VERSION 2.23.0)
project(FastFileTransfer VERSION 2.26.0)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_C_STANDARD 11)
@@ -68,6 +68,16 @@ if(NOT ZSTD_LIBRARY)
message(FATAL_ERROR "zstd library not found. Ensure it is in your nix-shell!")
endif()
find_library(ZLIB_LIBRARY z)
if(NOT ZLIB_LIBRARY)
message(FATAL_ERROR "zlib library not found. Ensure zlib1g-dev / nix zlib is available!")
endif()
find_library(LZ4_LIBRARY lz4)
if(NOT LZ4_LIBRARY)
message(FATAL_ERROR "lz4 library not found. Ensure liblz4-dev / nix lz4 is available!")
endif()
find_package(OpenSSL REQUIRED)
# --- Explicit source lists ---
@@ -136,8 +146,8 @@ set(CLIENT_MAIN_SRCS src/client/client_cli.c)
# --- Library targets ---
add_library(fastsync_shared STATIC ${SHARED_SRCS})
target_include_directories(fastsync_shared PUBLIC src/shared)
target_link_libraries(fastsync_shared PUBLIC Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL
OpenSSL::Crypto xxhash)
target_link_libraries(fastsync_shared PUBLIC Threads::Threads ${ZSTD_LIBRARY} ${ZLIB_LIBRARY}
${LZ4_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
add_library(fastsync_client_core STATIC ${CLIENT_CORE_SRCS})
target_include_directories(fastsync_client_core PUBLIC src/client)
@@ -276,7 +286,7 @@ if(ENABLE_FUZZ)
target_include_directories(${FUZZ_NAME} PRIVATE tests src/shared src/server)
target_compile_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined -fno-omit-frame-pointer)
target_link_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined)
target_link_libraries(${FUZZ_NAME} PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL
OpenSSL::Crypto xxhash)
target_link_libraries(${FUZZ_NAME} PRIVATE Threads::Threads ${ZSTD_LIBRARY} ${ZLIB_LIBRARY}
${LZ4_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
endforeach()
endif()
+2
View File
@@ -38,6 +38,8 @@ pkgs.mkShell {
buildInputs = with pkgs; [
zstd
zlib
lz4
openssl
];
+102 -33
View File
@@ -150,47 +150,96 @@ static int set_positive_int_option(int* dest, const char* value, const char* opt
}
/* Set and validate the compression algorithm selected by the client. rsync
* 3.4.1 can be built with zstd, none, lz4, zlibx, zlib and auto; FastSync only
* implements zstd (and no compression). "auto" is accepted as the default
* zstd choice; any other rsync choice is rejected by name instead of being
* silently accepted and ignored. */
* 3.4.1 can be built with zstd, none, lz4, zlibx, zlib and auto; all of those
* names are accepted and mapped to a real codec here. "auto" resolves through
* FastSync's compiled-in preference order (rsync 3.4.1's list). An unknown
* name is a hard error with rsync's exit code 4, never a silent no-op. */
static int set_compression_choice(Config* config, const char* value) {
/* rsync's "auto" is normalized to the canonical "zstd" at parse time (like
--checksum-choice=auto), so the value that crosses the wire is always one
the receiver accepts. */
const char* canonical = strcmp(value, "auto") == 0 ? "zstd" : value;
if (strcmp(canonical, "zstd") != 0 && strcmp(canonical, "none") != 0) {
log_message(LOG_LEVEL_ERROR,
"--compress-choice '%s' is not implemented; FastSync supports zstd, none or auto "
"(rsync's lz4/zlib/zlibx are rejected, never silently ignored)",
value);
if (!value) {
config->cli_exit_code = 4;
return -1;
}
int algo;
if (strcasecmp(value, "auto") == 0)
algo = (int)compression_negotiate_default();
else
algo = compression_algo_from_name(value);
if (algo < 0) {
log_message(LOG_LEVEL_ERROR,
"--compress-choice '%s' is not a supported algorithm; FastSync supports zstd, "
"lz4, zlib, zlibx, none or auto",
value);
config->cli_exit_code = 4;
return -1;
}
const char* canonical = compression_algo_name((CompressionAlgo)algo);
if (set_string_option(&config->compress_choice, canonical, "--compress-choice") != 0)
return -1;
config->use_compression = strcmp(canonical, "none") != 0;
config->compression_algo = algo;
config->use_compression = (algo != (int)COMPRESSION_ALGO_NONE);
return 0;
}
/* Validate and store the --checksum-choice/--cc algorithm. Only the algorithms
* the engine genuinely supports are accepted (xxh64/xxhash, xxh3, xxh128, md5);
* rsync's compiled-in choices that FastSync does not implement (md4, sha1,
* none) and the two-name transfer/pre-transfer syntax are a clear error, never
* a silent no-op. "auto" (rsync's default automatic choice) selects FastSync's
* default algorithm. */
static int set_checksum_choice(Config* config, const char* value) {
if (strcasecmp(value, "auto") == 0)
/* Store one algorithm name into *out. Returns 0 for a valid name, 1 for
* "auto" (caller resolves it), -1 for an unknown/too-long name. */
static int resolve_checksum_name(const char* name, size_t len, int* out) {
char buf[64];
if (len == 0 || len >= sizeof(buf))
return -1;
memcpy(buf, name, len);
buf[len] = '\0';
if (strcasecmp(buf, "auto") == 0)
return 1;
int algo = checksum_algo_from_name(buf);
if (algo < 0)
return -1;
*out = algo;
return 0;
int algo = checksum_algo_from_name(value);
if (algo < 0) {
log_message(LOG_LEVEL_ERROR,
"--checksum-choice '%s' is not implemented; FastSync supports xxh64 (or xxhash), "
"xxh3, xxh128, md5 or auto (rsync's md4/sha1/none and the two-name "
"transfer,pre-transfer form are rejected, never silently ignored)",
value);
}
/* Validate and store the --checksum-choice/--cc algorithm. rsync 3.4.1 accepts
* a single name (used for both the transfer and pre-transfer checksums) or the
* two-name "TRANSFER,PRE-TRANSFER" form (only one comma is significant). The
* pre-transfer half is FastSync's whole-file digest; the transfer half is
* validated for parity and, when "none", forces --whole-file like rsync. An
* unknown name (including an empty half or a second comma) is exit 4. "auto"
* resolves to FastSync's negotiated default (xxh128). */
static int set_checksum_choice(Config* config, const char* value) {
if (!value) {
config->cli_exit_code = 4;
return -1;
}
config->checksum_algo = algo;
const char* comma = strchr(value, ',');
const char* name1 = value;
size_t len1 = comma ? (size_t)(comma - value) : strlen(value);
const char* name2 = comma ? comma + 1 : NULL;
size_t len2 = name2 ? strlen(name2) : 0;
int transfer = -1;
int pre = -1;
int rc1 = resolve_checksum_name(name1, len1, &transfer);
int rc2 = name2 ? resolve_checksum_name(name2, len2, &pre) : 1;
if (rc1 < 0 || rc2 < 0) {
log_message(LOG_LEVEL_ERROR,
"--checksum-choice '%s' is invalid; FastSync supports xxh64 (or xxhash), xxh128, "
"xxh3, md5, md4, sha1, none or auto, optionally as 'transfer,pre-transfer'",
value);
config->cli_exit_code = 4;
return -1;
}
ChecksumAlgo negotiated = checksum_negotiate_default();
if (rc1 == 1)
transfer = (int)negotiated;
if (!name2)
pre = transfer;
else if (rc2 == 1)
pre = (int)negotiated;
config->checksum_algo = pre;
config->checksum_transfer_algo = transfer;
/* rsync: "none" for the transfer checksum forces --whole-file. */
if (transfer == (int)CHECKSUM_ALGO_NONE)
config->whole_file = true;
return 0;
}
@@ -2326,8 +2375,23 @@ static bool cli_handle_outbuf_option(CliParseCtx* ctx) {
* -1 on error. */
static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool no_incremental) {
set_log_level(config->quiet ? LOG_LEVEL_ERROR : (verbose ? LOG_LEVEL_DEBUG : LOG_LEVEL_WARNING));
if (config->compress_choice)
config->use_compression = strcmp(config->compress_choice, "none") != 0;
if (config->compress_choice) {
int algo = compression_algo_from_name(config->compress_choice);
if (algo >= 0) {
config->compression_algo = algo;
config->use_compression = (algo != (int)COMPRESSION_ALGO_NONE);
}
}
if (config->use_compression && config->compression_algo == (int)COMPRESSION_ALGO_NONE)
config->compression_algo = (int)compression_negotiate_default();
/* rsync parity: "none" as the pre-transfer checksum cannot be combined with
* --checksum (exit 4). The check runs here because --checksum may appear on
* either side of --checksum-choice. */
if (config->checksum && config->checksum_algo == (int)CHECKSUM_ALGO_NONE) {
log_message(LOG_LEVEL_ERROR, "Invalid checksum-choice for --checksum: none");
config->cli_exit_code = 4;
return -1;
}
/* rsync randomizes the checksum seed for every transfer when the user did not
* supply one (a seed of 0, including an explicit --checksum-seed=0), using
@@ -2786,7 +2850,7 @@ int main(int argc, char* argv[]) {
int parse_ret = parse_args(config, argc, argv, positional_args, &positional_count);
if (parse_ret != 0) {
if (parse_ret < 0)
exit_code = 1;
exit_code = config->cli_exit_code ? config->cli_exit_code : 1;
goto cleanup;
}
@@ -2876,6 +2940,11 @@ int main(int argc, char* argv[]) {
goto cleanup;
}
/* Install the negotiated codec for this process before any transfer thread
* is spawned; the compressed frames are self-describing, so the receiver's
* decompressor does not need this, but the sender compressor does. */
compression_set_algo((CompressionAlgo)config->compression_algo);
/* --iconv: install the sender-side local->wire conversion before any path is
scanned or serialized (the scanner and the chunk/data path read windows are
all driven from this process, so one global initialization covers every
+6 -5
View File
@@ -138,10 +138,10 @@ void print_usage(void) {
printf(" --link-dest <dir> Like --copy-dest, but hard-links the unchanged file from DIR\n");
printf(" into the destination (repeatable; earlier DIRs win)\n");
printf(" --checksum-choice, --cc <alg> Whole-file checksum algorithm for --incremental/\n");
printf(" --checksum compares. Accepted: xxh64 (aka xxhash), xxh3,\n");
printf(" xxh128, md5, or auto (default xxh64). rsync choices FastSync\n");
printf(" does not implement (md4, sha1, none) and the two-name\n");
printf(" transfer,pre-transfer form are rejected by name\n");
printf(" --checksum compares. Accepted: xxh128 (default), xxh3, xxh64\n");
printf(" (aka xxhash), md5, md4, sha1, or none. A two-name\n");
printf(" 'transfer,pre-transfer' form is accepted like rsync; 'none' as\n");
printf(" the pre-transfer algorithm is rejected with --checksum\n");
printf(" --checksum-seed <num> Seed for the whole-file xxHash digest (and the delta\n");
printf(" block strong hash, low 32 bits); md5 ignores the seed. A seed\n");
printf(" of 0 (the default) is randomized per transfer, exactly like\n");
@@ -168,7 +168,8 @@ void print_usage(void) {
printf(" SSH argv is already built injection-safe)\n");
printf(" --sendfile Enable sendfile zero-copy (TCP only; long form only;\n");
printf(" -f is bound to --filter, not --sendfile)\n");
printf(" --compress-choice <alg> Compression algorithm (default: zstd)\n");
printf(" --compress-choice <alg> Compression algorithm: zstd (default), lz4, zlib,\n");
printf(" zlibx, none, or auto\n");
printf(" --zc <alg> Alias for --compress-choice\n");
printf(" -v, --verbose Enable debug logging\n");
printf(" -q, --quiet Suppress non-error output\n");
+4
View File
@@ -741,6 +741,10 @@ void handler(int file_descriptor) {
* received config. */
if (gate_ctx.super_mode_override != -1)
config->super_mode = (SuperMode)gate_ctx.super_mode_override;
/* Install the codec this connection negotiated before the receiver/writer
* threads start (the server forks per connection, so the process-global
* codec is private to this session). */
compression_set_algo((CompressionAlgo)config->compression_algo);
/* If the client requested ownership but the effective super mode forbids it
* (operator --no-super, a privileged standalone receiver's secure default, or
* a daemon module without `client owner = yes`), say so ONCE per connection so
+209 -20
View File
@@ -11,6 +11,160 @@
#define XXH_STATIC_LINKING_ONLY
#include <xxhash.h>
/* ---------------------------------------------------------------------------
* Self-contained MD4 (RFC 1320). OpenSSL's MD4 lives in the legacy provider
* and is not guaranteed present, so FastSync carries its own implementation to
* keep --checksum-choice=md4 working on every build.
* ------------------------------------------------------------------------- */
typedef struct {
uint32_t state[4];
uint64_t bit_count;
uint8_t buffer[64];
size_t buffer_len;
} Md4Ctx;
static uint32_t md4_rotl(uint32_t x, int n) {
return (x << n) | (x >> (32 - n));
}
static void md4_transform(uint32_t state[4], const uint8_t block[64]) {
uint32_t x[16];
for (int i = 0; i < 16; i++)
x[i] = (uint32_t)block[i * 4] | ((uint32_t)block[i * 4 + 1] << 8) |
((uint32_t)block[i * 4 + 2] << 16) | ((uint32_t)block[i * 4 + 3] << 24);
uint32_t a = state[0], b = state[1], c = state[2], d = state[3];
#define F(x, y, z) (((x) & (y)) | (~(x) & (z)))
#define G(x, y, z) (((x) & (y)) | ((x) & (z)) | ((y) & (z)))
#define H(x, y, z) ((x) ^ (y) ^ (z))
#define ROUND1(a, b, c, d, k, s) a = md4_rotl(a + F(b, c, d) + x[k], s)
#define ROUND2(a, b, c, d, k, s) a = md4_rotl(a + G(b, c, d) + x[k] + 0x5a827999u, s)
#define ROUND3(a, b, c, d, k, s) a = md4_rotl(a + H(b, c, d) + x[k] + 0x6ed9eba1u, s)
ROUND1(a, b, c, d, 0, 3);
ROUND1(d, a, b, c, 1, 7);
ROUND1(c, d, a, b, 2, 11);
ROUND1(b, c, d, a, 3, 19);
ROUND1(a, b, c, d, 4, 3);
ROUND1(d, a, b, c, 5, 7);
ROUND1(c, d, a, b, 6, 11);
ROUND1(b, c, d, a, 7, 19);
ROUND1(a, b, c, d, 8, 3);
ROUND1(d, a, b, c, 9, 7);
ROUND1(c, d, a, b, 10, 11);
ROUND1(b, c, d, a, 11, 19);
ROUND1(a, b, c, d, 12, 3);
ROUND1(d, a, b, c, 13, 7);
ROUND1(c, d, a, b, 14, 11);
ROUND1(b, c, d, a, 15, 19);
ROUND2(a, b, c, d, 0, 3);
ROUND2(d, a, b, c, 4, 5);
ROUND2(c, d, a, b, 8, 9);
ROUND2(b, c, d, a, 12, 13);
ROUND2(a, b, c, d, 1, 3);
ROUND2(d, a, b, c, 5, 5);
ROUND2(c, d, a, b, 9, 9);
ROUND2(b, c, d, a, 13, 13);
ROUND2(a, b, c, d, 2, 3);
ROUND2(d, a, b, c, 6, 5);
ROUND2(c, d, a, b, 10, 9);
ROUND2(b, c, d, a, 14, 13);
ROUND2(a, b, c, d, 3, 3);
ROUND2(d, a, b, c, 7, 5);
ROUND2(c, d, a, b, 11, 9);
ROUND2(b, c, d, a, 15, 13);
ROUND3(a, b, c, d, 0, 3);
ROUND3(d, a, b, c, 8, 9);
ROUND3(c, d, a, b, 4, 11);
ROUND3(b, c, d, a, 12, 15);
ROUND3(a, b, c, d, 2, 3);
ROUND3(d, a, b, c, 10, 9);
ROUND3(c, d, a, b, 6, 11);
ROUND3(b, c, d, a, 14, 15);
ROUND3(a, b, c, d, 1, 3);
ROUND3(d, a, b, c, 9, 9);
ROUND3(c, d, a, b, 5, 11);
ROUND3(b, c, d, a, 13, 15);
ROUND3(a, b, c, d, 3, 3);
ROUND3(d, a, b, c, 11, 9);
ROUND3(c, d, a, b, 7, 11);
ROUND3(b, c, d, a, 15, 15);
#undef F
#undef G
#undef H
#undef ROUND1
#undef ROUND2
#undef ROUND3
state[0] += a;
state[1] += b;
state[2] += c;
state[3] += d;
}
static void md4_init(Md4Ctx* ctx) {
ctx->state[0] = 0x67452301u;
ctx->state[1] = 0xefcdab89u;
ctx->state[2] = 0x98badcfeu;
ctx->state[3] = 0x10325476u;
ctx->bit_count = 0;
ctx->buffer_len = 0;
}
static void md4_update(Md4Ctx* ctx, const uint8_t* data, size_t len) {
ctx->bit_count += (uint64_t)len * 8;
while (len > 0) {
size_t space = sizeof(ctx->buffer) - ctx->buffer_len;
size_t take = len < space ? len : space;
memcpy(ctx->buffer + ctx->buffer_len, data, take);
ctx->buffer_len += take;
data += take;
len -= take;
if (ctx->buffer_len == sizeof(ctx->buffer)) {
md4_transform(ctx->state, ctx->buffer);
ctx->buffer_len = 0;
}
}
}
static void md4_final(Md4Ctx* ctx, uint8_t out[16]) {
uint64_t bit_count = ctx->bit_count;
uint8_t pad = 0x80;
md4_update(ctx, &pad, 1);
uint8_t zero = 0;
while (ctx->buffer_len != 56)
md4_update(ctx, &zero, 1);
uint8_t length_le[8];
for (int i = 0; i < 8; i++)
length_le[i] = (uint8_t)((bit_count >> (8 * i)) & 0xff);
md4_update(ctx, length_le, sizeof(length_le));
for (int i = 0; i < 4; i++) {
out[i * 4] = (uint8_t)(ctx->state[i] & 0xff);
out[i * 4 + 1] = (uint8_t)((ctx->state[i] >> 8) & 0xff);
out[i * 4 + 2] = (uint8_t)((ctx->state[i] >> 16) & 0xff);
out[i * 4 + 3] = (uint8_t)((ctx->state[i] >> 24) & 0xff);
}
}
/* One-shot EVP digest (md5/sha1). Returns false when OpenSSL refuses. */
static bool evp_digest(const EVP_MD* md, const void* data, size_t size, uint8_t* out,
size_t out_capacity, size_t* out_len) {
static const uint8_t empty = 0;
const void* input = data ? data : &empty;
unsigned int digest_len = 0;
if (EVP_Digest(input, size, out, &digest_len, md, NULL) != 1)
return false;
if (digest_len > out_capacity)
return false;
*out_len = digest_len;
return true;
}
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
size_t out_capacity, size_t* out_len) {
if (!out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
@@ -18,43 +172,45 @@ bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t
if (data == NULL && size != 0)
return false;
if (algo == CHECKSUM_ALGO_XXH64) {
switch (algo) {
case CHECKSUM_ALGO_XXH64: {
uint64_t digest = XXH64(data, size, seed);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
return true;
}
if (algo == CHECKSUM_ALGO_XXH3) {
case CHECKSUM_ALGO_XXH3: {
uint64_t digest = XXH3_64bits_withSeed(data, size, seed);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
return true;
}
if (algo == CHECKSUM_ALGO_XXH128) {
case CHECKSUM_ALGO_XXH128: {
XXH128_hash_t digest = XXH3_128bits_withSeed(data, size, seed);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
return true;
}
if (algo == CHECKSUM_ALGO_MD5) {
case CHECKSUM_ALGO_MD5:
/* md5 takes no seed; the caller's seed is deliberately ignored (documented
* in RSYNC_COMPAT.md). OpenSSL's one-shot EVP_Digest needs a non-NULL
* buffer even for an empty input, so map a NULL data + size==0 to an empty
* buffer. */
static const uint8_t empty = 0;
const void* input = data ? data : &empty;
unsigned int digest_len = 0;
if (EVP_Digest(input, size, out, &digest_len, EVP_md5(), NULL) != 1)
return false;
if (digest_len > out_capacity)
return false;
*out_len = digest_len;
* in RSYNC_COMPAT.md). */
return evp_digest(EVP_md5(), data, size, out, out_capacity, out_len);
case CHECKSUM_ALGO_MD4: {
Md4Ctx ctx;
md4_init(&ctx);
md4_update(&ctx, (const uint8_t*)data, size);
md4_final(&ctx, out);
*out_len = 16;
return true;
}
case CHECKSUM_ALGO_SHA1:
/* sha1 takes no seed; the caller's seed is deliberately ignored. */
return evp_digest(EVP_sha1(), data, size, out, out_capacity, out_len);
case CHECKSUM_ALGO_NONE:
/* No checksum requested: an empty digest is the successful result. */
*out_len = 0;
return true;
}
return false;
}
@@ -162,6 +318,12 @@ int checksum_algo_from_name(const char* name) {
return (int)CHECKSUM_ALGO_XXH128;
if (strcasecmp(name, "md5") == 0)
return (int)CHECKSUM_ALGO_MD5;
if (strcasecmp(name, "md4") == 0)
return (int)CHECKSUM_ALGO_MD4;
if (strcasecmp(name, "sha1") == 0)
return (int)CHECKSUM_ALGO_SHA1;
if (strcasecmp(name, "none") == 0)
return (int)CHECKSUM_ALGO_NONE;
return -1;
}
@@ -175,13 +337,21 @@ const char* checksum_algo_name(ChecksumAlgo algo) {
return "xxh128";
case CHECKSUM_ALGO_MD5:
return "md5";
case CHECKSUM_ALGO_MD4:
return "md4";
case CHECKSUM_ALGO_SHA1:
return "sha1";
case CHECKSUM_ALGO_NONE:
return "none";
}
return "<unknown>";
}
bool checksum_algo_valid(int algo) {
return algo == (int)CHECKSUM_ALGO_XXH64 || algo == (int)CHECKSUM_ALGO_MD5 ||
algo == (int)CHECKSUM_ALGO_XXH3 || algo == (int)CHECKSUM_ALGO_XXH128;
algo == (int)CHECKSUM_ALGO_XXH3 || algo == (int)CHECKSUM_ALGO_XXH128 ||
algo == (int)CHECKSUM_ALGO_MD4 || algo == (int)CHECKSUM_ALGO_SHA1 ||
algo == (int)CHECKSUM_ALGO_NONE;
}
uint8_t checksum_digest_len(ChecksumAlgo algo) {
@@ -191,7 +361,26 @@ uint8_t checksum_digest_len(ChecksumAlgo algo) {
return 8;
case CHECKSUM_ALGO_XXH128:
case CHECKSUM_ALGO_MD5:
case CHECKSUM_ALGO_MD4:
return 16;
case CHECKSUM_ALGO_SHA1:
return 20;
case CHECKSUM_ALGO_NONE:
return 0;
}
return 0;
}
ChecksumAlgo checksum_negotiate_default(void) {
/* rsync 3.4.1 default preference order; every entry is compiled in, so this
* resolves to xxh128. */
static const ChecksumAlgo preference[] = {
CHECKSUM_ALGO_XXH128, CHECKSUM_ALGO_XXH3, CHECKSUM_ALGO_XXH64, CHECKSUM_ALGO_MD5,
CHECKSUM_ALGO_MD4, CHECKSUM_ALGO_SHA1, CHECKSUM_ALGO_NONE,
};
for (size_t i = 0; i < sizeof(preference) / sizeof(preference[0]); i++) {
if (checksum_algo_valid((int)preference[i]))
return preference[i];
}
return CHECKSUM_ALGO_XXH64;
}
+30 -14
View File
@@ -8,25 +8,35 @@
/* Whole-file content-digest algorithms selectable with --checksum-choice and
* seeded with --checksum-seed. The ids are the values actually placed on the
* wire (config frame), so they must be kept stable and validated on receive.
* CHECKSUM_ALGO_XXH64 == 0 is the default and is byte-for-byte what FastSync
* computed before these options existed (xxHash64 with seed 0). The set mirrors
* the algorithms rsync 3.4.1 can be built with; the ones FastSync does not
* implement (md4, sha1, none) are rejected by name at parse time. */
* CHECKSUM_ALGO_XXH64 == 0 is the historical FastSync default and its numeric
* value is preserved. The full set mirrors the algorithms rsync 3.4.1 can be
* built with; every one of them is implemented here. */
typedef enum {
CHECKSUM_ALGO_XXH64 = 0,
CHECKSUM_ALGO_MD5 = 1,
CHECKSUM_ALGO_XXH3 = 2,
CHECKSUM_ALGO_XXH128 = 3
CHECKSUM_ALGO_XXH128 = 3,
CHECKSUM_ALGO_MD4 = 4,
CHECKSUM_ALGO_SHA1 = 5,
CHECKSUM_ALGO_NONE = 6
} ChecksumAlgo;
/* xxh128 digest is 16 bytes, the longest supported. */
#define CHECKSUM_MAX_DIGEST_LEN 16
/* FastSync's negotiated default (rsync 3.4.1 auto-negotiates xxh128 first).
* The wire default for Config->checksum_algo is this value. */
#define CHECKSUM_ALGO_DEFAULT CHECKSUM_ALGO_XXH128
/* sha1 digest is 20 bytes, the longest supported. */
#define CHECKSUM_MAX_DIGEST_LEN 20
/* Compute the whole-file digest of the first `size` bytes of `data`.
*
* - CHECKSUM_ALGO_XXH64: xxHash64(data, size, seed) (full 64-bit seed).
* - CHECKSUM_ALGO_MD5: md5(data, size) via OpenSSL EVP.
* md5 has no seed, so `seed` is ignored (documented).
* - CHECKSUM_ALGO_XXH3: XXH3_64bits_withSeed(data, size, seed).
* - CHECKSUM_ALGO_XXH128: XXH3_128bits_withSeed(data, size, seed).
* - CHECKSUM_ALGO_MD5: md5(data, size) via OpenSSL EVP (seed ignored).
* - CHECKSUM_ALGO_MD4: md4(data, size), self-contained RFC 1320 (seed ignored).
* - CHECKSUM_ALGO_SHA1: sha1(data, size) via OpenSSL EVP (seed ignored).
* - CHECKSUM_ALGO_NONE: no digest; *out_len is 0 and nothing is written.
* - `size == 0` hashes the empty input (plus its seed), not a NULL input.
*
* Writes up to `out_capacity` bytes into `out`, storing the digest length in
@@ -41,10 +51,10 @@ bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t
bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, uint8_t* out,
size_t out_capacity, size_t* out_len);
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id. * Accepts
* "xxh64"/"xxhash", "xxh3", "xxh128" and "md5". "auto", rsync's default automatic choice, is
* resolved to the default by the caller (it is not a distinct algorithm here). Returns -1 for any
* name FastSync does not implement (md4/sha1/none included). */
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id.
* Accepts "xxh64"/"xxhash", "xxh3", "xxh128", "md5", "md4", "sha1", "none".
* "auto" is not an algorithm here; the caller resolves it to the negotiated
* default. Returns -1 for any unrecognized name. */
int checksum_algo_from_name(const char* name);
/* Canonical name of an algorithm (used in CLI error messages). */
@@ -53,7 +63,13 @@ const char* checksum_algo_name(ChecksumAlgo algo);
/* True when `algo` is a supported id (used by config receive validation). */
bool checksum_algo_valid(int algo);
/* Digest length in bytes for an algorithm (xxh64/xxh3 = 8, md5/xxh128 = 16). */
/* Digest length in bytes for an algorithm (xxh64/xxh3 = 8,
* md5/md4/xxh128 = 16, sha1 = 20, none = 0). */
uint8_t checksum_digest_len(ChecksumAlgo algo);
/* Pick the first algorithm from FastSync's compiled-in preference list that is
* supported on this build (rsync 3.4.1's `--version` order:
* xxh128 xxh3 xxh64 md5 md4 sha1 none). Used to resolve "auto". */
ChecksumAlgo checksum_negotiate_default(void);
#endif /* CHECKSUM_H */
+298 -24
View File
@@ -3,12 +3,15 @@
#include "log.h"
#include "protocol.h"
#include <limits.h>
#include <lz4.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <threads.h>
#include <unistd.h>
#include <zlib.h>
#include <zstd.h>
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
@@ -29,6 +32,13 @@
"z " \
"zip zst"
/* Self-describing compressed frames: the first byte is the CompressionAlgo id.
* zlib/lz4 store the uncompressed size as a little-endian uint32 after the
* codec byte so decompression can be exactly pre-sized and bounded. */
#define LZ4_SIZE_PREFIX_LEN 4
static _Atomic int g_compression_algo = COMPRESSION_ALGO_ZSTD;
/* Case-insensitive match of a bare suffix (no leading dot) against a
* space-separated suffix list. */
static bool suffix_in_list(const char* name, const char* list) {
@@ -67,6 +77,75 @@ bool compression_should_skip_with_suffixes(const char* path, char* const* suffix
return false;
}
CompressionAlgo compression_default_algo(void) {
return COMPRESSION_ALGO_ZSTD;
}
int compression_algo_from_name(const char* name) {
if (!name)
return -1;
if (strcasecmp(name, "zstd") == 0)
return (int)COMPRESSION_ALGO_ZSTD;
if (strcasecmp(name, "lz4") == 0)
return (int)COMPRESSION_ALGO_LZ4;
if (strcasecmp(name, "zlib") == 0)
return (int)COMPRESSION_ALGO_ZLIB;
if (strcasecmp(name, "zlibx") == 0)
return (int)COMPRESSION_ALGO_ZLIBX;
if (strcasecmp(name, "none") == 0)
return (int)COMPRESSION_ALGO_NONE;
return -1;
}
const char* compression_algo_name(CompressionAlgo algo) {
switch (algo) {
case COMPRESSION_ALGO_NONE:
return "none";
case COMPRESSION_ALGO_ZSTD:
return "zstd";
case COMPRESSION_ALGO_LZ4:
return "lz4";
case COMPRESSION_ALGO_ZLIB:
return "zlib";
case COMPRESSION_ALGO_ZLIBX:
return "zlibx";
}
return "<unknown>";
}
bool compression_algo_valid(int algo) {
return algo == (int)COMPRESSION_ALGO_NONE || algo == (int)COMPRESSION_ALGO_ZSTD ||
algo == (int)COMPRESSION_ALGO_LZ4 || algo == (int)COMPRESSION_ALGO_ZLIB ||
algo == (int)COMPRESSION_ALGO_ZLIBX;
}
bool compression_algo_enabled(CompressionAlgo algo) {
return algo != COMPRESSION_ALGO_NONE;
}
CompressionAlgo compression_negotiate_default(void) {
/* rsync 3.4.1 default preference order; every entry is compiled in, so this
* resolves to zstd. */
static const CompressionAlgo preference[] = {
COMPRESSION_ALGO_ZSTD, COMPRESSION_ALGO_LZ4, COMPRESSION_ALGO_ZLIBX,
COMPRESSION_ALGO_ZLIB, COMPRESSION_ALGO_NONE,
};
for (size_t i = 0; i < sizeof(preference) / sizeof(preference[0]); i++) {
if (compression_algo_valid((int)preference[i]))
return preference[i];
}
return COMPRESSION_ALGO_ZSTD;
}
void compression_set_algo(CompressionAlgo algo) {
if (compression_algo_valid((int)algo))
atomic_store(&g_compression_algo, (int)algo);
}
CompressionAlgo compression_get_algo(void) {
return (CompressionAlgo)atomic_load(&g_compression_algo);
}
/* Per-thread cache of zstd contexts plus the grow-only compression scratch
* buffer. zstd contexts are stateful and not safe to share between threads,
* so each thread keeps its own (see compression_get_thread_ctx). The cache is
@@ -157,17 +236,25 @@ static void compression_ctx_put(CompressionThreadCtx* ctx) {
compression_ctx_free(ctx);
}
Data* data_compress(Data* data_to_compress, int compression_level) {
return data_compress_with_threads(data_to_compress, compression_level, 0);
/* Build a frame consisting of a copy of `src` prefixed by `codec`. */
static Data* frame_with_codec(const void* src, size_t size, CompressionAlgo codec) {
if (size > SIZE_MAX - 1)
return NULL;
Data* out = data_create_empty(size + 1);
if (!out)
return NULL;
((uint8_t*)out->data)[0] = (uint8_t)codec;
if (size > 0)
memcpy((uint8_t*)out->data + 1, src, size);
out->size = size + 1;
return out;
}
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
int compression_threads) {
if (!data_to_compress || (!data_to_compress->data && data_to_compress->size != 0) ||
compression_threads < 0 || compression_threads > COMPRESSION_MAX_THREADS)
static Data* zstd_compress(Data* in, int compression_level, int compression_threads) {
size_t dst_size = ZSTD_compressBound(in->size);
if (dst_size > SIZE_MAX - 1)
return NULL;
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
size_t dst_size = ZSTD_compressBound(data_to_compress->size);
dst_size += 1; /* codec prefix */
CompressionThreadCtx* ctx = compression_get_thread_ctx();
if (ctx == NULL) {
@@ -218,7 +305,7 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
if (available_threads > 0) {
/* Streaming compression needs the source size before threaded mode can end a frame. */
size_t zret = ZSTD_CCtx_setPledgedSrcSize(ctx->cctx, data_to_compress->size);
size_t zret = ZSTD_CCtx_setPledgedSrcSize(ctx->cctx, in->size);
if (ZSTD_isError(zret)) {
log_message(LOG_LEVEL_ERROR, "Failed to set compression source size: %s",
ZSTD_getErrorName(zret));
@@ -236,8 +323,8 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
ctx->out_cap = dst_size;
}
ZSTD_inBuffer input = {data_to_compress->data, data_to_compress->size, 0};
ZSTD_outBuffer output = {ctx->out_buf, dst_size, 0};
ZSTD_inBuffer input = {in->data, in->size, 0};
ZSTD_outBuffer output = {(uint8_t*)ctx->out_buf + 1, dst_size - 1, 0};
size_t ret;
do {
@@ -250,30 +337,192 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
/* Hand off an exactly-sized copy; the scratch buffer stays cached so the next
* call does not reallocate a ZSTD_compressBound-sized block. */
compressed_data = data_create_empty(output.pos);
compressed_data = data_create_empty(output.pos + 1);
if (compressed_data == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to allocate compressed data");
goto cleanup;
}
((uint8_t*)compressed_data->data)[0] = (uint8_t)COMPRESSION_ALGO_ZSTD;
if (output.pos > 0)
memcpy(compressed_data->data, ctx->out_buf, output.pos);
compressed_data->size = output.pos;
memcpy((uint8_t*)compressed_data->data + 1, (uint8_t*)ctx->out_buf + 1, output.pos);
compressed_data->size = output.pos + 1;
log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu",
data_to_compress->size, compressed_data->size);
log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu", in->size,
compressed_data->size);
cleanup:
compression_ctx_put(ctx);
return compressed_data;
}
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
if (!compressed_data || (!compressed_data->data && compressed_data->size != 0) ||
maximum_size == 0)
static Data* lz4_compress(Data* in) {
int bound = LZ4_compressBound((int)in->size);
if (bound < 0 || in->size > (size_t)INT_MAX)
return NULL;
Data* out = data_create_empty((size_t)bound + 1 + LZ4_SIZE_PREFIX_LEN);
if (!out)
return NULL;
uint32_t raw_size = (uint32_t)in->size;
uint8_t* p = (uint8_t*)out->data;
p[0] = (uint8_t)COMPRESSION_ALGO_LZ4;
for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++)
p[1 + i] = (uint8_t)((raw_size >> (8 * i)) & 0xff);
int written = 0;
if (in->size > 0) {
written = LZ4_compress_default((const char*)in->data, (char*)p + 1 + LZ4_SIZE_PREFIX_LEN,
(int)in->size, bound);
if (written <= 0) {
data_destroy(out);
return NULL;
}
}
out->size = (size_t)written + 1 + LZ4_SIZE_PREFIX_LEN;
return out;
}
static Data* zlib_compress(Data* in, CompressionAlgo algo, int compression_level) {
int level = compression_level;
if (level < 1)
level = Z_DEFAULT_COMPRESSION;
if (level > 9)
level = 9;
uLong bound = compressBound((uLong)in->size);
if (in->size > (size_t)ULONG_MAX)
return NULL;
Data* out = data_create_empty((size_t)bound + 1 + LZ4_SIZE_PREFIX_LEN);
if (!out)
return NULL;
uint32_t raw_size = (uint32_t)in->size;
uint8_t* p = (uint8_t*)out->data;
p[0] = (uint8_t)algo;
for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++)
p[1 + i] = (uint8_t)((raw_size >> (8 * i)) & 0xff);
uLongf dest_len = bound;
int rc = compress2(p + 1 + LZ4_SIZE_PREFIX_LEN, &dest_len, (const Bytef*)in->data,
(uLong)in->size, level);
if (rc != Z_OK) {
data_destroy(out);
return NULL;
}
out->size = (size_t)dest_len + 1 + LZ4_SIZE_PREFIX_LEN;
return out;
}
Data* data_compress_codec(Data* data_to_compress, CompressionAlgo algo, int compression_level,
int compression_threads) {
if (!data_to_compress || (!data_to_compress->data && data_to_compress->size != 0) ||
compression_threads < 0 || compression_threads > COMPRESSION_MAX_THREADS)
return NULL;
if (!compression_algo_valid((int)algo))
return NULL;
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
switch (algo) {
case COMPRESSION_ALGO_NONE:
return frame_with_codec(data_to_compress->data, data_to_compress->size, COMPRESSION_ALGO_NONE);
case COMPRESSION_ALGO_ZSTD:
return zstd_compress(data_to_compress, compression_level, compression_threads);
case COMPRESSION_ALGO_LZ4:
return lz4_compress(data_to_compress);
case COMPRESSION_ALGO_ZLIB:
case COMPRESSION_ALGO_ZLIBX:
return zlib_compress(data_to_compress, algo, compression_level);
}
return NULL;
}
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
int compression_threads) {
return data_compress_codec(data_to_compress, compression_get_algo(), compression_level,
compression_threads);
}
Data* data_compress(Data* data_to_compress, int compression_level) {
return data_compress_codec(data_to_compress, compression_get_algo(), compression_level, 0);
}
static Data* decompress_none(const Data* compressed_data, size_t maximum_size) {
size_t size = compressed_data->size - 1;
if (size > maximum_size)
return NULL;
Data* out = data_create_empty(size);
if (!out)
return NULL;
if (size > 0)
memcpy(out->data, (const uint8_t*)compressed_data->data + 1, size);
out->size = size;
return out;
}
/* Read the 4-byte little-endian raw size stored after the codec byte. */
static bool read_raw_size(const Data* in, uint32_t* raw_size) {
if (in->size < 1 + LZ4_SIZE_PREFIX_LEN)
return false;
const uint8_t* p = (const uint8_t*)in->data;
uint32_t v = 0;
for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++)
v |= (uint32_t)p[1 + i] << (8 * i);
*raw_size = v;
return true;
}
static Data* lz4_decompress(Data* compressed_data, size_t maximum_size, size_t hard_limit) {
uint32_t raw_size = 0;
if (!read_raw_size(compressed_data, &raw_size))
return NULL;
if (raw_size > hard_limit || raw_size > maximum_size)
return NULL;
size_t comp_size = compressed_data->size - 1 - LZ4_SIZE_PREFIX_LEN;
Data* out = data_create_empty(raw_size);
if (!out)
return NULL;
if (raw_size == 0) {
out->size = 0;
return out;
}
int rc = LZ4_decompress_safe((const char*)compressed_data->data + 1 + LZ4_SIZE_PREFIX_LEN,
(char*)out->data, (int)comp_size, (int)raw_size);
if (rc < 0 || (uint32_t)rc != raw_size) {
log_message(LOG_LEVEL_ERROR, "LZ4 decompression failed");
data_destroy(out);
return NULL;
}
out->size = raw_size;
return out;
}
static Data* zlib_decompress(Data* compressed_data, size_t maximum_size, size_t hard_limit) {
uint32_t raw_size = 0;
if (!read_raw_size(compressed_data, &raw_size))
return NULL;
if (raw_size > hard_limit || raw_size > maximum_size)
return NULL;
size_t comp_size = compressed_data->size - 1 - LZ4_SIZE_PREFIX_LEN;
Data* out = data_create_empty(raw_size);
if (!out)
return NULL;
if (raw_size == 0) {
out->size = 0;
return out;
}
uLongf dest_len = raw_size;
int rc =
uncompress((Bytef*)out->data, &dest_len,
(const Bytef*)compressed_data->data + 1 + LZ4_SIZE_PREFIX_LEN, (uLong)comp_size);
if (rc != Z_OK || dest_len != raw_size) {
log_message(LOG_LEVEL_ERROR, "zlib decompression failed");
data_destroy(out);
return NULL;
}
out->size = raw_size;
return out;
}
static Data* zstd_decompress(Data* compressed_data, size_t maximum_size) {
/* The zstd frame starts after the codec byte. */
const void* frame = (const uint8_t*)compressed_data->data + 1;
size_t frame_size = compressed_data->size - 1;
log_debug_message(LOG_DEBUG_UTIL, "Start to decompress data");
unsigned long long dst_size =
ZSTD_getFrameContentSize(compressed_data->data, compressed_data->size);
unsigned long long dst_size = ZSTD_getFrameContentSize(frame, frame_size);
/* ZSTD_isError() is also true for ZSTD_CONTENTSIZE_ERROR and
* ZSTD_CONTENTSIZE_UNKNOWN (both are encoded near (size_t)-1), so test the
* sentinels explicitly instead of blanket-rejecting every error-ish value:
@@ -287,9 +536,9 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
// ZSTD_CONTENTSIZE_UNKNOWN (~2^64) can cause massive allocation;
// fall back to a conservative estimate (3x compressed size) when unknown.
if (dst_size == ZSTD_CONTENTSIZE_UNKNOWN) {
if (compressed_data->size > ULLONG_MAX / 3)
if (frame_size > ULLONG_MAX / 3)
return NULL;
dst_size = compressed_data->size * 3;
dst_size = frame_size * 3;
if (dst_size < INITIAL_DECOMPRESS_BUF_SIZE)
dst_size = INITIAL_DECOMPRESS_BUF_SIZE;
}
@@ -326,7 +575,7 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
goto cleanup;
}
ZSTD_inBuffer input = {compressed_data->data, compressed_data->size, 0};
ZSTD_inBuffer input = {frame, frame_size, 0};
ZSTD_outBuffer output = {uncompressed_data->data, buf_size, 0};
size_t ret;
@@ -385,6 +634,31 @@ cleanup:
return uncompressed_data;
}
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
if (!compressed_data || (!compressed_data->data && compressed_data->size != 0) ||
maximum_size == 0)
return NULL;
if (compressed_data->size < 1)
return NULL;
unsigned long long hard_limit =
maximum_size < MAX_DECOMPRESSED_SIZE ? maximum_size : MAX_DECOMPRESSED_SIZE;
uint8_t codec = ((const uint8_t*)compressed_data->data)[0];
if (!compression_algo_valid(codec))
return NULL;
switch ((CompressionAlgo)codec) {
case COMPRESSION_ALGO_NONE:
return decompress_none(compressed_data, (size_t)hard_limit);
case COMPRESSION_ALGO_ZSTD:
return zstd_decompress(compressed_data, (size_t)hard_limit);
case COMPRESSION_ALGO_LZ4:
return lz4_decompress(compressed_data, maximum_size, (size_t)hard_limit);
case COMPRESSION_ALGO_ZLIB:
case COMPRESSION_ALGO_ZLIBX:
return zlib_decompress(compressed_data, maximum_size, (size_t)hard_limit);
}
return NULL;
}
Data* data_decompress(Data* compressed_data) {
return data_decompress_limited(compressed_data, MAX_DECOMPRESSED_SIZE);
}
+51 -1
View File
@@ -6,11 +6,61 @@
#define COMPRESSION_MAX_THREADS 64
/* Compression algorithms selectable with --compress-choice / -z. The ids are
* the values placed on the wire (Config->compression_algo), so they must be
* kept stable. NONE is "no compression"; ZSTD is the historical FastSync
* default and the negotiated "auto" choice. ZLIBX is rsync's zlib-without-
* matched-data variant: FastSync compresses only the delta/token bytes (it does
* not put matched file data in the compression stream), so its zlib codec is
* already the "x" form and zlib/zlibx share the same implementation, recorded
* under distinct ids. */
typedef enum {
COMPRESSION_ALGO_NONE = 0,
COMPRESSION_ALGO_ZSTD = 1,
COMPRESSION_ALGO_LZ4 = 2,
COMPRESSION_ALGO_ZLIB = 3,
COMPRESSION_ALGO_ZLIBX = 4
} CompressionAlgo;
CompressionAlgo compression_default_algo(void);
/* Resolve a --compress-choice string (case-insensitive) to an algorithm id.
* Accepts "zstd", "lz4", "zlib", "zlibx", "none". "auto" is not an algorithm
* here; the caller resolves it to the negotiated default. Returns -1 for any
* unrecognized name. */
int compression_algo_from_name(const char* name);
const char* compression_algo_name(CompressionAlgo algo);
bool compression_algo_valid(int algo);
/* Pick the first algorithm from FastSync's compiled-in preference list
* (rsync 3.4.1's `--version` order: zstd lz4 zlibx zlib none). Resolves
* "auto". */
CompressionAlgo compression_negotiate_default(void);
/* True when the algorithm actually compresses (i.e. is not NONE). */
bool compression_algo_enabled(CompressionAlgo algo);
/* Select the process-wide codec used by the legacy wrappers below. Each
* process serves exactly one transfer config (the server forks per connection,
* the client configures itself before spawning transfer threads), so a
* process-global default is sufficient and constant for the lifetime of a
* transfer. Defaults to ZSTD when never set. Thread-safe. */
void compression_set_algo(CompressionAlgo algo);
CompressionAlgo compression_get_algo(void);
/* Codec-aware primitives. The compressed buffer is self-describing: its first
* byte is the CompressionAlgo id, so decompression never needs the codec passed
* separately (this keeps every existing Decompress call site source-compatible).
* `data_compress_codec` returns NULL on invalid input or an unsupported codec. */
Data* data_compress_codec(Data* data_to_compress, CompressionAlgo algo, int compression_level,
int compression_threads);
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
/* Legacy zstd-default wrappers retained for existing callers/tests. */
Data* data_compress(Data* data_to_compress, int compression_level);
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
int compression_threads);
Data* data_decompress(Data* compressed_data);
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
/* Release the calling thread's cached zstd contexts (compressor, decompressor
+66 -18
View File
@@ -14,6 +14,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <limits.h>
#include <errno.h>
@@ -67,6 +68,8 @@ static void config_set_defaults(Config* config) {
config->human_readable = false;
config->ignore_errors = false;
config->ignore_missing_args = false;
config->checksum_transfer_algo = CHECKSUM_ALGO_DEFAULT;
config->cli_exit_code = 0;
config->filters = NULL;
config->files_from = NULL;
config->files_from_set = NULL;
@@ -196,12 +199,13 @@ static bool validate_received_config(const Config* config) {
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
valid_wire_bool(config->dry_run) && checksum_algo_valid(config->checksum_algo) &&
identity_wire_valid(config) && valid_wire_bool(config->preserve_atimes) &&
valid_wire_bool(config->preserve_crtimes) && valid_wire_bool(config->omit_dir_times) &&
valid_wire_bool(config->omit_link_times) && valid_wire_bool(config->preserve_perms) &&
valid_wire_bool(config->preserve_times) && valid_wire_bool(config->preserve_owner) &&
valid_wire_bool(config->preserve_group) && valid_wire_bool(config->munge_links) &&
valid_wire_bool(config->keep_dirlinks) && valid_wire_bool(config->fake_super) &&
compression_algo_valid(config->compression_algo) && identity_wire_valid(config) &&
valid_wire_bool(config->preserve_atimes) && valid_wire_bool(config->preserve_crtimes) &&
valid_wire_bool(config->omit_dir_times) && valid_wire_bool(config->omit_link_times) &&
valid_wire_bool(config->preserve_perms) && valid_wire_bool(config->preserve_times) &&
valid_wire_bool(config->preserve_owner) && valid_wire_bool(config->preserve_group) &&
valid_wire_bool(config->munge_links) && valid_wire_bool(config->keep_dirlinks) &&
valid_wire_bool(config->fake_super) &&
(!config->copy_as_set || (config->copy_as_uid >= 0 && config->copy_as_gid >= 0)) &&
(!config->use_compression ||
(config->compression_level >= 1 && config->compression_level <= 22)) &&
@@ -894,6 +898,14 @@ static bool config_receive_checksum_algo(int fd, int* value) {
return true;
}
static bool config_receive_compression_algo(int fd, int* value) {
int algo;
if (!receive_int(fd, &algo) || !compression_algo_valid(algo))
return false;
*value = algo;
return true;
}
static bool config_receive_super_mode(int fd, SuperMode* value) {
int mode;
if (!receive_int(fd, &mode) || mode < SUPER_MODE_AUTO || mode > SUPER_MODE_OFF)
@@ -1095,6 +1107,9 @@ fail:
#define CONFIG_SEND_INT_CHECKSUM_ALGO(name) send_int(fd, c->name)
#define CONFIG_RECV_INT_CHECKSUM_ALGO(name) config_receive_checksum_algo(fd, &c->name)
#define CONFIG_SEND_INT_COMPRESSION_ALGO(name) send_int(fd, c->name)
#define CONFIG_RECV_INT_COMPRESSION_ALGO(name) config_receive_compression_algo(fd, &c->name)
#define CONFIG_SEND_SUPERMODE(name) send_int(fd, (int)c->name)
#define CONFIG_RECV_SUPERMODE(name) config_receive_super_mode(fd, &c->name)
@@ -1170,6 +1185,7 @@ CONFIG_DEFINE_SEND(send_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
CONFIG_DEFINE_SEND(send_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
CONFIG_DEFINE_SEND(send_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
CONFIG_DEFINE_SEND(send_output_options, CONFIG_WIRE_OUTPUT_FIELDS)
CONFIG_DEFINE_SEND(send_codec_options, CONFIG_WIRE_CODEC_FIELDS)
CONFIG_DEFINE_RECV(receive_core_fields, CONFIG_WIRE_CORE_FIELDS)
CONFIG_DEFINE_RECV(receive_delta_fields, CONFIG_WIRE_DELTA_FIELDS)
@@ -1189,6 +1205,7 @@ CONFIG_DEFINE_RECV(receive_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
CONFIG_DEFINE_RECV(receive_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
CONFIG_DEFINE_RECV(receive_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
CONFIG_DEFINE_RECV(receive_output_options, CONFIG_WIRE_OUTPUT_FIELDS)
CONFIG_DEFINE_RECV(receive_codec_options, CONFIG_WIRE_CODEC_FIELDS)
#undef XSEND
#undef XRECV
@@ -1306,7 +1323,8 @@ bool config_send_wire_block(int file_descriptor, const Config* config) {
send_iconv_spec(file_descriptor, config) &&
send_privilege_options(file_descriptor, config) &&
send_copy_as_options(file_descriptor, config) &&
send_output_options(file_descriptor, config);
send_output_options(file_descriptor, config) &&
send_codec_options(file_descriptor, config);
}
bool config_send(int file_descriptor, const Config* config) {
@@ -1377,29 +1395,59 @@ Config* config_receive_with_validate(int file_descriptor, ConfigValidateFunc val
!receive_iconv_spec(file_descriptor, config, &budget) ||
!receive_privilege_options(file_descriptor, config, &budget) ||
!receive_copy_as_options(file_descriptor, config, &budget) ||
!receive_output_options(file_descriptor, config, &budget))
!receive_output_options(file_descriptor, config, &budget) ||
!receive_codec_options(file_descriptor, config, &budget))
goto error;
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
strcmp(config->compress_choice, "none") != 0 &&
strcmp(config->compress_choice, "auto") != 0) {
/* Validate/normalize the negotiated codec. compress_choice is the human
* spelling (NULL or "" when -z was not given); compression_algo is the
* concrete codec id the sender used. They must agree, and "auto" is
* canonicalized to FastSync's negotiated default so the stored spelling is
* always concrete (a hostile/older client may still send "auto"). */
if (config->compress_choice && config->compress_choice[0] != '\0') {
int choice_algo = compression_algo_from_name(config->compress_choice);
if (choice_algo < 0 && strcasecmp(config->compress_choice, "auto") != 0) {
char* escaped_choice = output_escape(config->compress_choice, config->eight_bit_output);
log_message(LOG_LEVEL_ERROR, "Unsupported compression choice: %s",
escaped_choice ? escaped_choice : "<allocation failed>");
char detail[128];
char detail[160];
snprintf(detail, sizeof(detail), "unsupported compression choice: %s",
escaped_choice ? escaped_choice : "<allocation failed>");
send_error_detail(file_descriptor, detail);
free(escaped_choice);
goto error;
}
/* Defensive: an older/hostile client may still send "auto"; canonicalize it
to zstd (its effective choice) so the stored value is always concrete. */
if (strcmp(config->compress_choice, "auto") == 0) {
char* canonical = str_dup("zstd");
if (!canonical)
if (choice_algo < 0)
choice_algo = (int)compression_negotiate_default();
if (strcasecmp(config->compress_choice, "auto") == 0 ||
choice_algo == (int)COMPRESSION_ALGO_NONE) {
const char* canonical = compression_algo_name((CompressionAlgo)choice_algo);
char* dup = str_dup(canonical);
if (!dup)
goto error;
free(config->compress_choice);
config->compress_choice = canonical;
config->compress_choice = dup;
}
if (config->compression_algo != choice_algo) {
log_message(LOG_LEVEL_ERROR, "Compression choice '%s' does not match codec id %d",
config->compress_choice, config->compression_algo);
send_error_detail(file_descriptor, "compression choice/codec mismatch");
goto error;
}
}
/* The concrete codec must exist only when compression is on. A client that
* left -z off has no codec in effect, but the field keeps whatever id it
* carried (the receiver never dispatches on it without use_compression), so
* the wire value round-trips untouched. */
if (config->use_compression && config->compression_algo == (int)COMPRESSION_ALGO_NONE) {
log_message(LOG_LEVEL_ERROR, "Compression requested with the 'none' codec");
send_error_detail(file_descriptor, "compression requested with the none codec");
goto error;
}
/* rsync: "none" as the pre-transfer checksum is invalid with --checksum. */
if (config->checksum && config->checksum_algo == (int)CHECKSUM_ALGO_NONE) {
log_message(LOG_LEVEL_ERROR, "Invalid checksum-choice for --checksum: none");
send_error_detail(file_descriptor, "checksum-choice 'none' cannot be used with --checksum");
goto error;
}
if (!validate_received_config(config)) {
log_message(LOG_LEVEL_ERROR, "Invalid configuration received from client");
+51 -6
View File
@@ -3,6 +3,7 @@
#include "array_list.h"
#include "checksum.h"
#include "compression.h"
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
@@ -81,7 +82,7 @@ typedef struct {
typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode;
/* ===========================================================================
* Config wire-field table (single source of truth for protocol 2.23.0).
* Config wire-field table (single source of truth for protocol 2.26.0).
*
* Every field below crosses the wire. The table is the ONLY place a
* serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare
@@ -203,7 +204,7 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
#define CONFIG_WIRE_FUZZY_FIELDS(X) X(fuzzy, bool, false, BOOL)
#define CONFIG_WIRE_CHECKSUM_FIELDS(X) \
X(checksum_algo, int, CHECKSUM_ALGO_XXH64, INT_CHECKSUM_ALGO) \
X(checksum_algo, int, CHECKSUM_ALGO_DEFAULT, INT_CHECKSUM_ALGO) \
X(checksum_seed, uint64_t, 0, RAW)
#define CONFIG_WIRE_IDENTITY_FIELDS(X) \
@@ -262,6 +263,27 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
#define CONFIG_WIRE_OUTPUT_FIELDS(X) \
X(report_dest_info, bool, false, BOOL) X(report_stats, bool, false, BOOL)
/* Codec-negotiation wave (protocol 2.26.0). compression_algo is the concrete
* codec the client selected for this transfer (a CompressionAlgo id) and is the
* value the receiver validates and installs. It is the resolved result of
* --compress-choice / the "auto" negotiation so both peers agree exactly.
*
* Negotiation model: FastSync enforces a strict same-version handshake, so both
* peers carry the identical compiled-in codec set. The client resolves the
* effective algorithm deterministically and serializes it here; "auto" picks
* the first entry of the rsync 3.4.1 preference order
* (compression: zstd lz4 zlibx zlib none; checksum: xxh128 xxh3 xxh64 md5 md4
* sha1 none), and an explicit request wins. The receiver rejects (before
* STATUS_OK) any algorithm outside its own supported set, which is rsync's
* "no common choice is an error" behavior. The same resolver runs on both
* sides (compression_negotiate_default / checksum_negotiate_default), so the
* fallback is consistent.
*
* The field is appended after the output block so every pre-2.26 field keeps
* its wire position. */
#define CONFIG_WIRE_CODEC_FIELDS(X) \
X(compression_algo, int, COMPRESSION_ALGO_ZSTD, INT_COMPRESSION_ALGO)
/* All serialized fields, in exact wire order. Concatenating the per-segment
* lists here is what keeps the declaration order = the wire order. */
#define CONFIG_WIRE_FIELDS(X) \
@@ -283,7 +305,8 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
CONFIG_WIRE_ICONV_FIELDS(X) \
CONFIG_WIRE_PRIVILEGE_FIELDS(X) \
CONFIG_WIRE_COPY_AS_FIELDS(X) \
CONFIG_WIRE_OUTPUT_FIELDS(X)
CONFIG_WIRE_OUTPUT_FIELDS(X) \
CONFIG_WIRE_CODEC_FIELDS(X)
typedef struct Config {
/* -j/--threads=N: number of parallel scanner worker threads for the -m
@@ -378,6 +401,16 @@ typedef struct Config {
* enters the keep-set. Implied by --delete-missing-args. */
bool ignore_missing_args;
/* Codec-negotiation CLI state (all client-only, never serialized). The
* effective pre-transfer checksum is Config->checksum_algo (serialized);
* checksum_transfer_algo is the rsync "transfer" half of a two-name
* --checksum-choice form (validated and used only to mirror rsync's
* whole-file forcing, since FastSync's per-block strong hash is fixed).
* cli_exit_code carries a parser-requested process exit status (rsync uses 4
* for an unsupported checksum/compress algorithm) so main() can mirror it. */
int checksum_transfer_algo;
int cli_exit_code;
// Issue #129: Advanced file selection. These fields are CLIENT-ONLY: they are
// never serialized to the wire (the receiver must not learn them).
ArrayList* filters; /* --filter=RULE rule strings, in order */
@@ -941,9 +974,21 @@ typedef struct Config {
* would have removed without deleting anything. The config frame gains one
* trailing report_stats bool and the receiver emits a new STATUS_STATS frame
* (carrying matched data, the deleted-file count and the would-delete path
* list) immediately before its terminal success status. Both a config-frame
* layout change and a frame-sequence change, hence the bump. */
#define PROTOCOL_VERSION "2.25.0"
* list) immediately before its terminal success status.
*
* (6) Codec breadth + negotiation (protocol 2.26.0): the config frame gains one
* trailing int, compression_algo (a CompressionAlgo id), appended after the
* output block. It is the negotiated/effective compression codec and is what
* the receiver's self-describing decompressor validates against its own
* supported set. The checksum_algo wire value now also accepts md4/sha1/none,
* and its default changes to the rsync 3.4.1 auto-negotiated xxh128.
*
* Any config-frame layout change must bump the protocol version: a peer that
* does not parse the new trailing bytes would desynchronize on the frame
* boundary, and the strict same-version handshake (config_receive rejects a
* mismatched version before parsing anything else) keeps mixed deployments from
* ever reaching that state. */
#define PROTOCOL_VERSION "2.26.0"
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
#define MAX_BASIS_DIRS 64
+218
View File
@@ -0,0 +1,218 @@
"""Differential tests for --checksum-choice / --compress-choice against rsync 3.4.1.
These pin the accepted/rejected algorithm matrix and exit codes to real rsync,
and verify that every codec FastSync now offers still transfers byte-exactly.
The rsync-based tests skip cleanly when rsync is not installed.
The FastSync server confines transfers to its authorized root (the project
directory when the shared test server is launched), so every scratch tree lives
under ``TEST_DATA_DIR`` rather than pytest's ``tmp_path``.
"""
import os
import shutil
import subprocess
import sys
import pytest
sys.path.insert(0, os.path.dirname(__file__))
from common import (
TEST_DATA_DIR,
run_client,
clean_dir,
get_dest_received_dir,
)
RSYNC = shutil.which("rsync")
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
CHECKSUM_NAMES = ["xxh128", "xxh3", "xxh64", "md5", "md4", "sha1"]
COMPRESS_NAMES = ["zstd", "lz4", "zlib", "zlibx"]
CODEC_ROOT = os.path.join(TEST_DATA_DIR, "codec_differential")
def _rsync(args):
env = dict(os.environ, LC_ALL="C")
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120)
def _scratch(tag):
"""A confined, uniquely named scratch directory under the project tree."""
path = os.path.join(CODEC_ROOT, tag)
clean_dir(path)
os.makedirs(path, exist_ok=True)
return path
def _make_corpus(root):
clean_dir(root)
os.makedirs(os.path.join(root, "sub"), exist_ok=True)
# Highly compressible payload so each codec is actually exercised.
with open(os.path.join(root, "big.bin"), "wb") as fh:
fh.write(b"FastSync codec payload " * 4096)
with open(os.path.join(root, "sub", "text.txt"), "wb") as fh:
fh.write(b"hello codec world\n" * 128)
with open(os.path.join(root, "empty"), "wb"):
pass
return root
def _tree_bytes(root):
out = {}
for dirpath, _dirs, files in os.walk(root):
for name in files:
path = os.path.join(dirpath, name)
with open(path, "rb") as fh:
out[os.path.relpath(path, root)] = fh.read()
return out
class TestCodecChoiceMatrix:
"""The CLI accept/reject set and exit codes must match rsync 3.4.1."""
@requires_rsync
@pytest.mark.ci
@pytest.mark.parametrize("name", CHECKSUM_NAMES)
def test_checksum_names_accepted_by_both(self, name, shared_server):
src = _make_corpus(_scratch(f"cc_src_{name}"))
rdst = _scratch(f"cc_rsync_{name}")
rsync_result = _rsync(["-a", f"--cc={name}", src + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
fdst = _scratch(f"cc_fs_{name}")
result, _ = run_client(src, fdst, flags=[f"--cc={name}"], port=shared_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
@requires_rsync
@pytest.mark.ci
@pytest.mark.parametrize("name", COMPRESS_NAMES)
def test_compress_names_accepted_by_both(self, name, shared_server):
src = _make_corpus(_scratch(f"zc_src_{name}"))
rdst = _scratch(f"zc_rsync_{name}")
rsync_result = _rsync(["-az", f"--zc={name}", src + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
fdst = _scratch(f"zc_fs_{name}")
result, _ = run_client(src, fdst, flags=["-z", f"--zc={name}"], port=shared_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
@requires_rsync
@pytest.mark.ci
@pytest.mark.parametrize("choice", ["md4,sha1", "sha1,md4", "auto,md5", "none,md5"])
def test_checksum_two_name_accepted_by_both(self, choice, shared_server):
tag = choice.replace(",", "_")
src = _make_corpus(_scratch(f"two_src_{tag}"))
rdst = _scratch(f"two_rsync_{tag}")
rsync_result = _rsync(["-a", "--checksum", f"--cc={choice}", src + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
fdst = _scratch(f"two_fs_{tag}")
result, _ = run_client(src, fdst, flags=["--checksum", f"--cc={choice}"],
port=shared_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:200]
@requires_rsync
@pytest.mark.ci
@pytest.mark.parametrize("name", ["sha256", "crc32", "md5,", "md4,md5,sha1"])
def test_unknown_checksum_rejected_exit_4_both(self, name, shared_server):
src = _make_corpus(_scratch(f"badcc_src_{name.replace(',', '_').replace(':', '_')}"))
rdst = _scratch(f"badcc_rsync_{name.replace(',', '_').replace(':', '_')}")
rsync_result = _rsync(["-a", f"--cc={name}", src + "/", rdst + "/"])
assert rsync_result.returncode == 4, rsync_result.stderr
fdst = _scratch(f"badcc_fs_{name.replace(',', '_').replace(':', '_')}")
result, _ = run_client(src, fdst, flags=[f"--cc={name}"], port=shared_server.port)
assert result.returncode == 4, (result.stderr or result.stdout)[:200]
@requires_rsync
@pytest.mark.ci
@pytest.mark.parametrize("choice", ["none", "md5,none"])
def test_checksum_none_with_checksum_rejected_exit_4_both(self, choice, shared_server):
tag = choice.replace(",", "_")
src = _make_corpus(_scratch(f"nonecc_src_{tag}"))
rdst = _scratch(f"nonecc_rsync_{tag}")
rsync_result = _rsync(["-a", "--checksum", f"--cc={choice}", src + "/", rdst + "/"])
assert rsync_result.returncode == 4, rsync_result.stderr
fdst = _scratch(f"nonecc_fs_{tag}")
result, _ = run_client(src, fdst, flags=["--checksum", f"--cc={choice}"],
port=shared_server.port)
assert result.returncode == 4, (result.stderr or result.stdout)[:200]
@requires_rsync
@pytest.mark.ci
@pytest.mark.parametrize("name", ["bogus", "zstd,lz4"])
def test_unknown_compress_rejected_exit_4_both(self, name, shared_server):
tag = name.replace(",", "_")
src = _make_corpus(_scratch(f"badzc_src_{tag}"))
rdst = _scratch(f"badzc_rsync_{tag}")
rsync_result = _rsync(["-az", f"--zc={name}", src + "/", rdst + "/"])
assert rsync_result.returncode == 4, rsync_result.stderr
fdst = _scratch(f"badzc_fs_{tag}")
result, _ = run_client(src, fdst, flags=["-z", f"--zc={name}"], port=shared_server.port)
assert result.returncode == 4, (result.stderr or result.stdout)[:200]
class TestCodecTransferDifferential:
"""Each codec lands the same bytes rsync lands."""
@requires_rsync
@pytest.mark.ci
@pytest.mark.parametrize("name", COMPRESS_NAMES + ["none"])
def test_compress_codec_matches_rsync_bytes(self, name, shared_server):
src = _make_corpus(_scratch(f"byteszc_src_{name}"))
rsync_dst = _scratch(f"byteszc_rsync_{name}")
rsync_result = _rsync(["-a", "-z", f"--zc={name}", src + "/", rsync_dst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
fs_dst = _scratch(f"byteszc_fs_{name}")
result, _ = run_client(src, fs_dst, flags=["-a", "-z", f"--zc={name}"],
port=shared_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
received = get_dest_received_dir(fs_dst, src)
assert _tree_bytes(received) == _tree_bytes(rsync_dst)
@requires_rsync
@pytest.mark.ci
@pytest.mark.parametrize("name", CHECKSUM_NAMES)
def test_checksum_codec_matches_rsync_bytes(self, name, shared_server):
src = _make_corpus(_scratch(f"bytescc_src_{name}"))
rsync_dst = _scratch(f"bytescc_rsync_{name}")
rsync_result = _rsync(["-a", "--checksum", f"--cc={name}", src + "/", rsync_dst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
fs_dst = _scratch(f"bytescc_fs_{name}")
result, _ = run_client(src, fs_dst, flags=["-a", "--checksum", f"--cc={name}"],
port=shared_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
received = get_dest_received_dir(fs_dst, src)
assert _tree_bytes(received) == _tree_bytes(rsync_dst)
class TestCodecNegotiationFallback:
"""FastSync's auto negotiation and deterministic fallback order."""
@pytest.mark.ci
def test_default_checksum_and_compression_agree(self, shared_server):
"""A default transfer (auto on both peers) succeeds; the negotiated
default is xxh128 + zstd."""
src = _make_corpus(_scratch("auto_src"))
fdst = _scratch("auto_fs")
result, _ = run_client(src, fdst, flags=["-a", "-z"], port=shared_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
received = get_dest_received_dir(fdst, src)
assert _tree_bytes(received) == _tree_bytes(src)
@pytest.mark.ci
def test_explicit_choice_overrides_auto(self, shared_server):
"""An explicit --zc/--cc wins over the negotiated default on both ends,
so the receiver decodes with the sender's codec."""
src = _make_corpus(_scratch("explicit_src"))
fdst = _scratch("explicit_fs")
result, _ = run_client(src, fdst, flags=["-a", "-z", "--zc=lz4", "--cc=sha1"],
port=shared_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
received = get_dest_received_dir(fdst, src)
assert _tree_bytes(received) == _tree_bytes(src)
+1 -1
View File
@@ -36,7 +36,7 @@ from common import ( # noqa: E402
verify_transfer,
)
PROTOCOL_VERSION = b"2.25.0"
PROTOCOL_VERSION = b"2.26.0"
STATUS_MANIFEST = 5
STATUS_OK = 0
+87 -6
View File
@@ -1491,20 +1491,101 @@ class TestChecksumChoice:
assert fh.read() == b"same content\n"
@pytest.mark.ci
def test_checksum_choice_md4_single_name_rejected(self, shared_server):
for bad in ("md4", "sha1", "none", "xxh64,md5"):
@pytest.mark.parametrize("algo", ["xxh128", "xxh3", "xxh64", "md5", "md4", "sha1"])
def test_checksum_choice_all_algorithms_transfer(self, shared_server, algo):
"""Every rsync 3.4.1 checksum algorithm is accepted and transfers
byte-exactly. 'none' is covered separately (it needs no digest)."""
clean_dir(DEST_DIR)
flags = ["--preserve", "--incremental", "--checksum", f"--checksum-choice={algo}"]
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
assert result.returncode == 0, \
f"checksum-choice={algo} failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
@pytest.mark.ci
def test_checksum_choice_two_name_form(self, shared_server):
"""The rsync 'TRANSFER,PRE-TRANSFER' form is accepted; FastSync uses the
second (pre-transfer) algorithm for its whole-file digest."""
clean_dir(DEST_DIR)
flags = ["--preserve", "--incremental", "--checksum", "--cc=md4,sha1"]
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
assert result.returncode == 0, \
f"two-name --cc=md4,sha1 failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing and not mismatches, f"missing={missing} mismatches={mismatches}"
@pytest.mark.ci
def test_checksum_choice_none_accepted_without_checksum(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["--preserve", "--incremental", "--cc=none"],
port=shared_server.port)
assert result.returncode == 0, \
f"--cc=none failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing and not mismatches, f"missing={missing} mismatches={mismatches}"
@pytest.mark.ci
def test_checksum_choice_none_rejected_with_checksum(self, shared_server):
"""rsync rejects 'none' as the pre-transfer checksum with --checksum and
exits 4; mirror both the rejection and the exit code."""
for choice in ("none", "md5,none"):
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["--checksum", f"--cc={choice}"],
port=shared_server.port)
assert result.returncode == 4, \
f"--cc={choice} --checksum must exit 4, got {result.returncode}: " \
f"{(result.stderr or result.stdout)[:200]}"
@pytest.mark.ci
def test_checksum_choice_unknown_rejected_exit_4(self, shared_server):
for bad in ("sha256", "bogus", "md5,", "md4,md5,sha1"):
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=[f"--checksum-choice={bad}"],
port=shared_server.port)
assert result.returncode != 0, f"{bad} must be rejected"
assert result.returncode == 4, \
f"--checksum-choice={bad} must exit 4, got {result.returncode}"
@pytest.mark.ci
def test_compress_choice_unsupported_rejected(self, shared_server):
for bad in ("lz4", "zlib", "zlibx"):
@pytest.mark.parametrize("algo", ["zstd", "lz4", "zlib", "zlibx"])
def test_compress_choice_all_algorithms_transfer(self, shared_server, algo):
"""Every rsync 3.4.1 compression codec is accepted and transfers
byte-exactly through its own codec."""
clean_dir(DEST_DIR)
flags = ["-z", f"--compress-choice={algo}"]
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
assert result.returncode == 0, \
f"--compress-choice={algo} failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
@pytest.mark.ci
def test_compress_choice_unknown_rejected_exit_4(self, shared_server):
for bad in ("bogus", "zstd,lz4", ""):
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=[f"--compress-choice={bad}"],
port=shared_server.port)
assert result.returncode != 0, f"{bad} must be rejected"
assert result.returncode == 4, \
f"--compress-choice={bad} must exit 4, got {result.returncode}"
@pytest.mark.ci
def test_compress_choice_none_disables_compression(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["-z", "--compress-choice=none"],
port=shared_server.port)
assert result.returncode == 0, \
f"--compress-choice=none failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing and not mismatches, f"missing={missing} mismatches={mismatches}"
@pytest.mark.ci
def test_compress_choice_auto_transfers(self, shared_server):
+2 -2
View File
@@ -94,14 +94,14 @@ def _seed_protocol_source(source):
class TestProtocol:
@pytest.mark.ci
def test_protocol_current_version_accepted(self, shared_server):
"""--protocol=2.25.0 (the current PROTOCOL_VERSION) is accepted and the
"""--protocol=2.26.0 (the current PROTOCOL_VERSION) is accepted and the
transfer completes normally."""
source = os.path.join(TEST_DATA_DIR, "proto_ok_src")
dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst")
shutil.rmtree(dest, ignore_errors=True)
os.makedirs(dest)
_seed_protocol_source(source)
result, _ = run_client(source, dest, flags=["--protocol=2.25.0"],
result, _ = run_client(source, dest, flags=["--protocol=2.26.0"],
port=shared_server.port)
assert result.returncode == 0, \
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
+75 -4
View File
@@ -91,6 +91,63 @@ static void test_checksum_md5_seed_ignored() {
EXPECT_TRUE(memcmp(a, b, alen) == 0);
}
static void test_checksum_md4_vectors() {
uint8_t out[CHECKSUM_MAX_DIGEST_LEN];
size_t len = 0;
/* RFC 1320 / RFC 1321 test vectors. */
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_MD4, 0, "", 0, out, sizeof(out), &len));
EXPECT_TRUE(len == (size_t)16);
const uint8_t expect_empty[16] = {0x31, 0xd6, 0xcf, 0xe0, 0xd1, 0x6a, 0xe9, 0x31,
0xb7, 0x3c, 0x59, 0xd7, 0xe0, 0xc0, 0x89, 0xc0};
EXPECT_TRUE(memcmp(out, expect_empty, 16) == 0);
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_MD4, 0, "abc", 3, out, sizeof(out), &len));
const uint8_t expect_abc[16] = {0xa4, 0x48, 0x01, 0x7a, 0xaf, 0x21, 0xd8, 0x52,
0x5f, 0xc1, 0x0a, 0xe8, 0x7a, 0xa6, 0x72, 0x9d};
EXPECT_TRUE(memcmp(out, expect_abc, 16) == 0);
/* A longer input exercises the block loop and the padding boundary. */
const char* msg =
"12345678901234567890123456789012345678901234567890123456789012345678901234567890";
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_MD4, 0, msg, strlen(msg), out, sizeof(out), &len));
const uint8_t expect_long[16] = {0xe3, 0x3b, 0x4d, 0xdc, 0x9c, 0x38, 0xf2, 0x19,
0x9c, 0x3e, 0x7b, 0x16, 0x4f, 0xcc, 0x05, 0x36};
EXPECT_TRUE(memcmp(out, expect_long, 16) == 0);
}
static void test_checksum_sha1_vectors() {
uint8_t out[CHECKSUM_MAX_DIGEST_LEN];
size_t len = 0;
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_SHA1, 0, "abc", 3, out, sizeof(out), &len));
EXPECT_TRUE(len == (size_t)20);
const uint8_t expect_abc[20] = {0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 0x81, 0x6a, 0xba, 0x3e,
0x25, 0x71, 0x78, 0x50, 0xc2, 0x6c, 0x9c, 0xd0, 0xd8, 0x9d};
EXPECT_TRUE(memcmp(out, expect_abc, 20) == 0);
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_SHA1, 0, "", 0, out, sizeof(out), &len));
EXPECT_TRUE(len == (size_t)20);
const uint8_t expect_empty[20] = {0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55,
0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09};
EXPECT_TRUE(memcmp(out, expect_empty, 20) == 0);
/* sha1 has no seed: the digest is seed-independent (documented). */
uint8_t seeded[CHECKSUM_MAX_DIGEST_LEN];
size_t seeded_len = 0;
EXPECT_TRUE(
checksum_digest(CHECKSUM_ALGO_SHA1, 12345, "abc", 3, seeded, sizeof(seeded), &seeded_len));
EXPECT_TRUE(seeded_len == (size_t)20);
EXPECT_TRUE(memcmp(expect_abc, seeded, 20) == 0);
}
/* "none" is a successful no-digest: length 0, nothing written. */
static void test_checksum_none_digest() {
uint8_t out[CHECKSUM_MAX_DIGEST_LEN];
size_t len = 99;
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_NONE, 0, "data", 4, out, sizeof(out), &len));
EXPECT_EQ_INT((int)len, 0);
EXPECT_EQ_INT((int)checksum_digest_len(CHECKSUM_ALGO_NONE), 0);
}
static void test_checksum_algo_name_mapping() {
EXPECT_EQ_INT(checksum_algo_from_name("xxh64"), (int)CHECKSUM_ALGO_XXH64);
EXPECT_EQ_INT(checksum_algo_from_name("XXH64"), (int)CHECKSUM_ALGO_XXH64);
@@ -102,12 +159,14 @@ static void test_checksum_algo_name_mapping() {
EXPECT_EQ_INT(checksum_algo_from_name("XXH3"), (int)CHECKSUM_ALGO_XXH3);
EXPECT_EQ_INT(checksum_algo_from_name("xxh128"), (int)CHECKSUM_ALGO_XXH128);
EXPECT_EQ_INT(checksum_algo_from_name("XXH128"), (int)CHECKSUM_ALGO_XXH128);
/* rsync choices FastSync does not implement are rejected by name. */
EXPECT_TRUE(checksum_algo_from_name("md4") < 0);
EXPECT_TRUE(checksum_algo_from_name("sha1") < 0);
EXPECT_EQ_INT(checksum_algo_from_name("md4"), (int)CHECKSUM_ALGO_MD4);
EXPECT_EQ_INT(checksum_algo_from_name("MD4"), (int)CHECKSUM_ALGO_MD4);
EXPECT_EQ_INT(checksum_algo_from_name("sha1"), (int)CHECKSUM_ALGO_SHA1);
EXPECT_EQ_INT(checksum_algo_from_name("SHA1"), (int)CHECKSUM_ALGO_SHA1);
EXPECT_EQ_INT(checksum_algo_from_name("none"), (int)CHECKSUM_ALGO_NONE);
/* Names rsync does not offer (or FastSync cannot compute) are rejected. */
EXPECT_TRUE(checksum_algo_from_name("sha256") < 0);
EXPECT_TRUE(checksum_algo_from_name("crc32") < 0);
EXPECT_TRUE(checksum_algo_from_name("none") < 0);
EXPECT_TRUE(checksum_algo_from_name("") < 0);
EXPECT_TRUE(checksum_algo_from_name(NULL) < 0);
@@ -115,11 +174,20 @@ static void test_checksum_algo_name_mapping() {
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_MD5));
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_XXH3));
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_XXH128));
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_MD4));
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_SHA1));
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_NONE));
EXPECT_FALSE(checksum_algo_valid(99));
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_XXH64), "xxh64");
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_MD5), "md5");
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_XXH3), "xxh3");
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_XXH128), "xxh128");
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_MD4), "md4");
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_SHA1), "sha1");
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_NONE), "none");
/* rsync 3.4.1 auto-negotiates xxh128 first. */
EXPECT_EQ_INT((int)checksum_negotiate_default(), (int)CHECKSUM_ALGO_XXH128);
}
/* xxh3 is 8 bytes and seed-aware; xxh128 is 16 bytes and differs from both
@@ -168,6 +236,9 @@ void test_checksum(void) {
test_checksum_xxh64_seed_changes_digest();
test_checksum_xxh64_seed_deterministic();
test_checksum_md5_vectors();
test_checksum_md4_vectors();
test_checksum_sha1_vectors();
test_checksum_none_digest();
test_checksum_algo_lengths_distinct();
test_checksum_md5_seed_ignored();
test_checksum_algo_name_mapping();
+93 -15
View File
@@ -317,7 +317,7 @@ static void test_parse_args_protocol_accept_current() {
Config* cfg = valid_client_config();
EXPECT_NOT_NULL(cfg);
char* argv_equals[] = {"fastsync", "--source-dir", "/src",
"--dest-dir", "/dst", "--protocol=2.25.0"};
"--dest-dir", "/dst", "--protocol=2.26.0"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 6, argv_equals, positional_args, &positional_count), 0);
@@ -327,7 +327,7 @@ static void test_parse_args_protocol_accept_current() {
cfg = valid_client_config();
EXPECT_NOT_NULL(cfg);
char* argv_space[] = {"fastsync", "--source-dir", "/src", "--dest-dir",
"/dst", "--protocol", "2.25.0"};
"/dst", "--protocol", "2.26.0"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 7, argv_space, positional_args, &positional_count), 0);
EXPECT_EQ_STR(cfg->version, PROTOCOL_VERSION);
@@ -1509,22 +1509,25 @@ static void test_parse_args_checksum_choice_equals_forms() {
config_delete(cfg);
}
/* An algorithm FastSync does not support must be rejected, never a silent
no-op. */
/* An algorithm FastSync does not support, an empty half, a lone/extra comma or
a malformed separator must be rejected, never a silent no-op. A single
md4/sha1/none name and the two-name transfer,pre-transfer form are valid. */
static void test_parse_args_checksum_choice_rejects_unsupported() {
static const char* const bad[] = {"md4", "sha1", "sha256", "crc32",
"none", "bogus", "xxh64,md5", "xxhash:md5"};
static const char* const bad[] = {"sha256", "crc32", "bogus", "xxhash:md5",
"md5,", ",md5", "md5,md4,sha1"};
for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--checksum-choice", (char*)bad[i], "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1);
EXPECT_EQ_INT(cfg->cli_exit_code, 4);
config_delete(cfg);
}
}
/* xxh3/xxh128 are accepted; "auto" keeps the default algorithm. */
/* xxh3/xxh128/md4/sha1/none and the two-name form are accepted; "auto"
resolves to FastSync's negotiated default xxh128. */
static void test_parse_args_checksum_choice_new_algos() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--checksum-choice=xxh3", "/src", "/dst"};
@@ -1545,7 +1548,69 @@ static void test_parse_args_checksum_choice_new_algos() {
char* argv3[] = {"fastsync", "--checksum-choice=auto", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv3, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_XXH64);
EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_XXH128);
config_delete(cfg);
/* A single md4/sha1 name selects it for both transfer and pre-transfer. */
static const int single[] = {(int)CHECKSUM_ALGO_MD4, (int)CHECKSUM_ALGO_SHA1};
static const char* const single_names[] = {"md4", "sha1"};
for (size_t i = 0; i < 2; i++) {
cfg = config_create();
char* arg = (char*)single_names[i];
char* argv4[] = {"fastsync", "--cc", arg, "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv4, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->checksum_algo, single[i]);
EXPECT_EQ_INT(cfg->checksum_transfer_algo, single[i]);
config_delete(cfg);
}
/* Two-name form: first is the transfer checksum, second the pre-transfer one
that FastSync actually uses. */
cfg = config_create();
char* argv5[] = {"fastsync", "--cc=sha1,md4", "/checksum/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv5, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->checksum_transfer_algo, (int)CHECKSUM_ALGO_SHA1);
EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_MD4);
config_delete(cfg);
/* "none" is accepted without --checksum but forces --whole-file like rsync. */
cfg = config_create();
char* argv6[] = {"fastsync", "--cc=none", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv6, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_NONE);
EXPECT_TRUE(cfg->whole_file);
config_delete(cfg);
}
/* rsync rejects "none" as the pre-transfer checksum with --checksum (exit 4),
regardless of option order. */
static void test_parse_args_checksum_none_with_checksum_rejected() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--checksum", "--cc=none", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1);
EXPECT_EQ_INT(cfg->cli_exit_code, 4);
config_delete(cfg);
cfg = config_create();
char* argv2[] = {"fastsync", "--checksum", "--cc=md5,none", "/checksum/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv2, positional_args, &positional_count), -1);
EXPECT_EQ_INT(cfg->cli_exit_code, 4);
config_delete(cfg);
/* "none" as the TRANSFER checksum with a real pre-transfer checksum is
accepted (rsync allows none,md5 with -c). */
cfg = config_create();
char* argv3[] = {"fastsync", "--checksum", "--cc=none,md5", "/checksum/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv3, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_MD5);
EXPECT_TRUE(cfg->whole_file);
config_delete(cfg);
}
@@ -1604,28 +1669,40 @@ static void test_parse_args_timeout_zero_and_no_forms() {
config_delete(cfg);
}
/* rsync's --compress-choice choices FastSync does not implement are rejected by
* name; zstd/none/auto are accepted. */
/* Every rsync 3.4.1 --compress-choice name is accepted and mapped to a real
* codec; "auto" resolves to the negotiated default (zstd). An unknown name is
* rejected with rsync's exit code 4. */
static void test_parse_args_compress_choice_parity() {
static const char* const good[] = {"zstd", "none", "auto"};
struct {
const char* name;
CompressionAlgo algo;
bool enabled;
} good[] = {
{"zstd", COMPRESSION_ALGO_ZSTD, true}, {"lz4", COMPRESSION_ALGO_LZ4, true},
{"zlib", COMPRESSION_ALGO_ZLIB, true}, {"zlibx", COMPRESSION_ALGO_ZLIBX, true},
{"none", COMPRESSION_ALGO_NONE, false}, {"auto", COMPRESSION_ALGO_ZSTD, true},
{"ZSTD", COMPRESSION_ALGO_ZSTD, true},
};
for (size_t i = 0; i < sizeof(good) / sizeof(good[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--compress-choice", (char*)good[i], "/src", "/dst"};
char* argv[] = {"fastsync", "--compress-choice", (char*)good[i].name, "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
/* "auto" is normalized to the canonical "zstd" the receiver accepts. */
EXPECT_EQ_STR(cfg->compress_choice, strcmp(good[i], "auto") == 0 ? "zstd" : good[i]);
EXPECT_EQ_INT(cfg->use_compression, strcmp(good[i], "none") != 0 ? 1 : 0);
EXPECT_EQ_STR(cfg->compress_choice, compression_algo_name(good[i].algo));
EXPECT_EQ_INT(cfg->compression_algo, (int)good[i].algo);
EXPECT_EQ_INT(cfg->use_compression, good[i].enabled ? 1 : 0);
config_delete(cfg);
}
static const char* const bad[] = {"lz4", "zlib", "zlibx", "bogus"};
static const char* const bad[] = {"bogus", "", "zstd,lz4"};
for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--compress-choice", (char*)bad[i], "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1);
EXPECT_EQ_INT(cfg->cli_exit_code, 4);
config_delete(cfg);
}
}
@@ -4450,6 +4527,7 @@ void test_client_cli() {
test_parse_args_checksum_choice_equals_forms();
test_parse_args_checksum_choice_rejects_unsupported();
test_parse_args_checksum_choice_new_algos();
test_parse_args_checksum_none_with_checksum_rejected();
test_parse_args_checksum_implies_incremental_only();
test_parse_args_no_whole_file();
test_parse_args_timeout_zero_and_no_forms();
+95 -6
View File
@@ -157,20 +157,22 @@ static void test_chunk_compress_decompress_roundtrip() {
/* Build a zstd frame whose header omits the content size (the content size
* flag is cleared), which ZSTD_getFrameContentSize reports as
* ZSTD_CONTENTSIZE_UNKNOWN. */
* ZSTD_CONTENTSIZE_UNKNOWN. The frame carries the codec-id prefix the
* decompressor dispatches on. */
static Data* make_unknown_size_frame(const void* src, size_t len) {
ZSTD_CCtx* cctx = ZSTD_createCCtx();
if (!cctx)
return NULL;
ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0);
size_t cap = ZSTD_compressBound(len);
Data* out = data_create_empty(cap);
Data* out = data_create_empty(cap + 1);
if (!out) {
ZSTD_freeCCtx(cctx);
return NULL;
}
((uint8_t*)out->data)[0] = (uint8_t)COMPRESSION_ALGO_ZSTD;
ZSTD_inBuffer in = {src, len, 0};
ZSTD_outBuffer ob = {out->data, cap, 0};
ZSTD_outBuffer ob = {(uint8_t*)out->data + 1, cap, 0};
size_t ret;
do {
ret = ZSTD_compressStream2(cctx, &ob, &in, ZSTD_e_end);
@@ -180,7 +182,7 @@ static Data* make_unknown_size_frame(const void* src, size_t len) {
return NULL;
}
} while (ret > 0);
out->size = ob.pos;
out->size = ob.pos + 1;
ZSTD_freeCCtx(cctx);
return out;
}
@@ -199,8 +201,9 @@ static void test_data_decompress_unknown_size_frame() {
Data* frame = make_unknown_size_frame(buf, len);
free(buf);
EXPECT_NOT_NULL(frame);
/* Guard the premise of the test: the frame really has no stored size. */
EXPECT_EQ_INT((int)ZSTD_getFrameContentSize(frame->data, frame->size),
/* Guard the premise of the test: the frame (after the codec byte) really has
* no stored size. */
EXPECT_EQ_INT((int)ZSTD_getFrameContentSize((uint8_t*)frame->data + 1, frame->size - 1),
(int)ZSTD_CONTENTSIZE_UNKNOWN);
Data* decompressed = data_decompress(frame);
@@ -326,6 +329,89 @@ static void test_data_decompress_truncated_frame_fails() {
data_destroy(input);
}
/* Every codec must round-trip byte-exactly through the self-describing frame,
* including the empty and a highly compressible large payload. */
static void codec_roundtrip(CompressionAlgo algo) {
const char* samples[] = {
"",
"Hello, World! This is test data for compression round-trip!",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
};
for (size_t s = 0; s < sizeof(samples) / sizeof(samples[0]); s++) {
size_t len = strlen(samples[s]);
Data* original = data_create_empty(len);
EXPECT_NOT_NULL(original);
if (len > 0)
memcpy(original->data, samples[s], len);
original->size = len;
Data* compressed = data_compress_codec(original, algo, 3, 0);
EXPECT_NOT_NULL(compressed);
EXPECT_EQ_INT((int)((uint8_t*)compressed->data)[0], (int)algo);
Data* decompressed = data_decompress(compressed);
EXPECT_NOT_NULL(decompressed);
EXPECT_EQ_INT((int)decompressed->size, (int)len);
EXPECT_EQ_INT(memcmp(decompressed->data, original->data, len), 0);
data_destroy(decompressed);
data_destroy(compressed);
data_destroy(original);
}
}
static void test_codec_roundtrips() {
codec_roundtrip(COMPRESSION_ALGO_NONE);
codec_roundtrip(COMPRESSION_ALGO_ZSTD);
codec_roundtrip(COMPRESSION_ALGO_LZ4);
codec_roundtrip(COMPRESSION_ALGO_ZLIB);
codec_roundtrip(COMPRESSION_ALGO_ZLIBX);
}
static void test_codec_name_mapping() {
EXPECT_EQ_INT(compression_algo_from_name("zstd"), (int)COMPRESSION_ALGO_ZSTD);
EXPECT_EQ_INT(compression_algo_from_name("ZSTD"), (int)COMPRESSION_ALGO_ZSTD);
EXPECT_EQ_INT(compression_algo_from_name("lz4"), (int)COMPRESSION_ALGO_LZ4);
EXPECT_EQ_INT(compression_algo_from_name("zlib"), (int)COMPRESSION_ALGO_ZLIB);
EXPECT_EQ_INT(compression_algo_from_name("zlibx"), (int)COMPRESSION_ALGO_ZLIBX);
EXPECT_EQ_INT(compression_algo_from_name("none"), (int)COMPRESSION_ALGO_NONE);
EXPECT_TRUE(compression_algo_from_name("bogus") < 0);
EXPECT_TRUE(compression_algo_from_name(NULL) < 0);
EXPECT_TRUE(compression_algo_valid((int)COMPRESSION_ALGO_LZ4));
EXPECT_TRUE(compression_algo_valid((int)COMPRESSION_ALGO_ZLIB));
EXPECT_TRUE(compression_algo_valid((int)COMPRESSION_ALGO_ZLIBX));
EXPECT_FALSE(compression_algo_valid(99));
EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_ZSTD), "zstd");
EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_LZ4), "lz4");
EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_ZLIB), "zlib");
EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_ZLIBX), "zlibx");
EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_NONE), "none");
/* rsync 3.4.1 auto-negotiates zstd first. */
EXPECT_EQ_INT((int)compression_negotiate_default(), (int)COMPRESSION_ALGO_ZSTD);
EXPECT_FALSE(compression_algo_enabled(COMPRESSION_ALGO_NONE));
EXPECT_TRUE(compression_algo_enabled(COMPRESSION_ALGO_ZSTD));
}
/* The process-global codec selects what the legacy wrappers produce. */
static void test_codec_global_selection() {
Data* original = data_create_empty(64);
EXPECT_NOT_NULL(original);
memset(original->data, 'q', 64);
original->size = 64;
compression_set_algo(COMPRESSION_ALGO_LZ4);
Data* compressed = data_compress(original, 3);
EXPECT_NOT_NULL(compressed);
EXPECT_EQ_INT((int)((uint8_t*)compressed->data)[0], (int)COMPRESSION_ALGO_LZ4);
Data* decompressed = data_decompress(compressed);
EXPECT_NOT_NULL(decompressed);
EXPECT_TRUE(memcmp(decompressed->data, original->data, 64) == 0);
data_destroy(decompressed);
data_destroy(compressed);
/* Restore the default so later tests are unaffected. */
compression_set_algo(COMPRESSION_ALGO_ZSTD);
data_destroy(original);
}
void test_compression() {
test_data_compress_decompress_roundtrip();
test_data_compress_decompress_large();
@@ -335,4 +421,7 @@ void test_compression() {
test_data_compress_with_threads_roundtrip();
test_data_compress_reused_contexts_multithreaded();
test_chunk_compress_decompress_roundtrip();
test_codec_roundtrips();
test_codec_name_mapping();
test_codec_global_selection();
}
+65 -5
View File
@@ -1368,6 +1368,61 @@ static void test_config_receive_rejects_invalid_checksum_algo() {
EXPECT_FALSE(roundtrip_config_ok(c));
config_delete(c);
}
/* The negotiated codec id and the human --compress-choice spelling must agree,
* and the id itself must be a known codec. */
static void test_config_receive_rejects_invalid_compression_algo() {
if (is_running_under_valgrind())
return;
Config* c = config_create();
EXPECT_NOT_NULL(c);
c->send_directory = str_dup("/src");
c->receive_root_directory = str_dup("/dst");
c->compression_algo = 99;
EXPECT_FALSE(roundtrip_config_ok(c));
config_delete(c);
}
static void test_config_receive_rejects_codec_mismatch() {
if (is_running_under_valgrind())
return;
Config* c = config_create();
EXPECT_NOT_NULL(c);
c->send_directory = str_dup("/src");
c->receive_root_directory = str_dup("/dst");
free(c->compress_choice);
c->compress_choice = str_dup("lz4");
c->use_compression = true;
c->compression_algo = (int)COMPRESSION_ALGO_ZSTD; /* does not match lz4 */
EXPECT_FALSE(roundtrip_config_ok(c));
config_delete(c);
}
static void test_config_receive_rejects_none_codec_with_compression() {
if (is_running_under_valgrind())
return;
Config* c = config_create();
EXPECT_NOT_NULL(c);
c->send_directory = str_dup("/src");
c->receive_root_directory = str_dup("/dst");
c->use_compression = true;
c->compression_algo = (int)COMPRESSION_ALGO_NONE;
EXPECT_FALSE(roundtrip_config_ok(c));
config_delete(c);
}
static void test_config_receive_rejects_checksum_none_with_checksum() {
if (is_running_under_valgrind())
return;
Config* c = config_create();
EXPECT_NOT_NULL(c);
c->send_directory = str_dup("/src");
c->receive_root_directory = str_dup("/dst");
c->checksum = true;
c->checksum_algo = (int)CHECKSUM_ALGO_NONE;
EXPECT_FALSE(roundtrip_config_ok(c));
config_delete(c);
}
/* The identity-mapping fields (--numeric-ids / --usermap / --groupmap /
--chown) cross the config wire unchanged: the receiver needs them to apply
ownership with the same policy the client requested. */
@@ -2549,6 +2604,7 @@ static bool basis_equal(const Config* a, const Config* b) {
#define CONFIG_CMP_STR_MODULE(a, b, name) str_opt_equal((a)->name, (b)->name)
#define CONFIG_CMP_STR_REDACTED_AUTH(a, b, name) str_opt_equal((a)->name, (b)->name)
#define CONFIG_CMP_INT_CHECKSUM_ALGO(a, b, name) ((a)->name == (b)->name)
#define CONFIG_CMP_INT_COMPRESSION_ALGO(a, b, name) ((a)->name == (b)->name)
#define CONFIG_CMP_SUPERMODE(a, b, name) ((a)->name == (b)->name)
#define CONFIG_CMP_INT_IDENTITY(a, b, name) ((a)->name == (b)->name)
#define CONFIG_CMP_INT_SKIPCOUNT(a, b, name) ((a)->name == (b)->name)
@@ -2775,14 +2831,14 @@ static void golden_config_populate(Config* c) {
c->copy_as_gid = 222;
}
/* The pinned golden frame (protocol 2.25.0). The values below are the only
/* The pinned golden frame (protocol 2.26.0). The values below are the only
* thing that ties the generated table to the historical wire format; update
* them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.24.0
* per-directory delete-plan wave changed only the version string in the config
* frame; the 2.25.0 wire-stats wave appends one report_stats bool. The
* delete-plan wave changed only the version string; 2.25.0 appended the
* report_stats bool and 2.26.0 appended the compression_algo int. The
* byte-exact values are recomputed for the merged layout. */
#define GOLDEN_WIRE_LEN 701
#define GOLDEN_WIRE_HASH 16170466870400670271ULL
#define GOLDEN_WIRE_LEN 705
#define GOLDEN_WIRE_HASH 4673424031554175633ULL
static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) {
unsigned long long h = 1469598103934665603ULL;
@@ -3159,6 +3215,10 @@ void test_config() {
test_config_basis_normalization();
test_config_checksum_options_wire_roundtrip();
test_config_receive_rejects_invalid_checksum_algo();
test_config_receive_rejects_invalid_compression_algo();
test_config_receive_rejects_codec_mismatch();
test_config_receive_rejects_none_codec_with_compression();
test_config_receive_rejects_checksum_none_with_checksum();
test_config_identity_wire_roundtrip();
test_config_receive_rejects_invalid_identity();
test_config_metadata_times_wire_roundtrip();
+16 -16
View File
@@ -18,10 +18,10 @@
/* P8 config-frame tail: super_mode (4) + copy-as presence (4) + uid (4) + gid (4). */
#define P8_TAIL_BYTES 16
/* Protocol 2.25.0 appends two trailing bools (report_dest_info, report_stats)
* AFTER the P8 tail, so the P8 fields sit this many bytes before the end of the
* frame. */
#define OUTPUT_TAIL_BYTES 8
/* Bytes after the P8 tail: report_dest_info (4), report_stats (4, wire-stats
* wave) and compression_algo (4, codec wave). The P8 fields sit this many
* bytes before the end of the frame. */
#define POST_P8_TAIL_BYTES 12
/* Smoke test for chunk_deserialize fuzz target */
static void test_fuzz_chunk_deserialize() {
@@ -324,7 +324,7 @@ static void test_fuzz_config_receive_p8_tail() {
size_t len = 0;
bool captured = capture_config_frame(c, &frame, &len);
config_delete(c);
if (!captured || len <= P8_TAIL_BYTES) {
if (!captured || len <= P8_TAIL_BYTES + POST_P8_TAIL_BYTES) {
free(frame);
EXPECT_TRUE(false);
return;
@@ -338,31 +338,31 @@ static void test_fuzz_config_receive_p8_tail() {
/* super_mode outside the 0..2 tri-state is refused. */
memcpy(mut, frame, len);
put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, 99);
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES, 99);
EXPECT_FALSE(receive_config_frame(mut, len));
put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, -1);
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES, -1);
EXPECT_FALSE(receive_config_frame(mut, len));
/* A negative (sentinel) and an extreme copy-as uid/gid are refused. */
memcpy(mut, frame, len);
put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, SUPER_MODE_AUTO);
put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 4, 1);
put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 8, -1);
put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 12, 0);
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES, SUPER_MODE_AUTO);
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 4, 1);
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 8, -1);
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 12, 0);
EXPECT_FALSE(receive_config_frame(mut, len));
put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 8, 0);
put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 12, INT32_MIN);
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 8, 0);
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 12, INT32_MIN);
EXPECT_FALSE(receive_config_frame(mut, len));
/* A presence int that is not a wire bool is refused. */
memcpy(mut, frame, len);
put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, SUPER_MODE_AUTO);
put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 4, 2);
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES, SUPER_MODE_AUTO);
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 4, 2);
EXPECT_FALSE(receive_config_frame(mut, len));
/* Truncating anywhere inside the P8 tail is refused. */
EXPECT_FALSE(receive_config_frame(frame, len - 2));
EXPECT_FALSE(receive_config_frame(frame, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES));
EXPECT_FALSE(receive_config_frame(frame, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES));
free(mut);
free(frame);
+3 -3
View File
@@ -1072,10 +1072,10 @@ static void test_incremental_check_basis_fifo_does_not_hang() {
EXPECT_TRUE(send_n_data(p[1], &mtime, sizeof(mtime)));
EXPECT_TRUE(send_n_data(p[1], &mtime_nsec, sizeof(mtime_nsec)));
/* config_has_basis() makes the request carry the source digest. */
uint8_t wire_len = 8;
uint8_t digest[8] = {0};
uint8_t wire_len = checksum_digest_len((ChecksumAlgo)cfg->checksum_algo);
uint8_t digest[CHECKSUM_MAX_DIGEST_LEN] = {0};
EXPECT_TRUE(send_n_data(p[1], &wire_len, sizeof(wire_len)));
EXPECT_TRUE(send_n_data(p[1], digest, sizeof(digest)));
EXPECT_TRUE(send_n_data(p[1], digest, wire_len));
Status s;
EXPECT_TRUE(receive_status(p[1], &s));
EXPECT_EQ_INT(s, STATUS_NEXT);