diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b7e10c..37620b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.22) -project(FastFileTransfer VERSION 2.23.0) +project(FastFileTransfer VERSION 2.26.0) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_C_STANDARD 11) @@ -68,6 +68,16 @@ if(NOT ZSTD_LIBRARY) message(FATAL_ERROR "zstd library not found. Ensure it is in your nix-shell!") endif() +find_library(ZLIB_LIBRARY z) +if(NOT ZLIB_LIBRARY) + message(FATAL_ERROR "zlib library not found. Ensure zlib1g-dev / nix zlib is available!") +endif() + +find_library(LZ4_LIBRARY lz4) +if(NOT LZ4_LIBRARY) + message(FATAL_ERROR "lz4 library not found. Ensure liblz4-dev / nix lz4 is available!") +endif() + find_package(OpenSSL REQUIRED) # --- Explicit source lists --- @@ -136,8 +146,8 @@ set(CLIENT_MAIN_SRCS src/client/client_cli.c) # --- Library targets --- add_library(fastsync_shared STATIC ${SHARED_SRCS}) target_include_directories(fastsync_shared PUBLIC src/shared) -target_link_libraries(fastsync_shared PUBLIC Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL - OpenSSL::Crypto xxhash) +target_link_libraries(fastsync_shared PUBLIC Threads::Threads ${ZSTD_LIBRARY} ${ZLIB_LIBRARY} + ${LZ4_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash) add_library(fastsync_client_core STATIC ${CLIENT_CORE_SRCS}) target_include_directories(fastsync_client_core PUBLIC src/client) @@ -276,7 +286,7 @@ if(ENABLE_FUZZ) target_include_directories(${FUZZ_NAME} PRIVATE tests src/shared src/server) target_compile_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined -fno-omit-frame-pointer) target_link_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined) - target_link_libraries(${FUZZ_NAME} PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL - OpenSSL::Crypto xxhash) + target_link_libraries(${FUZZ_NAME} PRIVATE Threads::Threads ${ZSTD_LIBRARY} ${ZLIB_LIBRARY} + ${LZ4_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash) endforeach() endif() diff --git a/shell.nix b/shell.nix index 6d03b8d..b32c25f 100644 --- a/shell.nix +++ b/shell.nix @@ -38,6 +38,8 @@ pkgs.mkShell { buildInputs = with pkgs; [ zstd + zlib + lz4 openssl ]; diff --git a/src/client/client_cli.c b/src/client/client_cli.c index eb5b41c..781bb76 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -150,47 +150,96 @@ static int set_positive_int_option(int* dest, const char* value, const char* opt } /* Set and validate the compression algorithm selected by the client. rsync - * 3.4.1 can be built with zstd, none, lz4, zlibx, zlib and auto; FastSync only - * implements zstd (and no compression). "auto" is accepted as the default - * zstd choice; any other rsync choice is rejected by name instead of being - * silently accepted and ignored. */ + * 3.4.1 can be built with zstd, none, lz4, zlibx, zlib and auto; all of those + * names are accepted and mapped to a real codec here. "auto" resolves through + * FastSync's compiled-in preference order (rsync 3.4.1's list). An unknown + * name is a hard error with rsync's exit code 4, never a silent no-op. */ static int set_compression_choice(Config* config, const char* value) { - /* rsync's "auto" is normalized to the canonical "zstd" at parse time (like - --checksum-choice=auto), so the value that crosses the wire is always one - the receiver accepts. */ - const char* canonical = strcmp(value, "auto") == 0 ? "zstd" : value; - if (strcmp(canonical, "zstd") != 0 && strcmp(canonical, "none") != 0) { - log_message(LOG_LEVEL_ERROR, - "--compress-choice '%s' is not implemented; FastSync supports zstd, none or auto " - "(rsync's lz4/zlib/zlibx are rejected, never silently ignored)", - value); + if (!value) { + config->cli_exit_code = 4; return -1; } + int algo; + if (strcasecmp(value, "auto") == 0) + algo = (int)compression_negotiate_default(); + else + algo = compression_algo_from_name(value); + if (algo < 0) { + log_message(LOG_LEVEL_ERROR, + "--compress-choice '%s' is not a supported algorithm; FastSync supports zstd, " + "lz4, zlib, zlibx, none or auto", + value); + config->cli_exit_code = 4; + return -1; + } + const char* canonical = compression_algo_name((CompressionAlgo)algo); if (set_string_option(&config->compress_choice, canonical, "--compress-choice") != 0) return -1; - config->use_compression = strcmp(canonical, "none") != 0; + config->compression_algo = algo; + config->use_compression = (algo != (int)COMPRESSION_ALGO_NONE); return 0; } -/* Validate and store the --checksum-choice/--cc algorithm. Only the algorithms - * the engine genuinely supports are accepted (xxh64/xxhash, xxh3, xxh128, md5); - * rsync's compiled-in choices that FastSync does not implement (md4, sha1, - * none) and the two-name transfer/pre-transfer syntax are a clear error, never - * a silent no-op. "auto" (rsync's default automatic choice) selects FastSync's - * default algorithm. */ +/* Store one algorithm name into *out. Returns 0 for a valid name, 1 for + * "auto" (caller resolves it), -1 for an unknown/too-long name. */ +static int resolve_checksum_name(const char* name, size_t len, int* out) { + char buf[64]; + if (len == 0 || len >= sizeof(buf)) + return -1; + memcpy(buf, name, len); + buf[len] = '\0'; + if (strcasecmp(buf, "auto") == 0) + return 1; + int algo = checksum_algo_from_name(buf); + if (algo < 0) + return -1; + *out = algo; + return 0; +} + +/* Validate and store the --checksum-choice/--cc algorithm. rsync 3.4.1 accepts + * a single name (used for both the transfer and pre-transfer checksums) or the + * two-name "TRANSFER,PRE-TRANSFER" form (only one comma is significant). The + * pre-transfer half is FastSync's whole-file digest; the transfer half is + * validated for parity and, when "none", forces --whole-file like rsync. An + * unknown name (including an empty half or a second comma) is exit 4. "auto" + * resolves to FastSync's negotiated default (xxh128). */ static int set_checksum_choice(Config* config, const char* value) { - if (strcasecmp(value, "auto") == 0) - return 0; - int algo = checksum_algo_from_name(value); - if (algo < 0) { - log_message(LOG_LEVEL_ERROR, - "--checksum-choice '%s' is not implemented; FastSync supports xxh64 (or xxhash), " - "xxh3, xxh128, md5 or auto (rsync's md4/sha1/none and the two-name " - "transfer,pre-transfer form are rejected, never silently ignored)", - value); + if (!value) { + config->cli_exit_code = 4; return -1; } - config->checksum_algo = algo; + const char* comma = strchr(value, ','); + const char* name1 = value; + size_t len1 = comma ? (size_t)(comma - value) : strlen(value); + const char* name2 = comma ? comma + 1 : NULL; + size_t len2 = name2 ? strlen(name2) : 0; + + int transfer = -1; + int pre = -1; + int rc1 = resolve_checksum_name(name1, len1, &transfer); + int rc2 = name2 ? resolve_checksum_name(name2, len2, &pre) : 1; + if (rc1 < 0 || rc2 < 0) { + log_message(LOG_LEVEL_ERROR, + "--checksum-choice '%s' is invalid; FastSync supports xxh64 (or xxhash), xxh128, " + "xxh3, md5, md4, sha1, none or auto, optionally as 'transfer,pre-transfer'", + value); + config->cli_exit_code = 4; + return -1; + } + ChecksumAlgo negotiated = checksum_negotiate_default(); + if (rc1 == 1) + transfer = (int)negotiated; + if (!name2) + pre = transfer; + else if (rc2 == 1) + pre = (int)negotiated; + + config->checksum_algo = pre; + config->checksum_transfer_algo = transfer; + /* rsync: "none" for the transfer checksum forces --whole-file. */ + if (transfer == (int)CHECKSUM_ALGO_NONE) + config->whole_file = true; return 0; } @@ -2326,8 +2375,23 @@ static bool cli_handle_outbuf_option(CliParseCtx* ctx) { * -1 on error. */ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool no_incremental) { set_log_level(config->quiet ? LOG_LEVEL_ERROR : (verbose ? LOG_LEVEL_DEBUG : LOG_LEVEL_WARNING)); - if (config->compress_choice) - config->use_compression = strcmp(config->compress_choice, "none") != 0; + if (config->compress_choice) { + int algo = compression_algo_from_name(config->compress_choice); + if (algo >= 0) { + config->compression_algo = algo; + config->use_compression = (algo != (int)COMPRESSION_ALGO_NONE); + } + } + if (config->use_compression && config->compression_algo == (int)COMPRESSION_ALGO_NONE) + config->compression_algo = (int)compression_negotiate_default(); + /* rsync parity: "none" as the pre-transfer checksum cannot be combined with + * --checksum (exit 4). The check runs here because --checksum may appear on + * either side of --checksum-choice. */ + if (config->checksum && config->checksum_algo == (int)CHECKSUM_ALGO_NONE) { + log_message(LOG_LEVEL_ERROR, "Invalid checksum-choice for --checksum: none"); + config->cli_exit_code = 4; + return -1; + } /* rsync randomizes the checksum seed for every transfer when the user did not * supply one (a seed of 0, including an explicit --checksum-seed=0), using @@ -2786,7 +2850,7 @@ int main(int argc, char* argv[]) { int parse_ret = parse_args(config, argc, argv, positional_args, &positional_count); if (parse_ret != 0) { if (parse_ret < 0) - exit_code = 1; + exit_code = config->cli_exit_code ? config->cli_exit_code : 1; goto cleanup; } @@ -2876,6 +2940,11 @@ int main(int argc, char* argv[]) { goto cleanup; } + /* Install the negotiated codec for this process before any transfer thread + * is spawned; the compressed frames are self-describing, so the receiver's + * decompressor does not need this, but the sender compressor does. */ + compression_set_algo((CompressionAlgo)config->compression_algo); + /* --iconv: install the sender-side local->wire conversion before any path is scanned or serialized (the scanner and the chunk/data path read windows are all driven from this process, so one global initialization covers every diff --git a/src/client/usage.c b/src/client/usage.c index e654e5f..e8f3e65 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -138,10 +138,10 @@ void print_usage(void) { printf(" --link-dest Like --copy-dest, but hard-links the unchanged file from DIR\n"); printf(" into the destination (repeatable; earlier DIRs win)\n"); printf(" --checksum-choice, --cc Whole-file checksum algorithm for --incremental/\n"); - printf(" --checksum compares. Accepted: xxh64 (aka xxhash), xxh3,\n"); - printf(" xxh128, md5, or auto (default xxh64). rsync choices FastSync\n"); - printf(" does not implement (md4, sha1, none) and the two-name\n"); - printf(" transfer,pre-transfer form are rejected by name\n"); + printf(" --checksum compares. Accepted: xxh128 (default), xxh3, xxh64\n"); + printf(" (aka xxhash), md5, md4, sha1, or none. A two-name\n"); + printf(" 'transfer,pre-transfer' form is accepted like rsync; 'none' as\n"); + printf(" the pre-transfer algorithm is rejected with --checksum\n"); printf(" --checksum-seed Seed for the whole-file xxHash digest (and the delta\n"); printf(" block strong hash, low 32 bits); md5 ignores the seed. A seed\n"); printf(" of 0 (the default) is randomized per transfer, exactly like\n"); @@ -168,7 +168,8 @@ void print_usage(void) { printf(" SSH argv is already built injection-safe)\n"); printf(" --sendfile Enable sendfile zero-copy (TCP only; long form only;\n"); printf(" -f is bound to --filter, not --sendfile)\n"); - printf(" --compress-choice Compression algorithm (default: zstd)\n"); + printf(" --compress-choice Compression algorithm: zstd (default), lz4, zlib,\n"); + printf(" zlibx, none, or auto\n"); printf(" --zc Alias for --compress-choice\n"); printf(" -v, --verbose Enable debug logging\n"); printf(" -q, --quiet Suppress non-error output\n"); diff --git a/src/server/server.c b/src/server/server.c index 58fa871..2decda7 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -741,6 +741,10 @@ void handler(int file_descriptor) { * received config. */ if (gate_ctx.super_mode_override != -1) config->super_mode = (SuperMode)gate_ctx.super_mode_override; + /* Install the codec this connection negotiated before the receiver/writer + * threads start (the server forks per connection, so the process-global + * codec is private to this session). */ + compression_set_algo((CompressionAlgo)config->compression_algo); /* If the client requested ownership but the effective super mode forbids it * (operator --no-super, a privileged standalone receiver's secure default, or * a daemon module without `client owner = yes`), say so ONCE per connection so diff --git a/src/shared/checksum.c b/src/shared/checksum.c index f4706b8..d12e339 100644 --- a/src/shared/checksum.c +++ b/src/shared/checksum.c @@ -11,6 +11,160 @@ #define XXH_STATIC_LINKING_ONLY #include +/* --------------------------------------------------------------------------- + * Self-contained MD4 (RFC 1320). OpenSSL's MD4 lives in the legacy provider + * and is not guaranteed present, so FastSync carries its own implementation to + * keep --checksum-choice=md4 working on every build. + * ------------------------------------------------------------------------- */ + +typedef struct { + uint32_t state[4]; + uint64_t bit_count; + uint8_t buffer[64]; + size_t buffer_len; +} Md4Ctx; + +static uint32_t md4_rotl(uint32_t x, int n) { + return (x << n) | (x >> (32 - n)); +} + +static void md4_transform(uint32_t state[4], const uint8_t block[64]) { + uint32_t x[16]; + for (int i = 0; i < 16; i++) + x[i] = (uint32_t)block[i * 4] | ((uint32_t)block[i * 4 + 1] << 8) | + ((uint32_t)block[i * 4 + 2] << 16) | ((uint32_t)block[i * 4 + 3] << 24); + + uint32_t a = state[0], b = state[1], c = state[2], d = state[3]; + +#define F(x, y, z) (((x) & (y)) | (~(x) & (z))) +#define G(x, y, z) (((x) & (y)) | ((x) & (z)) | ((y) & (z))) +#define H(x, y, z) ((x) ^ (y) ^ (z)) +#define ROUND1(a, b, c, d, k, s) a = md4_rotl(a + F(b, c, d) + x[k], s) +#define ROUND2(a, b, c, d, k, s) a = md4_rotl(a + G(b, c, d) + x[k] + 0x5a827999u, s) +#define ROUND3(a, b, c, d, k, s) a = md4_rotl(a + H(b, c, d) + x[k] + 0x6ed9eba1u, s) + + ROUND1(a, b, c, d, 0, 3); + ROUND1(d, a, b, c, 1, 7); + ROUND1(c, d, a, b, 2, 11); + ROUND1(b, c, d, a, 3, 19); + ROUND1(a, b, c, d, 4, 3); + ROUND1(d, a, b, c, 5, 7); + ROUND1(c, d, a, b, 6, 11); + ROUND1(b, c, d, a, 7, 19); + ROUND1(a, b, c, d, 8, 3); + ROUND1(d, a, b, c, 9, 7); + ROUND1(c, d, a, b, 10, 11); + ROUND1(b, c, d, a, 11, 19); + ROUND1(a, b, c, d, 12, 3); + ROUND1(d, a, b, c, 13, 7); + ROUND1(c, d, a, b, 14, 11); + ROUND1(b, c, d, a, 15, 19); + + ROUND2(a, b, c, d, 0, 3); + ROUND2(d, a, b, c, 4, 5); + ROUND2(c, d, a, b, 8, 9); + ROUND2(b, c, d, a, 12, 13); + ROUND2(a, b, c, d, 1, 3); + ROUND2(d, a, b, c, 5, 5); + ROUND2(c, d, a, b, 9, 9); + ROUND2(b, c, d, a, 13, 13); + ROUND2(a, b, c, d, 2, 3); + ROUND2(d, a, b, c, 6, 5); + ROUND2(c, d, a, b, 10, 9); + ROUND2(b, c, d, a, 14, 13); + ROUND2(a, b, c, d, 3, 3); + ROUND2(d, a, b, c, 7, 5); + ROUND2(c, d, a, b, 11, 9); + ROUND2(b, c, d, a, 15, 13); + + ROUND3(a, b, c, d, 0, 3); + ROUND3(d, a, b, c, 8, 9); + ROUND3(c, d, a, b, 4, 11); + ROUND3(b, c, d, a, 12, 15); + ROUND3(a, b, c, d, 2, 3); + ROUND3(d, a, b, c, 10, 9); + ROUND3(c, d, a, b, 6, 11); + ROUND3(b, c, d, a, 14, 15); + ROUND3(a, b, c, d, 1, 3); + ROUND3(d, a, b, c, 9, 9); + ROUND3(c, d, a, b, 5, 11); + ROUND3(b, c, d, a, 13, 15); + ROUND3(a, b, c, d, 3, 3); + ROUND3(d, a, b, c, 11, 9); + ROUND3(c, d, a, b, 7, 11); + ROUND3(b, c, d, a, 15, 15); + +#undef F +#undef G +#undef H +#undef ROUND1 +#undef ROUND2 +#undef ROUND3 + + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; +} + +static void md4_init(Md4Ctx* ctx) { + ctx->state[0] = 0x67452301u; + ctx->state[1] = 0xefcdab89u; + ctx->state[2] = 0x98badcfeu; + ctx->state[3] = 0x10325476u; + ctx->bit_count = 0; + ctx->buffer_len = 0; +} + +static void md4_update(Md4Ctx* ctx, const uint8_t* data, size_t len) { + ctx->bit_count += (uint64_t)len * 8; + while (len > 0) { + size_t space = sizeof(ctx->buffer) - ctx->buffer_len; + size_t take = len < space ? len : space; + memcpy(ctx->buffer + ctx->buffer_len, data, take); + ctx->buffer_len += take; + data += take; + len -= take; + if (ctx->buffer_len == sizeof(ctx->buffer)) { + md4_transform(ctx->state, ctx->buffer); + ctx->buffer_len = 0; + } + } +} + +static void md4_final(Md4Ctx* ctx, uint8_t out[16]) { + uint64_t bit_count = ctx->bit_count; + uint8_t pad = 0x80; + md4_update(ctx, &pad, 1); + uint8_t zero = 0; + while (ctx->buffer_len != 56) + md4_update(ctx, &zero, 1); + uint8_t length_le[8]; + for (int i = 0; i < 8; i++) + length_le[i] = (uint8_t)((bit_count >> (8 * i)) & 0xff); + md4_update(ctx, length_le, sizeof(length_le)); + for (int i = 0; i < 4; i++) { + out[i * 4] = (uint8_t)(ctx->state[i] & 0xff); + out[i * 4 + 1] = (uint8_t)((ctx->state[i] >> 8) & 0xff); + out[i * 4 + 2] = (uint8_t)((ctx->state[i] >> 16) & 0xff); + out[i * 4 + 3] = (uint8_t)((ctx->state[i] >> 24) & 0xff); + } +} + +/* One-shot EVP digest (md5/sha1). Returns false when OpenSSL refuses. */ +static bool evp_digest(const EVP_MD* md, const void* data, size_t size, uint8_t* out, + size_t out_capacity, size_t* out_len) { + static const uint8_t empty = 0; + const void* input = data ? data : ∅ + unsigned int digest_len = 0; + if (EVP_Digest(input, size, out, &digest_len, md, NULL) != 1) + return false; + if (digest_len > out_capacity) + return false; + *out_len = digest_len; + return true; +} + bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out, size_t out_capacity, size_t* out_len) { if (!out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN) @@ -18,43 +172,45 @@ bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t if (data == NULL && size != 0) return false; - if (algo == CHECKSUM_ALGO_XXH64) { + switch (algo) { + case CHECKSUM_ALGO_XXH64: { uint64_t digest = XXH64(data, size, seed); memcpy(out, &digest, sizeof(digest)); *out_len = sizeof(digest); return true; } - - if (algo == CHECKSUM_ALGO_XXH3) { + case CHECKSUM_ALGO_XXH3: { uint64_t digest = XXH3_64bits_withSeed(data, size, seed); memcpy(out, &digest, sizeof(digest)); *out_len = sizeof(digest); return true; } - - if (algo == CHECKSUM_ALGO_XXH128) { + case CHECKSUM_ALGO_XXH128: { XXH128_hash_t digest = XXH3_128bits_withSeed(data, size, seed); memcpy(out, &digest, sizeof(digest)); *out_len = sizeof(digest); return true; } - - if (algo == CHECKSUM_ALGO_MD5) { + case CHECKSUM_ALGO_MD5: /* md5 takes no seed; the caller's seed is deliberately ignored (documented - * in RSYNC_COMPAT.md). OpenSSL's one-shot EVP_Digest needs a non-NULL - * buffer even for an empty input, so map a NULL data + size==0 to an empty - * buffer. */ - static const uint8_t empty = 0; - const void* input = data ? data : ∅ - unsigned int digest_len = 0; - if (EVP_Digest(input, size, out, &digest_len, EVP_md5(), NULL) != 1) - return false; - if (digest_len > out_capacity) - return false; - *out_len = digest_len; + * in RSYNC_COMPAT.md). */ + return evp_digest(EVP_md5(), data, size, out, out_capacity, out_len); + case CHECKSUM_ALGO_MD4: { + Md4Ctx ctx; + md4_init(&ctx); + md4_update(&ctx, (const uint8_t*)data, size); + md4_final(&ctx, out); + *out_len = 16; + return true; + } + case CHECKSUM_ALGO_SHA1: + /* sha1 takes no seed; the caller's seed is deliberately ignored. */ + return evp_digest(EVP_sha1(), data, size, out, out_capacity, out_len); + case CHECKSUM_ALGO_NONE: + /* No checksum requested: an empty digest is the successful result. */ + *out_len = 0; return true; } - return false; } @@ -162,6 +318,12 @@ int checksum_algo_from_name(const char* name) { return (int)CHECKSUM_ALGO_XXH128; if (strcasecmp(name, "md5") == 0) return (int)CHECKSUM_ALGO_MD5; + if (strcasecmp(name, "md4") == 0) + return (int)CHECKSUM_ALGO_MD4; + if (strcasecmp(name, "sha1") == 0) + return (int)CHECKSUM_ALGO_SHA1; + if (strcasecmp(name, "none") == 0) + return (int)CHECKSUM_ALGO_NONE; return -1; } @@ -175,13 +337,21 @@ const char* checksum_algo_name(ChecksumAlgo algo) { return "xxh128"; case CHECKSUM_ALGO_MD5: return "md5"; + case CHECKSUM_ALGO_MD4: + return "md4"; + case CHECKSUM_ALGO_SHA1: + return "sha1"; + case CHECKSUM_ALGO_NONE: + return "none"; } return ""; } bool checksum_algo_valid(int algo) { return algo == (int)CHECKSUM_ALGO_XXH64 || algo == (int)CHECKSUM_ALGO_MD5 || - algo == (int)CHECKSUM_ALGO_XXH3 || algo == (int)CHECKSUM_ALGO_XXH128; + algo == (int)CHECKSUM_ALGO_XXH3 || algo == (int)CHECKSUM_ALGO_XXH128 || + algo == (int)CHECKSUM_ALGO_MD4 || algo == (int)CHECKSUM_ALGO_SHA1 || + algo == (int)CHECKSUM_ALGO_NONE; } uint8_t checksum_digest_len(ChecksumAlgo algo) { @@ -191,7 +361,26 @@ uint8_t checksum_digest_len(ChecksumAlgo algo) { return 8; case CHECKSUM_ALGO_XXH128: case CHECKSUM_ALGO_MD5: + case CHECKSUM_ALGO_MD4: return 16; + case CHECKSUM_ALGO_SHA1: + return 20; + case CHECKSUM_ALGO_NONE: + return 0; } return 0; -} \ No newline at end of file +} + +ChecksumAlgo checksum_negotiate_default(void) { + /* rsync 3.4.1 default preference order; every entry is compiled in, so this + * resolves to xxh128. */ + static const ChecksumAlgo preference[] = { + CHECKSUM_ALGO_XXH128, CHECKSUM_ALGO_XXH3, CHECKSUM_ALGO_XXH64, CHECKSUM_ALGO_MD5, + CHECKSUM_ALGO_MD4, CHECKSUM_ALGO_SHA1, CHECKSUM_ALGO_NONE, + }; + for (size_t i = 0; i < sizeof(preference) / sizeof(preference[0]); i++) { + if (checksum_algo_valid((int)preference[i])) + return preference[i]; + } + return CHECKSUM_ALGO_XXH64; +} diff --git a/src/shared/checksum.h b/src/shared/checksum.h index fe32218..36deebc 100644 --- a/src/shared/checksum.h +++ b/src/shared/checksum.h @@ -8,25 +8,35 @@ /* Whole-file content-digest algorithms selectable with --checksum-choice and * seeded with --checksum-seed. The ids are the values actually placed on the * wire (config frame), so they must be kept stable and validated on receive. - * CHECKSUM_ALGO_XXH64 == 0 is the default and is byte-for-byte what FastSync - * computed before these options existed (xxHash64 with seed 0). The set mirrors - * the algorithms rsync 3.4.1 can be built with; the ones FastSync does not - * implement (md4, sha1, none) are rejected by name at parse time. */ + * CHECKSUM_ALGO_XXH64 == 0 is the historical FastSync default and its numeric + * value is preserved. The full set mirrors the algorithms rsync 3.4.1 can be + * built with; every one of them is implemented here. */ typedef enum { CHECKSUM_ALGO_XXH64 = 0, CHECKSUM_ALGO_MD5 = 1, CHECKSUM_ALGO_XXH3 = 2, - CHECKSUM_ALGO_XXH128 = 3 + CHECKSUM_ALGO_XXH128 = 3, + CHECKSUM_ALGO_MD4 = 4, + CHECKSUM_ALGO_SHA1 = 5, + CHECKSUM_ALGO_NONE = 6 } ChecksumAlgo; -/* xxh128 digest is 16 bytes, the longest supported. */ -#define CHECKSUM_MAX_DIGEST_LEN 16 +/* FastSync's negotiated default (rsync 3.4.1 auto-negotiates xxh128 first). + * The wire default for Config->checksum_algo is this value. */ +#define CHECKSUM_ALGO_DEFAULT CHECKSUM_ALGO_XXH128 + +/* sha1 digest is 20 bytes, the longest supported. */ +#define CHECKSUM_MAX_DIGEST_LEN 20 /* Compute the whole-file digest of the first `size` bytes of `data`. * * - CHECKSUM_ALGO_XXH64: xxHash64(data, size, seed) (full 64-bit seed). - * - CHECKSUM_ALGO_MD5: md5(data, size) via OpenSSL EVP. - * md5 has no seed, so `seed` is ignored (documented). + * - CHECKSUM_ALGO_XXH3: XXH3_64bits_withSeed(data, size, seed). + * - CHECKSUM_ALGO_XXH128: XXH3_128bits_withSeed(data, size, seed). + * - CHECKSUM_ALGO_MD5: md5(data, size) via OpenSSL EVP (seed ignored). + * - CHECKSUM_ALGO_MD4: md4(data, size), self-contained RFC 1320 (seed ignored). + * - CHECKSUM_ALGO_SHA1: sha1(data, size) via OpenSSL EVP (seed ignored). + * - CHECKSUM_ALGO_NONE: no digest; *out_len is 0 and nothing is written. * - `size == 0` hashes the empty input (plus its seed), not a NULL input. * * Writes up to `out_capacity` bytes into `out`, storing the digest length in @@ -41,10 +51,10 @@ bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, uint8_t* out, size_t out_capacity, size_t* out_len); -/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id. * Accepts - * "xxh64"/"xxhash", "xxh3", "xxh128" and "md5". "auto", rsync's default automatic choice, is - * resolved to the default by the caller (it is not a distinct algorithm here). Returns -1 for any - * name FastSync does not implement (md4/sha1/none included). */ +/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id. + * Accepts "xxh64"/"xxhash", "xxh3", "xxh128", "md5", "md4", "sha1", "none". + * "auto" is not an algorithm here; the caller resolves it to the negotiated + * default. Returns -1 for any unrecognized name. */ int checksum_algo_from_name(const char* name); /* Canonical name of an algorithm (used in CLI error messages). */ @@ -53,7 +63,13 @@ const char* checksum_algo_name(ChecksumAlgo algo); /* True when `algo` is a supported id (used by config receive validation). */ bool checksum_algo_valid(int algo); -/* Digest length in bytes for an algorithm (xxh64/xxh3 = 8, md5/xxh128 = 16). */ +/* Digest length in bytes for an algorithm (xxh64/xxh3 = 8, + * md5/md4/xxh128 = 16, sha1 = 20, none = 0). */ uint8_t checksum_digest_len(ChecksumAlgo algo); -#endif /* CHECKSUM_H */ \ No newline at end of file +/* Pick the first algorithm from FastSync's compiled-in preference list that is + * supported on this build (rsync 3.4.1's `--version` order: + * xxh128 xxh3 xxh64 md5 md4 sha1 none). Used to resolve "auto". */ +ChecksumAlgo checksum_negotiate_default(void); + +#endif /* CHECKSUM_H */ diff --git a/src/shared/compression.c b/src/shared/compression.c index 70a8bad..1c89fc0 100644 --- a/src/shared/compression.c +++ b/src/shared/compression.c @@ -3,12 +3,15 @@ #include "log.h" #include "protocol.h" #include +#include +#include #include #include #include #include #include #include +#include #include #define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024) @@ -29,6 +32,13 @@ "z " \ "zip zst" +/* Self-describing compressed frames: the first byte is the CompressionAlgo id. + * zlib/lz4 store the uncompressed size as a little-endian uint32 after the + * codec byte so decompression can be exactly pre-sized and bounded. */ +#define LZ4_SIZE_PREFIX_LEN 4 + +static _Atomic int g_compression_algo = COMPRESSION_ALGO_ZSTD; + /* Case-insensitive match of a bare suffix (no leading dot) against a * space-separated suffix list. */ static bool suffix_in_list(const char* name, const char* list) { @@ -67,6 +77,75 @@ bool compression_should_skip_with_suffixes(const char* path, char* const* suffix return false; } +CompressionAlgo compression_default_algo(void) { + return COMPRESSION_ALGO_ZSTD; +} + +int compression_algo_from_name(const char* name) { + if (!name) + return -1; + if (strcasecmp(name, "zstd") == 0) + return (int)COMPRESSION_ALGO_ZSTD; + if (strcasecmp(name, "lz4") == 0) + return (int)COMPRESSION_ALGO_LZ4; + if (strcasecmp(name, "zlib") == 0) + return (int)COMPRESSION_ALGO_ZLIB; + if (strcasecmp(name, "zlibx") == 0) + return (int)COMPRESSION_ALGO_ZLIBX; + if (strcasecmp(name, "none") == 0) + return (int)COMPRESSION_ALGO_NONE; + return -1; +} + +const char* compression_algo_name(CompressionAlgo algo) { + switch (algo) { + case COMPRESSION_ALGO_NONE: + return "none"; + case COMPRESSION_ALGO_ZSTD: + return "zstd"; + case COMPRESSION_ALGO_LZ4: + return "lz4"; + case COMPRESSION_ALGO_ZLIB: + return "zlib"; + case COMPRESSION_ALGO_ZLIBX: + return "zlibx"; + } + return ""; +} + +bool compression_algo_valid(int algo) { + return algo == (int)COMPRESSION_ALGO_NONE || algo == (int)COMPRESSION_ALGO_ZSTD || + algo == (int)COMPRESSION_ALGO_LZ4 || algo == (int)COMPRESSION_ALGO_ZLIB || + algo == (int)COMPRESSION_ALGO_ZLIBX; +} + +bool compression_algo_enabled(CompressionAlgo algo) { + return algo != COMPRESSION_ALGO_NONE; +} + +CompressionAlgo compression_negotiate_default(void) { + /* rsync 3.4.1 default preference order; every entry is compiled in, so this + * resolves to zstd. */ + static const CompressionAlgo preference[] = { + COMPRESSION_ALGO_ZSTD, COMPRESSION_ALGO_LZ4, COMPRESSION_ALGO_ZLIBX, + COMPRESSION_ALGO_ZLIB, COMPRESSION_ALGO_NONE, + }; + for (size_t i = 0; i < sizeof(preference) / sizeof(preference[0]); i++) { + if (compression_algo_valid((int)preference[i])) + return preference[i]; + } + return COMPRESSION_ALGO_ZSTD; +} + +void compression_set_algo(CompressionAlgo algo) { + if (compression_algo_valid((int)algo)) + atomic_store(&g_compression_algo, (int)algo); +} + +CompressionAlgo compression_get_algo(void) { + return (CompressionAlgo)atomic_load(&g_compression_algo); +} + /* Per-thread cache of zstd contexts plus the grow-only compression scratch * buffer. zstd contexts are stateful and not safe to share between threads, * so each thread keeps its own (see compression_get_thread_ctx). The cache is @@ -157,17 +236,25 @@ static void compression_ctx_put(CompressionThreadCtx* ctx) { compression_ctx_free(ctx); } -Data* data_compress(Data* data_to_compress, int compression_level) { - return data_compress_with_threads(data_to_compress, compression_level, 0); +/* Build a frame consisting of a copy of `src` prefixed by `codec`. */ +static Data* frame_with_codec(const void* src, size_t size, CompressionAlgo codec) { + if (size > SIZE_MAX - 1) + return NULL; + Data* out = data_create_empty(size + 1); + if (!out) + return NULL; + ((uint8_t*)out->data)[0] = (uint8_t)codec; + if (size > 0) + memcpy((uint8_t*)out->data + 1, src, size); + out->size = size + 1; + return out; } -Data* data_compress_with_threads(Data* data_to_compress, int compression_level, - int compression_threads) { - if (!data_to_compress || (!data_to_compress->data && data_to_compress->size != 0) || - compression_threads < 0 || compression_threads > COMPRESSION_MAX_THREADS) +static Data* zstd_compress(Data* in, int compression_level, int compression_threads) { + size_t dst_size = ZSTD_compressBound(in->size); + if (dst_size > SIZE_MAX - 1) return NULL; - log_message(LOG_LEVEL_DEBUG, "Starting to compress data"); - size_t dst_size = ZSTD_compressBound(data_to_compress->size); + dst_size += 1; /* codec prefix */ CompressionThreadCtx* ctx = compression_get_thread_ctx(); if (ctx == NULL) { @@ -218,7 +305,7 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level, if (available_threads > 0) { /* Streaming compression needs the source size before threaded mode can end a frame. */ - size_t zret = ZSTD_CCtx_setPledgedSrcSize(ctx->cctx, data_to_compress->size); + size_t zret = ZSTD_CCtx_setPledgedSrcSize(ctx->cctx, in->size); if (ZSTD_isError(zret)) { log_message(LOG_LEVEL_ERROR, "Failed to set compression source size: %s", ZSTD_getErrorName(zret)); @@ -236,8 +323,8 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level, ctx->out_cap = dst_size; } - ZSTD_inBuffer input = {data_to_compress->data, data_to_compress->size, 0}; - ZSTD_outBuffer output = {ctx->out_buf, dst_size, 0}; + ZSTD_inBuffer input = {in->data, in->size, 0}; + ZSTD_outBuffer output = {(uint8_t*)ctx->out_buf + 1, dst_size - 1, 0}; size_t ret; do { @@ -250,30 +337,192 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level, /* Hand off an exactly-sized copy; the scratch buffer stays cached so the next * call does not reallocate a ZSTD_compressBound-sized block. */ - compressed_data = data_create_empty(output.pos); + compressed_data = data_create_empty(output.pos + 1); if (compressed_data == NULL) { log_message(LOG_LEVEL_ERROR, "Failed to allocate compressed data"); goto cleanup; } + ((uint8_t*)compressed_data->data)[0] = (uint8_t)COMPRESSION_ALGO_ZSTD; if (output.pos > 0) - memcpy(compressed_data->data, ctx->out_buf, output.pos); - compressed_data->size = output.pos; + memcpy((uint8_t*)compressed_data->data + 1, (uint8_t*)ctx->out_buf + 1, output.pos); + compressed_data->size = output.pos + 1; - log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu", - data_to_compress->size, compressed_data->size); + log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu", in->size, + compressed_data->size); cleanup: compression_ctx_put(ctx); return compressed_data; } -Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) { - if (!compressed_data || (!compressed_data->data && compressed_data->size != 0) || - maximum_size == 0) +static Data* lz4_compress(Data* in) { + int bound = LZ4_compressBound((int)in->size); + if (bound < 0 || in->size > (size_t)INT_MAX) return NULL; + Data* out = data_create_empty((size_t)bound + 1 + LZ4_SIZE_PREFIX_LEN); + if (!out) + return NULL; + uint32_t raw_size = (uint32_t)in->size; + uint8_t* p = (uint8_t*)out->data; + p[0] = (uint8_t)COMPRESSION_ALGO_LZ4; + for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++) + p[1 + i] = (uint8_t)((raw_size >> (8 * i)) & 0xff); + int written = 0; + if (in->size > 0) { + written = LZ4_compress_default((const char*)in->data, (char*)p + 1 + LZ4_SIZE_PREFIX_LEN, + (int)in->size, bound); + if (written <= 0) { + data_destroy(out); + return NULL; + } + } + out->size = (size_t)written + 1 + LZ4_SIZE_PREFIX_LEN; + return out; +} + +static Data* zlib_compress(Data* in, CompressionAlgo algo, int compression_level) { + int level = compression_level; + if (level < 1) + level = Z_DEFAULT_COMPRESSION; + if (level > 9) + level = 9; + uLong bound = compressBound((uLong)in->size); + if (in->size > (size_t)ULONG_MAX) + return NULL; + Data* out = data_create_empty((size_t)bound + 1 + LZ4_SIZE_PREFIX_LEN); + if (!out) + return NULL; + uint32_t raw_size = (uint32_t)in->size; + uint8_t* p = (uint8_t*)out->data; + p[0] = (uint8_t)algo; + for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++) + p[1 + i] = (uint8_t)((raw_size >> (8 * i)) & 0xff); + uLongf dest_len = bound; + int rc = compress2(p + 1 + LZ4_SIZE_PREFIX_LEN, &dest_len, (const Bytef*)in->data, + (uLong)in->size, level); + if (rc != Z_OK) { + data_destroy(out); + return NULL; + } + out->size = (size_t)dest_len + 1 + LZ4_SIZE_PREFIX_LEN; + return out; +} + +Data* data_compress_codec(Data* data_to_compress, CompressionAlgo algo, int compression_level, + int compression_threads) { + if (!data_to_compress || (!data_to_compress->data && data_to_compress->size != 0) || + compression_threads < 0 || compression_threads > COMPRESSION_MAX_THREADS) + return NULL; + if (!compression_algo_valid((int)algo)) + return NULL; + log_message(LOG_LEVEL_DEBUG, "Starting to compress data"); + switch (algo) { + case COMPRESSION_ALGO_NONE: + return frame_with_codec(data_to_compress->data, data_to_compress->size, COMPRESSION_ALGO_NONE); + case COMPRESSION_ALGO_ZSTD: + return zstd_compress(data_to_compress, compression_level, compression_threads); + case COMPRESSION_ALGO_LZ4: + return lz4_compress(data_to_compress); + case COMPRESSION_ALGO_ZLIB: + case COMPRESSION_ALGO_ZLIBX: + return zlib_compress(data_to_compress, algo, compression_level); + } + return NULL; +} + +Data* data_compress_with_threads(Data* data_to_compress, int compression_level, + int compression_threads) { + return data_compress_codec(data_to_compress, compression_get_algo(), compression_level, + compression_threads); +} + +Data* data_compress(Data* data_to_compress, int compression_level) { + return data_compress_codec(data_to_compress, compression_get_algo(), compression_level, 0); +} + +static Data* decompress_none(const Data* compressed_data, size_t maximum_size) { + size_t size = compressed_data->size - 1; + if (size > maximum_size) + return NULL; + Data* out = data_create_empty(size); + if (!out) + return NULL; + if (size > 0) + memcpy(out->data, (const uint8_t*)compressed_data->data + 1, size); + out->size = size; + return out; +} + +/* Read the 4-byte little-endian raw size stored after the codec byte. */ +static bool read_raw_size(const Data* in, uint32_t* raw_size) { + if (in->size < 1 + LZ4_SIZE_PREFIX_LEN) + return false; + const uint8_t* p = (const uint8_t*)in->data; + uint32_t v = 0; + for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++) + v |= (uint32_t)p[1 + i] << (8 * i); + *raw_size = v; + return true; +} + +static Data* lz4_decompress(Data* compressed_data, size_t maximum_size, size_t hard_limit) { + uint32_t raw_size = 0; + if (!read_raw_size(compressed_data, &raw_size)) + return NULL; + if (raw_size > hard_limit || raw_size > maximum_size) + return NULL; + size_t comp_size = compressed_data->size - 1 - LZ4_SIZE_PREFIX_LEN; + Data* out = data_create_empty(raw_size); + if (!out) + return NULL; + if (raw_size == 0) { + out->size = 0; + return out; + } + int rc = LZ4_decompress_safe((const char*)compressed_data->data + 1 + LZ4_SIZE_PREFIX_LEN, + (char*)out->data, (int)comp_size, (int)raw_size); + if (rc < 0 || (uint32_t)rc != raw_size) { + log_message(LOG_LEVEL_ERROR, "LZ4 decompression failed"); + data_destroy(out); + return NULL; + } + out->size = raw_size; + return out; +} + +static Data* zlib_decompress(Data* compressed_data, size_t maximum_size, size_t hard_limit) { + uint32_t raw_size = 0; + if (!read_raw_size(compressed_data, &raw_size)) + return NULL; + if (raw_size > hard_limit || raw_size > maximum_size) + return NULL; + size_t comp_size = compressed_data->size - 1 - LZ4_SIZE_PREFIX_LEN; + Data* out = data_create_empty(raw_size); + if (!out) + return NULL; + if (raw_size == 0) { + out->size = 0; + return out; + } + uLongf dest_len = raw_size; + int rc = + uncompress((Bytef*)out->data, &dest_len, + (const Bytef*)compressed_data->data + 1 + LZ4_SIZE_PREFIX_LEN, (uLong)comp_size); + if (rc != Z_OK || dest_len != raw_size) { + log_message(LOG_LEVEL_ERROR, "zlib decompression failed"); + data_destroy(out); + return NULL; + } + out->size = raw_size; + return out; +} + +static Data* zstd_decompress(Data* compressed_data, size_t maximum_size) { + /* The zstd frame starts after the codec byte. */ + const void* frame = (const uint8_t*)compressed_data->data + 1; + size_t frame_size = compressed_data->size - 1; log_debug_message(LOG_DEBUG_UTIL, "Start to decompress data"); - unsigned long long dst_size = - ZSTD_getFrameContentSize(compressed_data->data, compressed_data->size); + unsigned long long dst_size = ZSTD_getFrameContentSize(frame, frame_size); /* ZSTD_isError() is also true for ZSTD_CONTENTSIZE_ERROR and * ZSTD_CONTENTSIZE_UNKNOWN (both are encoded near (size_t)-1), so test the * sentinels explicitly instead of blanket-rejecting every error-ish value: @@ -287,9 +536,9 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) { // ZSTD_CONTENTSIZE_UNKNOWN (~2^64) can cause massive allocation; // fall back to a conservative estimate (3x compressed size) when unknown. if (dst_size == ZSTD_CONTENTSIZE_UNKNOWN) { - if (compressed_data->size > ULLONG_MAX / 3) + if (frame_size > ULLONG_MAX / 3) return NULL; - dst_size = compressed_data->size * 3; + dst_size = frame_size * 3; if (dst_size < INITIAL_DECOMPRESS_BUF_SIZE) dst_size = INITIAL_DECOMPRESS_BUF_SIZE; } @@ -326,7 +575,7 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) { goto cleanup; } - ZSTD_inBuffer input = {compressed_data->data, compressed_data->size, 0}; + ZSTD_inBuffer input = {frame, frame_size, 0}; ZSTD_outBuffer output = {uncompressed_data->data, buf_size, 0}; size_t ret; @@ -385,6 +634,31 @@ cleanup: return uncompressed_data; } +Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) { + if (!compressed_data || (!compressed_data->data && compressed_data->size != 0) || + maximum_size == 0) + return NULL; + if (compressed_data->size < 1) + return NULL; + unsigned long long hard_limit = + maximum_size < MAX_DECOMPRESSED_SIZE ? maximum_size : MAX_DECOMPRESSED_SIZE; + uint8_t codec = ((const uint8_t*)compressed_data->data)[0]; + if (!compression_algo_valid(codec)) + return NULL; + switch ((CompressionAlgo)codec) { + case COMPRESSION_ALGO_NONE: + return decompress_none(compressed_data, (size_t)hard_limit); + case COMPRESSION_ALGO_ZSTD: + return zstd_decompress(compressed_data, (size_t)hard_limit); + case COMPRESSION_ALGO_LZ4: + return lz4_decompress(compressed_data, maximum_size, (size_t)hard_limit); + case COMPRESSION_ALGO_ZLIB: + case COMPRESSION_ALGO_ZLIBX: + return zlib_decompress(compressed_data, maximum_size, (size_t)hard_limit); + } + return NULL; +} + Data* data_decompress(Data* compressed_data) { return data_decompress_limited(compressed_data, MAX_DECOMPRESSED_SIZE); } diff --git a/src/shared/compression.h b/src/shared/compression.h index 2c3753c..ff5bbb5 100644 --- a/src/shared/compression.h +++ b/src/shared/compression.h @@ -6,11 +6,61 @@ #define COMPRESSION_MAX_THREADS 64 +/* Compression algorithms selectable with --compress-choice / -z. The ids are + * the values placed on the wire (Config->compression_algo), so they must be + * kept stable. NONE is "no compression"; ZSTD is the historical FastSync + * default and the negotiated "auto" choice. ZLIBX is rsync's zlib-without- + * matched-data variant: FastSync compresses only the delta/token bytes (it does + * not put matched file data in the compression stream), so its zlib codec is + * already the "x" form and zlib/zlibx share the same implementation, recorded + * under distinct ids. */ +typedef enum { + COMPRESSION_ALGO_NONE = 0, + COMPRESSION_ALGO_ZSTD = 1, + COMPRESSION_ALGO_LZ4 = 2, + COMPRESSION_ALGO_ZLIB = 3, + COMPRESSION_ALGO_ZLIBX = 4 +} CompressionAlgo; + +CompressionAlgo compression_default_algo(void); + +/* Resolve a --compress-choice string (case-insensitive) to an algorithm id. + * Accepts "zstd", "lz4", "zlib", "zlibx", "none". "auto" is not an algorithm + * here; the caller resolves it to the negotiated default. Returns -1 for any + * unrecognized name. */ +int compression_algo_from_name(const char* name); +const char* compression_algo_name(CompressionAlgo algo); +bool compression_algo_valid(int algo); + +/* Pick the first algorithm from FastSync's compiled-in preference list + * (rsync 3.4.1's `--version` order: zstd lz4 zlibx zlib none). Resolves + * "auto". */ +CompressionAlgo compression_negotiate_default(void); + +/* True when the algorithm actually compresses (i.e. is not NONE). */ +bool compression_algo_enabled(CompressionAlgo algo); + +/* Select the process-wide codec used by the legacy wrappers below. Each + * process serves exactly one transfer config (the server forks per connection, + * the client configures itself before spawning transfer threads), so a + * process-global default is sufficient and constant for the lifetime of a + * transfer. Defaults to ZSTD when never set. Thread-safe. */ +void compression_set_algo(CompressionAlgo algo); +CompressionAlgo compression_get_algo(void); + +/* Codec-aware primitives. The compressed buffer is self-describing: its first + * byte is the CompressionAlgo id, so decompression never needs the codec passed + * separately (this keeps every existing Decompress call site source-compatible). + * `data_compress_codec` returns NULL on invalid input or an unsupported codec. */ +Data* data_compress_codec(Data* data_to_compress, CompressionAlgo algo, int compression_level, + int compression_threads); +Data* data_decompress_limited(Data* compressed_data, size_t maximum_size); + +/* Legacy zstd-default wrappers retained for existing callers/tests. */ Data* data_compress(Data* data_to_compress, int compression_level); Data* data_compress_with_threads(Data* data_to_compress, int compression_level, int compression_threads); Data* data_decompress(Data* compressed_data); -Data* data_decompress_limited(Data* compressed_data, size_t maximum_size); bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count); /* Release the calling thread's cached zstd contexts (compressor, decompressor diff --git a/src/shared/config.c b/src/shared/config.c index 0a0b09d..6adc800 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -67,6 +68,8 @@ static void config_set_defaults(Config* config) { config->human_readable = false; config->ignore_errors = false; config->ignore_missing_args = false; + config->checksum_transfer_algo = CHECKSUM_ALGO_DEFAULT; + config->cli_exit_code = 0; config->filters = NULL; config->files_from = NULL; config->files_from_set = NULL; @@ -196,12 +199,13 @@ static bool validate_received_config(const Config* config) { valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) && valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) && valid_wire_bool(config->dry_run) && checksum_algo_valid(config->checksum_algo) && - identity_wire_valid(config) && valid_wire_bool(config->preserve_atimes) && - valid_wire_bool(config->preserve_crtimes) && valid_wire_bool(config->omit_dir_times) && - valid_wire_bool(config->omit_link_times) && valid_wire_bool(config->preserve_perms) && - valid_wire_bool(config->preserve_times) && valid_wire_bool(config->preserve_owner) && - valid_wire_bool(config->preserve_group) && valid_wire_bool(config->munge_links) && - valid_wire_bool(config->keep_dirlinks) && valid_wire_bool(config->fake_super) && + compression_algo_valid(config->compression_algo) && identity_wire_valid(config) && + valid_wire_bool(config->preserve_atimes) && valid_wire_bool(config->preserve_crtimes) && + valid_wire_bool(config->omit_dir_times) && valid_wire_bool(config->omit_link_times) && + valid_wire_bool(config->preserve_perms) && valid_wire_bool(config->preserve_times) && + valid_wire_bool(config->preserve_owner) && valid_wire_bool(config->preserve_group) && + valid_wire_bool(config->munge_links) && valid_wire_bool(config->keep_dirlinks) && + valid_wire_bool(config->fake_super) && (!config->copy_as_set || (config->copy_as_uid >= 0 && config->copy_as_gid >= 0)) && (!config->use_compression || (config->compression_level >= 1 && config->compression_level <= 22)) && @@ -894,6 +898,14 @@ static bool config_receive_checksum_algo(int fd, int* value) { return true; } +static bool config_receive_compression_algo(int fd, int* value) { + int algo; + if (!receive_int(fd, &algo) || !compression_algo_valid(algo)) + return false; + *value = algo; + return true; +} + static bool config_receive_super_mode(int fd, SuperMode* value) { int mode; if (!receive_int(fd, &mode) || mode < SUPER_MODE_AUTO || mode > SUPER_MODE_OFF) @@ -1095,6 +1107,9 @@ fail: #define CONFIG_SEND_INT_CHECKSUM_ALGO(name) send_int(fd, c->name) #define CONFIG_RECV_INT_CHECKSUM_ALGO(name) config_receive_checksum_algo(fd, &c->name) +#define CONFIG_SEND_INT_COMPRESSION_ALGO(name) send_int(fd, c->name) +#define CONFIG_RECV_INT_COMPRESSION_ALGO(name) config_receive_compression_algo(fd, &c->name) + #define CONFIG_SEND_SUPERMODE(name) send_int(fd, (int)c->name) #define CONFIG_RECV_SUPERMODE(name) config_receive_super_mode(fd, &c->name) @@ -1170,6 +1185,7 @@ CONFIG_DEFINE_SEND(send_iconv_spec, CONFIG_WIRE_ICONV_FIELDS) CONFIG_DEFINE_SEND(send_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS) CONFIG_DEFINE_SEND(send_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS) CONFIG_DEFINE_SEND(send_output_options, CONFIG_WIRE_OUTPUT_FIELDS) +CONFIG_DEFINE_SEND(send_codec_options, CONFIG_WIRE_CODEC_FIELDS) CONFIG_DEFINE_RECV(receive_core_fields, CONFIG_WIRE_CORE_FIELDS) CONFIG_DEFINE_RECV(receive_delta_fields, CONFIG_WIRE_DELTA_FIELDS) @@ -1189,6 +1205,7 @@ CONFIG_DEFINE_RECV(receive_iconv_spec, CONFIG_WIRE_ICONV_FIELDS) CONFIG_DEFINE_RECV(receive_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS) CONFIG_DEFINE_RECV(receive_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS) CONFIG_DEFINE_RECV(receive_output_options, CONFIG_WIRE_OUTPUT_FIELDS) +CONFIG_DEFINE_RECV(receive_codec_options, CONFIG_WIRE_CODEC_FIELDS) #undef XSEND #undef XRECV @@ -1306,7 +1323,8 @@ bool config_send_wire_block(int file_descriptor, const Config* config) { send_iconv_spec(file_descriptor, config) && send_privilege_options(file_descriptor, config) && send_copy_as_options(file_descriptor, config) && - send_output_options(file_descriptor, config); + send_output_options(file_descriptor, config) && + send_codec_options(file_descriptor, config); } bool config_send(int file_descriptor, const Config* config) { @@ -1377,29 +1395,59 @@ Config* config_receive_with_validate(int file_descriptor, ConfigValidateFunc val !receive_iconv_spec(file_descriptor, config, &budget) || !receive_privilege_options(file_descriptor, config, &budget) || !receive_copy_as_options(file_descriptor, config, &budget) || - !receive_output_options(file_descriptor, config, &budget)) + !receive_output_options(file_descriptor, config, &budget) || + !receive_codec_options(file_descriptor, config, &budget)) goto error; - if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 && - strcmp(config->compress_choice, "none") != 0 && - strcmp(config->compress_choice, "auto") != 0) { - char* escaped_choice = output_escape(config->compress_choice, config->eight_bit_output); - log_message(LOG_LEVEL_ERROR, "Unsupported compression choice: %s", - escaped_choice ? escaped_choice : ""); - char detail[128]; - snprintf(detail, sizeof(detail), "unsupported compression choice: %s", - escaped_choice ? escaped_choice : ""); - send_error_detail(file_descriptor, detail); - free(escaped_choice); + /* Validate/normalize the negotiated codec. compress_choice is the human + * spelling (NULL or "" when -z was not given); compression_algo is the + * concrete codec id the sender used. They must agree, and "auto" is + * canonicalized to FastSync's negotiated default so the stored spelling is + * always concrete (a hostile/older client may still send "auto"). */ + if (config->compress_choice && config->compress_choice[0] != '\0') { + int choice_algo = compression_algo_from_name(config->compress_choice); + if (choice_algo < 0 && strcasecmp(config->compress_choice, "auto") != 0) { + char* escaped_choice = output_escape(config->compress_choice, config->eight_bit_output); + log_message(LOG_LEVEL_ERROR, "Unsupported compression choice: %s", + escaped_choice ? escaped_choice : ""); + char detail[160]; + snprintf(detail, sizeof(detail), "unsupported compression choice: %s", + escaped_choice ? escaped_choice : ""); + send_error_detail(file_descriptor, detail); + free(escaped_choice); + goto error; + } + if (choice_algo < 0) + choice_algo = (int)compression_negotiate_default(); + if (strcasecmp(config->compress_choice, "auto") == 0 || + choice_algo == (int)COMPRESSION_ALGO_NONE) { + const char* canonical = compression_algo_name((CompressionAlgo)choice_algo); + char* dup = str_dup(canonical); + if (!dup) + goto error; + free(config->compress_choice); + config->compress_choice = dup; + } + if (config->compression_algo != choice_algo) { + log_message(LOG_LEVEL_ERROR, "Compression choice '%s' does not match codec id %d", + config->compress_choice, config->compression_algo); + send_error_detail(file_descriptor, "compression choice/codec mismatch"); + goto error; + } + } + /* The concrete codec must exist only when compression is on. A client that + * left -z off has no codec in effect, but the field keeps whatever id it + * carried (the receiver never dispatches on it without use_compression), so + * the wire value round-trips untouched. */ + if (config->use_compression && config->compression_algo == (int)COMPRESSION_ALGO_NONE) { + log_message(LOG_LEVEL_ERROR, "Compression requested with the 'none' codec"); + send_error_detail(file_descriptor, "compression requested with the none codec"); goto error; } - /* Defensive: an older/hostile client may still send "auto"; canonicalize it - to zstd (its effective choice) so the stored value is always concrete. */ - if (strcmp(config->compress_choice, "auto") == 0) { - char* canonical = str_dup("zstd"); - if (!canonical) - goto error; - free(config->compress_choice); - config->compress_choice = canonical; + /* rsync: "none" as the pre-transfer checksum is invalid with --checksum. */ + if (config->checksum && config->checksum_algo == (int)CHECKSUM_ALGO_NONE) { + log_message(LOG_LEVEL_ERROR, "Invalid checksum-choice for --checksum: none"); + send_error_detail(file_descriptor, "checksum-choice 'none' cannot be used with --checksum"); + goto error; } if (!validate_received_config(config)) { log_message(LOG_LEVEL_ERROR, "Invalid configuration received from client"); diff --git a/src/shared/config.h b/src/shared/config.h index 8e7c0fe..e9fe715 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -3,6 +3,7 @@ #include "array_list.h" #include "checksum.h" +#include "compression.h" #include #include #include @@ -81,7 +82,7 @@ typedef struct { typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode; /* =========================================================================== - * Config wire-field table (single source of truth for protocol 2.23.0). + * Config wire-field table (single source of truth for protocol 2.26.0). * * Every field below crosses the wire. The table is the ONLY place a * serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare @@ -203,7 +204,7 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF #define CONFIG_WIRE_FUZZY_FIELDS(X) X(fuzzy, bool, false, BOOL) #define CONFIG_WIRE_CHECKSUM_FIELDS(X) \ - X(checksum_algo, int, CHECKSUM_ALGO_XXH64, INT_CHECKSUM_ALGO) \ + X(checksum_algo, int, CHECKSUM_ALGO_DEFAULT, INT_CHECKSUM_ALGO) \ X(checksum_seed, uint64_t, 0, RAW) #define CONFIG_WIRE_IDENTITY_FIELDS(X) \ @@ -262,6 +263,27 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF #define CONFIG_WIRE_OUTPUT_FIELDS(X) \ X(report_dest_info, bool, false, BOOL) X(report_stats, bool, false, BOOL) +/* Codec-negotiation wave (protocol 2.26.0). compression_algo is the concrete + * codec the client selected for this transfer (a CompressionAlgo id) and is the + * value the receiver validates and installs. It is the resolved result of + * --compress-choice / the "auto" negotiation so both peers agree exactly. + * + * Negotiation model: FastSync enforces a strict same-version handshake, so both + * peers carry the identical compiled-in codec set. The client resolves the + * effective algorithm deterministically and serializes it here; "auto" picks + * the first entry of the rsync 3.4.1 preference order + * (compression: zstd lz4 zlibx zlib none; checksum: xxh128 xxh3 xxh64 md5 md4 + * sha1 none), and an explicit request wins. The receiver rejects (before + * STATUS_OK) any algorithm outside its own supported set, which is rsync's + * "no common choice is an error" behavior. The same resolver runs on both + * sides (compression_negotiate_default / checksum_negotiate_default), so the + * fallback is consistent. + * + * The field is appended after the output block so every pre-2.26 field keeps + * its wire position. */ +#define CONFIG_WIRE_CODEC_FIELDS(X) \ + X(compression_algo, int, COMPRESSION_ALGO_ZSTD, INT_COMPRESSION_ALGO) + /* All serialized fields, in exact wire order. Concatenating the per-segment * lists here is what keeps the declaration order = the wire order. */ #define CONFIG_WIRE_FIELDS(X) \ @@ -283,7 +305,8 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF CONFIG_WIRE_ICONV_FIELDS(X) \ CONFIG_WIRE_PRIVILEGE_FIELDS(X) \ CONFIG_WIRE_COPY_AS_FIELDS(X) \ - CONFIG_WIRE_OUTPUT_FIELDS(X) + CONFIG_WIRE_OUTPUT_FIELDS(X) \ + CONFIG_WIRE_CODEC_FIELDS(X) typedef struct Config { /* -j/--threads=N: number of parallel scanner worker threads for the -m @@ -378,6 +401,16 @@ typedef struct Config { * enters the keep-set. Implied by --delete-missing-args. */ bool ignore_missing_args; + /* Codec-negotiation CLI state (all client-only, never serialized). The + * effective pre-transfer checksum is Config->checksum_algo (serialized); + * checksum_transfer_algo is the rsync "transfer" half of a two-name + * --checksum-choice form (validated and used only to mirror rsync's + * whole-file forcing, since FastSync's per-block strong hash is fixed). + * cli_exit_code carries a parser-requested process exit status (rsync uses 4 + * for an unsupported checksum/compress algorithm) so main() can mirror it. */ + int checksum_transfer_algo; + int cli_exit_code; + // Issue #129: Advanced file selection. These fields are CLIENT-ONLY: they are // never serialized to the wire (the receiver must not learn them). ArrayList* filters; /* --filter=RULE rule strings, in order */ @@ -941,9 +974,21 @@ typedef struct Config { * would have removed without deleting anything. The config frame gains one * trailing report_stats bool and the receiver emits a new STATUS_STATS frame * (carrying matched data, the deleted-file count and the would-delete path - * list) immediately before its terminal success status. Both a config-frame - * layout change and a frame-sequence change, hence the bump. */ -#define PROTOCOL_VERSION "2.25.0" + * list) immediately before its terminal success status. + * + * (6) Codec breadth + negotiation (protocol 2.26.0): the config frame gains one + * trailing int, compression_algo (a CompressionAlgo id), appended after the + * output block. It is the negotiated/effective compression codec and is what + * the receiver's self-describing decompressor validates against its own + * supported set. The checksum_algo wire value now also accepts md4/sha1/none, + * and its default changes to the rsync 3.4.1 auto-negotiated xxh128. + * + * Any config-frame layout change must bump the protocol version: a peer that + * does not parse the new trailing bytes would desynchronize on the frame + * boundary, and the strict same-version handshake (config_receive rejects a + * mismatched version before parsing anything else) keeps mixed deployments from + * ever reaching that state. */ +#define PROTOCOL_VERSION "2.26.0" #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) /* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */ #define MAX_BASIS_DIRS 64 diff --git a/tests/integration/test_codecs.py b/tests/integration/test_codecs.py new file mode 100644 index 0000000..0e0dfbe --- /dev/null +++ b/tests/integration/test_codecs.py @@ -0,0 +1,218 @@ +"""Differential tests for --checksum-choice / --compress-choice against rsync 3.4.1. + +These pin the accepted/rejected algorithm matrix and exit codes to real rsync, +and verify that every codec FastSync now offers still transfers byte-exactly. +The rsync-based tests skip cleanly when rsync is not installed. + +The FastSync server confines transfers to its authorized root (the project +directory when the shared test server is launched), so every scratch tree lives +under ``TEST_DATA_DIR`` rather than pytest's ``tmp_path``. +""" +import os +import shutil +import subprocess +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(__file__)) +from common import ( + TEST_DATA_DIR, + run_client, + clean_dir, + get_dest_received_dir, +) + +RSYNC = shutil.which("rsync") +requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed") + +CHECKSUM_NAMES = ["xxh128", "xxh3", "xxh64", "md5", "md4", "sha1"] +COMPRESS_NAMES = ["zstd", "lz4", "zlib", "zlibx"] + +CODEC_ROOT = os.path.join(TEST_DATA_DIR, "codec_differential") + + +def _rsync(args): + env = dict(os.environ, LC_ALL="C") + return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120) + + +def _scratch(tag): + """A confined, uniquely named scratch directory under the project tree.""" + path = os.path.join(CODEC_ROOT, tag) + clean_dir(path) + os.makedirs(path, exist_ok=True) + return path + + +def _make_corpus(root): + clean_dir(root) + os.makedirs(os.path.join(root, "sub"), exist_ok=True) + # Highly compressible payload so each codec is actually exercised. + with open(os.path.join(root, "big.bin"), "wb") as fh: + fh.write(b"FastSync codec payload " * 4096) + with open(os.path.join(root, "sub", "text.txt"), "wb") as fh: + fh.write(b"hello codec world\n" * 128) + with open(os.path.join(root, "empty"), "wb"): + pass + return root + + +def _tree_bytes(root): + out = {} + for dirpath, _dirs, files in os.walk(root): + for name in files: + path = os.path.join(dirpath, name) + with open(path, "rb") as fh: + out[os.path.relpath(path, root)] = fh.read() + return out + + +class TestCodecChoiceMatrix: + """The CLI accept/reject set and exit codes must match rsync 3.4.1.""" + + @requires_rsync + @pytest.mark.ci + @pytest.mark.parametrize("name", CHECKSUM_NAMES) + def test_checksum_names_accepted_by_both(self, name, shared_server): + src = _make_corpus(_scratch(f"cc_src_{name}")) + rdst = _scratch(f"cc_rsync_{name}") + rsync_result = _rsync(["-a", f"--cc={name}", src + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + + fdst = _scratch(f"cc_fs_{name}") + result, _ = run_client(src, fdst, flags=[f"--cc={name}"], port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + + @requires_rsync + @pytest.mark.ci + @pytest.mark.parametrize("name", COMPRESS_NAMES) + def test_compress_names_accepted_by_both(self, name, shared_server): + src = _make_corpus(_scratch(f"zc_src_{name}")) + rdst = _scratch(f"zc_rsync_{name}") + rsync_result = _rsync(["-az", f"--zc={name}", src + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + + fdst = _scratch(f"zc_fs_{name}") + result, _ = run_client(src, fdst, flags=["-z", f"--zc={name}"], port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + + @requires_rsync + @pytest.mark.ci + @pytest.mark.parametrize("choice", ["md4,sha1", "sha1,md4", "auto,md5", "none,md5"]) + def test_checksum_two_name_accepted_by_both(self, choice, shared_server): + tag = choice.replace(",", "_") + src = _make_corpus(_scratch(f"two_src_{tag}")) + rdst = _scratch(f"two_rsync_{tag}") + rsync_result = _rsync(["-a", "--checksum", f"--cc={choice}", src + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + + fdst = _scratch(f"two_fs_{tag}") + result, _ = run_client(src, fdst, flags=["--checksum", f"--cc={choice}"], + port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:200] + + @requires_rsync + @pytest.mark.ci + @pytest.mark.parametrize("name", ["sha256", "crc32", "md5,", "md4,md5,sha1"]) + def test_unknown_checksum_rejected_exit_4_both(self, name, shared_server): + src = _make_corpus(_scratch(f"badcc_src_{name.replace(',', '_').replace(':', '_')}")) + rdst = _scratch(f"badcc_rsync_{name.replace(',', '_').replace(':', '_')}") + rsync_result = _rsync(["-a", f"--cc={name}", src + "/", rdst + "/"]) + assert rsync_result.returncode == 4, rsync_result.stderr + + fdst = _scratch(f"badcc_fs_{name.replace(',', '_').replace(':', '_')}") + result, _ = run_client(src, fdst, flags=[f"--cc={name}"], port=shared_server.port) + assert result.returncode == 4, (result.stderr or result.stdout)[:200] + + @requires_rsync + @pytest.mark.ci + @pytest.mark.parametrize("choice", ["none", "md5,none"]) + def test_checksum_none_with_checksum_rejected_exit_4_both(self, choice, shared_server): + tag = choice.replace(",", "_") + src = _make_corpus(_scratch(f"nonecc_src_{tag}")) + rdst = _scratch(f"nonecc_rsync_{tag}") + rsync_result = _rsync(["-a", "--checksum", f"--cc={choice}", src + "/", rdst + "/"]) + assert rsync_result.returncode == 4, rsync_result.stderr + + fdst = _scratch(f"nonecc_fs_{tag}") + result, _ = run_client(src, fdst, flags=["--checksum", f"--cc={choice}"], + port=shared_server.port) + assert result.returncode == 4, (result.stderr or result.stdout)[:200] + + @requires_rsync + @pytest.mark.ci + @pytest.mark.parametrize("name", ["bogus", "zstd,lz4"]) + def test_unknown_compress_rejected_exit_4_both(self, name, shared_server): + tag = name.replace(",", "_") + src = _make_corpus(_scratch(f"badzc_src_{tag}")) + rdst = _scratch(f"badzc_rsync_{tag}") + rsync_result = _rsync(["-az", f"--zc={name}", src + "/", rdst + "/"]) + assert rsync_result.returncode == 4, rsync_result.stderr + + fdst = _scratch(f"badzc_fs_{tag}") + result, _ = run_client(src, fdst, flags=["-z", f"--zc={name}"], port=shared_server.port) + assert result.returncode == 4, (result.stderr or result.stdout)[:200] + + +class TestCodecTransferDifferential: + """Each codec lands the same bytes rsync lands.""" + + @requires_rsync + @pytest.mark.ci + @pytest.mark.parametrize("name", COMPRESS_NAMES + ["none"]) + def test_compress_codec_matches_rsync_bytes(self, name, shared_server): + src = _make_corpus(_scratch(f"byteszc_src_{name}")) + rsync_dst = _scratch(f"byteszc_rsync_{name}") + rsync_result = _rsync(["-a", "-z", f"--zc={name}", src + "/", rsync_dst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + + fs_dst = _scratch(f"byteszc_fs_{name}") + result, _ = run_client(src, fs_dst, flags=["-a", "-z", f"--zc={name}"], + port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + received = get_dest_received_dir(fs_dst, src) + assert _tree_bytes(received) == _tree_bytes(rsync_dst) + + @requires_rsync + @pytest.mark.ci + @pytest.mark.parametrize("name", CHECKSUM_NAMES) + def test_checksum_codec_matches_rsync_bytes(self, name, shared_server): + src = _make_corpus(_scratch(f"bytescc_src_{name}")) + rsync_dst = _scratch(f"bytescc_rsync_{name}") + rsync_result = _rsync(["-a", "--checksum", f"--cc={name}", src + "/", rsync_dst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + + fs_dst = _scratch(f"bytescc_fs_{name}") + result, _ = run_client(src, fs_dst, flags=["-a", "--checksum", f"--cc={name}"], + port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + received = get_dest_received_dir(fs_dst, src) + assert _tree_bytes(received) == _tree_bytes(rsync_dst) + + +class TestCodecNegotiationFallback: + """FastSync's auto negotiation and deterministic fallback order.""" + + @pytest.mark.ci + def test_default_checksum_and_compression_agree(self, shared_server): + """A default transfer (auto on both peers) succeeds; the negotiated + default is xxh128 + zstd.""" + src = _make_corpus(_scratch("auto_src")) + fdst = _scratch("auto_fs") + result, _ = run_client(src, fdst, flags=["-a", "-z"], port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + received = get_dest_received_dir(fdst, src) + assert _tree_bytes(received) == _tree_bytes(src) + + @pytest.mark.ci + def test_explicit_choice_overrides_auto(self, shared_server): + """An explicit --zc/--cc wins over the negotiated default on both ends, + so the receiver decodes with the sender's codec.""" + src = _make_corpus(_scratch("explicit_src")) + fdst = _scratch("explicit_fs") + result, _ = run_client(src, fdst, flags=["-a", "-z", "--zc=lz4", "--cc=sha1"], + port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + received = get_dest_received_dir(fdst, src) + assert _tree_bytes(received) == _tree_bytes(src) diff --git a/tests/integration/test_fault_injection.py b/tests/integration/test_fault_injection.py index 1c2fd22..80d6a8d 100644 --- a/tests/integration/test_fault_injection.py +++ b/tests/integration/test_fault_injection.py @@ -36,7 +36,7 @@ from common import ( # noqa: E402 verify_transfer, ) -PROTOCOL_VERSION = b"2.25.0" +PROTOCOL_VERSION = b"2.26.0" STATUS_MANIFEST = 5 STATUS_OK = 0 diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index 2a382d0..2f9e686 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -1491,20 +1491,101 @@ class TestChecksumChoice: assert fh.read() == b"same content\n" @pytest.mark.ci - def test_checksum_choice_md4_single_name_rejected(self, shared_server): - for bad in ("md4", "sha1", "none", "xxh64,md5"): + @pytest.mark.parametrize("algo", ["xxh128", "xxh3", "xxh64", "md5", "md4", "sha1"]) + def test_checksum_choice_all_algorithms_transfer(self, shared_server, algo): + """Every rsync 3.4.1 checksum algorithm is accepted and transfers + byte-exactly. 'none' is covered separately (it needs no digest).""" + clean_dir(DEST_DIR) + flags = ["--preserve", "--incremental", "--checksum", f"--checksum-choice={algo}"] + result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port) + assert result.returncode == 0, \ + f"checksum-choice={algo} failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) + mismatches, missing = verify_transfer(SOURCE_DIR, received) + assert not missing, f"Missing: {missing}" + assert not mismatches, f"Mismatch: {mismatches}" + + @pytest.mark.ci + def test_checksum_choice_two_name_form(self, shared_server): + """The rsync 'TRANSFER,PRE-TRANSFER' form is accepted; FastSync uses the + second (pre-transfer) algorithm for its whole-file digest.""" + clean_dir(DEST_DIR) + flags = ["--preserve", "--incremental", "--checksum", "--cc=md4,sha1"] + result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port) + assert result.returncode == 0, \ + f"two-name --cc=md4,sha1 failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) + mismatches, missing = verify_transfer(SOURCE_DIR, received) + assert not missing and not mismatches, f"missing={missing} mismatches={mismatches}" + + @pytest.mark.ci + def test_checksum_choice_none_accepted_without_checksum(self, shared_server): + clean_dir(DEST_DIR) + result, _ = run_client(SOURCE_DIR, DEST_DIR, + flags=["--preserve", "--incremental", "--cc=none"], + port=shared_server.port) + assert result.returncode == 0, \ + f"--cc=none failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) + mismatches, missing = verify_transfer(SOURCE_DIR, received) + assert not missing and not mismatches, f"missing={missing} mismatches={mismatches}" + + @pytest.mark.ci + def test_checksum_choice_none_rejected_with_checksum(self, shared_server): + """rsync rejects 'none' as the pre-transfer checksum with --checksum and + exits 4; mirror both the rejection and the exit code.""" + for choice in ("none", "md5,none"): + result, _ = run_client(SOURCE_DIR, DEST_DIR, + flags=["--checksum", f"--cc={choice}"], + port=shared_server.port) + assert result.returncode == 4, \ + f"--cc={choice} --checksum must exit 4, got {result.returncode}: " \ + f"{(result.stderr or result.stdout)[:200]}" + + @pytest.mark.ci + def test_checksum_choice_unknown_rejected_exit_4(self, shared_server): + for bad in ("sha256", "bogus", "md5,", "md4,md5,sha1"): result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=[f"--checksum-choice={bad}"], port=shared_server.port) - assert result.returncode != 0, f"{bad} must be rejected" + assert result.returncode == 4, \ + f"--checksum-choice={bad} must exit 4, got {result.returncode}" @pytest.mark.ci - def test_compress_choice_unsupported_rejected(self, shared_server): - for bad in ("lz4", "zlib", "zlibx"): + @pytest.mark.parametrize("algo", ["zstd", "lz4", "zlib", "zlibx"]) + def test_compress_choice_all_algorithms_transfer(self, shared_server, algo): + """Every rsync 3.4.1 compression codec is accepted and transfers + byte-exactly through its own codec.""" + clean_dir(DEST_DIR) + flags = ["-z", f"--compress-choice={algo}"] + result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port) + assert result.returncode == 0, \ + f"--compress-choice={algo} failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) + mismatches, missing = verify_transfer(SOURCE_DIR, received) + assert not missing, f"Missing: {missing}" + assert not mismatches, f"Mismatch: {mismatches}" + + @pytest.mark.ci + def test_compress_choice_unknown_rejected_exit_4(self, shared_server): + for bad in ("bogus", "zstd,lz4", ""): result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=[f"--compress-choice={bad}"], port=shared_server.port) - assert result.returncode != 0, f"{bad} must be rejected" + assert result.returncode == 4, \ + f"--compress-choice={bad} must exit 4, got {result.returncode}" + + @pytest.mark.ci + def test_compress_choice_none_disables_compression(self, shared_server): + clean_dir(DEST_DIR) + result, _ = run_client(SOURCE_DIR, DEST_DIR, + flags=["-z", "--compress-choice=none"], + port=shared_server.port) + assert result.returncode == 0, \ + f"--compress-choice=none failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) + mismatches, missing = verify_transfer(SOURCE_DIR, received) + assert not missing and not mismatches, f"missing={missing} mismatches={mismatches}" @pytest.mark.ci def test_compress_choice_auto_transfers(self, shared_server): diff --git a/tests/integration/test_preflight.py b/tests/integration/test_preflight.py index 01aeaaf..b78c808 100644 --- a/tests/integration/test_preflight.py +++ b/tests/integration/test_preflight.py @@ -94,14 +94,14 @@ def _seed_protocol_source(source): class TestProtocol: @pytest.mark.ci def test_protocol_current_version_accepted(self, shared_server): - """--protocol=2.25.0 (the current PROTOCOL_VERSION) is accepted and the + """--protocol=2.26.0 (the current PROTOCOL_VERSION) is accepted and the transfer completes normally.""" source = os.path.join(TEST_DATA_DIR, "proto_ok_src") dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst") shutil.rmtree(dest, ignore_errors=True) os.makedirs(dest) _seed_protocol_source(source) - result, _ = run_client(source, dest, flags=["--protocol=2.25.0"], + result, _ = run_client(source, dest, flags=["--protocol=2.26.0"], port=shared_server.port) assert result.returncode == 0, \ f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}" diff --git a/tests/test_checksum.c b/tests/test_checksum.c index 3c37e1b..bf781ae 100644 --- a/tests/test_checksum.c +++ b/tests/test_checksum.c @@ -91,6 +91,63 @@ static void test_checksum_md5_seed_ignored() { EXPECT_TRUE(memcmp(a, b, alen) == 0); } +static void test_checksum_md4_vectors() { + uint8_t out[CHECKSUM_MAX_DIGEST_LEN]; + size_t len = 0; + /* RFC 1320 / RFC 1321 test vectors. */ + EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_MD4, 0, "", 0, out, sizeof(out), &len)); + EXPECT_TRUE(len == (size_t)16); + const uint8_t expect_empty[16] = {0x31, 0xd6, 0xcf, 0xe0, 0xd1, 0x6a, 0xe9, 0x31, + 0xb7, 0x3c, 0x59, 0xd7, 0xe0, 0xc0, 0x89, 0xc0}; + EXPECT_TRUE(memcmp(out, expect_empty, 16) == 0); + + EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_MD4, 0, "abc", 3, out, sizeof(out), &len)); + const uint8_t expect_abc[16] = {0xa4, 0x48, 0x01, 0x7a, 0xaf, 0x21, 0xd8, 0x52, + 0x5f, 0xc1, 0x0a, 0xe8, 0x7a, 0xa6, 0x72, 0x9d}; + EXPECT_TRUE(memcmp(out, expect_abc, 16) == 0); + + /* A longer input exercises the block loop and the padding boundary. */ + const char* msg = + "12345678901234567890123456789012345678901234567890123456789012345678901234567890"; + EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_MD4, 0, msg, strlen(msg), out, sizeof(out), &len)); + const uint8_t expect_long[16] = {0xe3, 0x3b, 0x4d, 0xdc, 0x9c, 0x38, 0xf2, 0x19, + 0x9c, 0x3e, 0x7b, 0x16, 0x4f, 0xcc, 0x05, 0x36}; + EXPECT_TRUE(memcmp(out, expect_long, 16) == 0); +} + +static void test_checksum_sha1_vectors() { + uint8_t out[CHECKSUM_MAX_DIGEST_LEN]; + size_t len = 0; + EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_SHA1, 0, "abc", 3, out, sizeof(out), &len)); + EXPECT_TRUE(len == (size_t)20); + const uint8_t expect_abc[20] = {0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 0x81, 0x6a, 0xba, 0x3e, + 0x25, 0x71, 0x78, 0x50, 0xc2, 0x6c, 0x9c, 0xd0, 0xd8, 0x9d}; + EXPECT_TRUE(memcmp(out, expect_abc, 20) == 0); + + EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_SHA1, 0, "", 0, out, sizeof(out), &len)); + EXPECT_TRUE(len == (size_t)20); + const uint8_t expect_empty[20] = {0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, + 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09}; + EXPECT_TRUE(memcmp(out, expect_empty, 20) == 0); + + /* sha1 has no seed: the digest is seed-independent (documented). */ + uint8_t seeded[CHECKSUM_MAX_DIGEST_LEN]; + size_t seeded_len = 0; + EXPECT_TRUE( + checksum_digest(CHECKSUM_ALGO_SHA1, 12345, "abc", 3, seeded, sizeof(seeded), &seeded_len)); + EXPECT_TRUE(seeded_len == (size_t)20); + EXPECT_TRUE(memcmp(expect_abc, seeded, 20) == 0); +} + +/* "none" is a successful no-digest: length 0, nothing written. */ +static void test_checksum_none_digest() { + uint8_t out[CHECKSUM_MAX_DIGEST_LEN]; + size_t len = 99; + EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_NONE, 0, "data", 4, out, sizeof(out), &len)); + EXPECT_EQ_INT((int)len, 0); + EXPECT_EQ_INT((int)checksum_digest_len(CHECKSUM_ALGO_NONE), 0); +} + static void test_checksum_algo_name_mapping() { EXPECT_EQ_INT(checksum_algo_from_name("xxh64"), (int)CHECKSUM_ALGO_XXH64); EXPECT_EQ_INT(checksum_algo_from_name("XXH64"), (int)CHECKSUM_ALGO_XXH64); @@ -102,12 +159,14 @@ static void test_checksum_algo_name_mapping() { EXPECT_EQ_INT(checksum_algo_from_name("XXH3"), (int)CHECKSUM_ALGO_XXH3); EXPECT_EQ_INT(checksum_algo_from_name("xxh128"), (int)CHECKSUM_ALGO_XXH128); EXPECT_EQ_INT(checksum_algo_from_name("XXH128"), (int)CHECKSUM_ALGO_XXH128); - /* rsync choices FastSync does not implement are rejected by name. */ - EXPECT_TRUE(checksum_algo_from_name("md4") < 0); - EXPECT_TRUE(checksum_algo_from_name("sha1") < 0); + EXPECT_EQ_INT(checksum_algo_from_name("md4"), (int)CHECKSUM_ALGO_MD4); + EXPECT_EQ_INT(checksum_algo_from_name("MD4"), (int)CHECKSUM_ALGO_MD4); + EXPECT_EQ_INT(checksum_algo_from_name("sha1"), (int)CHECKSUM_ALGO_SHA1); + EXPECT_EQ_INT(checksum_algo_from_name("SHA1"), (int)CHECKSUM_ALGO_SHA1); + EXPECT_EQ_INT(checksum_algo_from_name("none"), (int)CHECKSUM_ALGO_NONE); + /* Names rsync does not offer (or FastSync cannot compute) are rejected. */ EXPECT_TRUE(checksum_algo_from_name("sha256") < 0); EXPECT_TRUE(checksum_algo_from_name("crc32") < 0); - EXPECT_TRUE(checksum_algo_from_name("none") < 0); EXPECT_TRUE(checksum_algo_from_name("") < 0); EXPECT_TRUE(checksum_algo_from_name(NULL) < 0); @@ -115,11 +174,20 @@ static void test_checksum_algo_name_mapping() { EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_MD5)); EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_XXH3)); EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_XXH128)); + EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_MD4)); + EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_SHA1)); + EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_NONE)); EXPECT_FALSE(checksum_algo_valid(99)); EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_XXH64), "xxh64"); EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_MD5), "md5"); EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_XXH3), "xxh3"); EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_XXH128), "xxh128"); + EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_MD4), "md4"); + EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_SHA1), "sha1"); + EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_NONE), "none"); + + /* rsync 3.4.1 auto-negotiates xxh128 first. */ + EXPECT_EQ_INT((int)checksum_negotiate_default(), (int)CHECKSUM_ALGO_XXH128); } /* xxh3 is 8 bytes and seed-aware; xxh128 is 16 bytes and differs from both @@ -168,6 +236,9 @@ void test_checksum(void) { test_checksum_xxh64_seed_changes_digest(); test_checksum_xxh64_seed_deterministic(); test_checksum_md5_vectors(); + test_checksum_md4_vectors(); + test_checksum_sha1_vectors(); + test_checksum_none_digest(); test_checksum_algo_lengths_distinct(); test_checksum_md5_seed_ignored(); test_checksum_algo_name_mapping(); diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index ace49fb..d3ef21a 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -317,7 +317,7 @@ static void test_parse_args_protocol_accept_current() { Config* cfg = valid_client_config(); EXPECT_NOT_NULL(cfg); char* argv_equals[] = {"fastsync", "--source-dir", "/src", - "--dest-dir", "/dst", "--protocol=2.25.0"}; + "--dest-dir", "/dst", "--protocol=2.26.0"}; int positional_args[2]; int positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 6, argv_equals, positional_args, &positional_count), 0); @@ -327,7 +327,7 @@ static void test_parse_args_protocol_accept_current() { cfg = valid_client_config(); EXPECT_NOT_NULL(cfg); char* argv_space[] = {"fastsync", "--source-dir", "/src", "--dest-dir", - "/dst", "--protocol", "2.25.0"}; + "/dst", "--protocol", "2.26.0"}; positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 7, argv_space, positional_args, &positional_count), 0); EXPECT_EQ_STR(cfg->version, PROTOCOL_VERSION); @@ -1509,22 +1509,25 @@ static void test_parse_args_checksum_choice_equals_forms() { config_delete(cfg); } -/* An algorithm FastSync does not support must be rejected, never a silent - no-op. */ +/* An algorithm FastSync does not support, an empty half, a lone/extra comma or + a malformed separator must be rejected, never a silent no-op. A single + md4/sha1/none name and the two-name transfer,pre-transfer form are valid. */ static void test_parse_args_checksum_choice_rejects_unsupported() { - static const char* const bad[] = {"md4", "sha1", "sha256", "crc32", - "none", "bogus", "xxh64,md5", "xxhash:md5"}; + static const char* const bad[] = {"sha256", "crc32", "bogus", "xxhash:md5", + "md5,", ",md5", "md5,md4,sha1"}; for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) { Config* cfg = config_create(); char* argv[] = {"fastsync", "--checksum-choice", (char*)bad[i], "/src", "/dst"}; int positional_args[2]; int positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1); + EXPECT_EQ_INT(cfg->cli_exit_code, 4); config_delete(cfg); } } -/* xxh3/xxh128 are accepted; "auto" keeps the default algorithm. */ +/* xxh3/xxh128/md4/sha1/none and the two-name form are accepted; "auto" + resolves to FastSync's negotiated default xxh128. */ static void test_parse_args_checksum_choice_new_algos() { Config* cfg = config_create(); char* argv[] = {"fastsync", "--checksum-choice=xxh3", "/src", "/dst"}; @@ -1545,7 +1548,69 @@ static void test_parse_args_checksum_choice_new_algos() { char* argv3[] = {"fastsync", "--checksum-choice=auto", "/src", "/dst"}; positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 4, argv3, positional_args, &positional_count), 0); - EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_XXH64); + EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_XXH128); + config_delete(cfg); + + /* A single md4/sha1 name selects it for both transfer and pre-transfer. */ + static const int single[] = {(int)CHECKSUM_ALGO_MD4, (int)CHECKSUM_ALGO_SHA1}; + static const char* const single_names[] = {"md4", "sha1"}; + for (size_t i = 0; i < 2; i++) { + cfg = config_create(); + char* arg = (char*)single_names[i]; + char* argv4[] = {"fastsync", "--cc", arg, "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv4, positional_args, &positional_count), 0); + EXPECT_EQ_INT(cfg->checksum_algo, single[i]); + EXPECT_EQ_INT(cfg->checksum_transfer_algo, single[i]); + config_delete(cfg); + } + + /* Two-name form: first is the transfer checksum, second the pre-transfer one + that FastSync actually uses. */ + cfg = config_create(); + char* argv5[] = {"fastsync", "--cc=sha1,md4", "/checksum/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv5, positional_args, &positional_count), 0); + EXPECT_EQ_INT(cfg->checksum_transfer_algo, (int)CHECKSUM_ALGO_SHA1); + EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_MD4); + config_delete(cfg); + + /* "none" is accepted without --checksum but forces --whole-file like rsync. */ + cfg = config_create(); + char* argv6[] = {"fastsync", "--cc=none", "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv6, positional_args, &positional_count), 0); + EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_NONE); + EXPECT_TRUE(cfg->whole_file); + config_delete(cfg); +} + +/* rsync rejects "none" as the pre-transfer checksum with --checksum (exit 4), + regardless of option order. */ +static void test_parse_args_checksum_none_with_checksum_rejected() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "--checksum", "--cc=none", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1); + EXPECT_EQ_INT(cfg->cli_exit_code, 4); + config_delete(cfg); + + cfg = config_create(); + char* argv2[] = {"fastsync", "--checksum", "--cc=md5,none", "/checksum/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv2, positional_args, &positional_count), -1); + EXPECT_EQ_INT(cfg->cli_exit_code, 4); + config_delete(cfg); + + /* "none" as the TRANSFER checksum with a real pre-transfer checksum is + accepted (rsync allows none,md5 with -c). */ + cfg = config_create(); + char* argv3[] = {"fastsync", "--checksum", "--cc=none,md5", "/checksum/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv3, positional_args, &positional_count), 0); + EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_MD5); + EXPECT_TRUE(cfg->whole_file); config_delete(cfg); } @@ -1604,28 +1669,40 @@ static void test_parse_args_timeout_zero_and_no_forms() { config_delete(cfg); } -/* rsync's --compress-choice choices FastSync does not implement are rejected by - * name; zstd/none/auto are accepted. */ +/* Every rsync 3.4.1 --compress-choice name is accepted and mapped to a real + * codec; "auto" resolves to the negotiated default (zstd). An unknown name is + * rejected with rsync's exit code 4. */ static void test_parse_args_compress_choice_parity() { - static const char* const good[] = {"zstd", "none", "auto"}; + struct { + const char* name; + CompressionAlgo algo; + bool enabled; + } good[] = { + {"zstd", COMPRESSION_ALGO_ZSTD, true}, {"lz4", COMPRESSION_ALGO_LZ4, true}, + {"zlib", COMPRESSION_ALGO_ZLIB, true}, {"zlibx", COMPRESSION_ALGO_ZLIBX, true}, + {"none", COMPRESSION_ALGO_NONE, false}, {"auto", COMPRESSION_ALGO_ZSTD, true}, + {"ZSTD", COMPRESSION_ALGO_ZSTD, true}, + }; for (size_t i = 0; i < sizeof(good) / sizeof(good[0]); i++) { Config* cfg = config_create(); - char* argv[] = {"fastsync", "--compress-choice", (char*)good[i], "/src", "/dst"}; + char* argv[] = {"fastsync", "--compress-choice", (char*)good[i].name, "/src", "/dst"}; int positional_args[2]; int positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); /* "auto" is normalized to the canonical "zstd" the receiver accepts. */ - EXPECT_EQ_STR(cfg->compress_choice, strcmp(good[i], "auto") == 0 ? "zstd" : good[i]); - EXPECT_EQ_INT(cfg->use_compression, strcmp(good[i], "none") != 0 ? 1 : 0); + EXPECT_EQ_STR(cfg->compress_choice, compression_algo_name(good[i].algo)); + EXPECT_EQ_INT(cfg->compression_algo, (int)good[i].algo); + EXPECT_EQ_INT(cfg->use_compression, good[i].enabled ? 1 : 0); config_delete(cfg); } - static const char* const bad[] = {"lz4", "zlib", "zlibx", "bogus"}; + static const char* const bad[] = {"bogus", "", "zstd,lz4"}; for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) { Config* cfg = config_create(); char* argv[] = {"fastsync", "--compress-choice", (char*)bad[i], "/src", "/dst"}; int positional_args[2]; int positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1); + EXPECT_EQ_INT(cfg->cli_exit_code, 4); config_delete(cfg); } } @@ -4450,6 +4527,7 @@ void test_client_cli() { test_parse_args_checksum_choice_equals_forms(); test_parse_args_checksum_choice_rejects_unsupported(); test_parse_args_checksum_choice_new_algos(); + test_parse_args_checksum_none_with_checksum_rejected(); test_parse_args_checksum_implies_incremental_only(); test_parse_args_no_whole_file(); test_parse_args_timeout_zero_and_no_forms(); diff --git a/tests/test_compression.c b/tests/test_compression.c index ebd762e..833835e 100644 --- a/tests/test_compression.c +++ b/tests/test_compression.c @@ -157,20 +157,22 @@ static void test_chunk_compress_decompress_roundtrip() { /* Build a zstd frame whose header omits the content size (the content size * flag is cleared), which ZSTD_getFrameContentSize reports as - * ZSTD_CONTENTSIZE_UNKNOWN. */ + * ZSTD_CONTENTSIZE_UNKNOWN. The frame carries the codec-id prefix the + * decompressor dispatches on. */ static Data* make_unknown_size_frame(const void* src, size_t len) { ZSTD_CCtx* cctx = ZSTD_createCCtx(); if (!cctx) return NULL; ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0); size_t cap = ZSTD_compressBound(len); - Data* out = data_create_empty(cap); + Data* out = data_create_empty(cap + 1); if (!out) { ZSTD_freeCCtx(cctx); return NULL; } + ((uint8_t*)out->data)[0] = (uint8_t)COMPRESSION_ALGO_ZSTD; ZSTD_inBuffer in = {src, len, 0}; - ZSTD_outBuffer ob = {out->data, cap, 0}; + ZSTD_outBuffer ob = {(uint8_t*)out->data + 1, cap, 0}; size_t ret; do { ret = ZSTD_compressStream2(cctx, &ob, &in, ZSTD_e_end); @@ -180,7 +182,7 @@ static Data* make_unknown_size_frame(const void* src, size_t len) { return NULL; } } while (ret > 0); - out->size = ob.pos; + out->size = ob.pos + 1; ZSTD_freeCCtx(cctx); return out; } @@ -199,8 +201,9 @@ static void test_data_decompress_unknown_size_frame() { Data* frame = make_unknown_size_frame(buf, len); free(buf); EXPECT_NOT_NULL(frame); - /* Guard the premise of the test: the frame really has no stored size. */ - EXPECT_EQ_INT((int)ZSTD_getFrameContentSize(frame->data, frame->size), + /* Guard the premise of the test: the frame (after the codec byte) really has + * no stored size. */ + EXPECT_EQ_INT((int)ZSTD_getFrameContentSize((uint8_t*)frame->data + 1, frame->size - 1), (int)ZSTD_CONTENTSIZE_UNKNOWN); Data* decompressed = data_decompress(frame); @@ -326,6 +329,89 @@ static void test_data_decompress_truncated_frame_fails() { data_destroy(input); } +/* Every codec must round-trip byte-exactly through the self-describing frame, + * including the empty and a highly compressible large payload. */ +static void codec_roundtrip(CompressionAlgo algo) { + const char* samples[] = { + "", + "Hello, World! This is test data for compression round-trip!", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }; + for (size_t s = 0; s < sizeof(samples) / sizeof(samples[0]); s++) { + size_t len = strlen(samples[s]); + Data* original = data_create_empty(len); + EXPECT_NOT_NULL(original); + if (len > 0) + memcpy(original->data, samples[s], len); + original->size = len; + + Data* compressed = data_compress_codec(original, algo, 3, 0); + EXPECT_NOT_NULL(compressed); + EXPECT_EQ_INT((int)((uint8_t*)compressed->data)[0], (int)algo); + Data* decompressed = data_decompress(compressed); + EXPECT_NOT_NULL(decompressed); + EXPECT_EQ_INT((int)decompressed->size, (int)len); + EXPECT_EQ_INT(memcmp(decompressed->data, original->data, len), 0); + data_destroy(decompressed); + data_destroy(compressed); + data_destroy(original); + } +} + +static void test_codec_roundtrips() { + codec_roundtrip(COMPRESSION_ALGO_NONE); + codec_roundtrip(COMPRESSION_ALGO_ZSTD); + codec_roundtrip(COMPRESSION_ALGO_LZ4); + codec_roundtrip(COMPRESSION_ALGO_ZLIB); + codec_roundtrip(COMPRESSION_ALGO_ZLIBX); +} + +static void test_codec_name_mapping() { + EXPECT_EQ_INT(compression_algo_from_name("zstd"), (int)COMPRESSION_ALGO_ZSTD); + EXPECT_EQ_INT(compression_algo_from_name("ZSTD"), (int)COMPRESSION_ALGO_ZSTD); + EXPECT_EQ_INT(compression_algo_from_name("lz4"), (int)COMPRESSION_ALGO_LZ4); + EXPECT_EQ_INT(compression_algo_from_name("zlib"), (int)COMPRESSION_ALGO_ZLIB); + EXPECT_EQ_INT(compression_algo_from_name("zlibx"), (int)COMPRESSION_ALGO_ZLIBX); + EXPECT_EQ_INT(compression_algo_from_name("none"), (int)COMPRESSION_ALGO_NONE); + EXPECT_TRUE(compression_algo_from_name("bogus") < 0); + EXPECT_TRUE(compression_algo_from_name(NULL) < 0); + EXPECT_TRUE(compression_algo_valid((int)COMPRESSION_ALGO_LZ4)); + EXPECT_TRUE(compression_algo_valid((int)COMPRESSION_ALGO_ZLIB)); + EXPECT_TRUE(compression_algo_valid((int)COMPRESSION_ALGO_ZLIBX)); + EXPECT_FALSE(compression_algo_valid(99)); + EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_ZSTD), "zstd"); + EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_LZ4), "lz4"); + EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_ZLIB), "zlib"); + EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_ZLIBX), "zlibx"); + EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_NONE), "none"); + /* rsync 3.4.1 auto-negotiates zstd first. */ + EXPECT_EQ_INT((int)compression_negotiate_default(), (int)COMPRESSION_ALGO_ZSTD); + EXPECT_FALSE(compression_algo_enabled(COMPRESSION_ALGO_NONE)); + EXPECT_TRUE(compression_algo_enabled(COMPRESSION_ALGO_ZSTD)); +} + +/* The process-global codec selects what the legacy wrappers produce. */ +static void test_codec_global_selection() { + Data* original = data_create_empty(64); + EXPECT_NOT_NULL(original); + memset(original->data, 'q', 64); + original->size = 64; + + compression_set_algo(COMPRESSION_ALGO_LZ4); + Data* compressed = data_compress(original, 3); + EXPECT_NOT_NULL(compressed); + EXPECT_EQ_INT((int)((uint8_t*)compressed->data)[0], (int)COMPRESSION_ALGO_LZ4); + Data* decompressed = data_decompress(compressed); + EXPECT_NOT_NULL(decompressed); + EXPECT_TRUE(memcmp(decompressed->data, original->data, 64) == 0); + data_destroy(decompressed); + data_destroy(compressed); + + /* Restore the default so later tests are unaffected. */ + compression_set_algo(COMPRESSION_ALGO_ZSTD); + data_destroy(original); +} + void test_compression() { test_data_compress_decompress_roundtrip(); test_data_compress_decompress_large(); @@ -335,4 +421,7 @@ void test_compression() { test_data_compress_with_threads_roundtrip(); test_data_compress_reused_contexts_multithreaded(); test_chunk_compress_decompress_roundtrip(); + test_codec_roundtrips(); + test_codec_name_mapping(); + test_codec_global_selection(); } diff --git a/tests/test_config.c b/tests/test_config.c index 668b7e5..cea9088 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -1368,6 +1368,61 @@ static void test_config_receive_rejects_invalid_checksum_algo() { EXPECT_FALSE(roundtrip_config_ok(c)); config_delete(c); } + +/* The negotiated codec id and the human --compress-choice spelling must agree, + * and the id itself must be a known codec. */ +static void test_config_receive_rejects_invalid_compression_algo() { + if (is_running_under_valgrind()) + return; + Config* c = config_create(); + EXPECT_NOT_NULL(c); + c->send_directory = str_dup("/src"); + c->receive_root_directory = str_dup("/dst"); + c->compression_algo = 99; + EXPECT_FALSE(roundtrip_config_ok(c)); + config_delete(c); +} + +static void test_config_receive_rejects_codec_mismatch() { + if (is_running_under_valgrind()) + return; + Config* c = config_create(); + EXPECT_NOT_NULL(c); + c->send_directory = str_dup("/src"); + c->receive_root_directory = str_dup("/dst"); + free(c->compress_choice); + c->compress_choice = str_dup("lz4"); + c->use_compression = true; + c->compression_algo = (int)COMPRESSION_ALGO_ZSTD; /* does not match lz4 */ + EXPECT_FALSE(roundtrip_config_ok(c)); + config_delete(c); +} + +static void test_config_receive_rejects_none_codec_with_compression() { + if (is_running_under_valgrind()) + return; + Config* c = config_create(); + EXPECT_NOT_NULL(c); + c->send_directory = str_dup("/src"); + c->receive_root_directory = str_dup("/dst"); + c->use_compression = true; + c->compression_algo = (int)COMPRESSION_ALGO_NONE; + EXPECT_FALSE(roundtrip_config_ok(c)); + config_delete(c); +} + +static void test_config_receive_rejects_checksum_none_with_checksum() { + if (is_running_under_valgrind()) + return; + Config* c = config_create(); + EXPECT_NOT_NULL(c); + c->send_directory = str_dup("/src"); + c->receive_root_directory = str_dup("/dst"); + c->checksum = true; + c->checksum_algo = (int)CHECKSUM_ALGO_NONE; + EXPECT_FALSE(roundtrip_config_ok(c)); + config_delete(c); +} /* The identity-mapping fields (--numeric-ids / --usermap / --groupmap / --chown) cross the config wire unchanged: the receiver needs them to apply ownership with the same policy the client requested. */ @@ -2549,6 +2604,7 @@ static bool basis_equal(const Config* a, const Config* b) { #define CONFIG_CMP_STR_MODULE(a, b, name) str_opt_equal((a)->name, (b)->name) #define CONFIG_CMP_STR_REDACTED_AUTH(a, b, name) str_opt_equal((a)->name, (b)->name) #define CONFIG_CMP_INT_CHECKSUM_ALGO(a, b, name) ((a)->name == (b)->name) +#define CONFIG_CMP_INT_COMPRESSION_ALGO(a, b, name) ((a)->name == (b)->name) #define CONFIG_CMP_SUPERMODE(a, b, name) ((a)->name == (b)->name) #define CONFIG_CMP_INT_IDENTITY(a, b, name) ((a)->name == (b)->name) #define CONFIG_CMP_INT_SKIPCOUNT(a, b, name) ((a)->name == (b)->name) @@ -2775,14 +2831,14 @@ static void golden_config_populate(Config* c) { c->copy_as_gid = 222; } -/* The pinned golden frame (protocol 2.25.0). The values below are the only +/* The pinned golden frame (protocol 2.26.0). The values below are the only * thing that ties the generated table to the historical wire format; update * them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.24.0 - * per-directory delete-plan wave changed only the version string in the config - * frame; the 2.25.0 wire-stats wave appends one report_stats bool. The + * delete-plan wave changed only the version string; 2.25.0 appended the + * report_stats bool and 2.26.0 appended the compression_algo int. The * byte-exact values are recomputed for the merged layout. */ -#define GOLDEN_WIRE_LEN 701 -#define GOLDEN_WIRE_HASH 16170466870400670271ULL +#define GOLDEN_WIRE_LEN 705 +#define GOLDEN_WIRE_HASH 4673424031554175633ULL static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) { unsigned long long h = 1469598103934665603ULL; @@ -3159,6 +3215,10 @@ void test_config() { test_config_basis_normalization(); test_config_checksum_options_wire_roundtrip(); test_config_receive_rejects_invalid_checksum_algo(); + test_config_receive_rejects_invalid_compression_algo(); + test_config_receive_rejects_codec_mismatch(); + test_config_receive_rejects_none_codec_with_compression(); + test_config_receive_rejects_checksum_none_with_checksum(); test_config_identity_wire_roundtrip(); test_config_receive_rejects_invalid_identity(); test_config_metadata_times_wire_roundtrip(); diff --git a/tests/test_fuzz_smoke.c b/tests/test_fuzz_smoke.c index 279ff76..ca5d38c 100644 --- a/tests/test_fuzz_smoke.c +++ b/tests/test_fuzz_smoke.c @@ -18,10 +18,10 @@ /* P8 config-frame tail: super_mode (4) + copy-as presence (4) + uid (4) + gid (4). */ #define P8_TAIL_BYTES 16 -/* Protocol 2.25.0 appends two trailing bools (report_dest_info, report_stats) - * AFTER the P8 tail, so the P8 fields sit this many bytes before the end of the - * frame. */ -#define OUTPUT_TAIL_BYTES 8 +/* Bytes after the P8 tail: report_dest_info (4), report_stats (4, wire-stats + * wave) and compression_algo (4, codec wave). The P8 fields sit this many + * bytes before the end of the frame. */ +#define POST_P8_TAIL_BYTES 12 /* Smoke test for chunk_deserialize fuzz target */ static void test_fuzz_chunk_deserialize() { @@ -324,7 +324,7 @@ static void test_fuzz_config_receive_p8_tail() { size_t len = 0; bool captured = capture_config_frame(c, &frame, &len); config_delete(c); - if (!captured || len <= P8_TAIL_BYTES) { + if (!captured || len <= P8_TAIL_BYTES + POST_P8_TAIL_BYTES) { free(frame); EXPECT_TRUE(false); return; @@ -338,31 +338,31 @@ static void test_fuzz_config_receive_p8_tail() { /* super_mode outside the 0..2 tri-state is refused. */ memcpy(mut, frame, len); - put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, 99); + put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES, 99); EXPECT_FALSE(receive_config_frame(mut, len)); - put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, -1); + put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES, -1); EXPECT_FALSE(receive_config_frame(mut, len)); /* A negative (sentinel) and an extreme copy-as uid/gid are refused. */ memcpy(mut, frame, len); - put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, SUPER_MODE_AUTO); - put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 4, 1); - put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 8, -1); - put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 12, 0); + put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES, SUPER_MODE_AUTO); + put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 4, 1); + put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 8, -1); + put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 12, 0); EXPECT_FALSE(receive_config_frame(mut, len)); - put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 8, 0); - put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 12, INT32_MIN); + put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 8, 0); + put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 12, INT32_MIN); EXPECT_FALSE(receive_config_frame(mut, len)); /* A presence int that is not a wire bool is refused. */ memcpy(mut, frame, len); - put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, SUPER_MODE_AUTO); - put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 4, 2); + put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES, SUPER_MODE_AUTO); + put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 4, 2); EXPECT_FALSE(receive_config_frame(mut, len)); /* Truncating anywhere inside the P8 tail is refused. */ EXPECT_FALSE(receive_config_frame(frame, len - 2)); - EXPECT_FALSE(receive_config_frame(frame, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES)); + EXPECT_FALSE(receive_config_frame(frame, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES)); free(mut); free(frame); diff --git a/tests/test_server.c b/tests/test_server.c index f4f7fc8..da4134b 100644 --- a/tests/test_server.c +++ b/tests/test_server.c @@ -1072,10 +1072,10 @@ static void test_incremental_check_basis_fifo_does_not_hang() { EXPECT_TRUE(send_n_data(p[1], &mtime, sizeof(mtime))); EXPECT_TRUE(send_n_data(p[1], &mtime_nsec, sizeof(mtime_nsec))); /* config_has_basis() makes the request carry the source digest. */ - uint8_t wire_len = 8; - uint8_t digest[8] = {0}; + uint8_t wire_len = checksum_digest_len((ChecksumAlgo)cfg->checksum_algo); + uint8_t digest[CHECKSUM_MAX_DIGEST_LEN] = {0}; EXPECT_TRUE(send_n_data(p[1], &wire_len, sizeof(wire_len))); - EXPECT_TRUE(send_n_data(p[1], digest, sizeof(digest))); + EXPECT_TRUE(send_n_data(p[1], digest, wire_len)); Status s; EXPECT_TRUE(receive_status(p[1], &s)); EXPECT_EQ_INT(s, STATUS_NEXT);