feat(codecs): per-codec level defaults, RSYNC_*_LIST auto, zlibx reclassify

- rsync 3.4.1 per-codec defaults (zstd 3, zlib/zlibx 6, lz4 level ignored)
  and per-codec clamping; explicit --zl still wins
- auto resolves via whitespace-separated RSYNC_COMPRESS_LIST /
  RSYNC_CHECKSUM_LIST (first supported wins; all-unknown exits 4)
- zlibx reclassified: FastSync's zlib stream already excludes matched data,
  so its tree/stdout/exit match zlib
- --compress/-z and --compress-choice flip to parity (113/12/32)
This commit is contained in:
2026-09-19 13:14:48 +02:00
parent c80098623f
commit eff9852038
17 changed files with 686 additions and 25 deletions
+213
View File
@@ -0,0 +1,213 @@
"""Differential tests for the client CLI's codec defaults and env lists.
Track 3a of the rsync-parity plan pins two rsync 3.4.1 behaviors that are
resolved entirely on the client:
* the per-codec default ``--compress-level`` (zstd 3, zlib/zlibx 6, lz4
ignored) applied when the user omits ``--compress-level``/``--zl``, with an
explicit level clamped to the codec's range; and
* the ``RSYNC_COMPRESS_LIST`` / ``RSYNC_CHECKSUM_LIST`` preference lists that
rsync's ``auto`` consults before its compiled-in order (whitespace-separated,
unknown names skipped, first supported wins, all-unknown is exit 4).
The rsync side is observed through ``--debug=NSTR1``; FastSync publishes its
resolved codec/level through ``--debug=util``. The checksum side is confirmed
byte-for-byte through ``--out-format %C``. The rsync-based tests skip cleanly
when rsync is not installed.
"""
import os
import re
import shutil
import subprocess
import sys
import pytest
sys.path.insert(0, os.path.dirname(__file__))
from common import (
TEST_DATA_DIR,
run_client,
clean_dir,
get_dest_received_dir,
)
RSYNC = shutil.which("rsync")
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
CODEC_ROOT = os.path.join(TEST_DATA_DIR, "cli_differential")
_COMPRESS_RE = re.compile(r"compress(?:ion)?: (\w+) \(level (-?\d+)\)")
def _rsync(args):
env = dict(os.environ, LC_ALL="C")
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120)
def _scratch(tag):
path = os.path.join(CODEC_ROOT, tag)
clean_dir(path)
os.makedirs(path, exist_ok=True)
return path
def _make_corpus(root):
clean_dir(root)
os.makedirs(root, exist_ok=True)
with open(os.path.join(root, "big.bin"), "wb") as fh:
fh.write(b"FastSync codec payload " * 4096)
with open(os.path.join(root, "small.txt"), "wb") as fh:
fh.write(b"hello codec world\n" * 32)
return root
def _rsync_compress_level(choice, level):
src = _make_corpus(_scratch(f"lvl_src_{choice}_{level}"))
dst = _scratch(f"lvl_rsync_{choice}_{level}")
args = ["-a", "-z", f"--zc={choice}"]
if level is not None:
args.append(f"--zl={level}")
args += ["--debug=NSTR1", src + "/", dst + "/"]
result = _rsync(args)
assert result.returncode == 0, result.stderr
match = _COMPRESS_RE.search(result.stdout + result.stderr)
assert match, (result.stdout, result.stderr)
return match.group(1), int(match.group(2))
def _fastsync_compress_level(choice, level, shared_server):
src = _make_corpus(_scratch(f"lvl_src_fs_{choice}_{level}"))
dst = _scratch(f"lvl_fs_{choice}_{level}")
args = ["-a", "-z", f"--zc={choice}"]
if level is not None:
args.append(f"--zl={level}")
args += ["-v", "--debug=util"]
result, _ = run_client(src, dst, flags=args, port=shared_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
match = _COMPRESS_RE.search(result.stdout)
assert match, result.stdout[:500]
return match.group(1), int(match.group(2))
class TestPerCodecCompressionLevelDefaults:
"""``--compress-level`` defaults and clamping match rsync per codec."""
# FastSync uses a positive lz4 placeholder because its "level > 0" gate
# enables compression; lz4_compress ignores the value, so rsync's level 0
# and FastSync's level 1 produce the same bytes.
CASES = [
("zstd", None, 3),
("zlib", None, 6),
("zlibx", None, 6),
("lz4", None, 1),
("zstd", 10, 10),
("zlib", 15, 9),
("zlib", 3, 3),
("lz4", 15, 1),
]
@requires_rsync
@pytest.mark.ci
@pytest.mark.parametrize("choice,level,fs_level", CASES)
def test_level_matches_rsync(self, choice, level, fs_level, shared_server):
rsync_algo, rsync_level = _rsync_compress_level(choice, level)
fs_algo, fs_level_actual = _fastsync_compress_level(choice, level, shared_server)
assert rsync_algo == choice
assert fs_algo == choice
if choice == "lz4":
assert rsync_level == 0 and fs_level_actual > 0
else:
assert rsync_level == fs_level
assert fs_level_actual == fs_level
class TestEnvPreferenceLists:
"""``RSYNC_COMPRESS_LIST`` / ``RSYNC_CHECKSUM_LIST`` drive auto like rsync."""
# (env value, expected codec, rsync level, FastSync level)
COMPRESS_CASES = [
("zlib lz4", "zlib", 6, 6),
("lz4 zstd", "lz4", 0, 1),
("bogus zstd zlib", "zstd", 3, 3),
(" ", "zstd", 3, 3),
]
@requires_rsync
@pytest.mark.ci
@pytest.mark.parametrize("env,algo,rsync_level,fs_level", COMPRESS_CASES)
def test_compress_list_matches_rsync(self, env, algo, rsync_level, fs_level, shared_server,
monkeypatch):
monkeypatch.setenv("RSYNC_COMPRESS_LIST", env)
src = _make_corpus(_scratch(f"envc_src_{algo}"))
rdst = _scratch(f"envc_rsync_{algo}")
rs = _rsync(["-a", "-z", "--debug=NSTR1", src + "/", rdst + "/"])
assert rs.returncode == 0, rs.stderr
rm = _COMPRESS_RE.search(rs.stdout + rs.stderr)
assert rm, (rs.stdout, rs.stderr)
assert rm.group(1) == algo
assert int(rm.group(2)) == rsync_level
fdst = _scratch(f"envc_fs_{algo}")
result, _ = run_client(src, fdst, flags=["-a", "-z", "-v", "--debug=util"],
port=shared_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
fm = _COMPRESS_RE.search(result.stdout)
assert fm, result.stdout[:500]
assert fm.group(1) == algo
assert int(fm.group(2)) == fs_level
received = get_dest_received_dir(fdst, src)
assert _tree_bytes(received) == _tree_bytes(src)
@requires_rsync
@pytest.mark.ci
@pytest.mark.parametrize("env,algo", [("md5", "md5"), ("sha1", "sha1"), ("xxh3 md5", "xxh3")])
def test_checksum_list_matches_rsync(self, env, algo, shared_server, monkeypatch):
monkeypatch.setenv("RSYNC_CHECKSUM_LIST", env)
src = _make_corpus(_scratch(f"envcc_src_{algo}"))
rdst = _scratch(f"envcc_rsync_{algo}")
rs = _rsync(["-a", "--checksum", "--out-format=%C %n", src + "/", rdst + "/"])
assert rs.returncode == 0, rs.stderr
rs_digests = _digests(rs.stdout)
fdst = _scratch(f"envcc_fs_{algo}")
result, _ = run_client(src, fdst, flags=["-a", "--checksum", "--out-format=%C %n"],
port=shared_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
assert _digests(result.stdout) == rs_digests
@requires_rsync
@pytest.mark.ci
def test_all_unknown_lists_fail_like_rsync(self, shared_server, monkeypatch):
src = _make_corpus(_scratch("envbad_src"))
monkeypatch.setenv("RSYNC_COMPRESS_LIST", "bogus")
rs = _rsync(["-a", "-z", src + "/", _scratch("envbad_rsync_c") + "/"])
assert rs.returncode == 4, rs.stderr
result, _ = run_client(src, _scratch("envbad_fs_c"), flags=["-a", "-z"],
port=shared_server.port)
assert result.returncode == 4, (result.stderr or result.stdout)[:200]
monkeypatch.setenv("RSYNC_CHECKSUM_LIST", "bogus")
rs = _rsync(["-a", src + "/", _scratch("envbad_rsync_s") + "/"])
assert rs.returncode == 4, rs.stderr
result, _ = run_client(src, _scratch("envbad_fs_s"), flags=["-a"],
port=shared_server.port)
assert result.returncode == 4, (result.stderr or result.stdout)[:200]
def _tree_bytes(root):
out = {}
for dirpath, _dirs, files in os.walk(root):
for name in files:
path = os.path.join(dirpath, name)
with open(path, "rb") as fh:
out[os.path.relpath(path, root)] = fh.read()
return out
def _digests(output):
out = {}
for line in output.splitlines():
parts = line.split()
if len(parts) == 2 and parts[0]:
out[parts[1]] = parts[0]
return out
+6
View File
@@ -43,6 +43,7 @@
#include "test_utils.h"
#include "test_xattr.h"
#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
// Define global test state variables
@@ -52,6 +53,11 @@ bool current_test_failed = false;
int main() {
signal(SIGPIPE, SIG_IGN);
/* The codec/checksum resolvers consult rsync's preference-list environment
* variables; clear them so a developer's shell cannot change test outcomes.
* The env-specific tests set and restore their own values. */
unsetenv("RSYNC_COMPRESS_LIST");
unsetenv("RSYNC_CHECKSUM_LIST");
printf("\033[1;36m=== RUNNING UNIT TESTS ===\033[0m\n\n");
RUN_TEST(test_queue);
+23
View File
@@ -265,6 +265,28 @@ static void test_checksum_digest_file_matches_oneshot(void) {
free(data);
}
/* RSYNC_CHECKSUM_LIST precedence, syntax and fallback. */
static void test_checksum_choice_env_list() {
unsetenv("RSYNC_CHECKSUM_LIST");
EXPECT_EQ_INT(checksum_choice_resolve(), (int)CHECKSUM_ALGO_XXH128);
EXPECT_EQ_INT((int)checksum_negotiate_default(), (int)CHECKSUM_ALGO_XXH128);
setenv("RSYNC_CHECKSUM_LIST", "bogus md5 xxh3", 1);
EXPECT_EQ_INT(checksum_choice_resolve(), (int)CHECKSUM_ALGO_MD5);
setenv("RSYNC_CHECKSUM_LIST", "SHA1", 1);
EXPECT_EQ_INT(checksum_choice_resolve(), (int)CHECKSUM_ALGO_SHA1);
/* Whitespace-separated only: comma is not a separator in rsync's syntax. */
setenv("RSYNC_CHECKSUM_LIST", "md5,xxh3", 1);
EXPECT_EQ_INT(checksum_choice_resolve(), -1);
setenv("RSYNC_CHECKSUM_LIST", " ", 1);
EXPECT_EQ_INT(checksum_choice_resolve(), (int)CHECKSUM_ALGO_XXH128);
unsetenv("RSYNC_CHECKSUM_LIST");
}
void test_checksum(void) {
test_checksum_xxh64_seed0();
test_checksum_xxh64_empty();
@@ -281,4 +303,5 @@ void test_checksum(void) {
test_checksum_truncated_buffer_rejected();
test_checksum_null_empty_digest();
test_checksum_digest_file_matches_oneshot();
test_checksum_choice_env_list();
}
+116
View File
@@ -2431,6 +2431,119 @@ static void test_parse_args_rejects_invalid_compression_choice() {
config_delete(cfg);
}
/* rsync gives each codec its own default --compress-level; an omitted level
* resolves to that default and an explicit one is clamped to the codec range. */
static void test_parse_args_per_codec_compression_level_defaults() {
unsetenv("RSYNC_COMPRESS_LIST");
struct {
const char* choice;
int level;
} cases[] = {
{"zstd", 3},
{"zlib", 6},
{"zlibx", 6},
{"lz4", 1},
};
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", "-z", "--compress-choice", (char*)cases[i].choice, "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 6, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->use_compression);
EXPECT_EQ_INT(cfg->compression_level, cases[i].level);
config_delete(cfg);
}
/* Bare -z resolves to the zstd default. */
Config* cfg = config_create();
char* bare[] = {"fastsync", "-z", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, bare, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->compression_level, 3);
config_delete(cfg);
/* An explicit level wins unchanged for zstd... */
cfg = config_create();
char* zv[] = {"fastsync", "-z", "--compress-level", "10", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, zv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->compression_level, 10);
config_delete(cfg);
/* ...but zlib clamps an over-range level to 9 like rsync. */
cfg = config_create();
char* zc[] = {"fastsync", "-z", "--compress-choice", "zlib", "--compress-level", "15",
"/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 8, zc, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->compression_level, 9);
config_delete(cfg);
}
/* RSYNC_COMPRESS_LIST drives the bare -z ("auto") resolution. */
static void test_parse_args_compression_env_list() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "-z", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
setenv("RSYNC_COMPRESS_LIST", "zlib lz4", 1);
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->compression_algo, (int)COMPRESSION_ALGO_ZLIB);
EXPECT_EQ_INT(cfg->compression_level, 6);
config_delete(cfg);
/* An explicit --compress-choice beats the env list. */
cfg = config_create();
char* explicit_argv[] = {"fastsync", "-z", "--compress-choice", "zstd", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 6, explicit_argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->compression_algo, (int)COMPRESSION_ALGO_ZSTD);
config_delete(cfg);
/* A list with no supported name is rsync's failed negotiation (exit 4). */
setenv("RSYNC_COMPRESS_LIST", "bogus", 1);
cfg = config_create();
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), -1);
EXPECT_EQ_INT(cfg->cli_exit_code, 4);
config_delete(cfg);
unsetenv("RSYNC_COMPRESS_LIST");
}
/* RSYNC_CHECKSUM_LIST drives the default checksum choice. */
static void test_parse_args_checksum_env_list() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--checksum", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
setenv("RSYNC_CHECKSUM_LIST", "md5", 1);
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_MD5);
EXPECT_EQ_INT(cfg->checksum_transfer_algo, (int)CHECKSUM_ALGO_MD5);
config_delete(cfg);
/* An explicit --cc wins. */
cfg = config_create();
char* cc_argv[] = {"fastsync", "--checksum", "--cc=sha1", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, cc_argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->checksum_algo, (int)CHECKSUM_ALGO_SHA1);
config_delete(cfg);
/* A list with no supported name is rsync's failed negotiation (exit 4). */
setenv("RSYNC_CHECKSUM_LIST", "bogus", 1);
cfg = config_create();
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), -1);
EXPECT_EQ_INT(cfg->cli_exit_code, 4);
config_delete(cfg);
unsetenv("RSYNC_CHECKSUM_LIST");
}
/* Every value-taking table option accepts an inline "--opt=value" form. */
static void test_parse_args_table_equals_size_options() {
Config* cfg = config_create();
@@ -4682,6 +4795,9 @@ void test_client_cli() {
test_parse_args_compression_alias_equals();
test_parse_args_rejects_invalid_compression_level_equals();
test_parse_args_rejects_invalid_compression_choice();
test_parse_args_per_codec_compression_level_defaults();
test_parse_args_compression_env_list();
test_parse_args_checksum_env_list();
test_parse_args_table_equals_size_options();
test_parse_args_table_equals_string_and_int_options();
test_parse_args_missing_argument_diagnostic();
+53
View File
@@ -4,6 +4,7 @@
#include "data.h"
#include "file.h"
#include "utils.h"
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/wait.h>
@@ -390,6 +391,56 @@ static void test_codec_name_mapping() {
EXPECT_TRUE(compression_algo_enabled(COMPRESSION_ALGO_ZSTD));
}
/* rsync 3.4.1's per-codec default levels and its clamping ranges. */
static void test_codec_level_defaults_and_clamp() {
EXPECT_EQ_INT(compression_default_level(COMPRESSION_ALGO_ZSTD), ZSTD_CLEVEL_DEFAULT);
EXPECT_EQ_INT(compression_default_level(COMPRESSION_ALGO_ZSTD), 3);
EXPECT_EQ_INT(compression_default_level(COMPRESSION_ALGO_ZLIB), 6);
EXPECT_EQ_INT(compression_default_level(COMPRESSION_ALGO_ZLIBX), 6);
/* lz4 has no tunable level; a positive placeholder keeps the codec engaged. */
EXPECT_TRUE(compression_default_level(COMPRESSION_ALGO_LZ4) > 0);
EXPECT_EQ_INT(compression_default_level(COMPRESSION_ALGO_NONE), 0);
EXPECT_EQ_INT(compression_clamp_level(COMPRESSION_ALGO_ZSTD, 1), 1);
EXPECT_EQ_INT(compression_clamp_level(COMPRESSION_ALGO_ZSTD, 22), 22);
EXPECT_EQ_INT(compression_clamp_level(COMPRESSION_ALGO_ZSTD, 23), 22);
EXPECT_EQ_INT(compression_clamp_level(COMPRESSION_ALGO_ZSTD, 0), 1);
EXPECT_EQ_INT(compression_clamp_level(COMPRESSION_ALGO_ZLIB, 15), 9);
EXPECT_EQ_INT(compression_clamp_level(COMPRESSION_ALGO_ZLIBX, 15), 9);
EXPECT_EQ_INT(compression_clamp_level(COMPRESSION_ALGO_ZLIB, 1), 1);
EXPECT_TRUE(compression_clamp_level(COMPRESSION_ALGO_LZ4, 20) > 0);
EXPECT_EQ_INT(compression_clamp_level(COMPRESSION_ALGO_NONE, 20), 0);
}
/* RSYNC_COMPRESS_LIST precedence, syntax and fallback. */
static void test_codec_choice_env_list() {
unsetenv("RSYNC_COMPRESS_LIST");
EXPECT_EQ_INT(compression_choice_resolve(), (int)COMPRESSION_ALGO_ZSTD);
EXPECT_EQ_INT((int)compression_negotiate_default(), (int)COMPRESSION_ALGO_ZSTD);
/* Unknown entries are skipped; the first supported wins. */
setenv("RSYNC_COMPRESS_LIST", "bogus zlib lz4", 1);
EXPECT_EQ_INT(compression_choice_resolve(), (int)COMPRESSION_ALGO_ZLIB);
/* Case-insensitive. */
setenv("RSYNC_COMPRESS_LIST", "ZSTD", 1);
EXPECT_EQ_INT(compression_choice_resolve(), (int)COMPRESSION_ALGO_ZSTD);
/* Whitespace-separated; the client half ends at '&'. */
setenv("RSYNC_COMPRESS_LIST", "lz4 zlib & zstd", 1);
EXPECT_EQ_INT(compression_choice_resolve(), (int)COMPRESSION_ALGO_LZ4);
/* Blank falls back to the compiled-in order. */
setenv("RSYNC_COMPRESS_LIST", " ", 1);
EXPECT_EQ_INT(compression_choice_resolve(), (int)COMPRESSION_ALGO_ZSTD);
/* rsync's syntax has no comma/colon separator: this is one unknown name. */
setenv("RSYNC_COMPRESS_LIST", "bogus,lz4", 1);
EXPECT_EQ_INT(compression_choice_resolve(), -1);
unsetenv("RSYNC_COMPRESS_LIST");
}
/* The process-global codec selects what the legacy wrappers produce. */
static void test_codec_global_selection() {
Data* original = data_create_empty(64);
@@ -423,5 +474,7 @@ void test_compression() {
test_chunk_compress_decompress_roundtrip();
test_codec_roundtrips();
test_codec_name_mapping();
test_codec_level_defaults_and_clamp();
test_codec_choice_env_list();
test_codec_global_selection();
}
+37
View File
@@ -10,6 +10,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <threads.h>
@@ -578,7 +579,43 @@ static void test_getdelim_bounded() {
fclose(fp);
}
static int test_env_resolver(const char* name) {
if (strcasecmp(name, "alpha") == 0)
return 10;
if (strcasecmp(name, "beta") == 0)
return 20;
return -1;
}
static void test_env_choice_first_parsing() {
const char* var = "FASTSYNC_TEST_CHOICE_LIST";
bool specified = true;
unsetenv(var);
EXPECT_EQ_INT(env_choice_first(var, test_env_resolver, &specified), -1);
EXPECT_FALSE(specified);
/* Unknown entries are skipped, case-insensitive, first supported wins. */
setenv(var, "bogus BETA alpha", 1);
EXPECT_EQ_INT(env_choice_first(var, test_env_resolver, &specified), 20);
EXPECT_TRUE(specified);
/* The client half ends at '&'. */
setenv(var, "alpha & beta", 1);
EXPECT_EQ_INT(env_choice_first(var, test_env_resolver, &specified), 10);
/* Blank means "unspecified"; all-unknown means "specified but no match". */
setenv(var, " ", 1);
EXPECT_EQ_INT(env_choice_first(var, test_env_resolver, &specified), -1);
EXPECT_FALSE(specified);
setenv(var, "nope,alpha", 1);
EXPECT_EQ_INT(env_choice_first(var, test_env_resolver, &specified), -1);
EXPECT_TRUE(specified);
unsetenv(var);
}
void test_shared_utils() {
test_env_choice_first_parsing();
test_path_index_bounded();
test_path_index_semantics();
test_getdelim_bounded();