From f099a6e20fcbe7b1ca1e349c391803efa886b5f5 Mon Sep 17 00:00:00 2001 From: TapTap Date: Sun, 6 Sep 2026 21:04:11 +0200 Subject: [PATCH 01/10] feat(config): add fuzzy flag and bump protocol to 2.9.0 -y/--fuzzy is receiver-side similar-file basis selection, so the receiver must learn the flag: add a bool to Config, serialize it as a trailing field on the config frame, and validate the received value. Because the config frame layout changed, PROTOCOL_VERSION moves to 2.9.0 (peers must match). --- src/shared/config.c | 23 ++++++++++++++++++----- src/shared/config.h | 10 +++++++++- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/shared/config.c b/src/shared/config.c index b839744..7f8b625 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -46,6 +46,7 @@ static void config_set_defaults(Config* config) { config->size_only = false; config->use_delta = false; config->whole_file = false; + config->fuzzy = false; config->modify_window = 0; config->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT; config->delta_max_file_size = DELTA_MAX_FILE_SIZE; @@ -152,9 +153,9 @@ static bool validate_received_config(const Config* config) { valid_wire_bool(config->use_delete) && valid_wire_bool(config->use_incremental) && valid_wire_bool(config->size_only) && valid_wire_bool(config->ignore_times) && valid_wire_bool(config->use_delta) && valid_wire_bool(config->backup) && - valid_wire_bool(config->remove_source_files) && valid_wire_bool(config->follow_symlinks) && - valid_wire_bool(config->copy_links) && valid_wire_bool(config->safe_links) && - valid_wire_bool(config->copy_unsafe_links) && + valid_wire_bool(config->fuzzy) && valid_wire_bool(config->remove_source_files) && + valid_wire_bool(config->follow_symlinks) && valid_wire_bool(config->copy_links) && + valid_wire_bool(config->safe_links) && valid_wire_bool(config->copy_unsafe_links) && valid_wire_bool(config->preserve_hard_links) && valid_wire_bool(config->preserve_acls) && valid_wire_bool(config->preserve_xattrs) && valid_wire_bool(config->preserve_devices) && valid_wire_bool(config->preserve_sparse) && valid_wire_bool(config->ignore_existing) && @@ -453,6 +454,12 @@ static bool send_basis_options(int fd, const Config* c) { return true; } +/* -y/--fuzzy (receiver-side similar-file basis selection). Trailing field on + * the config frame; protocol 2.9.0. */ +static bool send_fuzzy_option(int fd, const Config* c) { + return send_int(fd, c->fuzzy); +} + static bool receive_core_fields(int fd, Config* c) { int value; if (!receive_wire_bool(fd, &c->eight_bit_output)) @@ -624,12 +631,17 @@ static bool receive_basis_options(int fd, Config* c) { return true; } +static bool receive_fuzzy_option(int fd, Config* c) { + return receive_wire_bool(fd, &c->fuzzy); +} + bool config_send(int file_descriptor, const Config* config) { protocol_session_set_max_alloc(NULL, config->max_alloc); if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) || !send_file_options(file_descriptor, config) || !send_selection_options(file_descriptor, config) || - !send_resume_options(file_descriptor, config) || !send_basis_options(file_descriptor, config)) + !send_resume_options(file_descriptor, config) || + !send_basis_options(file_descriptor, config) || !send_fuzzy_option(file_descriptor, config)) return false; Status status; if (!receive_status(file_descriptor, &status)) @@ -662,7 +674,8 @@ Config* config_receive(int file_descriptor) { !receive_file_options(file_descriptor, config) || !receive_selection_options(file_descriptor, config) || !receive_resume_options(file_descriptor, config) || - !receive_basis_options(file_descriptor, config)) + !receive_basis_options(file_descriptor, config) || + !receive_fuzzy_option(file_descriptor, config)) goto error; if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 && strcmp(config->compress_choice, "none") != 0) { diff --git a/src/shared/config.h b/src/shared/config.h index 4ee1caa..7858f9b 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -63,6 +63,14 @@ typedef struct Config { bool size_only; bool use_delta; bool whole_file; + /* -y/--fuzzy: when a file must be transferred and the destination holds no + * usable file at the exact path, the receiver may reuse a SIMILAR-named + * existing regular file in the same destination directory as the delta + * basis so the sender transmits only the differences. Crosses the wire + * (the receiver performs the candidate search); the CLI implies + * --incremental + --delta because the similar-basis only matters on the + * receiver-driven delta path. Off by default. */ + bool fuzzy; int modify_window; uint32_t delta_block_size; unsigned long long delta_max_file_size; @@ -206,7 +214,7 @@ typedef struct Config { DelayUpdatesContext* delay_context; } Config; -#define PROTOCOL_VERSION "2.8.0" +#define PROTOCOL_VERSION "2.9.0" #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) /* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */ #define MAX_BASIS_DIRS 64 From b753efcecc2644da3eca144e2fcef13fc9b4e189 Mon Sep 17 00:00:00 2001 From: TapTap Date: Sun, 6 Sep 2026 21:04:14 +0200 Subject: [PATCH 02/10] feat(cli): parse -y/--fuzzy and --no-fuzzy Register --fuzzy (alias -y) as a boolean option and fuzzy as negatable so --no-fuzzy works through the generic negation machinery. FastSync's delta machinery is off by default (unlike rsync, where --fuzzy implies nothing because delta is the default), so --fuzzy implies --incremental and --delta unless --whole-file or an explicit --no-delta switched delta off (leaving fuzzy inert, matching rsync where -W makes fuzzy irrelevant). Add help text. --- src/client/client_cli.c | 20 ++++++++++++++++++++ src/client/usage.c | 5 +++++ 2 files changed, 25 insertions(+) diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 5778f41..1760226 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -418,6 +418,7 @@ static const OptionEntry OPTION_TABLE[] = { {"--modify-window", "-@", OPT_NONNEG_INT, offsetof(Config, modify_window)}, {"--delta", NULL, OPT_FLAG, offsetof(Config, use_delta)}, {"--whole-file", "-W", OPT_FLAG, offsetof(Config, whole_file)}, + {"--fuzzy", "-y", OPT_FLAG, offsetof(Config, fuzzy)}, {"--save-to-disk", NULL, OPT_FLAG, offsetof(Config, save_to_disk)}, {"--progress", NULL, OPT_FLAG, offsetof(Config, show_progress)}, {"--tls", NULL, OPT_FLAG, offsetof(Config, use_tls)}, @@ -487,6 +488,7 @@ static const NegatableOption NEGATABLE_OPTIONS[] = { {"delete", NULL, offsetof(Config, use_delete)}, {"incremental", NULL, offsetof(Config, use_incremental)}, {"delta", NULL, offsetof(Config, use_delta)}, + {"fuzzy", NULL, offsetof(Config, fuzzy)}, {"save-to-disk", NULL, offsetof(Config, save_to_disk)}, {"progress", NULL, offsetof(Config, show_progress)}, {"tls", NULL, offsetof(Config, use_tls)}, @@ -608,6 +610,9 @@ static int apply_table_option(Config* config, const OptionEntry* entry, const ch int parse_args(Config* config, int argc, char* argv[], int* positional_args, int* positional_count) { bool verbose = false; + /* --no-delta seen on the command line: the user explicitly switched the + delta machinery off, so the --fuzzy implication must not override it. */ + bool no_delta = false; protocol_set_8_bit_output(config->eight_bit_output); /* Apply output controls before processing other options so their order is irrelevant. */ @@ -637,6 +642,8 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args, continue; } if (strncmp(argv[i], "--no-", strlen("--no-")) == 0) { + if (strcmp(argv[i], "--no-delta") == 0) + no_delta = true; if (apply_negation(config, argv[i]) != 0) return -1; continue; @@ -1068,6 +1075,19 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args, if (config_has_basis(config)) config->use_incremental = true; + /* -y/--fuzzy reuses an existing similar-named destination file as the delta + * basis, so it is meaningless without the receiver-driven delta path: + * imply --incremental, and --delta unless --whole-file (or an explicit + * --no-delta) switched the delta machinery off. FastSync has delta OFF by + * default (unlike rsync), so a bare --fuzzy must turn it on or it would be + * a silent no-op. --whole-file/--no-delta after --fuzzy therefore leave + * fuzzy inert, matching rsync where --fuzzy only affects delta transfers. */ + if (config->fuzzy) { + config->use_incremental = true; + if (!config->whole_file && !no_delta) + config->use_delta = true; + } + /* Incremental and delta transfers need metadata unless the user disabled it. */ if ((config->use_incremental || config->use_delta) && !config->use_metadata && !config->metadata_explicitly_disabled) { diff --git a/src/client/usage.c b/src/client/usage.c index 0cb0d76..268299b 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -81,6 +81,11 @@ void print_usage(void) { "used)\n"); printf(" --delta Delta transfer for changed files (requires --incremental)\n"); printf(" -W, --whole-file Transfer changed files without delta processing\n"); + printf(" -y, --fuzzy Use a similar-named file already in the destination\n"); + printf(" directory as the delta basis when the destination has no\n"); + printf(" usable file at the exact path (saves bandwidth; implies\n"); + printf(" --incremental and --delta; inert with --whole-file)\n"); + printf(" --no-fuzzy Disable --fuzzy\n"); printf(" --delta-block Delta block size in bytes (default: %d)\n", DELTA_BLOCK_SIZE_DEFAULT); printf(" --delta-max Max file size for delta transfer (default: %llu)\n", From cebd23a23907f85e652b43dcb4f86261a87882d9 Mon Sep 17 00:00:00 2001 From: TapTap Date: Sun, 6 Sep 2026 21:04:19 +0200 Subject: [PATCH 03/10] feat(receiver): --fuzzy similar-file delta basis When a file must be transferred and the destination holds no usable content at the exact path (file absent, or outside the delta engine's size bounds), the receiver now searches the target's own destination directory for an existing regular file with a similar basename and uses it as the delta basis via the existing receiver-driven STATUS_DELTA_SIGNATURE handshake. The sender never learns the basis was another file, so no wire change beyond the new config flag was required. Heuristic (deterministic, simpler than rsync's, documented): candidates are sibling entries confined below the root and opened O_NOFOLLOW (symlinks are never followed, nothing outside the destination root is read); dotfiles, directories, the target's own name and stage/temp names are excluded; size gate is delta_should_attempt; name gate is a Levenshtein distance <= half the longer basename; the closest candidate (size tie-break, then lexical) is loaded; the scan is capped at 4096 entries. Byte-exactness is independent of the basis: block matches are verified by Adler-32 + xxHash32, delta_apply validates every reference, and a basis that shares nothing makes the sender reply with a whole-file transfer. When no candidate qualifies the normal whole-file transfer runs unchanged. --- src/shared/file_receive.c | 237 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) diff --git a/src/shared/file_receive.c b/src/shared/file_receive.c index 0e63375..a934a0c 100644 --- a/src/shared/file_receive.c +++ b/src/shared/file_receive.c @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -660,6 +661,208 @@ static bool basis_match_find(const Config* config, const char* check_path, return false; } +/* --------------------------------------------------------------------------- + * -y/--fuzzy similar-file delta basis. + * + * When a file must be transferred and the destination holds no usable content + * at the exact path (the destination file is absent, or is outside the delta + * engine's size bounds), --fuzzy lets the receiver reuse an EXISTING regular + * file in the SAME destination directory as the delta basis, so the sender + * transmits only the differences instead of the whole file. This is the + * rsync "find a similar file to use as a basis for a transfer" case (e.g. a + * file recreated under a new name whose old-named sibling is still present). + * + * The delta handshake is unchanged and receiver-driven, so the sender never + * learns the basis was a different file and needs no new protocol. Byte + * exactness never depends on which bytes the basis holds: the delta protocol + * only references basis blocks whose Adler-32 + xxHash32 checksums match the + * source, delta_apply validates every reference against the basis size, and a + * basis that shares nothing simply makes the sender reply STATUS_NEXT (full + * transfer). A fuzzy basis can therefore waste bandwidth but never corrupt a + * file. + * + * Similarity heuristic (deterministic, deliberately simpler than rsync's): + * * candidates are the target's sibling entries in its destination + * directory, opened through the confined root (file_open_secure_parent + + * openat O_NOFOLLOW, fstatat AT_SYMLINK_NOFOLLOW) -- symlinks are never + * followed and nothing outside the destination root is ever read; + * * dotfiles, directories, the target's own name, and the .fastsync-stage / + * temp scratch names are never candidates; + * * size gate = the delta engine's own bounds (delta_should_attempt: both + * files >= DELTA_MIN_FILE_SIZE, <= delta_max_file_size, ratio <= 10x), + * NOT rsync's ~1.5x size window; + * * name gate = Levenshtein edit distance between the basenames, accepted + * only when distance <= half the length of the longer basename; + * * the single best candidate (smallest distance; tie-break: size closest + * to the incoming file, then lexicographically smaller basename) is read + * and returned as the basis. + * ------------------------------------------------------------------------- */ + +/* A directory scan is linear in the number of entries; the fuzzy search stops + * after this many so a pathological huge directory cannot stall a transfer. */ +#define FUZZY_MAX_DIRECTORY_SCAN 4096 +/* Names longer than this never take part in fuzzy matching: the edit-distance + * DP below is O(len^2), so over-long names are bounded out of the search. */ +#define FUZZY_NAME_LIMIT 192 + +typedef struct { + char name[FUZZY_NAME_LIMIT + 1]; + unsigned long long size; + size_t distance; + unsigned long long size_gap; +} FuzzyCandidate; + +/* Levenshtein edit distance, or SIZE_MAX when the operands are too long or the + * DP could not be allocated. */ +static size_t fuzzy_edit_distance(const char* a, size_t la, const char* b, size_t lb) { + if (la > FUZZY_NAME_LIMIT || lb > FUZZY_NAME_LIMIT) + return SIZE_MAX; + size_t* prev = malloc((lb + 1) * sizeof(size_t)); + size_t* cur = malloc((lb + 1) * sizeof(size_t)); + if (!prev || !cur) { + free(prev); + free(cur); + return SIZE_MAX; + } + for (size_t j = 0; j <= lb; j++) + prev[j] = j; + for (size_t i = 1; i <= la; i++) { + cur[0] = i; + for (size_t j = 1; j <= lb; j++) { + size_t cost = a[i - 1] == b[j - 1] ? 0 : 1; + size_t del = prev[j] + 1; + size_t ins = cur[j - 1] + 1; + size_t sub = prev[j - 1] + cost; + size_t m = del < ins ? del : ins; + cur[j] = m < sub ? m : sub; + } + size_t* tmp = prev; + prev = cur; + cur = tmp; + } + size_t distance = prev[lb]; + free(prev); + free(cur); + return distance; +} + +/* Deterministic ordering of two fuzzy candidates: smallest edit distance, + * then the size closest to the incoming file, then the lexical basename. */ +static bool fuzzy_candidate_better(const FuzzyCandidate* cand, const FuzzyCandidate* best) { + if (!best->name[0]) + return true; + if (cand->distance != best->distance) + return cand->distance < best->distance; + if (cand->size_gap != best->size_gap) + return cand->size_gap < best->size_gap; + return strcmp(cand->name, best->name) < 0; +} + +/* Search the destination directory that will contain `check_path` for a + * similar regular file usable as a --fuzzy delta basis and return its full + * content in a malloc'd (protocol_alloc) buffer. Returns NULL (with *out_size + * = 0) when no candidate qualifies, which means the caller performs the normal + * whole-file transfer. */ +static void* fuzzy_basis_find_and_load(const Config* config, const char* check_path, + unsigned long long check_size, + unsigned long long* out_size) { + *out_size = 0; + if (!config || !config->receive_root_directory || !config->fuzzy || !config->use_delta || + !check_path || check_size < DELTA_MIN_FILE_SIZE || check_size > config->delta_max_file_size || + check_size > MAX_RECEIVE_WHOLE_FILE_SIZE) + return NULL; + + char* full_path = path_cat(config->receive_root_directory, check_path); + if (!full_path) + return NULL; + char* leaf = NULL; + int dir_fd = file_open_secure_parent(full_path, &leaf, false); + if (dir_fd < 0 || !leaf) { + free(leaf); + free(full_path); + return NULL; + } + size_t target_len = strlen(leaf); + + int scanfd = dup(dir_fd); + if (scanfd < 0) { + close(dir_fd); + free(leaf); + free(full_path); + return NULL; + } + DIR* dir = fdopendir(scanfd); + if (!dir) { + close(scanfd); + close(dir_fd); + free(leaf); + free(full_path); + return NULL; + } + + FuzzyCandidate best; + memset(&best, 0, sizeof(best)); + const struct dirent* entry; + size_t scanned = 0; + while (scanned < FUZZY_MAX_DIRECTORY_SCAN && (entry = readdir(dir)) != NULL) { + scanned++; + const char* name = entry->d_name; + size_t name_len = strlen(name); + if (name[0] == '.' || name_len == 0 || name_len > FUZZY_NAME_LIMIT || strcmp(name, leaf) == 0) + continue; + struct stat st; + if (fstatat(dir_fd, name, &st, AT_SYMLINK_NOFOLLOW) != 0 || !S_ISREG(st.st_mode)) + continue; + unsigned long long cand_size = (unsigned long long)st.st_size; + if (cand_size == 0 || cand_size > MAX_RECEIVE_WHOLE_FILE_SIZE || + !delta_should_attempt(cand_size, check_size, config->delta_max_file_size)) + continue; + size_t distance = fuzzy_edit_distance(leaf, target_len, name, name_len); + size_t longer = target_len > name_len ? target_len : name_len; + if (distance == SIZE_MAX || distance * 2 > longer) + continue; + FuzzyCandidate cand; + memcpy(cand.name, name, name_len + 1); + cand.size = cand_size; + cand.distance = distance; + cand.size_gap = cand_size > check_size ? cand_size - check_size : check_size - cand_size; + if (fuzzy_candidate_better(&cand, &best)) + best = cand; + } + closedir(dir); + free(leaf); + + void* basis = NULL; + if (best.name[0]) { + int fd = openat(dir_fd, best.name, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (fd >= 0) { + struct stat st; + if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode) && + (unsigned long long)st.st_size == best.size && best.size <= SIZE_MAX) { + basis = protocol_alloc((size_t)best.size); + if (basis) { + size_t got = 0; + while (got < (size_t)best.size) { + ssize_t n = read(fd, (char*)basis + got, (size_t)best.size - got); + if (n <= 0) { + free(basis); + basis = NULL; + break; + } + got += (size_t)n; + } + } + } + close(fd); + } + } + close(dir_fd); + free(full_path); + if (basis) + *out_size = best.size; + return basis; +} + File* receive_incremental_check(int fd, const Config* config, bool* skipped) { if (!config || !skipped) { send_status(fd, STATUS_ERROR); @@ -891,6 +1094,40 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) { free(old_data); old_data = NULL; + /* ---- -y/--fuzzy similar-file delta basis ---- + * Reaching this point means the file must be transferred and the + * destination's own content at the exact path could not serve as a delta + * basis (it is absent, outside the delta size bounds, or unreadable). With + * --fuzzy the receiver tries an existing similar-named file in the same + * destination directory instead. receive_delta_file performs the whole + * handshake: when the sender judges the delta not worthwhile it replies + * STATUS_NEXT and the full content is received there, so a fuzzy attempt + * can only improve bandwidth, never fall through into the plain transfer + * below (that path is reserved for "no usable candidate was found"). */ + if (config->fuzzy && config->use_delta) { + unsigned long long fuzzy_size = 0; + void* fuzzy_basis = fuzzy_basis_find_and_load(config, check_path, check_size, &fuzzy_size); + if (fuzzy_basis != NULL) { + bool fuzzy_failed = false; + File* fuzzy_file = + receive_delta_file(fd, config, check_path, fuzzy_basis, fuzzy_size, &fuzzy_failed); + fuzzy_basis = NULL; /* receive_delta_file consumes the buffer on every path */ + if (fuzzy_file) { + close(old_fd); + free(full_path); + free(check_path); + return fuzzy_file; + } + if (fuzzy_failed) { + close(old_fd); + free(full_path); + free(check_path); + return NULL; + } + } + free(fuzzy_basis); + } + if (!send_status(fd, STATUS_NEXT)) { close(old_fd); free(full_path); From 9c1ec6758cad25418ae80ae8583478b76a78bfbd Mon Sep 17 00:00:00 2001 From: TapTap Date: Sun, 6 Sep 2026 21:04:23 +0200 Subject: [PATCH 04/10] test: --fuzzy coverage (CLI, config wire, integration) Unit: parse -y/--fuzzy and --no-fuzzy negation (order-independent), -W and --no-delta leave fuzzy inert, --fuzzy implies --incremental/--delta, and validate_config rejects fuzzy with -s (chunk serialization) and -f (sendfile). Config: fuzzy survives the socketpair wire round-trip. Integration (TestFuzzy, byte counts via a new CountingProxy helper): the rename case transfers a 2 MiB file in a few percent of its size with byte- exact output under --fuzzy, while the no-fuzzy, no-candidate, dissimilar- sibling and --whole-file runs send the whole file; -y alias; -m parity; dest-holds-unsuitable-file falls back to the sibling; --delay-updates and --remove-source-files keep their semantics on fuzzy transfers. --- tests/integration/common.py | 71 +++++++++ tests/integration/test_features.py | 244 ++++++++++++++++++++++++++++- tests/test_client_cli.c | 119 ++++++++++++++ tests/test_config.c | 3 + 4 files changed, 436 insertions(+), 1 deletion(-) diff --git a/tests/integration/common.py b/tests/integration/common.py index 2e9653a..09b737a 100644 --- a/tests/integration/common.py +++ b/tests/integration/common.py @@ -6,6 +6,7 @@ import socket import subprocess import sys import tempfile +import threading import time PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) @@ -53,6 +54,76 @@ class ServerManager: self.stop() +class CountingProxy: + """One-shot TCP forwarder that counts the bytes flowing in each direction + between one client and the real server. + + Client output and --stats report SOURCE lengths, so a delta/fuzzy transfer + that moves only a few percent of the file is invisible in normal output. + Routing the client through this proxy makes the actual wire usage + observable: client_to_server counts every byte the client sent (config, + paths, and file/delta payloads), server_to_client counts the reply bytes + (including the receiver's delta signatures). + """ + + def __init__(self, target_port): + self.target_port = target_port + self._listener = socket.socket() + self._listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._listener.bind(("127.0.0.1", 0)) + self._listener.listen(1) + self._listener.settimeout(30) + self.port = self._listener.getsockname()[1] + self.client_to_server = 0 + self.server_to_client = 0 + + @staticmethod + def _pump(src, dst, counter): + while True: + try: + data = src.recv(65536) + except OSError: + return + if not data: + try: + dst.shutdown(socket.SHUT_WR) + except OSError: + pass + return + try: + dst.sendall(data) + except OSError: + return + counter[0] += len(data) + + def run(self, cmd): + """Forward one client run (the full command list) to the real server and + return the CompletedProcess after the counts have settled.""" + + def serve(): + try: + client_sock, _ = self._listener.accept() + server_sock = socket.create_connection(("127.0.0.1", self.target_port), + timeout=10) + except OSError: + return + c2s, s2c = [0], [0] + a = threading.Thread(target=self._pump, args=(client_sock, server_sock, c2s)) + b = threading.Thread(target=self._pump, args=(server_sock, client_sock, s2c)) + a.start() + b.start() + a.join() + b.join() + self.client_to_server = c2s[0] + self.server_to_client = s2c[0] + + thread = threading.Thread(target=serve) + thread.start() + result = subprocess.run(cmd, capture_output=True, text=True, timeout=180) + thread.join(20) + return result + + def run_client(source_dir, dest_dir, flags=None, port=None, extra_args=None): """Run the client and return (result, duration).""" cmd = CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir, "--save-to-disk"] diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index a42bf29..8425b4a 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -1,6 +1,7 @@ """Feature tests: incremental sync, bandwidth limiting, dry run, metadata, filters.""" import filecmp import os +import random import shutil import subprocess import sys @@ -10,7 +11,7 @@ import pytest sys.path.insert(0, os.path.dirname(__file__)) from common import ( PROJECT_ROOT, BUILD_DIR, TEST_DATA_DIR, - run_client, + run_client, CountingProxy, generate_test_files, verify_transfer, clean_dir, make_result, get_dest_received_dir, CLIENT_CMD, ServerManager, ) @@ -2479,3 +2480,244 @@ class TestBasisDestDirs: received = get_dest_received_dir(dest, source) assert not os.path.exists(received), \ "over-limit basis run transferred files before failing" + + +def _random_payloads(size=2 * 1024 * 1024, changed=64 * 1024, seed=1234): + """Return (old, new) byte strings of equal length where `new` differs from + `old` only in one contiguous `changed`-byte region. Incompressible (random) + data keeps the whole-file wire cost near the file size, so a delta transfer + is distinguishable from a whole-file one by its wire bytes.""" + r = random.Random(seed) + data = bytearray(r.randbytes(size)) + old = bytes(data) + off = size // 3 + for i in range(off, off + changed): + data[i] = r.randrange(256) + return old, bytes(data) + + +class TestFuzzy: + """-y/--fuzzy similar-file delta basis. + + Scenario modelled on rsync's --fuzzy: a file is recreated under a similar + NEW basename in the same directory. The destination still holds the + old-named file (nothing deleted it), but the new path has no content of its + own at the destination, so without --fuzzy the receiver has no delta basis + and the whole file is sent. With --fuzzy the receiver searches the + destination directory, picks the similar-named sibling as the delta basis, + sends its block signature, and the sender transmits only the differences. + The reconstructed file must be byte-identical to the source in every mode; + only the wire usage changes (observed through CountingProxy, because client + --stats report source lengths, not wire bytes). + """ + + OLD_NAME = "report-2025.dat" + NEW_NAME = "report-2026.dat" + + def _client_via_proxy(self, source, dest, flags, proxy): + cmd = (CLIENT_CMD + ["--source-dir", source, "--dest-dir", dest, + "--save-to-disk", "--server-port", str(proxy.port)] + flags) + return proxy.run(cmd) + + def _seed_dest(self, source, dest, files, port): + """Write `files` {rel: bytes} into source and mirror them to dest.""" + for rel, content in files.items(): + full = os.path.join(source, rel) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "wb") as fh: + fh.write(content) + clean_dir(dest) + result, _ = run_client(source, dest, port=port) + assert result.returncode == 0, f"seed failed: {(result.stderr or result.stdout)[:300]}" + + def _prepare(self, tag): + source = os.path.join(TEST_DATA_DIR, f"fuzzy_{tag}_src") + dest = os.path.join(TEST_DATA_DIR, f"fuzzy_{tag}_dst") + clean_dir(source) + return source, dest + + def _run_measured(self, source, dest, flags, port): + """Run a transfer through a byte-counting proxy. Returns (result, proxy).""" + proxy = CountingProxy(port) + result = self._client_via_proxy(source, dest, flags, proxy) + return result, proxy + + def test_fuzzy_uses_similar_sibling_as_delta_basis(self, shared_server): + source, dest = self._prepare("basis") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + # Recreate the file under a similar new name; the old sibling stays on + # the destination (nothing deletes it). + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + + result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy rename transfer failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \ + "fuzzy reconstruction is not byte-exact" + # The wire carried the delta, not the 2 MiB whole file. + assert proxy.client_to_server < len(new_bytes) // 4, \ + f"fuzzy transfer sent {proxy.client_to_server} bytes; expected a delta" + + def test_without_fuzzy_sends_the_whole_file(self, shared_server): + source, dest = self._prepare("whole") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + + result, proxy = self._run_measured(source, dest, ["--incremental", "--delta"], shared_server.port) + assert result.returncode == 0, f"no-fuzzy rename failed: {result.stderr[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + # No similar basis: the whole file goes over the wire. + assert proxy.client_to_server > len(new_bytes) // 2, \ + f"expected a whole-file transfer, got {proxy.client_to_server} bytes" + + @pytest.mark.parametrize("mt", [False, True]) + def test_fuzzy_byte_exact_single_and_multithreaded(self, shared_server, mt): + source, dest = self._prepare(f"mt{'1' if mt else '0'}") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + + flags = ["--fuzzy"] + (["-m"] if mt else []) + result, proxy = self._run_measured(source, dest, flags, shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy {'-m ' if mt else ''}rename failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \ + f"fuzzy {'-m ' if mt else ''}reconstruction is not byte-exact" + assert proxy.client_to_server < len(new_bytes) // 4 + + def test_no_candidate_falls_back_to_whole_file(self, shared_server): + # A brand-new destination directory holds no sibling at all, so --fuzzy + # finds nothing and the file is transferred whole (and correctly). + source, dest = self._prepare("nocand") + clean_dir(dest) + os.makedirs(source, exist_ok=True) + _, new_bytes = _random_payloads() + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy no-candidate fallback failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + assert proxy.client_to_server > len(new_bytes) // 2, \ + "no-candidate fuzzy run should have sent the whole file" + + def test_dissimilar_sibling_is_not_used(self, shared_server): + # The destination holds a large sibling whose basename is too different + # from the incoming name; the name gate must reject it and fall back to + # a whole-file transfer. + source, dest = self._prepare("dissim") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {"totally-unrelated-notes.bin": old_bytes}, + shared_server.port) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy dissimilar-sibling run failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + assert proxy.client_to_server > len(new_bytes) // 2, \ + "a dissimilar-named sibling must not be used as a fuzzy basis" + + def test_fuzzy_helps_when_dest_holds_an_unsuitable_file(self, shared_server): + # The destination DOES hold the exact new name, but it is a tiny stale + # file (below the delta engine's minimum, ratio far outside its window), + # so it cannot serve as the basis. --fuzzy then falls back to the + # similar-named sibling. + source, dest = self._prepare("unsuitable") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, + {self.OLD_NAME: old_bytes, self.NEW_NAME: b"stale small file\n"}, + shared_server.port) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy unsuitable-dest run failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \ + "byte-exactness broken when the destination file was unsuitable" + assert proxy.client_to_server < len(new_bytes) // 4, \ + "fuzzy should have reused the similar sibling as the basis" + + def test_whole_file_makes_fuzzy_inert(self, shared_server): + # -W/--whole-file switches the delta machinery off, so --fuzzy has + # nothing to attach to and the file is transferred whole (rsync parity). + source, dest = self._prepare("wholefile") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + result, proxy = self._run_measured(source, dest, ["--fuzzy", "-W"], shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy -W run failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + assert proxy.client_to_server > len(new_bytes) // 2, \ + "--whole-file must disable the fuzzy delta basis" + + def test_fuzzy_via_short_y_alias(self, shared_server): + source, dest = self._prepare("shorty") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + result, proxy = self._run_measured(source, dest, ["-y"], shared_server.port) + assert result.returncode == 0, f"-y rename failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + assert proxy.client_to_server < len(new_bytes) // 4, "-y did not enable fuzzy" + + def test_fuzzy_with_delay_updates_publishes_cleanly(self, shared_server): + # A fuzzy-reconstructed file goes through the normal store engine, so + # --delay-updates must stage and publish it with no staging leftovers. + source, dest = self._prepare("delay") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + result, _ = run_client(source, dest, + flags=["--fuzzy", "--delay-updates"], + port=shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy --delay-updates failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + assert not os.path.isdir(os.path.join(dest, ".fastsync-stage")), \ + "delay-updates staging tree was not cleaned up" + + def test_fuzzy_source_removed_after_transfer(self, shared_server): + # A fuzzy transfer is a real transfer (not a skip), so + # --remove-source-files must remove the renamed source file. + source, dest = self._prepare("rm") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(new_bytes) + result, _ = run_client(source, dest, + flags=["--fuzzy", "--remove-source-files"], + port=shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy --remove-source-files failed: {(result.stderr or result.stdout)[:300]}" + assert not os.path.exists(os.path.join(source, self.NEW_NAME)), \ + "a fuzzy-transferred source should have been removed" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + assert _read_file(os.path.join(received, self.OLD_NAME)) == old_bytes + diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index 3d21fc6..45170e8 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -1169,6 +1169,120 @@ static void test_parse_args_whole_file() { config_delete(cfg); } +/* -y/--fuzzy reuses a similar destination file as a delta basis, so it implies + * the receiver-driven delta path (--incremental + --delta): FastSync's delta + * machinery is OFF by default, so without the implication a bare --fuzzy would + * be a silent no-op. Both spellings behave identically. */ +static void test_parse_args_fuzzy_implies_delta() { + static const char* const spellings[] = {"--fuzzy", "-y"}; + for (size_t i = 0; i < sizeof(spellings) / sizeof(spellings[0]); i++) { + Config* cfg = config_create(); + char* argv[] = {"fastsync", (char*)spellings[i], "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->fuzzy); + EXPECT_TRUE(cfg->use_incremental); + EXPECT_TRUE(cfg->use_delta); + EXPECT_TRUE(cfg->use_metadata); + config_delete(cfg); + } +} + +/* --no-fuzzy turns the flag back off; the incremental/delta implication must + * only fire when the FINAL value of the flag is true (order-independent). */ +static void test_parse_args_fuzzy_negation() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "--fuzzy", "--no-fuzzy", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_FALSE(cfg->fuzzy); + EXPECT_FALSE(cfg->use_delta); + EXPECT_FALSE(cfg->use_incremental); + config_delete(cfg); + + cfg = config_create(); + char* reordered[] = {"fastsync", "--no-fuzzy", "--fuzzy", "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, reordered, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->fuzzy); + EXPECT_TRUE(cfg->use_incremental); + EXPECT_TRUE(cfg->use_delta); + config_delete(cfg); +} + +/* -W/--whole-file switches the delta machinery off, so --fuzzy is inert (the + * per-file quick check still needs --incremental, which stays implied). */ +static void test_parse_args_fuzzy_with_whole_file() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "--fuzzy", "-W", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->fuzzy); + EXPECT_TRUE(cfg->whole_file); + EXPECT_TRUE(cfg->use_incremental); + EXPECT_FALSE(cfg->use_delta); + config_delete(cfg); +} + +/* An explicit --no-delta is respected by the --fuzzy implication in either + * argument order (a user who switched delta off does not want it forced on). */ +static void test_parse_args_fuzzy_respects_no_delta() { + static const char* const combos[][2] = { + {"--fuzzy", "--no-delta"}, + {"--no-delta", "--fuzzy"}, + }; + for (size_t i = 0; i < sizeof(combos) / sizeof(combos[0]); i++) { + Config* cfg = config_create(); + char* argv[] = {"fastsync", (char*)combos[i][0], (char*)combos[i][1], "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->fuzzy); + EXPECT_FALSE(cfg->use_delta); + config_delete(cfg); + } +} + +/* --fuzzy requires the delta machinery, which the chunk-serialization (-s) and + * sendfile (-f) modes reject -- mirroring the --delta constraint checks. */ +static void test_validate_config_fuzzy_incompatible_modes() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "--fuzzy", "-s", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + EXPECT_TRUE(cfg->use_incremental); + EXPECT_FALSE(validate_config(cfg)); + config_delete(cfg); + + cfg = config_create(); + char* sendfile_argv[] = {"fastsync", "--fuzzy", "-f", "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, sendfile_argv, positional_args, &positional_count), 0); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + EXPECT_TRUE(cfg->use_delta); + EXPECT_FALSE(validate_config(cfg)); + config_delete(cfg); + + /* A plain --fuzzy run is a valid configuration. */ + cfg = config_create(); + char* ok_argv[] = {"fastsync", "--fuzzy", "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, ok_argv, positional_args, &positional_count), 0); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + EXPECT_TRUE(cfg->use_delta); + EXPECT_TRUE(cfg->use_incremental); + EXPECT_TRUE(validate_config(cfg)); + config_delete(cfg); +} + /* -x and --one-file-system enable client-side single-filesystem scanning. */ static void test_parse_args_one_file_system() { Config* cfg = config_create(); @@ -1713,6 +1827,11 @@ void test_client_cli() { test_parse_args_secluded_args(); test_parse_args_short_s_remains_chunk_serialization(); test_parse_args_whole_file(); + test_parse_args_fuzzy_implies_delta(); + test_parse_args_fuzzy_negation(); + test_parse_args_fuzzy_with_whole_file(); + test_parse_args_fuzzy_respects_no_delta(); + test_validate_config_fuzzy_incompatible_modes(); test_parse_args_one_file_system(); test_parse_args_compression_aliases(); test_parse_args_compression_equals_and_none(); diff --git a/tests/test_config.c b/tests/test_config.c index b0c8ceb..2cbacbd 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -128,6 +128,7 @@ static void test_config_send_receive() { send_cfg->use_executability = true; send_cfg->use_delta = true; send_cfg->whole_file = true; + send_cfg->fuzzy = true; send_cfg->ignore_times = true; send_cfg->size_only = true; send_cfg->compression_level = 5; @@ -188,6 +189,8 @@ static void test_config_send_receive() { ok = false; if (recv_cfg->use_delta) ok = false; + if (!recv_cfg->fuzzy) + ok = false; if (recv_cfg->modify_window != 4) ok = false; if (!recv_cfg->existing) From 2bc43d6084c979f4e4171999ec8f74f904788408 Mon Sep 17 00:00:00 2001 From: TapTap Date: Sun, 6 Sep 2026 21:04:26 +0200 Subject: [PATCH 05/10] docs: mark -y/--fuzzy/--no-fuzzy implemented in RSYNC_COMPAT Document the receiver-side decision location, the exact deterministic similarity heuristic, the byte-exactness argument, when fuzzy applies (and when it deliberately does not), the 2.8.0 -> 2.9.0 protocol bump, and the divergences from rsync. Update the basis-dir rows' protocol references to the now-current 2.9.0. --- RSYNC_COMPAT.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/RSYNC_COMPAT.md b/RSYNC_COMPAT.md index c4da818..db73d88 100644 --- a/RSYNC_COMPAT.md +++ b/RSYNC_COMPAT.md @@ -210,10 +210,10 @@ order-independent because it runs over the fully parsed config. |------|-------------------|-----------------|-------| | `--checksum` | Skip based on checksum | ✅ Implemented | With `--incremental`, compares xxHash64 content checksums; `-c` remains compression | | `--checksum-choice=STR` | Choose checksum algorithm | ❌ Not Implemented | xxHash used internally | -| `--compare-dest=DIR` | Compare dest files relative to DIR | ✅ Implemented | DIR is a receiver-side basis relative to the destination root (confined below it; absolute/`..`/`.` rejected, `//` collapsed and trailing `/` dropped). On the receiver's per-file check (implies `--incremental`) an exact match = same size + mtime (unless `--size-only`; `-I` disables matching) **and** equal xxHash64 of the sender's file; a match suppresses the data transfer. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. Divergences: when the destination already holds a *different* version rsync deletes it but FastSync instead transfers the data (keeps the mirror complete; never deletes without `--delete`); attribute-only differences on a match are not re-applied (data is skipped so the sender never sends metadata); content is verified by xxHash64, stricter than rsync's default quick check. Sizing: FastSync's whole-file payload limit is 256 MiB on **every** transfer path (not basis-specific); rsync applies basis dirs to arbitrary sizes, so FastSync refuses a basis run whose source contains a larger file up front with a clear error before any transfer. Wire: a basis-count field is always present on the config frame (protocol bumped to 2.8.0, so clients and servers must both be 2.8.0) | -| `--copy-dest=DIR` | Include copies of unchanged files | ✅ Implemented | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the normal atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Repeatable; command-line order = priority. Content is xxHash64-verified before the copy. Divergences: a basis-hit destination keeps the basis file's own mode/uid/gid and mtime (the sender sends no metadata on a skip), so with `--size-only` its mtime can differ from the source and attribute-only differences are copied with the basis attributes rather than rsync's "copy + fix attributes". Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.8.0 | -| `--link-dest=DIR` | Hardlink to files when unchanged | ✅ Implemented | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy, never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Content is xxHash64-verified before linking. Divergences and caveats: an already up-to-date destination file is not re-linked to a basis file (only files that would otherwise be written are linked); a link keeps the basis inode's own mode/uid/gid and mtime — metadata is never written through the shared inode (that would mutate the basis file), so a later `--inplace` run that rewrites such a destination path **will mutate the basis snapshot** through the shared inode (use `--copy-dest` when the destination must stay independently writable); with `--size-only` the linked mtime can differ from the source; a `--remove-source-files` source satisfied by a basis dir is treated as skipped and therefore **retained** (never removed); basis dirs are excluded from `--delete`. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.8.0 | -| `--fuzzy`, `--no-fuzzy` | Find similar file for basis | ❌ Not Implemented | | +| `--compare-dest=DIR` | Compare dest files relative to DIR | ✅ Implemented | DIR is a receiver-side basis relative to the destination root (confined below it; absolute/`..`/`.` rejected, `//` collapsed and trailing `/` dropped). On the receiver's per-file check (implies `--incremental`) an exact match = same size + mtime (unless `--size-only`; `-I` disables matching) **and** equal xxHash64 of the sender's file; a match suppresses the data transfer. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. Divergences: when the destination already holds a *different* version rsync deletes it but FastSync instead transfers the data (keeps the mirror complete; never deletes without `--delete`); attribute-only differences on a match are not re-applied (data is skipped so the sender never sends metadata); content is verified by xxHash64, stricter than rsync's default quick check. Sizing: FastSync's whole-file payload limit is 256 MiB on **every** transfer path (not basis-specific); rsync applies basis dirs to arbitrary sizes, so FastSync refuses a basis run whose source contains a larger file up front with a clear error before any transfer. Wire: a basis-count field is always present on the config frame (protocol 2.9.0, so clients and servers must both be 2.9.0) | +| `--copy-dest=DIR` | Include copies of unchanged files | ✅ Implemented | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the normal atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Repeatable; command-line order = priority. Content is xxHash64-verified before the copy. Divergences: a basis-hit destination keeps the basis file's own mode/uid/gid and mtime (the sender sends no metadata on a skip), so with `--size-only` its mtime can differ from the source and attribute-only differences are copied with the basis attributes rather than rsync's "copy + fix attributes". Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 | +| `--link-dest=DIR` | Hardlink to files when unchanged | ✅ Implemented | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy, never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Content is xxHash64-verified before linking. Divergences and caveats: an already up-to-date destination file is not re-linked to a basis file (only files that would otherwise be written are linked); a link keeps the basis inode's own mode/uid/gid and mtime — metadata is never written through the shared inode (that would mutate the basis file), so a later `--inplace` run that rewrites such a destination path **will mutate the basis snapshot** through the shared inode (use `--copy-dest` when the destination must stay independently writable); with `--size-only` the linked mtime can differ from the source; a `--remove-source-files` source satisfied by a basis dir is treated as skipped and therefore **retained** (never removed); basis dirs are excluded from `--delete`. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 | +| `-y`, `--fuzzy`, `--no-fuzzy` | Find similar file for basis | ✅ Implemented | `-y/--fuzzy` is a pure bandwidth optimization on the existing receiver-driven delta path: when a file must be transferred and the destination holds no usable content at the exact path (file absent, or the destination file is outside the delta engine's size bounds), the receiver searches the SAME destination directory for an existing regular file whose basename is similar to the incoming name and uses it as the delta basis, so the sender transmits only the differences instead of the whole file. The output is always byte-exact regardless of which (or whether any) basis is chosen. Decision location: the receiver performs the candidate search inside `receive_incremental_check` and sends the normal `STATUS_DELTA_SIGNATURE`; the sender never learns the basis was a different file, so no new frame type or sender logic was needed — only the config frame grew a `fuzzy` boolean, so `PROTOCOL_VERSION` was bumped **2.8.0 → 2.9.0** (peers must match). Similarity heuristic (deterministic, simpler than rsync's deliberately-fuzzy matching, and documented precisely): candidates are the target's sibling entries in its destination directory, opened `O_NOFOLLOW`/`AT_SYMLINK_NOFOLLOW` under the confined root (symlinks never followed; nothing outside the destination root is ever read or hashed); dotfiles, directories, the target's own name, and the `.fastsync-stage`/temp scratch names are excluded; the size gate is the delta engine's own bounds (both files ≥ 16 KiB, ≤ `--delta-max`, ratio ≤ 10×) rather than rsync's ~1.5× size window; the name gate is a Levenshtein edit distance between the basenames accepted only when ≤ half the length of the longer basename; the single best candidate (smallest distance, tie-break size closest to the incoming file then lexicographically smaller basename) is read; the directory scan is capped at 4096 entries so a pathological directory cannot stall a transfer. When fuzzy applies: only to files the receiver would otherwise send whole — the destination's own file is always preferred as the delta basis when it exists and fits the delta size bounds, so fuzzy does NOT replace an existing-but-different destination basis; FastSync's 10× delta size-ratio bound means an existing destination file that is too far away in size still lets the fuzzy search run. When no similar candidate exists the transfer falls back to the normal whole-file transfer. rsync-divergence note: rsync's own matching uses a fuzzy name/size rule set; FastSync implements the closest safe deterministic approximation above. Because FastSync's delta machinery is off by default (rsync's is on), `--fuzzy` implies `--incremental` + `--delta` (unless `--whole-file`/`-W` or an explicit `--no-delta` switched delta off, in which case fuzzy is inert — matching rsync where `--whole-file` makes fuzzy irrelevant); `--no-fuzzy` negates it. All surrounding semantics are untouched: a fuzzy-reconstructed file is stored as a normal file, so `--remove-source-files`, itemize/`-i`, `--stats`, `--backup`, `--delay-updates`, `--existing`/`--ignore-existing`/`--update` behave exactly as for a whole-file transfer (the fuzzy delta does not skip the file) | ## 12. Compression From be37a509a5ef05c33a004db0f3198c39e9043aa6 Mon Sep 17 00:00:00 2001 From: TapTap Date: Sun, 6 Sep 2026 23:05:32 +0200 Subject: [PATCH 06/10] perf(receiver): bound the fuzzy edit-distance cost, harden the open Review follow-ups on the --fuzzy candidate scan: - Allocate the two DP rows once per directory scan instead of once per candidate (4096-entry directories no longer do thousands of malloc pairs). - Pre-prune before the DP with two cheap lower bounds on the edit distance: the name length gap and the count of characters of one basename absent from the other; a candidate whose gate (distance*2 <= longer) already fails on the max of those bounds is skipped without running the DP. - Trim the common prefix and non-overlapping common suffix before the DP so it only runs over the differing middles. - Note that the 4096 readdir cap bounds iterations, not per-entry DP cost, and that the seen set is filesystem-order dependent (winner stays deterministic via the total comparator). - Open the chosen candidate with O_NONBLOCK so a name raced to a FIFO cannot block the receive thread forever in open(2); the existing fstat S_ISREG gate still rejects non-regular files. (The pre-existing basis_open_regular has the same latent FIFO pattern and is intentionally left unchanged.) - Free old_data in receive_delta_file's defensive NULL guard. --- src/shared/file_receive.c | 144 ++++++++++++++++++++++++++++++++------ 1 file changed, 121 insertions(+), 23 deletions(-) diff --git a/src/shared/file_receive.c b/src/shared/file_receive.c index a934a0c..b014f98 100644 --- a/src/shared/file_receive.c +++ b/src/shared/file_receive.c @@ -336,6 +336,7 @@ fail: static File* receive_delta_file(int fd, const Config* config, const char* check_path, void* old_data, unsigned long long old_size, bool* failed) { if (!old_data) { + free(old_data); /* defensive: old_data is always non-NULL today */ *failed = true; return NULL; } @@ -699,7 +700,12 @@ static bool basis_match_find(const Config* config, const char* check_path, * ------------------------------------------------------------------------- */ /* A directory scan is linear in the number of entries; the fuzzy search stops - * after this many so a pathological huge directory cannot stall a transfer. */ + * after this many so a pathological huge directory cannot stall a transfer. + * The cap bounds the readdir() ITERATIONS, not the per-entry work: every + * entry that survives the (cheap) size and pre-name gates still runs an + * edit-distance DP, so the per-entry DP cost is separately bounded below by + * pre-pruning on the name length gap and the absent-character bound, and by + * trimming the common prefix/suffix before the DP runs on the middles only. */ #define FUZZY_MAX_DIRECTORY_SCAN 4096 /* Names longer than this never take part in fuzzy matching: the edit-distance * DP below is O(len^2), so over-long names are bounded out of the search. */ @@ -712,24 +718,80 @@ typedef struct { unsigned long long size_gap; } FuzzyCandidate; -/* Levenshtein edit distance, or SIZE_MAX when the operands are too long or the - * DP could not be allocated. */ -static size_t fuzzy_edit_distance(const char* a, size_t la, const char* b, size_t lb) { - if (la > FUZZY_NAME_LIMIT || lb > FUZZY_NAME_LIMIT) - return SIZE_MAX; - size_t* prev = malloc((lb + 1) * sizeof(size_t)); - size_t* cur = malloc((lb + 1) * sizeof(size_t)); - if (!prev || !cur) { - free(prev); - free(cur); - return SIZE_MAX; +/* Two-row DP scratch, allocated once per directory scan (not per candidate) so + * a 4096-entry directory never performs 4096 malloc/free pairs. */ +typedef struct { + size_t* prev; + size_t* cur; +} FuzzyEditBuffer; + +static bool fuzzy_edit_buffer_init(FuzzyEditBuffer* buf) { + buf->prev = malloc((FUZZY_NAME_LIMIT + 1) * sizeof(size_t)); + buf->cur = malloc((FUZZY_NAME_LIMIT + 1) * sizeof(size_t)); + if (!buf->prev || !buf->cur) { + free(buf->prev); + free(buf->cur); + buf->prev = NULL; + buf->cur = NULL; + return false; } - for (size_t j = 0; j <= lb; j++) + return true; +} + +static void fuzzy_edit_buffer_destroy(FuzzyEditBuffer* buf) { + free(buf->prev); + free(buf->cur); + buf->prev = NULL; + buf->cur = NULL; +} + +/* Cheap lower bounds used to reject a candidate BEFORE the DP: + * - any edit script must at least absorb the length gap: d >= |la - lb|; + * - any character of `a` that does not occur in `b` at all must be deleted or + * substituted at its own position: d >= (count of such characters). + * The acceptance gate is d*2 <= longer, so a candidate whose max of these two + * bounds already violates it can be skipped without computing the distance. */ +static size_t fuzzy_absent_char_bound(const char* a, size_t la, const char* b, size_t lb) { + if (lb == 0) + return la; + bool present[256] = {false}; + for (size_t i = 0; i < lb; i++) + present[(uint8_t)b[i]] = true; + size_t absent = 0; + for (size_t i = 0; i < la; i++) + if (!present[(uint8_t)a[i]]) + absent++; + return absent; +} + +/* Levenshtein edit distance between the two basenames. A shared prefix and a + * (non-overlapping) shared suffix can always be aligned at no cost, so the DP + * only runs over the differing middles; its two rows come from `buf` (allocated + * once by the caller). Callers enforce la, lb <= FUZZY_NAME_LIMIT. */ +static size_t fuzzy_edit_distance(FuzzyEditBuffer* buf, const char* a, size_t la, const char* b, + size_t lb) { + size_t p = 0; + while (p < la && p < lb && a[p] == b[p]) + p++; + size_t s = 0; + while (s < la - p && s < lb - p && a[la - 1 - s] == b[lb - 1 - s]) + s++; + size_t ma = la - p - s; + size_t mb = lb - p - s; + if (ma == 0) + return mb; + if (mb == 0) + return ma; + const char* A = a + p; + const char* B = b + p; + size_t* prev = buf->prev; + size_t* cur = buf->cur; + for (size_t j = 0; j <= mb; j++) prev[j] = j; - for (size_t i = 1; i <= la; i++) { + for (size_t i = 1; i <= ma; i++) { cur[0] = i; - for (size_t j = 1; j <= lb; j++) { - size_t cost = a[i - 1] == b[j - 1] ? 0 : 1; + for (size_t j = 1; j <= mb; j++) { + size_t cost = A[i - 1] == B[j - 1] ? 0 : 1; size_t del = prev[j] + 1; size_t ins = cur[j - 1] + 1; size_t sub = prev[j - 1] + cost; @@ -740,10 +802,7 @@ static size_t fuzzy_edit_distance(const char* a, size_t la, const char* b, size_ prev = cur; cur = tmp; } - size_t distance = prev[lb]; - free(prev); - free(cur); - return distance; + return prev[mb]; } /* Deterministic ordering of two fuzzy candidates: smallest edit distance, @@ -783,6 +842,14 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p return NULL; } size_t target_len = strlen(leaf); + /* A target basename longer than FUZZY_NAME_LIMIT can never pass the name gate + (every candidate name is bounded by the same limit), so skip the scan. */ + if (target_len > FUZZY_NAME_LIMIT) { + close(dir_fd); + free(leaf); + free(full_path); + return NULL; + } int scanfd = dup(dir_fd); if (scanfd < 0) { @@ -800,10 +867,24 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p return NULL; } + /* The DP scratch rows are allocated once per scan (not once per candidate). */ + FuzzyEditBuffer ebuf; + if (!fuzzy_edit_buffer_init(&ebuf)) { + closedir(dir); + close(dir_fd); + free(leaf); + free(full_path); + return NULL; + } + FuzzyCandidate best; memset(&best, 0, sizeof(best)); const struct dirent* entry; size_t scanned = 0; + /* readdir() yields entries in filesystem-dependent order, so the SET of + candidates seen is order-dependent; the winner is still deterministic + because every candidate is compared with the total ordering in + fuzzy_candidate_better (acceptable for a heuristic). */ while (scanned < FUZZY_MAX_DIRECTORY_SCAN && (entry = readdir(dir)) != NULL) { scanned++; const char* name = entry->d_name; @@ -817,9 +898,21 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p if (cand_size == 0 || cand_size > MAX_RECEIVE_WHOLE_FILE_SIZE || !delta_should_attempt(cand_size, check_size, config->delta_max_file_size)) continue; - size_t distance = fuzzy_edit_distance(leaf, target_len, name, name_len); + /* Cheap pre-name gates run BEFORE the edit-distance DP. The edit distance + is bounded below by the length gap |la-lb| and by the number of + characters of one basename that are absent from the other (each such + position costs at least one op), so a candidate whose acceptance gate + (distance*2 <= longer) already fails on the max of those bounds is + skipped without running the DP. */ size_t longer = target_len > name_len ? target_len : name_len; - if (distance == SIZE_MAX || distance * 2 > longer) + size_t bound = longer - (target_len < name_len ? target_len : name_len); + size_t absent = fuzzy_absent_char_bound(leaf, target_len, name, name_len); + if (absent > bound) + bound = absent; + if (bound * 2 > longer) + continue; + size_t distance = fuzzy_edit_distance(&ebuf, leaf, target_len, name, name_len); + if (distance * 2 > longer) continue; FuzzyCandidate cand; memcpy(cand.name, name, name_len + 1); @@ -831,10 +924,15 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p } closedir(dir); free(leaf); + fuzzy_edit_buffer_destroy(&ebuf); void* basis = NULL; if (best.name[0]) { - int fd = openat(dir_fd, best.name, O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + /* O_NONBLOCK: a name raced to a FIFO between the fstatat gate and this open + would otherwise block the receive thread forever on open(2); with it the + open fails (ENXIO) and the fstat/S_ISREG gate below would reject it too. + A regular file opened with O_NONBLOCK is unaffected. */ + int fd = openat(dir_fd, best.name, O_RDONLY | O_NOFOLLOW | O_NONBLOCK | O_CLOEXEC); if (fd >= 0) { struct stat st; if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode) && From c1aabc68dec7667e041e0936da907bcad179bbd3 Mon Sep 17 00:00:00 2001 From: TapTap Date: Sun, 6 Sep 2026 23:05:35 +0200 Subject: [PATCH 07/10] feat(cli): --fuzzy honors an explicit --no-incremental The --fuzzy implication previously forced use_incremental back on even when the user passed --no-incremental, while --no-delta and -W were honored. Track a --no-incremental latch (like the no_delta latch): with it set, do not force the handshake on, and because delta needs the handshake, also suppress the delta implication so no invalid '--delta requires --incremental' config results. A --fuzzy --no-incremental run is therefore a plain default-mode transfer (fuzzy inert), consistent with -W/--no-delta. Documented in the usage text (this deliberately differs from the basis-dir options, which still force incremental unconditionally). --- src/client/client_cli.c | 27 ++++++++++++++++++--------- src/client/usage.c | 3 ++- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 1760226..e52cf09 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -610,9 +610,11 @@ static int apply_table_option(Config* config, const OptionEntry* entry, const ch int parse_args(Config* config, int argc, char* argv[], int* positional_args, int* positional_count) { bool verbose = false; - /* --no-delta seen on the command line: the user explicitly switched the - delta machinery off, so the --fuzzy implication must not override it. */ + /* Explicit --no-delta / --no-incremental seen on the command line: the user + switched part of the delta machinery off, so the --fuzzy implication must + not silently turn it back on. */ bool no_delta = false; + bool no_incremental = false; protocol_set_8_bit_output(config->eight_bit_output); /* Apply output controls before processing other options so their order is irrelevant. */ @@ -644,6 +646,8 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args, if (strncmp(argv[i], "--no-", strlen("--no-")) == 0) { if (strcmp(argv[i], "--no-delta") == 0) no_delta = true; + else if (strcmp(argv[i], "--no-incremental") == 0) + no_incremental = true; if (apply_negation(config, argv[i]) != 0) return -1; continue; @@ -1077,14 +1081,19 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args, /* -y/--fuzzy reuses an existing similar-named destination file as the delta * basis, so it is meaningless without the receiver-driven delta path: - * imply --incremental, and --delta unless --whole-file (or an explicit - * --no-delta) switched the delta machinery off. FastSync has delta OFF by - * default (unlike rsync), so a bare --fuzzy must turn it on or it would be - * a silent no-op. --whole-file/--no-delta after --fuzzy therefore leave - * fuzzy inert, matching rsync where --fuzzy only affects delta transfers. */ + * imply --incremental and --delta unless --whole-file or an explicit + * --no-delta / --no-incremental switched the machinery off. FastSync has + * delta OFF by default (unlike rsync), so a bare --fuzzy must turn it on or + * it would be a silent no-op. -W/--no-delta/--no-incremental therefore + * leave fuzzy inert, matching rsync where --whole-file makes fuzzy + * irrelevant (note: unlike the basis-dir options, --fuzzy honors an + * explicit --no-incremental instead of forcing the handshake back on). */ if (config->fuzzy) { - config->use_incremental = true; - if (!config->whole_file && !no_delta) + if (!no_incremental) + config->use_incremental = true; + /* Delta needs the incremental per-file handshake, so an explicit + * --no-incremental also suppresses the delta implication. */ + if (!config->whole_file && !no_delta && !no_incremental) config->use_delta = true; } diff --git a/src/client/usage.c b/src/client/usage.c index 268299b..95dbc41 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -84,7 +84,8 @@ void print_usage(void) { printf(" -y, --fuzzy Use a similar-named file already in the destination\n"); printf(" directory as the delta basis when the destination has no\n"); printf(" usable file at the exact path (saves bandwidth; implies\n"); - printf(" --incremental and --delta; inert with --whole-file)\n"); + printf(" --incremental and --delta; inert with --whole-file,\n"); + printf(" --no-delta, or --no-incremental)\n"); printf(" --no-fuzzy Disable --fuzzy\n"); printf(" --delta-block Delta block size in bytes (default: %d)\n", DELTA_BLOCK_SIZE_DEFAULT); From c379e2dbdddb31acec5ee980614e43e44daa35c9 Mon Sep 17 00:00:00 2001 From: TapTap Date: Sun, 6 Sep 2026 23:05:39 +0200 Subject: [PATCH 08/10] docs: fuzzy oracle note and --no-incremental asymmetry Review follow-ups on the --fuzzy row: note that the receiver's block signature is derived from a sibling file it may not otherwise send, exposing destination sibling files to the sender at block granularity (the same known-plaintext information class as the ordinary delta path); and document that --fuzzy honors an explicit --no-incremental (unlike the basis-dir options, which force it), with the delta implication suppressed accordingly. --- RSYNC_COMPAT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RSYNC_COMPAT.md b/RSYNC_COMPAT.md index db73d88..bd3850d 100644 --- a/RSYNC_COMPAT.md +++ b/RSYNC_COMPAT.md @@ -213,7 +213,7 @@ order-independent because it runs over the fully parsed config. | `--compare-dest=DIR` | Compare dest files relative to DIR | ✅ Implemented | DIR is a receiver-side basis relative to the destination root (confined below it; absolute/`..`/`.` rejected, `//` collapsed and trailing `/` dropped). On the receiver's per-file check (implies `--incremental`) an exact match = same size + mtime (unless `--size-only`; `-I` disables matching) **and** equal xxHash64 of the sender's file; a match suppresses the data transfer. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. Divergences: when the destination already holds a *different* version rsync deletes it but FastSync instead transfers the data (keeps the mirror complete; never deletes without `--delete`); attribute-only differences on a match are not re-applied (data is skipped so the sender never sends metadata); content is verified by xxHash64, stricter than rsync's default quick check. Sizing: FastSync's whole-file payload limit is 256 MiB on **every** transfer path (not basis-specific); rsync applies basis dirs to arbitrary sizes, so FastSync refuses a basis run whose source contains a larger file up front with a clear error before any transfer. Wire: a basis-count field is always present on the config frame (protocol 2.9.0, so clients and servers must both be 2.9.0) | | `--copy-dest=DIR` | Include copies of unchanged files | ✅ Implemented | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the normal atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Repeatable; command-line order = priority. Content is xxHash64-verified before the copy. Divergences: a basis-hit destination keeps the basis file's own mode/uid/gid and mtime (the sender sends no metadata on a skip), so with `--size-only` its mtime can differ from the source and attribute-only differences are copied with the basis attributes rather than rsync's "copy + fix attributes". Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 | | `--link-dest=DIR` | Hardlink to files when unchanged | ✅ Implemented | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy, never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Content is xxHash64-verified before linking. Divergences and caveats: an already up-to-date destination file is not re-linked to a basis file (only files that would otherwise be written are linked); a link keeps the basis inode's own mode/uid/gid and mtime — metadata is never written through the shared inode (that would mutate the basis file), so a later `--inplace` run that rewrites such a destination path **will mutate the basis snapshot** through the shared inode (use `--copy-dest` when the destination must stay independently writable); with `--size-only` the linked mtime can differ from the source; a `--remove-source-files` source satisfied by a basis dir is treated as skipped and therefore **retained** (never removed); basis dirs are excluded from `--delete`. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 | -| `-y`, `--fuzzy`, `--no-fuzzy` | Find similar file for basis | ✅ Implemented | `-y/--fuzzy` is a pure bandwidth optimization on the existing receiver-driven delta path: when a file must be transferred and the destination holds no usable content at the exact path (file absent, or the destination file is outside the delta engine's size bounds), the receiver searches the SAME destination directory for an existing regular file whose basename is similar to the incoming name and uses it as the delta basis, so the sender transmits only the differences instead of the whole file. The output is always byte-exact regardless of which (or whether any) basis is chosen. Decision location: the receiver performs the candidate search inside `receive_incremental_check` and sends the normal `STATUS_DELTA_SIGNATURE`; the sender never learns the basis was a different file, so no new frame type or sender logic was needed — only the config frame grew a `fuzzy` boolean, so `PROTOCOL_VERSION` was bumped **2.8.0 → 2.9.0** (peers must match). Similarity heuristic (deterministic, simpler than rsync's deliberately-fuzzy matching, and documented precisely): candidates are the target's sibling entries in its destination directory, opened `O_NOFOLLOW`/`AT_SYMLINK_NOFOLLOW` under the confined root (symlinks never followed; nothing outside the destination root is ever read or hashed); dotfiles, directories, the target's own name, and the `.fastsync-stage`/temp scratch names are excluded; the size gate is the delta engine's own bounds (both files ≥ 16 KiB, ≤ `--delta-max`, ratio ≤ 10×) rather than rsync's ~1.5× size window; the name gate is a Levenshtein edit distance between the basenames accepted only when ≤ half the length of the longer basename; the single best candidate (smallest distance, tie-break size closest to the incoming file then lexicographically smaller basename) is read; the directory scan is capped at 4096 entries so a pathological directory cannot stall a transfer. When fuzzy applies: only to files the receiver would otherwise send whole — the destination's own file is always preferred as the delta basis when it exists and fits the delta size bounds, so fuzzy does NOT replace an existing-but-different destination basis; FastSync's 10× delta size-ratio bound means an existing destination file that is too far away in size still lets the fuzzy search run. When no similar candidate exists the transfer falls back to the normal whole-file transfer. rsync-divergence note: rsync's own matching uses a fuzzy name/size rule set; FastSync implements the closest safe deterministic approximation above. Because FastSync's delta machinery is off by default (rsync's is on), `--fuzzy` implies `--incremental` + `--delta` (unless `--whole-file`/`-W` or an explicit `--no-delta` switched delta off, in which case fuzzy is inert — matching rsync where `--whole-file` makes fuzzy irrelevant); `--no-fuzzy` negates it. All surrounding semantics are untouched: a fuzzy-reconstructed file is stored as a normal file, so `--remove-source-files`, itemize/`-i`, `--stats`, `--backup`, `--delay-updates`, `--existing`/`--ignore-existing`/`--update` behave exactly as for a whole-file transfer (the fuzzy delta does not skip the file) | +| `-y`, `--fuzzy`, `--no-fuzzy` | Find similar file for basis | ✅ Implemented | `-y/--fuzzy` is a pure bandwidth optimization on the existing receiver-driven delta path: when a file must be transferred and the destination holds no usable content at the exact path (file absent, or the destination file is outside the delta engine's size bounds), the receiver searches the SAME destination directory for an existing regular file whose basename is similar to the incoming name and uses it as the delta basis, so the sender transmits only the differences instead of the whole file. The output is always byte-exact regardless of which (or whether any) basis is chosen. Decision location: the receiver performs the candidate search inside `receive_incremental_check` and sends the normal `STATUS_DELTA_SIGNATURE`; the sender never learns the basis was a different file, so no new frame type or sender logic was needed — only the config frame grew a `fuzzy` boolean, so `PROTOCOL_VERSION` was bumped **2.8.0 → 2.9.0** (peers must match). Similarity heuristic (deterministic, simpler than rsync's deliberately-fuzzy matching, and documented precisely): candidates are the target's sibling entries in its destination directory, opened `O_NOFOLLOW`/`AT_SYMLINK_NOFOLLOW` under the confined root (symlinks never followed; nothing outside the destination root is ever read or hashed); dotfiles, directories, the target's own name, and the `.fastsync-stage`/temp scratch names are excluded; like the ordinary delta path, the block signature the receiver transmits is derived from on-disk content it may not otherwise send, so a negotiated `--fuzzy` run exposes the destination's sibling files (at block granularity) to the sender as a known-plaintext oracle — the same information class as the normal delta handshake over the file being replaced; the size gate is the delta engine's own bounds (both files ≥ 16 KiB, ≤ `--delta-max`, ratio ≤ 10×) rather than rsync's ~1.5× size window; the name gate is a Levenshtein edit distance between the basenames accepted only when ≤ half the length of the longer basename; the single best candidate (smallest distance, tie-break size closest to the incoming file then lexicographically smaller basename) is read; the directory scan is capped at 4096 entries so a pathological directory cannot stall a transfer. When fuzzy applies: only to files the receiver would otherwise send whole — the destination's own file is always preferred as the delta basis when it exists and fits the delta size bounds, so fuzzy does NOT replace an existing-but-different destination basis; FastSync's 10× delta size-ratio bound means an existing destination file that is too far away in size still lets the fuzzy search run. When no similar candidate exists the transfer falls back to the normal whole-file transfer. rsync-divergence note: rsync's own matching uses a fuzzy name/size rule set; FastSync implements the closest safe deterministic approximation above. Because FastSync's delta machinery is off by default (rsync's is on), `--fuzzy` implies `--incremental` + `--delta` (unless `--whole-file`/`-W` or an explicit `--no-delta` switched delta off, in which case fuzzy is inert — matching rsync where `--whole-file` makes fuzzy irrelevant). Unlike the basis-dir options, `--fuzzy` honors an explicit `--no-incremental` (it does not force the handshake back on); an explicit `--no-incremental` also suppresses the delta implication so no invalid `--delta requires --incremental` config results. `--no-fuzzy` negates it. All surrounding semantics are untouched: a fuzzy-reconstructed file is stored as a normal file, so `--remove-source-files`, itemize/`-i`, `--stats`, `--backup`, `--delay-updates`, `--existing`/`--ignore-existing`/`--update` behave exactly as for a whole-file transfer (the fuzzy delta does not skip the file) | ## 12. Compression From 741d1f3b93ebe9e6e67d90c201cc809bb823163b Mon Sep 17 00:00:00 2001 From: TapTap Date: Sun, 6 Sep 2026 23:05:43 +0200 Subject: [PATCH 09/10] test: review follow-up coverage for --fuzzy Unit (CLI): --fuzzy --no-incremental (either order) stays a valid plain-mode config -- no forced handshake, no delta implication; --fuzzy --no-delta stays covered. Integration (TestFuzzy): - worthless basis: a sibling that passes the name+size gates but shares no blocks makes the sender reply STATUS_NEXT; the whole file is consumed inside the delta handshake with byte-exact output (no protocol desync). - non-displacement: an existing exact-path destination file INSIDE the delta size bounds (same size, different content, older mtime) is used as the delta basis instead of a byte-identical similar sibling (whole-file wire cost). - asymmetric bases: basis larger than source (prefix reuse) and basis smaller than source (appended tail as literals) both reconstruct byte-exactly with a small delta. - --no-fuzzy end-to-end equals the no-flag whole-file behavior. CountingProxy closes its listener socket (fd hygiene). --- tests/integration/common.py | 2 + tests/integration/test_features.py | 111 +++++++++++++++++++++++++++++ tests/test_client_cli.c | 27 +++++++ 3 files changed, 140 insertions(+) diff --git a/tests/integration/common.py b/tests/integration/common.py index 09b737a..9fc93ad 100644 --- a/tests/integration/common.py +++ b/tests/integration/common.py @@ -106,6 +106,7 @@ class CountingProxy: server_sock = socket.create_connection(("127.0.0.1", self.target_port), timeout=10) except OSError: + self._listener.close() return c2s, s2c = [0], [0] a = threading.Thread(target=self._pump, args=(client_sock, server_sock, c2s)) @@ -116,6 +117,7 @@ class CountingProxy: b.join() self.client_to_server = c2s[0] self.server_to_client = s2c[0] + self._listener.close() thread = threading.Thread(target=serve) thread.start() diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index 8425b4a..82c223f 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -2513,6 +2513,8 @@ class TestFuzzy: OLD_NAME = "report-2025.dat" NEW_NAME = "report-2026.dat" + TS = 1577836800 # 2020-01-01, used to pin stale destination mtimes + SIZE = 2 * 1024 * 1024 def _client_via_proxy(self, source, dest, flags, proxy): cmd = (CLIENT_CMD + ["--source-dir", source, "--dest-dir", dest, @@ -2721,3 +2723,112 @@ class TestFuzzy: assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes assert _read_file(os.path.join(received, self.OLD_NAME)) == old_bytes + @staticmethod + def _rand_bytes(size, seed): + return random.Random(seed).randbytes(size) + + def _replace_source_file(self, source, old_name, new_name, new_bytes): + """Remove old_name from source and add new_name with new_bytes.""" + os.unlink(os.path.join(source, old_name)) + with open(os.path.join(source, new_name), "wb") as fh: + fh.write(new_bytes) + + def test_worthless_fuzzy_basis_falls_back_inside_handshake(self, shared_server): + # The sibling passes the name AND size gates but shares no blocks with + # the incoming file, so the sender's delta is not worthwhile: it replies + # STATUS_NEXT and the receiver consumes the WHOLE file inside the delta + # handshake. This proves a bad fuzzy basis cannot desync the protocol + # or corrupt the result. + source, dest = self._prepare("worthless") + basis = self._rand_bytes(self.SIZE, 424242) + target = self._rand_bytes(self.SIZE, 777777) + self._seed_dest(source, dest, {self.OLD_NAME: basis}, shared_server.port) + self._replace_source_file(source, self.OLD_NAME, self.NEW_NAME, target) + result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy worthless-basis run failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == target, \ + "whole-file fallback after a worthless fuzzy basis is not byte-exact" + assert proxy.client_to_server > self.SIZE // 2, \ + "a worthless basis should have made the sender fall back to the whole file" + + def test_existing_dest_file_preferred_over_fuzzy_sibling(self, shared_server): + # Non-displacement: the destination holds a file at the exact path that + # is inside the delta size bounds (same size, different content, older + # mtime). FastSync must delta against THAT file -- even though it + # shares nothing with the source -- and must NOT reuse a similar-named + # sibling that is byte-identical to the source. + source, dest = self._prepare("nondisp") + sibling = self._rand_bytes(self.SIZE, 111) # will equal the incoming file + stale = self._rand_bytes(self.SIZE, 333) # worthless exact-path file + self._seed_dest(source, dest, + {self.OLD_NAME: sibling, self.NEW_NAME: stale}, + shared_server.port) + # Force the exact-path destination file's mtime into the past so the + # quick check deterministically decides to transfer it. + os.utime(os.path.join(get_dest_received_dir(dest, source), self.NEW_NAME), + (self.TS, self.TS)) + os.unlink(os.path.join(source, self.OLD_NAME)) + with open(os.path.join(source, self.NEW_NAME), "wb") as fh: + fh.write(sibling) + result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy non-displacement run failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == sibling + assert proxy.client_to_server > self.SIZE // 2, \ + "the exact-path destination file must be the delta basis, not the fuzzy sibling" + + def test_fuzzy_basis_larger_than_source(self, shared_server): + # The similar sibling is LARGER than the incoming file (within the delta + # engine's 10x ratio); the new file is an exact prefix of the basis, so + # every block matches and only a tiny delta travels. + source, dest = self._prepare("largerbasis") + big = self._rand_bytes(1536 * 1024, 1) + prefix = big[:1024 * 1024] + self._seed_dest(source, dest, {self.OLD_NAME: big}, shared_server.port) + self._replace_source_file(source, self.OLD_NAME, self.NEW_NAME, prefix) + result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy larger-basis run failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == prefix, \ + "shrunken file reconstructed from a larger fuzzy basis is not byte-exact" + assert proxy.client_to_server < len(prefix) // 4, \ + "larger fuzzy basis should have carried most of the file as block matches" + + def test_fuzzy_basis_smaller_than_source(self, shared_server): + # The similar sibling is SMALLER than the incoming file; the new file + # appends data past the basis, so the appended tail travels as literals + # while the shared prefix is block-matched. + source, dest = self._prepare("smallerbasis") + base = self._rand_bytes(self.SIZE, 2) + tail = self._rand_bytes(64 * 1024, 3) + new_bytes = base + tail + self._seed_dest(source, dest, {self.OLD_NAME: base}, shared_server.port) + self._replace_source_file(source, self.OLD_NAME, self.NEW_NAME, new_bytes) + result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port) + assert result.returncode == 0, \ + f"--fuzzy smaller-basis run failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes, \ + "grown file reconstructed from a smaller fuzzy basis is not byte-exact" + assert proxy.client_to_server < len(new_bytes) // 4, \ + "smaller fuzzy basis should have block-matched the shared prefix" + + def test_no_fuzzy_end_to_end_equals_no_flag(self, shared_server): + # --no-fuzzy must not enable anything: a run with it behaves exactly + # like a run without it (whole-file transfer, byte-exact output). + source, dest = self._prepare("nofuzzye2e") + old_bytes, new_bytes = _random_payloads() + self._seed_dest(source, dest, {self.OLD_NAME: old_bytes}, shared_server.port) + self._replace_source_file(source, self.OLD_NAME, self.NEW_NAME, new_bytes) + result, proxy = self._run_measured(source, dest, ["--no-fuzzy"], shared_server.port) + assert result.returncode == 0, \ + f"--no-fuzzy run failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes + assert proxy.client_to_server > len(new_bytes) // 2, \ + "--no-fuzzy should leave the default whole-file behavior intact" + diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index 45170e8..807a14c 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -1246,6 +1246,32 @@ static void test_parse_args_fuzzy_respects_no_delta() { } } +/* An explicit --no-incremental is respected by the --fuzzy implication in + * either argument order (unlike the basis-dir options, --fuzzy does not force + * the incremental handshake back on). Because delta needs the handshake, the + * delta implication is suppressed too, so the run is a plain (default-mode) + * transfer rather than an invalid "--delta requires --incremental" config. */ +static void test_parse_args_fuzzy_respects_no_incremental() { + static const char* const combos[][2] = { + {"--fuzzy", "--no-incremental"}, + {"--no-incremental", "--fuzzy"}, + }; + for (size_t i = 0; i < sizeof(combos) / sizeof(combos[0]); i++) { + Config* cfg = config_create(); + char* argv[] = {"fastsync", (char*)combos[i][0], (char*)combos[i][1], "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->fuzzy); + EXPECT_FALSE(cfg->use_incremental); + EXPECT_FALSE(cfg->use_delta); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + EXPECT_TRUE(validate_config(cfg)); + config_delete(cfg); + } +} + /* --fuzzy requires the delta machinery, which the chunk-serialization (-s) and * sendfile (-f) modes reject -- mirroring the --delta constraint checks. */ static void test_validate_config_fuzzy_incompatible_modes() { @@ -1831,6 +1857,7 @@ void test_client_cli() { test_parse_args_fuzzy_negation(); test_parse_args_fuzzy_with_whole_file(); test_parse_args_fuzzy_respects_no_delta(); + test_parse_args_fuzzy_respects_no_incremental(); test_validate_config_fuzzy_incompatible_modes(); test_parse_args_one_file_system(); test_parse_args_compression_aliases(); From 87b58975de34e579d5b5ec4f75b2a52261652447 Mon Sep 17 00:00:00 2001 From: TapTap Date: Sun, 6 Sep 2026 23:11:01 +0200 Subject: [PATCH 10/10] style(receiver): rework fuzzy DP suffix trim, silence cppcheck FP The suffix-trim loop using computed end offsets tripped cppcheck's knownConditionTrueFalse value-range analysis (it unsoundly concluded the trims always consume the whole middle). Rewrite it with explicit moving end indices and add an inline suppression with a rationale for the residual false positive; cppcheck --error-exitcode=1 is clean again. The trimming logic is unchanged and was verified against a full DP reference over 200k random name pairs. --- src/shared/file_receive.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/shared/file_receive.c b/src/shared/file_receive.c index b014f98..bf19a28 100644 --- a/src/shared/file_receive.c +++ b/src/shared/file_receive.c @@ -773,11 +773,19 @@ static size_t fuzzy_edit_distance(FuzzyEditBuffer* buf, const char* a, size_t la size_t p = 0; while (p < la && p < lb && a[p] == b[p]) p++; - size_t s = 0; - while (s < la - p && s < lb - p && a[la - 1 - s] == b[lb - 1 - s]) - s++; - size_t ma = la - p - s; - size_t mb = lb - p - s; + /* Trim the common suffix (never overlapping the prefix). Working with two + moving end indices keeps the region arithmetic explicit and safe. */ + size_t ae = la; + size_t be = lb; + while (ae > p && be > p && a[ae - 1] == b[be - 1]) { + ae--; + be--; + } + size_t ma = ae - p; + size_t mb = be - p; + /* cppcheck-suppress knownConditionTrueFalse -- the prefix/suffix trims above + only run while the corresponding ends match, so a middle can remain; the + analysis unsoundly concludes the trims always consume everything. */ if (ma == 0) return mb; if (mb == 0)