Merge feat/p3-fuzzy: --fuzzy/-y/--no-fuzzy similar-file delta basis

This commit is contained in:
2026-09-06 23:52:24 +02:00
10 changed files with 982 additions and 8 deletions
+29
View File
@@ -418,6 +418,7 @@ static const OptionEntry OPTION_TABLE[] = {
{"--modify-window", "-@", OPT_NONNEG_INT, offsetof(Config, modify_window)},
{"--delta", NULL, OPT_FLAG, offsetof(Config, use_delta)},
{"--whole-file", "-W", OPT_FLAG, offsetof(Config, whole_file)},
{"--fuzzy", "-y", OPT_FLAG, offsetof(Config, fuzzy)},
{"--save-to-disk", NULL, OPT_FLAG, offsetof(Config, save_to_disk)},
{"--progress", NULL, OPT_FLAG, offsetof(Config, show_progress)},
{"--tls", NULL, OPT_FLAG, offsetof(Config, use_tls)},
@@ -492,6 +493,7 @@ static const NegatableOption NEGATABLE_OPTIONS[] = {
{"delete", NULL, offsetof(Config, use_delete)},
{"incremental", NULL, offsetof(Config, use_incremental)},
{"delta", NULL, offsetof(Config, use_delta)},
{"fuzzy", NULL, offsetof(Config, fuzzy)},
{"save-to-disk", NULL, offsetof(Config, save_to_disk)},
{"progress", NULL, offsetof(Config, show_progress)},
{"tls", NULL, offsetof(Config, use_tls)},
@@ -613,6 +615,11 @@ static int apply_table_option(Config* config, const OptionEntry* entry, const ch
int parse_args(Config* config, int argc, char* argv[], int* positional_args,
int* positional_count) {
bool verbose = false;
/* Explicit --no-delta / --no-incremental seen on the command line: the user
switched part of the delta machinery off, so the --fuzzy implication must
not silently turn it back on. */
bool no_delta = false;
bool no_incremental = false;
protocol_set_8_bit_output(config->eight_bit_output);
/* Apply output controls before processing other options so their order is irrelevant. */
@@ -642,6 +649,10 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
continue;
}
if (strncmp(argv[i], "--no-", strlen("--no-")) == 0) {
if (strcmp(argv[i], "--no-delta") == 0)
no_delta = true;
else if (strcmp(argv[i], "--no-incremental") == 0)
no_incremental = true;
if (apply_negation(config, argv[i]) != 0)
return -1;
continue;
@@ -1073,6 +1084,24 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
if (config_has_basis(config))
config->use_incremental = true;
/* -y/--fuzzy reuses an existing similar-named destination file as the delta
* basis, so it is meaningless without the receiver-driven delta path:
* imply --incremental and --delta unless --whole-file or an explicit
* --no-delta / --no-incremental switched the machinery off. FastSync has
* delta OFF by default (unlike rsync), so a bare --fuzzy must turn it on or
* it would be a silent no-op. -W/--no-delta/--no-incremental therefore
* leave fuzzy inert, matching rsync where --whole-file makes fuzzy
* irrelevant (note: unlike the basis-dir options, --fuzzy honors an
* explicit --no-incremental instead of forcing the handshake back on). */
if (config->fuzzy) {
if (!no_incremental)
config->use_incremental = true;
/* Delta needs the incremental per-file handshake, so an explicit
* --no-incremental also suppresses the delta implication. */
if (!config->whole_file && !no_delta && !no_incremental)
config->use_delta = true;
}
/* Incremental and delta transfers need metadata unless the user disabled it. */
if ((config->use_incremental || config->use_delta) && !config->use_metadata &&
!config->metadata_explicitly_disabled) {
+6
View File
@@ -95,6 +95,12 @@ void print_usage(void) {
"used)\n");
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
printf(" -W, --whole-file Transfer changed files without delta processing\n");
printf(" -y, --fuzzy Use a similar-named file already in the destination\n");
printf(" directory as the delta basis when the destination has no\n");
printf(" usable file at the exact path (saves bandwidth; implies\n");
printf(" --incremental and --delta; inert with --whole-file,\n");
printf(" --no-delta, or --no-incremental)\n");
printf(" --no-fuzzy Disable --fuzzy\n");
printf(" --delta-block <n> Delta block size in bytes (default: %d)\n",
DELTA_BLOCK_SIZE_DEFAULT);
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
+18 -5
View File
@@ -46,6 +46,7 @@ static void config_set_defaults(Config* config) {
config->size_only = false;
config->use_delta = false;
config->whole_file = false;
config->fuzzy = false;
config->modify_window = 0;
config->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT;
config->delta_max_file_size = DELTA_MAX_FILE_SIZE;
@@ -154,9 +155,9 @@ static bool validate_received_config(const Config* config) {
valid_wire_bool(config->use_delete) && valid_wire_bool(config->use_incremental) &&
valid_wire_bool(config->size_only) && valid_wire_bool(config->ignore_times) &&
valid_wire_bool(config->use_delta) && valid_wire_bool(config->backup) &&
valid_wire_bool(config->remove_source_files) && valid_wire_bool(config->follow_symlinks) &&
valid_wire_bool(config->copy_links) && valid_wire_bool(config->safe_links) &&
valid_wire_bool(config->copy_unsafe_links) &&
valid_wire_bool(config->fuzzy) && valid_wire_bool(config->remove_source_files) &&
valid_wire_bool(config->follow_symlinks) && valid_wire_bool(config->copy_links) &&
valid_wire_bool(config->safe_links) && valid_wire_bool(config->copy_unsafe_links) &&
valid_wire_bool(config->preserve_hard_links) && valid_wire_bool(config->preserve_acls) &&
valid_wire_bool(config->preserve_xattrs) && valid_wire_bool(config->preserve_devices) &&
valid_wire_bool(config->preserve_sparse) && valid_wire_bool(config->ignore_existing) &&
@@ -455,6 +456,12 @@ static bool send_basis_options(int fd, const Config* c) {
return true;
}
/* -y/--fuzzy (receiver-side similar-file basis selection). Trailing field on
* the config frame; protocol 2.9.0. */
static bool send_fuzzy_option(int fd, const Config* c) {
return send_int(fd, c->fuzzy);
}
static bool receive_core_fields(int fd, Config* c) {
int value;
if (!receive_wire_bool(fd, &c->eight_bit_output))
@@ -626,12 +633,17 @@ static bool receive_basis_options(int fd, Config* c) {
return true;
}
static bool receive_fuzzy_option(int fd, Config* c) {
return receive_wire_bool(fd, &c->fuzzy);
}
bool config_send(int file_descriptor, const Config* config) {
protocol_session_set_max_alloc(NULL, config->max_alloc);
if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) ||
!send_file_options(file_descriptor, config) ||
!send_selection_options(file_descriptor, config) ||
!send_resume_options(file_descriptor, config) || !send_basis_options(file_descriptor, config))
!send_resume_options(file_descriptor, config) ||
!send_basis_options(file_descriptor, config) || !send_fuzzy_option(file_descriptor, config))
return false;
Status status;
if (!receive_status(file_descriptor, &status))
@@ -664,7 +676,8 @@ Config* config_receive(int file_descriptor) {
!receive_file_options(file_descriptor, config) ||
!receive_selection_options(file_descriptor, config) ||
!receive_resume_options(file_descriptor, config) ||
!receive_basis_options(file_descriptor, config))
!receive_basis_options(file_descriptor, config) ||
!receive_fuzzy_option(file_descriptor, config))
goto error;
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
strcmp(config->compress_choice, "none") != 0) {
+8
View File
@@ -63,6 +63,14 @@ typedef struct Config {
bool size_only;
bool use_delta;
bool whole_file;
/* -y/--fuzzy: when a file must be transferred and the destination holds no
* usable file at the exact path, the receiver may reuse a SIMILAR-named
* existing regular file in the same destination directory as the delta
* basis so the sender transmits only the differences. Crosses the wire
* (the receiver performs the candidate search); the CLI implies
* --incremental + --delta because the similar-basis only matters on the
* receiver-driven delta path. Off by default. */
bool fuzzy;
int modify_window;
uint32_t delta_block_size;
unsigned long long delta_max_file_size;
+343
View File
@@ -1,4 +1,5 @@
#include <errno.h>
#include <dirent.h>
#include <fcntl.h>
#include <libgen.h>
#include <stdio.h>
@@ -349,6 +350,7 @@ fail:
static File* receive_delta_file(int fd, const Config* config, const char* check_path,
void* old_data, unsigned long long old_size, bool* failed) {
if (!old_data) {
free(old_data); /* defensive: old_data is always non-NULL today */
*failed = true;
return NULL;
}
@@ -674,6 +676,313 @@ static bool basis_match_find(const Config* config, const char* check_path,
return false;
}
/* ---------------------------------------------------------------------------
* -y/--fuzzy similar-file delta basis.
*
* When a file must be transferred and the destination holds no usable content
* at the exact path (the destination file is absent, or is outside the delta
* engine's size bounds), --fuzzy lets the receiver reuse an EXISTING regular
* file in the SAME destination directory as the delta basis, so the sender
* transmits only the differences instead of the whole file. This is the
* rsync "find a similar file to use as a basis for a transfer" case (e.g. a
* file recreated under a new name whose old-named sibling is still present).
*
* The delta handshake is unchanged and receiver-driven, so the sender never
* learns the basis was a different file and needs no new protocol. Byte
* exactness never depends on which bytes the basis holds: the delta protocol
* only references basis blocks whose Adler-32 + xxHash32 checksums match the
* source, delta_apply validates every reference against the basis size, and a
* basis that shares nothing simply makes the sender reply STATUS_NEXT (full
* transfer). A fuzzy basis can therefore waste bandwidth but never corrupt a
* file.
*
* Similarity heuristic (deterministic, deliberately simpler than rsync's):
* * candidates are the target's sibling entries in its destination
* directory, opened through the confined root (file_open_secure_parent +
* openat O_NOFOLLOW, fstatat AT_SYMLINK_NOFOLLOW) -- symlinks are never
* followed and nothing outside the destination root is ever read;
* * dotfiles, directories, the target's own name, and the .fastsync-stage /
* temp scratch names are never candidates;
* * size gate = the delta engine's own bounds (delta_should_attempt: both
* files >= DELTA_MIN_FILE_SIZE, <= delta_max_file_size, ratio <= 10x),
* NOT rsync's ~1.5x size window;
* * name gate = Levenshtein edit distance between the basenames, accepted
* only when distance <= half the length of the longer basename;
* * the single best candidate (smallest distance; tie-break: size closest
* to the incoming file, then lexicographically smaller basename) is read
* and returned as the basis.
* ------------------------------------------------------------------------- */
/* A directory scan is linear in the number of entries; the fuzzy search stops
* after this many so a pathological huge directory cannot stall a transfer.
* The cap bounds the readdir() ITERATIONS, not the per-entry work: every
* entry that survives the (cheap) size and pre-name gates still runs an
* edit-distance DP, so the per-entry DP cost is separately bounded below by
* pre-pruning on the name length gap and the absent-character bound, and by
* trimming the common prefix/suffix before the DP runs on the middles only. */
#define FUZZY_MAX_DIRECTORY_SCAN 4096
/* Names longer than this never take part in fuzzy matching: the edit-distance
* DP below is O(len^2), so over-long names are bounded out of the search. */
#define FUZZY_NAME_LIMIT 192
typedef struct {
char name[FUZZY_NAME_LIMIT + 1];
unsigned long long size;
size_t distance;
unsigned long long size_gap;
} FuzzyCandidate;
/* Two-row DP scratch, allocated once per directory scan (not per candidate) so
* a 4096-entry directory never performs 4096 malloc/free pairs. */
typedef struct {
size_t* prev;
size_t* cur;
} FuzzyEditBuffer;
static bool fuzzy_edit_buffer_init(FuzzyEditBuffer* buf) {
buf->prev = malloc((FUZZY_NAME_LIMIT + 1) * sizeof(size_t));
buf->cur = malloc((FUZZY_NAME_LIMIT + 1) * sizeof(size_t));
if (!buf->prev || !buf->cur) {
free(buf->prev);
free(buf->cur);
buf->prev = NULL;
buf->cur = NULL;
return false;
}
return true;
}
static void fuzzy_edit_buffer_destroy(FuzzyEditBuffer* buf) {
free(buf->prev);
free(buf->cur);
buf->prev = NULL;
buf->cur = NULL;
}
/* Cheap lower bounds used to reject a candidate BEFORE the DP:
* - any edit script must at least absorb the length gap: d >= |la - lb|;
* - any character of `a` that does not occur in `b` at all must be deleted or
* substituted at its own position: d >= (count of such characters).
* The acceptance gate is d*2 <= longer, so a candidate whose max of these two
* bounds already violates it can be skipped without computing the distance. */
static size_t fuzzy_absent_char_bound(const char* a, size_t la, const char* b, size_t lb) {
if (lb == 0)
return la;
bool present[256] = {false};
for (size_t i = 0; i < lb; i++)
present[(uint8_t)b[i]] = true;
size_t absent = 0;
for (size_t i = 0; i < la; i++)
if (!present[(uint8_t)a[i]])
absent++;
return absent;
}
/* Levenshtein edit distance between the two basenames. A shared prefix and a
* (non-overlapping) shared suffix can always be aligned at no cost, so the DP
* only runs over the differing middles; its two rows come from `buf` (allocated
* once by the caller). Callers enforce la, lb <= FUZZY_NAME_LIMIT. */
static size_t fuzzy_edit_distance(FuzzyEditBuffer* buf, const char* a, size_t la, const char* b,
size_t lb) {
size_t p = 0;
while (p < la && p < lb && a[p] == b[p])
p++;
/* Trim the common suffix (never overlapping the prefix). Working with two
moving end indices keeps the region arithmetic explicit and safe. */
size_t ae = la;
size_t be = lb;
while (ae > p && be > p && a[ae - 1] == b[be - 1]) {
ae--;
be--;
}
size_t ma = ae - p;
size_t mb = be - p;
/* cppcheck-suppress knownConditionTrueFalse -- the prefix/suffix trims above
only run while the corresponding ends match, so a middle can remain; the
analysis unsoundly concludes the trims always consume everything. */
if (ma == 0)
return mb;
if (mb == 0)
return ma;
const char* A = a + p;
const char* B = b + p;
size_t* prev = buf->prev;
size_t* cur = buf->cur;
for (size_t j = 0; j <= mb; j++)
prev[j] = j;
for (size_t i = 1; i <= ma; i++) {
cur[0] = i;
for (size_t j = 1; j <= mb; j++) {
size_t cost = A[i - 1] == B[j - 1] ? 0 : 1;
size_t del = prev[j] + 1;
size_t ins = cur[j - 1] + 1;
size_t sub = prev[j - 1] + cost;
size_t m = del < ins ? del : ins;
cur[j] = m < sub ? m : sub;
}
size_t* tmp = prev;
prev = cur;
cur = tmp;
}
return prev[mb];
}
/* Deterministic ordering of two fuzzy candidates: smallest edit distance,
* then the size closest to the incoming file, then the lexical basename. */
static bool fuzzy_candidate_better(const FuzzyCandidate* cand, const FuzzyCandidate* best) {
if (!best->name[0])
return true;
if (cand->distance != best->distance)
return cand->distance < best->distance;
if (cand->size_gap != best->size_gap)
return cand->size_gap < best->size_gap;
return strcmp(cand->name, best->name) < 0;
}
/* Search the destination directory that will contain `check_path` for a
* similar regular file usable as a --fuzzy delta basis and return its full
* content in a malloc'd (protocol_alloc) buffer. Returns NULL (with *out_size
* = 0) when no candidate qualifies, which means the caller performs the normal
* whole-file transfer. */
static void* fuzzy_basis_find_and_load(const Config* config, const char* check_path,
unsigned long long check_size,
unsigned long long* out_size) {
*out_size = 0;
if (!config || !config->receive_root_directory || !config->fuzzy || !config->use_delta ||
!check_path || check_size < DELTA_MIN_FILE_SIZE || check_size > config->delta_max_file_size ||
check_size > MAX_RECEIVE_WHOLE_FILE_SIZE)
return NULL;
char* full_path = path_cat(config->receive_root_directory, check_path);
if (!full_path)
return NULL;
char* leaf = NULL;
int dir_fd = file_open_secure_parent(full_path, &leaf, false);
if (dir_fd < 0 || !leaf) {
free(leaf);
free(full_path);
return NULL;
}
size_t target_len = strlen(leaf);
/* A target basename longer than FUZZY_NAME_LIMIT can never pass the name gate
(every candidate name is bounded by the same limit), so skip the scan. */
if (target_len > FUZZY_NAME_LIMIT) {
close(dir_fd);
free(leaf);
free(full_path);
return NULL;
}
int scanfd = dup(dir_fd);
if (scanfd < 0) {
close(dir_fd);
free(leaf);
free(full_path);
return NULL;
}
DIR* dir = fdopendir(scanfd);
if (!dir) {
close(scanfd);
close(dir_fd);
free(leaf);
free(full_path);
return NULL;
}
/* The DP scratch rows are allocated once per scan (not once per candidate). */
FuzzyEditBuffer ebuf;
if (!fuzzy_edit_buffer_init(&ebuf)) {
closedir(dir);
close(dir_fd);
free(leaf);
free(full_path);
return NULL;
}
FuzzyCandidate best;
memset(&best, 0, sizeof(best));
const struct dirent* entry;
size_t scanned = 0;
/* readdir() yields entries in filesystem-dependent order, so the SET of
candidates seen is order-dependent; the winner is still deterministic
because every candidate is compared with the total ordering in
fuzzy_candidate_better (acceptable for a heuristic). */
while (scanned < FUZZY_MAX_DIRECTORY_SCAN && (entry = readdir(dir)) != NULL) {
scanned++;
const char* name = entry->d_name;
size_t name_len = strlen(name);
if (name[0] == '.' || name_len == 0 || name_len > FUZZY_NAME_LIMIT || strcmp(name, leaf) == 0)
continue;
struct stat st;
if (fstatat(dir_fd, name, &st, AT_SYMLINK_NOFOLLOW) != 0 || !S_ISREG(st.st_mode))
continue;
unsigned long long cand_size = (unsigned long long)st.st_size;
if (cand_size == 0 || cand_size > MAX_RECEIVE_WHOLE_FILE_SIZE ||
!delta_should_attempt(cand_size, check_size, config->delta_max_file_size))
continue;
/* Cheap pre-name gates run BEFORE the edit-distance DP. The edit distance
is bounded below by the length gap |la-lb| and by the number of
characters of one basename that are absent from the other (each such
position costs at least one op), so a candidate whose acceptance gate
(distance*2 <= longer) already fails on the max of those bounds is
skipped without running the DP. */
size_t longer = target_len > name_len ? target_len : name_len;
size_t bound = longer - (target_len < name_len ? target_len : name_len);
size_t absent = fuzzy_absent_char_bound(leaf, target_len, name, name_len);
if (absent > bound)
bound = absent;
if (bound * 2 > longer)
continue;
size_t distance = fuzzy_edit_distance(&ebuf, leaf, target_len, name, name_len);
if (distance * 2 > longer)
continue;
FuzzyCandidate cand;
memcpy(cand.name, name, name_len + 1);
cand.size = cand_size;
cand.distance = distance;
cand.size_gap = cand_size > check_size ? cand_size - check_size : check_size - cand_size;
if (fuzzy_candidate_better(&cand, &best))
best = cand;
}
closedir(dir);
free(leaf);
fuzzy_edit_buffer_destroy(&ebuf);
void* basis = NULL;
if (best.name[0]) {
/* O_NONBLOCK: a name raced to a FIFO between the fstatat gate and this open
would otherwise block the receive thread forever on open(2); with it the
open fails (ENXIO) and the fstat/S_ISREG gate below would reject it too.
A regular file opened with O_NONBLOCK is unaffected. */
int fd = openat(dir_fd, best.name, O_RDONLY | O_NOFOLLOW | O_NONBLOCK | O_CLOEXEC);
if (fd >= 0) {
struct stat st;
if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode) &&
(unsigned long long)st.st_size == best.size && best.size <= SIZE_MAX) {
basis = protocol_alloc((size_t)best.size);
if (basis) {
size_t got = 0;
while (got < (size_t)best.size) {
ssize_t n = read(fd, (char*)basis + got, (size_t)best.size - got);
if (n <= 0) {
free(basis);
basis = NULL;
break;
}
got += (size_t)n;
}
}
}
close(fd);
}
}
close(dir_fd);
free(full_path);
if (basis)
*out_size = best.size;
return basis;
}
File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
if (!config || !skipped) {
send_status(fd, STATUS_ERROR);
@@ -905,6 +1214,40 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
free(old_data);
old_data = NULL;
/* ---- -y/--fuzzy similar-file delta basis ----
* Reaching this point means the file must be transferred and the
* destination's own content at the exact path could not serve as a delta
* basis (it is absent, outside the delta size bounds, or unreadable). With
* --fuzzy the receiver tries an existing similar-named file in the same
* destination directory instead. receive_delta_file performs the whole
* handshake: when the sender judges the delta not worthwhile it replies
* STATUS_NEXT and the full content is received there, so a fuzzy attempt
* can only improve bandwidth, never fall through into the plain transfer
* below (that path is reserved for "no usable candidate was found"). */
if (config->fuzzy && config->use_delta) {
unsigned long long fuzzy_size = 0;
void* fuzzy_basis = fuzzy_basis_find_and_load(config, check_path, check_size, &fuzzy_size);
if (fuzzy_basis != NULL) {
bool fuzzy_failed = false;
File* fuzzy_file =
receive_delta_file(fd, config, check_path, fuzzy_basis, fuzzy_size, &fuzzy_failed);
fuzzy_basis = NULL; /* receive_delta_file consumes the buffer on every path */
if (fuzzy_file) {
close(old_fd);
free(full_path);
free(check_path);
return fuzzy_file;
}
if (fuzzy_failed) {
close(old_fd);
free(full_path);
free(check_path);
return NULL;
}
}
free(fuzzy_basis);
}
if (!send_status(fd, STATUS_NEXT)) {
close(old_fd);
free(full_path);