src/shared/file_receive.c

@@ -1,4 +1,5 @@
 #include <errno.h>
+#include <ctype.h>
 #include <dirent.h>
 #include <fcntl.h>
 #include <libgen.h>
@@ -1401,7 +1402,8 @@ static bool basis_match_find(const Config* config, const char* check_path,
  * transfer).  A fuzzy basis can therefore waste bandwidth but never corrupt a
  * file.
  *
- * Similarity heuristic (deterministic, deliberately simpler than rsync's):
+ * Similarity heuristic (rsync 3.4.1 parity, util1.c fuzzy_distance /
+ * find_filename_suffix + generator.c find_fuzzy):
  *   * candidates are the target's sibling entries in its destination
  *     directory, opened through the confined root (file_open_secure_parent +
  *     openat O_NOFOLLOW, fstatat AT_SYMLINK_NOFOLLOW) -- symlinks are never
@@ -1410,12 +1412,15 @@ static bool basis_match_find(const Config* config, const char* check_path,
  *     temp scratch names are never candidates;
  *   * size gate = the delta engine's own bounds (delta_should_attempt: both
  *     files >= DELTA_MIN_FILE_SIZE, <= delta_max_file_size, ratio <= 10x),
- *     NOT rsync's ~1.5x size window;
+ *     because FastSync's delta engine cannot use a basis outside them;
- *   * name gate = Levenshtein edit distance between the basenames, accepted
+ *   * first pass = an exact size+mtime match wins regardless of name (rsync's
- *     only when distance <= half the length of the longer basename;
+ *     "fuzzy size/modtime match");
- *   * the single best candidate (smallest distance; tie-break: size closest
+ *   * otherwise the winner minimizes rsync's weighted Levenshtein distance
- *     to the incoming file, then lexicographically smaller basename) is read
+ *     (substitution ± byte difference, insertion UNIT+byte, 16.16 fixed point)
- *     and returned as the basis.
+ *     plus ten times the suffix distance, accepted only when <= 25*UNIT; the
+ *     tie-break (smallest size gap, then lexical name) keeps the result
+ *     deterministic across filesystem readdir order (rsync leaves equal
+ *     distances to its file-list order).
  * ------------------------------------------------------------------------- */
 
 /* A directory scan is linear in the number of entries; the fuzzy search stops
@@ -1433,107 +1438,109 @@ static bool basis_match_find(const Config* config, const char* check_path,
 typedef struct {
   char name[FUZZY_NAME_LIMIT + 1];
   unsigned long long size;
-  size_t distance;
+  uint32_t distance;
   unsigned long long size_gap;
 } FuzzyCandidate;
 
-/* Two-row DP scratch, allocated once per directory scan (not per candidate) so
+/* rsync's fuzzy distance is a weighted Levenshtein variant in 16.16 fixed point
- * a 4096-entry directory never performs 4096 malloc/free pairs. */
+ * (util1.c fuzzy_distance): a substitution costs UNIT +/- the byte difference
-typedef struct {
+ * and an insertion costs UNIT + the inserted byte, so similar names score low.
-  size_t* prev;
+ * The search keeps only distances <= 25*UNIT.  Ported verbatim for parity. */
-  size_t* cur;
+#define FUZZY_DIST_UNIT (1u << 16)
-} FuzzyEditBuffer;
+#define FUZZY_DIST_REJECT (0xFFFFu * FUZZY_DIST_UNIT + 1)
+#define FUZZY_DIST_LIMIT (25u * FUZZY_DIST_UNIT)
 
-static bool fuzzy_edit_buffer_init(FuzzyEditBuffer* buf) {
+static uint32_t fuzzy_distance(const char* s1, unsigned len1, const char* s2, unsigned len2,
-  buf->prev = malloc((FUZZY_NAME_LIMIT + 1) * sizeof(size_t));
+                               uint32_t upperlimit, uint32_t* scratch) {
-  buf->cur = malloc((FUZZY_NAME_LIMIT + 1) * sizeof(size_t));
+  if ((len1 > len2 ? len1 - len2 : len2 - len1) * FUZZY_DIST_UNIT > upperlimit)
-  if (!buf->prev || !buf->cur) {
+    return FUZZY_DIST_REJECT;
-    free(buf->prev);
+  if (!len1 || !len2) {
-    free(buf->cur);
+    if (!len1) {
-    buf->prev = NULL;
+      s1 = s2;
-    buf->cur = NULL;
+      len1 = len2;
-    return false;
     }
-  return true;
+    uint32_t cost = 0;
+    for (unsigned i = 0; i < len1; i++)
+      cost += (uint8_t)s1[i];
+    return (uint32_t)len1 * FUZZY_DIST_UNIT + cost;
+  }
+  uint32_t* a = scratch;
+  for (unsigned i2 = 0; i2 < len2; i2++)
+    a[i2] = (i2 + 1) * FUZZY_DIST_UNIT;
+  for (unsigned i1 = 0; i1 < len1; i1++) {
+    uint32_t diag = i1 * FUZZY_DIST_UNIT;
+    uint32_t above = (i1 + 1) * FUZZY_DIST_UNIT;
+    for (unsigned i2 = 0; i2 < len2; i2++) {
+      uint32_t left = a[i2];
+      int32_t cost = (int32_t)(uint8_t)s1[i1] - (int32_t)(uint8_t)s2[i2];
+      if (cost != 0)
+        cost = cost < 0 ? (int32_t)(FUZZY_DIST_UNIT - (uint32_t)(-cost))
+                        : (int32_t)(FUZZY_DIST_UNIT + (uint32_t)cost);
+      uint32_t diag_inc = diag + (uint32_t)cost;
+      uint32_t left_inc = left + FUZZY_DIST_UNIT + (uint8_t)s1[i1];
+      uint32_t above_inc = above + FUZZY_DIST_UNIT + (uint8_t)s2[i2];
+      a[i2] = above = left < above ? (left_inc < diag_inc ? left_inc : diag_inc)
+                                   : (above_inc < diag_inc ? above_inc : diag_inc);
+      diag = left;
+    }
+  }
+  return a[len2 - 1];
 }
 
-static void fuzzy_edit_buffer_destroy(FuzzyEditBuffer* buf) {
+/* rsync's find_filename_suffix (util1.c): return the last significant filename
-  free(buf->prev);
+ * suffix (its dot included).  Leading dots are not a suffix; a trailing "~" is
-  free(buf->cur);
+ * ignored; .bak/.old/.orig and a "~/<num>" backup marker are skipped. */
-  buf->prev = NULL;
+static const char* fuzzy_find_suffix(const char* fn, int fn_len, int* len_ptr) {
-  buf->cur = NULL;
+  const char* suf;
+  const char* s;
+  bool had_tilde;
+  int s_len;
+
+  while (fn_len && *fn == '.') {
+    fn++;
+    fn_len--;
+  }
+  if (fn_len > 1 && fn[fn_len - 1] == '~') {
+    fn_len--;
+    had_tilde = true;
+  } else {
+    had_tilde = false;
+  }
+  suf = "";
+  *len_ptr = 0;
+  for (s = fn + fn_len; fn_len > 1;) {
+    while (--s != fn && *s != '.') {
+    }
+    if (s == fn)
+      break;
+    s_len = fn_len - (int)(s - fn);
+    fn_len = (int)(s - fn);
+    if (s_len == 4) {
+      if (strcmp(s + 1, "bak") == 0 || strcmp(s + 1, "old") == 0)
+        continue;
+    } else if (s_len == 5) {
+      if (strcmp(s + 1, "orig") == 0)
+        continue;
+    } else if (s_len > 2 && had_tilde && s[1] == '~' && isdigit((unsigned char)s[2])) {
+      continue;
+    }
+    *len_ptr = s_len;
+    suf = s;
+    if (s_len == 1)
+      break;
+    for (s++, s_len--; s_len > 0; s++, s_len--) {
+      if (!isdigit((unsigned char)*s))
+        return suf;
+    }
+    s = suf;
+  }
+  return suf;
 }
 
-/* Cheap lower bounds used to reject a candidate BEFORE the DP:
- *  - any edit script must at least absorb the length gap: d >= |la - lb|;
- *  - any character of `a` that does not occur in `b` at all must be deleted or
- *    substituted at its own position: d >= (count of such characters).
- * The acceptance gate is d*2 <= longer, so a candidate whose max of these two
- * bounds already violates it can be skipped without computing the distance. */
-static size_t fuzzy_absent_char_bound(const char* a, size_t la, const char* b, size_t lb) {
-  if (lb == 0)
-    return la;
-  bool present[256] = {false};
-  for (size_t i = 0; i < lb; i++)
-    present[(uint8_t)b[i]] = true;
-  size_t absent = 0;
-  for (size_t i = 0; i < la; i++)
-    if (!present[(uint8_t)a[i]])
-      absent++;
-  return absent;
-}
 
-/* Levenshtein edit distance between the two basenames.  A shared prefix and a
+/* Deterministic ordering of two fuzzy candidates with equal rsync distance:
- * (non-overlapping) shared suffix can always be aligned at no cost, so the DP
+ * smallest size gap, then the lexical basename (rsync itself takes the last
- * only runs over the differing middles; its two rows come from `buf` (allocated
+ * equal-distance candidate in file-list order). */
- * once by the caller).  Callers enforce la, lb <= FUZZY_NAME_LIMIT. */
-static size_t fuzzy_edit_distance(FuzzyEditBuffer* buf, const char* a, size_t la, const char* b,
-                                  size_t lb) {
-  size_t p = 0;
-  while (p < la && p < lb && a[p] == b[p])
-    p++;
-  /* Trim the common suffix (never overlapping the prefix).  Working with two
-     moving end indices keeps the region arithmetic explicit and safe. */
-  size_t ae = la;
-  size_t be = lb;
-  while (ae > p && be > p && a[ae - 1] == b[be - 1]) {
-    ae--;
-    be--;
-  }
-  size_t ma = ae - p;
-  size_t mb = be - p;
-  /* cppcheck-suppress knownConditionTrueFalse -- the prefix/suffix trims above
-     only run while the corresponding ends match, so a middle can remain; the
-     analysis unsoundly concludes the trims always consume everything. */
-  if (ma == 0)
-    return mb;
-  if (mb == 0)
-    return ma;
-  const char* A = a + p;
-  const char* B = b + p;
-  size_t* prev = buf->prev;
-  size_t* cur = buf->cur;
-  for (size_t j = 0; j <= mb; j++)
-    prev[j] = j;
-  for (size_t i = 1; i <= ma; i++) {
-    cur[0] = i;
-    for (size_t j = 1; j <= mb; j++) {
-      size_t cost = A[i - 1] == B[j - 1] ? 0 : 1;
-      size_t del = prev[j] + 1;
-      size_t ins = cur[j - 1] + 1;
-      size_t sub = prev[j - 1] + cost;
-      size_t m = del < ins ? del : ins;
-      cur[j] = m < sub ? m : sub;
-    }
-    size_t* tmp = prev;
-    prev = cur;
-    cur = tmp;
-  }
-  return prev[mb];
-}
-
-/* Deterministic ordering of two fuzzy candidates: smallest edit distance,
- * then the size closest to the incoming file, then the lexical basename. */
 static bool fuzzy_candidate_better(const FuzzyCandidate* cand, const FuzzyCandidate* best) {
   if (!best->name[0])
     return true;
@@ -1550,8 +1557,8 @@ static bool fuzzy_candidate_better(const FuzzyCandidate* cand, const FuzzyCandid
  * = 0) when no candidate qualifies, which means the caller performs the normal
  * whole-file transfer. */
 static void* fuzzy_basis_find_and_load(const Config* config, const char* check_path,
-                                       unsigned long long check_size,
+                                       unsigned long long check_size, time_t check_mtime,
-                                       unsigned long long* out_size) {
+                                       long check_mtime_nsec, unsigned long long* out_size) {
   *out_size = 0;
   if (!config || !config->receive_root_directory || !config->fuzzy || !config->use_delta ||
       !check_path || check_size < DELTA_MIN_FILE_SIZE || check_size > config->delta_max_file_size ||
@@ -1594,18 +1601,28 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
     return NULL;
   }
 
-  /* The DP scratch rows are allocated once per scan (not once per candidate). */
+  /* The weighted-distance scratch row is allocated once per scan (not once per
-  FuzzyEditBuffer ebuf;
+     candidate). */
-  if (!fuzzy_edit_buffer_init(&ebuf)) {
+  uint32_t* dist_scratch = malloc((FUZZY_NAME_LIMIT + 1) * sizeof(uint32_t));
+  if (!dist_scratch) {
     closedir(dir);
     close(dir_fd);
     free(leaf);
     free(full_path);
     return NULL;
   }
+  int fname_suf_len = 0;
+  const char* fname_suf = fuzzy_find_suffix(leaf, (int)target_len, &fname_suf_len);
 
   FuzzyCandidate best;
   memset(&best, 0, sizeof(best));
+  uint32_t lowest_dist = FUZZY_DIST_LIMIT;
+  /* rsync's fuzzy search runs an exact size+mtime pass before the name-distance
+     pass; such a candidate is almost certainly the same content and wins
+     regardless of how dissimilar its name is.  The first one (directory order,
+     deterministic) is kept. */
+  FuzzyCandidate exact;
+  memset(&exact, 0, sizeof(exact));
   const struct dirent* entry;
   size_t scanned = 0;
   /* readdir() yields entries in filesystem-dependent order, so the SET of
@@ -1625,22 +1642,32 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
     if (cand_size == 0 || cand_size > MAX_RECEIVE_WHOLE_FILE_SIZE ||
         !delta_should_attempt(cand_size, check_size, config->delta_max_file_size))
       continue;
-    /* Cheap pre-name gates run BEFORE the edit-distance DP.  The edit distance
+    long cand_nsec = 0;
-       is bounded below by the length gap |la-lb| and by the number of
+#ifdef __linux__
-       characters of one basename that are absent from the other (each such
+    cand_nsec = st.st_mtim.tv_nsec;
-       position costs at least one op), so a candidate whose acceptance gate
+#endif
-       (distance*2 <= longer) already fails on the max of those bounds is
+    if (!exact.name[0] && cand_size == check_size &&
-       skipped without running the DP. */
+        metadata_mtime_matches(st.st_mtime, cand_nsec, check_mtime, check_mtime_nsec,
-    size_t longer = target_len > name_len ? target_len : name_len;
+                               config->modify_window)) {
-    size_t bound = longer - (target_len < name_len ? target_len : name_len);
+      memcpy(exact.name, name, name_len + 1);
-    size_t absent = fuzzy_absent_char_bound(leaf, target_len, name, name_len);
+      exact.size = cand_size;
-    if (absent > bound)
+      exact.size_gap = 0;
-      bound = absent;
-    if (bound * 2 > longer)
       continue;
-    size_t distance = fuzzy_edit_distance(&ebuf, leaf, target_len, name, name_len);
+    }
-    if (distance * 2 > longer)
+    /* rsync's name-distance pass: a weighted Levenshtein distance over the full
+       basenames, plus ten times the same distance over the filename suffixes,
+       accepted only when it does not exceed the running lowest distance. */
+    int name_suf_len = 0;
+    const char* name_suf = fuzzy_find_suffix(name, (int)name_len, &name_suf_len);
+    uint32_t distance =
+        fuzzy_distance(name, (unsigned)name_len, leaf, (unsigned)target_len, lowest_dist, dist_scratch);
+    if (distance < 0xFFFF0000U)
+      distance += fuzzy_distance(name_suf, (unsigned)name_suf_len, fname_suf, (unsigned)fname_suf_len,
+                                 0xFFFF0000U, dist_scratch) *
+                  10;
+    if (distance > lowest_dist)
       continue;
+    lowest_dist = distance;
     FuzzyCandidate cand;
     memcpy(cand.name, name, name_len + 1);
     cand.size = cand_size;
@@ -1651,7 +1678,11 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
   }
   closedir(dir);
   free(leaf);
-  fuzzy_edit_buffer_destroy(&ebuf);
+  free(dist_scratch);
+
+  /* Prefer the exact size+mtime candidate over any name-distance winner. */
+  if (exact.name[0])
+    best = exact;
 
   void* basis = NULL;
   if (best.name[0]) {
@@ -2373,7 +2404,8 @@ static IncrementalCheckOutcome incremental_check_try_fuzzy(IncrementalCheckState
     return INCREMENTAL_CONTINUE;
   unsigned long long fuzzy_size = 0;
   void* fuzzy_basis =
-      fuzzy_basis_find_and_load(config, state->check_path, state->check_size, &fuzzy_size);
+      fuzzy_basis_find_and_load(config, state->check_path, state->check_size,
+                                (time_t)state->check_mtime, (long)state->check_mtime_nsec, &fuzzy_size);
   if (fuzzy_basis != NULL) {
     bool fuzzy_failed = false;
     File* fuzzy_file = receive_delta_file(state->fd, config, state->check_path, fuzzy_basis,

tests/integration/test_features.py

@@ -4963,15 +4963,20 @@ class TestFuzzy:
             "no-candidate fuzzy run should have sent the whole file"
 
     def test_dissimilar_sibling_is_not_used(self, shared_server):
-        # The destination holds a large sibling whose basename is too different
+        # A sibling whose basename is too different from the incoming name is
-        # from the incoming name; the name gate must reject it and fall back to
+        # rejected by rsync's fuzzy distance window (the length gap exceeds
-        # a whole-file transfer.
+        # 25), so the run falls back to a whole-file transfer.  A distinct
+        # mtime keeps rsync's exact size+mtime first pass from accepting it.
         source, dest = self._prepare("dissim")
         old_bytes, new_bytes = _random_payloads()
-        self._seed_dest(source, dest, {"totally-unrelated-notes.bin": old_bytes},
+        long_name = "totally-unrelated-notes-with-a-very-long-name.bin"
+        self._seed_dest(source, dest, {long_name: old_bytes},
                            shared_server.port)
         with open(os.path.join(source, self.NEW_NAME), "wb") as fh:
             fh.write(new_bytes)
+        received_dir = get_dest_received_dir(dest, source)
+        os.utime(os.path.join(received_dir, long_name), (self.TS, self.TS))
+        os.utime(os.path.join(source, self.NEW_NAME), (self.TS + 100000, self.TS + 100000))
         result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port)
         assert result.returncode == 0, \
             f"--fuzzy dissimilar-sibling run failed: {(result.stderr or result.stdout)[:300]}"
@@ -4980,6 +4985,28 @@ class TestFuzzy:
         assert proxy.client_to_server > len(new_bytes) // 2, \
             "a dissimilar-named sibling must not be used as a fuzzy basis"
 
+    def test_exact_size_mtime_sibling_is_used(self, shared_server):
+        # rsync's fuzzy first pass accepts a sibling with an exact size+mtime
+        # match regardless of how unrelated its name is (its content is almost
+        # certainly the same).
+        source, dest = self._prepare("exact")
+        old_bytes, new_bytes = _random_payloads()
+        self._seed_dest(source, dest, {"unrelated-blob.bin": old_bytes},
+                           shared_server.port)
+        with open(os.path.join(source, self.NEW_NAME), "wb") as fh:
+            fh.write(new_bytes)
+        received_dir = get_dest_received_dir(dest, source)
+        ts = 1600000000
+        os.utime(os.path.join(received_dir, "unrelated-blob.bin"), (ts, ts))
+        os.utime(os.path.join(source, self.NEW_NAME), (ts, ts))
+        result, proxy = self._run_measured(source, dest, ["--fuzzy"], shared_server.port)
+        assert result.returncode == 0, \
+            f"--fuzzy exact size+mtime run failed: {(result.stderr or result.stdout)[:300]}"
+        received = get_dest_received_dir(dest, source)
+        assert _read_file(os.path.join(received, self.NEW_NAME)) == new_bytes
+        assert proxy.client_to_server < len(new_bytes) // 4, \
+            "an exact size+mtime sibling should be used as a fuzzy basis"
+
     def test_fuzzy_helps_when_dest_holds_an_unsuitable_file(self, shared_server):
         # The destination DOES hold the exact new name, but it is a tiny stale
         # file (below the delta engine's minimum, ratio far outside its window),