feat(parity): real --info=del deletion lines + rsync throttle pacing

- Wire: config frame gains report_deletes (protocol 2.26.0 -> 2.27.0); the receiver lists actually-removed paths in the STATUS_STATS path list, so the sender prints rsync's `deleting PATH` / `*deleting PATH` lines for a real --delete run (and -i/out-format). Observers threaded through the manifest, missing-args and per-directory delete engines; golden wire len/hash updated. - bwlimit: throttle now paces like rsync 3.4.1 -- ~100ms burst capacity and the sleep is no longer credited as refill, so 4 MiB at 1024/2048 KiB/s matches rsync within ~4% (was ~2x too fast).
2026-09-18 21:06:20 +02:00
parent f3ac4df4d0
commit 6a129b54d4
20 changed files with 332 additions and 89 deletions
@@ -259,9 +259,17 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
 * for -n/--dry-run --delete, the destination-relative paths it WOULD have
 * deleted.  It is set by the client only when --stats, --progress/-P, an
 * --out-format token needs a wire counter (%b/%c), or a dry-run carries
+ * --delete; the transfer decision itself is unchanged.
+ *
+ * --info wave (protocol 2.27.0).  report_deletes tells the receiver to include
+ * the destination-relative paths it ACTUALLY removed in its terminal
+ * STATUS_STATS record (the same path-list field the dry-run would-delete report
+ * uses), so the sender can print rsync's `deleting PATH`/`*deleting` lines for a
+ * real (non-dry-run) deletion.  It is set only when --info=del is requested with
 * --delete; the transfer decision itself is unchanged. */
 #define CONFIG_WIRE_OUTPUT_FIELDS(X)                                                               \
-  X(report_dest_info, bool, false, BOOL) X(report_stats, bool, false, BOOL)
+  X(report_dest_info, bool, false, BOOL) X(report_stats, bool, false, BOOL)                        \
+      X(report_deletes, bool, false, BOOL)

 /* Codec-negotiation wave (protocol 2.26.0).  compression_algo is the concrete
 * codec the client selected for this transfer (a CompressionAlgo id) and is the
@@ -988,7 +996,14 @@ typedef struct Config {
 * boundary, and the strict same-version handshake (config_receive rejects a
 * mismatched version before parsing anything else) keeps mixed deployments from
 * ever reaching that state. */
-#define PROTOCOL_VERSION "2.26.0"
+/* (7) --info=del report (protocol 2.27.0): the config frame gains one trailing
+ * bool, report_deletes, appended after report_stats.  When set, the receiver
+ * lists the paths it actually removed in the terminal STATUS_STATS path list
+ * (the same count-delimited list the -n/--dry-run would-delete report uses), so
+ * the sender can print rsync's `deleting PATH` lines for a real deletion.  No
+ * change to the fixed STATUS_STATS record itself; only a new trailing config
+ * bool, which still requires the version bump for the strict lockstep. */
+#define PROTOCOL_VERSION "2.27.0"
 #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
 /* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
 #define MAX_BASIS_DIRS 64
@@ -426,8 +426,16 @@ struct DeletePlanSession {
  ArrayList* size_skipped;
  ArrayList* missing;
  ArrayList* deferred;
+  DeletePathObserver observer;
+  void* observer_context;
 };

+/* Report one path the session truly removed (no-op without an observer). */
+static void notify_deleted(DeletePlanSession* session, const char* rel) {
+  if (session && session->observer && rel)
+    session->observer(session->observer_context, rel);
+}
+
 DeletePlanSession* delete_plan_session_create(const Config* config) {
  if (!config)
    return NULL;
@@ -642,6 +650,7 @@ static bool process_extra_dir(int dirfd, const char* name, const char* child_rel
    session->deleted++;
    session->planned++;
    log_deleted(child_rel);
+    notify_deleted(session, child_rel);
    *removed = true;
    return true;
  }
@@ -667,6 +676,7 @@ static bool process_extra_file(int dirfd, const char* name, const char* child_re
    session->deleted++;
    session->planned++;
    log_deleted(child_rel);
+    notify_deleted(session, child_rel);
  } else if (errno != ENOENT) {
    return false;
  }
@@ -781,8 +791,9 @@ static bool apply_missing(DeletePlanSession* session, const Config* config) {
  size_t deleted = 0;
  size_t skipped = 0;
  bool limit = false;
-  bool ok = manifest_delete_missing_args_limited(config, &manifest, remaining, &deleted, &skipped,
-                                                 &limit);
+  bool ok = manifest_delete_missing_args_limited_observed(
+      config, &manifest, remaining, &deleted, &skipped, &limit, session->observer,
+      session->observer_context);
  session->deleted += deleted;
  session->planned += deleted;
  session->skipped += skipped;
@@ -875,12 +886,21 @@ static bool apply_deferred_path(DeletePlanSession* session, const Config* config
  if (rc == 0) {
    session->deleted++;
    log_deleted(rel);
+    notify_deleted(session, rel);
  }
  close(parent_fd);
  free(leaf);
  return ok;
 }

+void delete_plan_session_set_delete_observer(DeletePlanSession* session, DeletePathObserver observer,
+                                             void* context) {
+  if (!session)
+    return;
+  session->observer = observer;
+  session->observer_context = context;
+}
+
 DeleteCommitResult delete_plan_session_commit(DeletePlanSession* session, const Config* config) {
  if (!session || !config)
    return DELETE_COMMIT_ERROR;
@@ -5,6 +5,7 @@
 #include "config.h"
 #include "file_receive.h"
 #include "protocol.h"
+#include "utils.h"
 #include <stdbool.h>

 /* Per-directory delete plans (protocol 2.24.0).
@@ -79,5 +80,11 @@ bool delete_plan_session_limit_reached(const DeletePlanSession* session);
 /* Number of destination entries the session's plans removed (or, for
   --delete-delay, snapshotted for removal), for the end-of-transfer stats. */
 size_t delete_plan_session_deleted(const DeletePlanSession* session);
+/* Install an observer invoked for every destination-relative path the session
+   truly removes (including the deferred --delete-delay commit), so the receiver
+   can report rsync's `deleting PATH` lines through the terminal STATUS_STATS
+   record.  Pass NULL/0 to clear. */
+void delete_plan_session_set_delete_observer(DeletePlanSession* session, DeletePathObserver observer,
+                                             void* context);

 #endif
@@ -3195,8 +3195,9 @@ char* file_receive_basis_delete_relative(const Config* config, const char* path)
   alternate basis directories are never destination content and are skipped at
   any depth.  Returns true unless a traversal/unlink error aborted the walk;
   the budget's limit_hit/skipped fields report a cap-stopped run. */
-static bool delete_extras_budgeted(const Config* config, DeleteManifest* manifest,
-                                   DeleteBudgetState* budget) {
+static bool delete_extras_budgeted_observed(const Config* config, DeleteManifest* manifest,
+                                            DeleteBudgetState* budget, DeletePathObserver observer,
+                                            void* observer_context) {
  if (!config || !manifest || !manifest->keeps)
    return false;
  fprintf(stderr, "Deleting files not in manifest...\n");
@@ -3261,8 +3262,9 @@ static bool delete_extras_budgeted(const Config* config, DeleteManifest* manifes
  size_t deleted = 0;
  size_t skipped = 0;
  DeleteWalkResult result =
-      delete_extras_limited(config->receive_root_directory, manifest->keeps, manifest->dirs,
-                            remaining, skips, used, &deleted, &skipped);
+      delete_extras_limited_observed(config->receive_root_directory, manifest->keeps, manifest->dirs,
+                                     remaining, skips, used, &deleted, &skipped, observer,
+                                     observer_context);
  if (owned_prefixes) {
    for (int i = 0; i < config->basis_count; i++)
      free(owned_prefixes[i]);
@@ -3282,6 +3284,31 @@ static bool delete_extras_budgeted(const Config* config, DeleteManifest* manifes
  return true;
 }

+static bool delete_extras_budgeted(const Config* config, DeleteManifest* manifest,
+                                   DeleteBudgetState* budget) {
+  return delete_extras_budgeted_observed(config, manifest, budget, NULL, NULL);
+}
+
+/* Prefixes every observed path with a fixed subtree root, so a nested walk
+   (a recursively removed missing-arg directory) reports receive-root-relative
+   names like the rest of the delete output. */
+typedef struct {
+  DeletePathObserver inner;
+  void* inner_context;
+  const char* prefix;
+} PrefixedDeleteObserver;
+
+static void prefixed_delete_observer(void* context, const char* rel) {
+  PrefixedDeleteObserver* prefixed = context;
+  if (!prefixed->inner || !rel)
+    return;
+  char* joined = path_cat((char*)prefixed->prefix, rel);
+  if (joined) {
+    prefixed->inner(prefixed->inner_context, joined);
+    free(joined);
+  }
+}
+
 /* --delete-missing-args exact-path deletions: each destination mirror in
   manifest->missing is an explicit user request, so it is removed even when the
   ordinary extras walk (with its protected prefixes) would leave it alone.  The
@@ -3295,8 +3322,10 @@ static bool delete_extras_budgeted(const Config* config, DeleteManifest* manifes
   --max-delete budget: once it is exhausted the remaining requests are skipped
   and counted.  Returns false only on a genuine error (a confinement failure on
   a validated path or an I/O error), which fails the run. */
-static bool delete_missing_args_budgeted(const Config* config, DeleteManifest* manifest,
-                                         DeleteBudgetState* budget) {
+static bool delete_missing_args_budgeted_observed(const Config* config, DeleteManifest* manifest,
+                                                  DeleteBudgetState* budget,
+                                                  DeletePathObserver observer,
+                                                  void* observer_context) {
  if (!config || !manifest)
    return false;
  if (!manifest->missing || manifest->missing->size == 0)
@@ -3413,9 +3442,12 @@ static bool delete_missing_args_budgeted(const Config* config, DeleteManifest* m
              budget->deleted >= budget->max_delete ? 0 : budget->max_delete - budget->deleted;
          size_t contents_deleted = 0;
          size_t contents_skipped = 0;
+          PrefixedDeleteObserver nested = {observer, observer_context, rel};
          DeleteWalkResult walk =
-              no_keeps ? delete_extras_limited(full, no_keeps, NULL, remaining, NULL, 0,
-                                               &contents_deleted, &contents_skipped)
+              no_keeps ? delete_extras_limited_observed(full, no_keeps, NULL, remaining, NULL, 0,
+                                                        &contents_deleted, &contents_skipped,
+                                                        observer ? prefixed_delete_observer : NULL,
+                                                        observer ? &nested : NULL)
                       : DELETE_WALK_ERROR;
          if (no_keeps)
            array_list_delete(no_keeps);
@@ -3455,6 +3487,8 @@ static bool delete_missing_args_budgeted(const Config* config, DeleteManifest* m
    }
    if (removed) {
      budget->deleted++;
+      if (observer)
+        observer(observer_context, rel);
      char* escaped = output_escape(rel, log_get_8_bit_output());
      fprintf(stderr, "  Deleted: %s\n", escaped ? escaped : "<allocation failed>");
      free(escaped);
@@ -3541,15 +3575,25 @@ bool manifest_delete_extras(const Config* config, DeleteManifest* manifest) {
 bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest) {
  DeleteBudgetState budget = {
      .max_delete = SIZE_MAX, .deleted = 0, .skipped = 0, .limit_hit = false};
-  return delete_missing_args_budgeted(config, manifest, &budget);
+  return delete_missing_args_budgeted_observed(config, manifest, &budget, NULL, NULL);
 }

 bool manifest_delete_missing_args_limited(const Config* config, DeleteManifest* manifest,
                                          size_t max_delete, size_t* deleted, size_t* skipped,
                                          bool* limit_hit) {
+  return manifest_delete_missing_args_limited_observed(config, manifest, max_delete, deleted,
+                                                       skipped, limit_hit, NULL, NULL);
+}
+
+bool manifest_delete_missing_args_limited_observed(const Config* config, DeleteManifest* manifest,
+                                                   size_t max_delete, size_t* deleted,
+                                                   size_t* skipped, bool* limit_hit,
+                                                   DeletePathObserver observer,
+                                                   void* observer_context) {
  DeleteBudgetState budget = {
      .max_delete = max_delete, .deleted = 0, .skipped = 0, .limit_hit = false};
-  bool ok = delete_missing_args_budgeted(config, manifest, &budget);
+  bool ok = delete_missing_args_budgeted_observed(config, manifest, &budget, observer,
+                                                  observer_context);
  if (deleted)
    *deleted = budget.deleted;
  if (skipped)
@@ -3572,6 +3616,12 @@ DeleteCommitResult manifest_delete_all(const Config* config, DeleteManifest* man

 DeleteCommitResult manifest_delete_all_counted(const Config* config, DeleteManifest* manifest,
                                               size_t* deleted) {
+  return manifest_delete_all_observed(config, manifest, deleted, NULL, NULL);
+}
+
+DeleteCommitResult manifest_delete_all_observed(const Config* config, DeleteManifest* manifest,
+                                                size_t* deleted, DeletePathObserver observer,
+                                                void* observer_context) {
  if (deleted)
    *deleted = 0;
  if (!config || !manifest)
@@ -3590,9 +3640,11 @@ DeleteCommitResult manifest_delete_all_counted(const Config* config, DeleteManif
                              .deleted = 0,
                              .skipped = 0,
                              .limit_hit = false};
-  if (config->delete_missing_args && !delete_missing_args_budgeted(config, manifest, &budget))
+  if (config->delete_missing_args &&
+      !delete_missing_args_budgeted_observed(config, manifest, &budget, observer, observer_context))
    return DELETE_COMMIT_ERROR;
-  if (config->use_delete && !delete_extras_budgeted(config, manifest, &budget))
+  if (config->use_delete &&
+      !delete_extras_budgeted_observed(config, manifest, &budget, observer, observer_context))
    return DELETE_COMMIT_ERROR;
  if (deleted)
    *deleted = budget.deleted;
@@ -3,6 +3,7 @@

 #include "config.h"
 #include "file_types.h"
+#include "utils.h"
 #include <stdbool.h>

 /* Server-side file receive/save path. */
@@ -126,6 +127,12 @@ bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest
 bool manifest_delete_missing_args_limited(const Config* config, DeleteManifest* manifest,
                                          size_t max_delete, size_t* deleted, size_t* skipped,
                                          bool* limit_hit);
+/* Observer-aware form of manifest_delete_missing_args_limited: `observer` (may
+   be NULL) is invoked for every destination-relative path truly removed. */
+bool manifest_delete_missing_args_limited_observed(const Config* config, DeleteManifest* manifest,
+                                                   size_t max_delete, size_t* deleted,
+                                                   size_t* skipped, bool* limit_hit,
+                                                   DeletePathObserver observer, void* observer_context);
 /* Outcome of committing a delete manifest.  LIMIT_REACHED reports rsync's
   partial --max-delete result: the budget allowed some deletions and the rest
   were skipped (the run still stores all file data but the client exits 25). */
@@ -146,6 +153,11 @@ DeleteCommitResult manifest_delete_all(const Config* config, DeleteManifest* man
   removed (for the end-of-transfer wire stats).  `deleted` may be NULL. */
 DeleteCommitResult manifest_delete_all_counted(const Config* config, DeleteManifest* manifest,
                                               size_t* deleted);
+/* Observer-aware form of manifest_delete_all_counted: `observer` (may be NULL)
+   is invoked for every destination-relative path truly removed. */
+DeleteCommitResult manifest_delete_all_observed(const Config* config, DeleteManifest* manifest,
+                                                size_t* deleted, DeletePathObserver observer,
+                                                void* observer_context);

 /* -n/--dry-run --delete would-delete reporting: walk the destination exactly as
   the delete pass would and append (strdup'd) destination-relative paths that
@@ -187,12 +187,24 @@ unsigned long long io_get_bwlimit(void) {
  return global_bwlimit();
 }

+/* rsync's throttle (io.c sleep_for_bwlimit) sleeps once its unslept debt
+ * reaches ~100 ms of bandwidth, so its effective initial burst is about 0.1 s
+ * worth of bytes, not a full second.  FastSync models the same with a token
+ * bucket whose capacity is bwlimit/10, so a throttled run paces like rsync
+ * instead of sending a full second's worth up front. */
+static long long bw_burst_capacity(unsigned long long bwlimit) {
+  if (bwlimit == 0)
+    return 0;
+  long long burst = (long long)(bwlimit / 10);
+  return burst > 0 ? burst : 1;
+}
+
 void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec) {
  if (!session)
    return;
  session->bwlimit =
      bytes_per_sec > (unsigned long long)LLONG_MAX ? (unsigned long long)LLONG_MAX : bytes_per_sec;
-  session->bw_tokens = (long long)session->bwlimit;
+  session->bw_tokens = bw_burst_capacity(session->bwlimit);
  struct timespec now;
  clock_gettime(CLOCK_MONOTONIC, &now);
  session->bw_last_refill_sec = now.tv_sec;
@@ -226,8 +238,9 @@ static void bw_throttle_session(ProtocolSession* session, size_t bytes_written)

  long long tokens_to_add = (long long)((double)session->bwlimit * elapsed_ns / 1000000000.0);
  session->bw_tokens += tokens_to_add;
-  if (session->bw_tokens > (long long)session->bwlimit)
-    session->bw_tokens = (long long)session->bwlimit;
+  long long burst = bw_burst_capacity(session->bwlimit);
+  if (session->bw_tokens > burst)
+    session->bw_tokens = burst;

  session->bw_tokens -= bytes_written;

@@ -238,7 +251,11 @@ static void bw_throttle_session(ProtocolSession* session, size_t bytes_written)
      poll(NULL, 0, (int)(deficit_us / 1000));
    else
      usleep((useconds_t)deficit_us);
+    /* Reset the bucket AFTER the sleep: crediting the sleep duration as elapsed
+       refill time would cancel half the throttle (the next call would see the
+       whole sleep as refill and immediately grant a fresh burst). */
    session->bw_tokens = 0;
+    clock_gettime(CLOCK_MONOTONIC, &now);
    session->bw_last_refill_sec = now.tv_sec;
    session->bw_last_refill_nsec = now.tv_nsec;
  }
@@ -616,7 +616,8 @@ static bool is_synced_dir(const PathIndex* dirs, const char* rel) {
 static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep,
                             const PathIndex* dirs, DeleteBudget* budget,
                             const DeleteSkipEntry* skips, int skip_count, bool parent_deletable,
-                             bool* all_removed) {
+                             bool* all_removed, DeletePathObserver observer,
+                             void* observer_context) {
  /* openat(dirfd, ".") opens an independent file description: a dup() would
     share dirfd's file offset and a prior pass could leave the stream drained. */
  int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
@@ -666,7 +667,7 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
      bool child_all_removed = false;
      if (childfd >= 0) {
        if (!delete_extras_fd(childfd, child_rel, keep, dirs, budget, skips, skip_count, deletable,
-                              &child_all_removed))
+                              &child_all_removed, observer, observer_context))
          operation_ok = false;
        close(childfd);
      } else if (errno != ENOENT) {
@@ -693,6 +694,8 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
          local_survives = true;
        } else {
          budget->deleted++;
+          if (observer)
+            observer(observer_context, child_rel);
        }
      } else {
        local_survives = true;
@@ -713,6 +716,8 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
        local_survives = true;
      } else {
        budget->deleted++;
+        if (observer)
+          observer(observer_context, child_rel);
        char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
        fprintf(stderr, "  Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
        free(escaped_path);
@@ -867,10 +872,11 @@ bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
  return ok;
 }

-DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
-                                       const ArrayList* synced_dirs, size_t max_delete,
-                                       const DeleteSkipEntry* skips, int skip_count,
-                                       size_t* deleted_out, size_t* skipped_out) {
+DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const ArrayList* manifest,
+                                                const ArrayList* synced_dirs, size_t max_delete,
+                                                const DeleteSkipEntry* skips, int skip_count,
+                                                size_t* deleted_out, size_t* skipped_out,
+                                                DeletePathObserver observer, void* observer_context) {
  if (deleted_out)
    *deleted_out = 0;
  if (skipped_out)
@@ -911,7 +917,7 @@ DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* m
  DeleteBudget budget = {.max_delete = max_delete, .deleted = 0, .skipped = 0, .limit_hit = false};
  bool all_removed = false;
  bool ok = delete_extras_fd(rootfd, "", &keep, have_dirs ? &dirs : NULL, &budget, skips,
-                             skip_count, false, &all_removed);
+                             skip_count, false, &all_removed, observer, observer_context);
  if (close(rootfd) != 0)
    ok = false;
  path_index_free(&keep);
@@ -926,6 +932,14 @@ DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* m
  return budget.limit_hit ? DELETE_WALK_LIMIT_REACHED : DELETE_WALK_OK;
 }

+DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
+                                       const ArrayList* synced_dirs, size_t max_delete,
+                                       const DeleteSkipEntry* skips, int skip_count,
+                                       size_t* deleted_out, size_t* skipped_out) {
+  return delete_extras_limited_observed(dest_root, manifest, synced_dirs, max_delete, skips,
+                                        skip_count, deleted_out, skipped_out, NULL, NULL);
+}
+
 bool delete_extras(const char* dest_root, const ArrayList* manifest) {
  return delete_extras_limited(dest_root, manifest, NULL, SIZE_MAX, NULL, 0, NULL, NULL) ==
         DELETE_WALK_OK;
@@ -134,6 +134,19 @@ bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSki
   cap and returns DELETE_WALK_LIMIT_REACHED when more extras remained.
   `deleted_out`/`skipped_out` optionally receive the number of entries removed
   and the number skipped because of the cap. */
+/* Optional per-deletion observer: called for each destination-relative path
+   actually removed (a file, symlink, or directory), in removal order, so the
+   receiver can stream rsync's `--info=del`/`--info=remove` lines. */
+typedef void (*DeletePathObserver)(void* context, const char* rel_path);
+
+/* `delete_extras_limited_observed` is delete_extras_limited with an optional
+   observer; the observer is invoked only for entries truly removed. */
+DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const ArrayList* manifest,
+                                                const ArrayList* synced_dirs, size_t max_delete,
+                                                const DeleteSkipEntry* skips, int skip_count,
+                                                size_t* deleted_out, size_t* skipped_out,
+                                                DeletePathObserver observer, void* observer_context);
+
 DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
                                       const ArrayList* synced_dirs, size_t max_delete,
                                       const DeleteSkipEntry* skips, int skip_count,