fix(delay-updates): unique staging dir and implied --delete-after ordering (#317)

tests/integration/test_differential_parity.py

@@ -106,6 +106,15 @@ def seed_backup(_src, rroot, froot):
         _mk(os.path.join(root, "a.txt"), b"OLD-CONTENT\n", _OLD_MTIME)
 
 
+def seed_delay_updates(_src, rroot, froot):
+    """A changed file plus an extra, so --delay-updates (and its implied
+    --delete-after) has both a publication and a deletion to order."""
+    for root in (rroot, froot):
+        _mk(os.path.join(root, "a.txt"), b"OLD-CONTENT\n", _OLD_MTIME)
+        _mk(os.path.join(root, "extra.txt"), b"extra\n", _OLD_MTIME)
+        _mk(os.path.join(root, "extradir", "z.txt"), b"z\n", _OLD_MTIME)
+
+
 def seed_size_only(_src, rroot, froot):
     for root in (rroot, froot):
         _mk(os.path.join(root, "a.txt"), b"XXXXXXXXXXX\n", _OLD_MTIME)
@@ -305,6 +314,14 @@ _CASES = [
     H.Case("delete_commit", "basic", ["-a", "--delete-after"], seed=seed_extras,
            fastsync_flags=["-a", "--delete-commit"], server_args=DELETE,
            ref="FastSync-only --delete-commit == rsync --delete-after"),
+    # #317: --delay-updates stages under a per-run unique name and publishes
+    # every update before the implied --delete-after removes extras.
+    H.Case("delay_updates", "basic", ["-a", "--delay-updates"],
+           seed=seed_delay_updates, ref="--delay-updates stages then publishes"),
+    H.Case("delay_updates_delete", "basic",
+           ["-a", "--delay-updates", "--delete"], seed=seed_delay_updates,
+           server_args=DELETE, ci=True,
+           ref="--delay-updates implies --delete-after (publish before delete)"),
     H.Case("delete_excluded", "filters",
            ["-a", "--delete", "--delete-excluded", "--exclude=*.log"],
            seed=seed_delete_excluded, server_args=DELETE, ref="--delete-excluded"),

tests/integration/test_features.py

@@ -3089,12 +3089,12 @@ class TestDelayUpdates:
             "staging directory left behind after a successful delayed transfer"
 
     @pytest.mark.skipif(shutil.which("rsync") is None, reason="rsync not installed")
-    def test_delay_updates_staging_name_collision_residual(self):
-        """Documented residual (RSYNC_COMPAT.md `--delay-updates` row): FastSync
-        uses a fixed `.fastsync-stage` staging name and wipes a pre-existing tree
-        of that name at the start of a delayed run (crash-leftover cleanup),
-        even without `--delete`; rsync leaves a genuine destination entry of that
-        name untouched.  Pins the divergence that keeps the row Divergent."""
+    def test_delay_updates_staging_name_collision_preserved(self):
+        """rsync parity (RSYNC_COMPAT.md `--delay-updates` row): the receiver
+        stages under a per-run unique name, so a genuine pre-existing
+        destination entry named like the reserved staging prefix (`.fastsync-
+        stage`) is never wiped -- even without `--delete`.  rsync likewise
+        leaves a real destination entry of its own temp name untouched."""
         source = self._make_source("delay_collide_src")
         rdst = os.path.join(TEST_DATA_DIR, "delay_collide_rdst")
         fdst = os.path.join(TEST_DATA_DIR, "delay_collide_fdst")
@@ -3120,8 +3120,11 @@ class TestDelayUpdates:
             result, _ = run_client(source, fdst, flags=["--delay-updates"],
                                    port=server.port)
         assert result.returncode == 0, (result.stderr or result.stdout)[:300]
-        assert not os.path.exists(os.path.join(fdst, self.STAGING)), \
-            "FastSync did not wipe the reserved staging name (residual changed)"
+        assert _read_file(os.path.join(fdst, self.STAGING, "keepme.txt")) == b"genuine user data\n", \
+            "FastSync destroyed a genuine destination entry named like the staging prefix"
+        # The per-run staging directory itself is removed after a clean run.
+        leftovers = [n for n in os.listdir(fdst) if n.startswith(self.STAGING + ".")]
+        assert leftovers == [], f"per-run staging directories left behind: {leftovers}"
 
     @pytest.mark.parametrize("mt", [False, True])
     def test_delay_updates_incremental_rerun_no_leftovers(self, shared_server, mt):
@@ -3161,10 +3164,11 @@ class TestDelayUpdates:
 
     @pytest.mark.parametrize("mt", [False, True])
     def test_delete_with_delay_updates(self, mt):
-        """--delete runs before publication, so the delete walker must not treat
-        the staging directory as a set of extras: a changed file must still be
-        published after genuine extras are removed.  Uses its own server started
-        with --allow-delete (the shared session server refuses deletion)."""
+        """rsync parity: --delay-updates implies --delete-after, so every staged
+        update is published first and the genuine extras are removed only after
+        that (the delete walker must never treat the staging directory as a set
+        of extras).  Uses its own server started with --allow-delete (the shared
+        session server refuses deletion)."""
         source = os.path.join(TEST_DATA_DIR, "delay_delete_src")
         dest = os.path.join(TEST_DATA_DIR, "delay_delete_dst")
         clean_dir(source)
@@ -3194,6 +3198,65 @@ class TestDelayUpdates:
             assert not os.path.exists(os.path.join(received, "extra.txt")), \
                 "genuine extra file was not deleted"
             assert not os.path.isdir(os.path.join(dest, self.STAGING))
+            assert [n for n in os.listdir(dest) if n.startswith(self.STAGING + ".")] == []
+
+    @pytest.mark.parametrize("mt", [False, True])
+    def test_delay_updates_delete_keeps_backup(self, mt):
+        """The --backup/--delay-updates interplay: the old destination file is
+        moved aside at publication, and that backup survives the implied
+        --delete-after pass (rsync never treats a backup file as an extra)."""
+        source = os.path.join(TEST_DATA_DIR, "delay_bak_del_src")
+        dest = os.path.join(TEST_DATA_DIR, "delay_bak_del_dst")
+        clean_dir(source)
+        clean_dir(dest)
+        with open(os.path.join(source, "f.txt"), "wb") as fh:
+            fh.write(b"NEW")
+        received = get_dest_received_dir(dest, source)
+        os.makedirs(received, exist_ok=True)
+        with open(os.path.join(received, "f.txt"), "wb") as fh:
+            fh.write(b"OLD")
+        os.utime(os.path.join(received, "f.txt"), (1_500_000_000, 1_500_000_000))
+        # A pre-existing backup-looking extra must also be shielded.
+        with open(os.path.join(received, "stale.txt~"), "wb") as fh:
+            fh.write(b"stale backup")
+        with ServerManager() as server:
+            server.start(extra_args=["--allow-delete"])
+            flags = ["--delete", "--backup", "--delay-updates"] + (["--threads"] if mt else [])
+            result, _ = run_client(source, dest, flags=flags, port=server.port)
+        assert result.returncode == 0, (result.stderr or result.stdout)[:300]
+        assert _read_file(os.path.join(received, "f.txt")) == b"NEW"
+        assert _read_file(os.path.join(received, "f.txt~")) == b"OLD", \
+            "the publication backup was removed by the delete-after pass"
+        assert os.path.exists(os.path.join(received, "stale.txt~")), \
+            "a pre-existing backup-suffixed entry was deleted"
+
+    @pytest.mark.parametrize("mt", [False, True])
+    def test_delay_updates_failed_run_leaves_no_staged_files(self, shared_server, mt):
+        """A run that fails before publication installs nothing and removes the
+        per-run staging directory (no staged leftovers)."""
+        source = os.path.join(TEST_DATA_DIR, "delay_fail_src")
+        dest = os.path.join(TEST_DATA_DIR, "delay_fail_dst")
+        clean_dir(source)
+        clean_dir(dest)
+        with open(os.path.join(source, "top.txt"), "wb") as fh:
+            fh.write(b"top\n")
+        os.makedirs(os.path.join(source, "sub"))
+        with open(os.path.join(source, "sub", "deep.txt"), "wb") as fh:
+            fh.write(b"deep\n")
+        # Plant a regular file where the "sub" directory must be created so the
+        # nested publish fails (the top-level file still publishes first).
+        received = get_dest_received_dir(dest, source)
+        os.makedirs(received)
+        with open(os.path.join(received, "sub"), "wb") as fh:
+            fh.write(b"blocker")
+
+        flags = ["--delay-updates"] + (["--threads"] if mt else [])
+        result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
+        assert result.returncode != 0, "a blocked nested publish must fail the run"
+        assert not os.path.lexists(os.path.join(received, "sub", "deep.txt")), \
+            "a staged file appeared despite the failed run"
+        assert [n for n in os.listdir(dest) if n.startswith(self.STAGING + ".")] == [], \
+            "the per-run staging directory survived a failed run"
 
     def test_delay_updates_rejects_reserved_backup_dir(self):
         """--backup-dir equal to the internal staging name must be rejected so

tests/test_client_cli.c

@@ -2977,6 +2977,48 @@ static void test_parse_args_delay_updates() {
   config_delete(cfg);
 }
 
+/* rsync parity: --delay-updates implies --delete-after when --delete is
+   active (all updates publish first, then extras are removed).  An explicit
+   other timing is overridden; without --delete no timing is set. */
+static void test_parse_args_delay_updates_implies_delete_after() {
+  Config* cfg = config_create();
+  char* argv[] = {"fastsync", "--delay-updates", "--delete", "/src", "/dst"};
+  int positional_args[2];
+  int positional_count = 0;
+  EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
+  EXPECT_TRUE(cfg->use_delete);
+  EXPECT_TRUE(cfg->delete_after);
+  EXPECT_FALSE(cfg->delete_before);
+  EXPECT_FALSE(cfg->delete_during);
+  EXPECT_FALSE(cfg->delete_delay);
+  cfg->send_directory = str_dup("/src");
+  cfg->receive_root_directory = str_dup("/dst");
+  EXPECT_TRUE(validate_config(cfg));
+  config_delete(cfg);
+
+  /* An explicit conflicting timing is normalized to delete-after. */
+  cfg = config_create();
+  char* argv_before[] = {"fastsync", "--delay-updates", "--delete-before", "/src", "/dst"};
+  positional_count = 0;
+  EXPECT_EQ_INT(parse_args(cfg, 5, argv_before, positional_args, &positional_count), 0);
+  EXPECT_TRUE(cfg->use_delete);
+  EXPECT_TRUE(cfg->delete_after);
+  EXPECT_FALSE(cfg->delete_before);
+  config_delete(cfg);
+
+  /* Without --delete there is no deletion, so no timing is selected. */
+  cfg = config_create();
+  char* argv_alone[] = {"fastsync", "--delay-updates", "/src", "/dst"};
+  positional_count = 0;
+  EXPECT_EQ_INT(parse_args(cfg, 4, argv_alone, positional_args, &positional_count), 0);
+  EXPECT_FALSE(cfg->use_delete);
+  EXPECT_FALSE(cfg->delete_after);
+  EXPECT_FALSE(cfg->delete_before);
+  EXPECT_FALSE(cfg->delete_during);
+  EXPECT_FALSE(cfg->delete_delay);
+  config_delete(cfg);
+}
+
 /* rsync rejects --delay-updates with --inplace; FastSync must too. */
 static void test_validate_config_delay_updates_rejects_inplace() {
   Config* cfg = valid_client_config();
@@ -5312,6 +5354,7 @@ void test_client_cli() {
   test_parse_args_checksum_seed();
   test_parse_args_temp_dir();
   test_parse_args_delay_updates();
+  test_parse_args_delay_updates_implies_delete_after();
   test_validate_config_delay_updates_rejects_inplace();
   test_validate_config_delay_updates_rejects_reserved_backup_dir();
   test_parse_args_files_from();

tests/test_delay_updates.c

@@ -87,8 +87,10 @@ static void test_delay_updates_no_final_before_publish() {
   const char* final_path = "test_delay_tmp/sub/file.txt";
   /* Before publication the final destination must not contain the file. */
   EXPECT_FALSE(file_path_exists_secure(final_path));
-  /* The complete staged copy must live inside the staging tree. */
-  char* staged = path_cat("test_delay_tmp/.fastsync-stage", "/sub/file.txt");
+  /* The complete staged copy must live inside the per-run staging tree. */
+  EXPECT_NOT_NULL(cfg->delay_context->staging_name);
+  EXPECT_EQ_INT(strncmp(cfg->delay_context->staging_name, ".fastsync-stage.", 16), 0);
+  char* staged = path_cat(cfg->delay_context->staging_root, "/sub/file.txt");
   EXPECT_NOT_NULL(staged);
   // cppcheck-suppress knownConditionTrueFalse
   if (staged) {
@@ -125,6 +127,8 @@ static void test_delay_updates_publish_installs_files() {
   const char* final_path = "test_delay_pub_tmp/sub/file.txt";
   EXPECT_FALSE(file_path_exists_secure(final_path));
 
+  char* staging_root = str_dup(cfg->delay_context->staging_root);
+  EXPECT_NOT_NULL(staging_root);
   EXPECT_TRUE(delay_updates_publish(cfg->delay_context, cfg));
   /* After a successful publish the file is installed and staging is gone. */
   char* content = read_all(final_path);
@@ -134,7 +138,8 @@ static void test_delay_updates_publish_installs_files() {
     EXPECT_EQ_STR(content, "published payload");
     free(content);
   }
-  EXPECT_FALSE(file_path_exists_secure("test_delay_pub_tmp/.fastsync-stage"));
+  EXPECT_FALSE(file_path_exists_secure(staging_root));
+  free(staging_root);
 
 out:
   file_destroy(f);
@@ -158,11 +163,17 @@ static void test_delay_updates_cleanup_removes_staged() {
     goto out;
 
   EXPECT_EQ_INT(file_save_to_disk_full(root, f, cfg), FILE_SAVE_WRITTEN);
-  EXPECT_TRUE(file_path_exists_secure("test_delay_clean_tmp/.fastsync-stage/sub/file.txt"));
+  char* staged_file = path_cat(cfg->delay_context->staging_root, "/sub/file.txt");
+  char* staging_root = str_dup(cfg->delay_context->staging_root);
+  EXPECT_NOT_NULL(staged_file);
+  EXPECT_NOT_NULL(staging_root);
+  EXPECT_TRUE(file_path_exists_secure(staged_file));
 
   delay_updates_cleanup(cfg->delay_context);
-  EXPECT_FALSE(file_path_exists_secure("test_delay_clean_tmp/.fastsync-stage"));
+  EXPECT_FALSE(file_path_exists_secure(staging_root));
   EXPECT_FALSE(file_path_exists_secure("test_delay_clean_tmp/sub/file.txt"));
+  free(staged_file);
+  free(staging_root);
 
 out:
   file_destroy(f);
@@ -274,6 +285,54 @@ out:
   remove_tree(root);
 }
 
+/* Every context picks its own staging directory name, so two delayed
+   transfers to the same root can never share (and corrupt) a staging tree. */
+static void test_delay_updates_unique_staging_name() {
+  DelayUpdatesContext* first = delay_updates_context_create("test_delay_uniq_tmp");
+  DelayUpdatesContext* second = delay_updates_context_create("test_delay_uniq_tmp");
+  EXPECT_NOT_NULL(first);
+  EXPECT_NOT_NULL(second);
+  if (first && second) {
+    EXPECT_EQ_INT(strncmp(first->staging_name, ".fastsync-stage.", 16), 0);
+    EXPECT_EQ_INT(strncmp(second->staging_name, ".fastsync-stage.", 16), 0);
+    EXPECT_TRUE(strcmp(first->staging_name, second->staging_name) != 0);
+    EXPECT_TRUE(strcmp(first->staging_root, second->staging_root) != 0);
+  }
+  delay_updates_context_destroy(first);
+  delay_updates_context_destroy(second);
+}
+
+/* A pre-existing destination entry at the exact (random) staging path is not
+   ours: prepare() must refuse rather than wipe it. */
+static void test_delay_updates_prepare_refuses_non_owned_collision() {
+  const char* root = "test_delay_collide_tmp";
+  remove_tree(root);
+  DelayUpdatesContext* context = delay_updates_context_create(root);
+  EXPECT_NOT_NULL(context);
+  // cppcheck-suppress knownConditionTrueFalse
+  if (!context)
+    return;
+  /* Plant a genuine directory with user data at the exact staging path. */
+  EXPECT_TRUE(file_ensure_directory_secure(context->staging_root));
+  char* inner = path_cat(context->staging_root, "keepme.txt");
+  EXPECT_NOT_NULL(inner);
+  // cppcheck-suppress knownConditionTrueFalse
+  if (inner) {
+    EXPECT_TRUE(file_write_to_disk(inner, "genuine", 7, false, false));
+    EXPECT_FALSE(delay_updates_prepare(context));
+    char* content = read_all(inner);
+    EXPECT_NOT_NULL(content);
+    // cppcheck-suppress knownConditionTrueFalse
+    if (content) {
+      EXPECT_EQ_STR(content, "genuine");
+      free(content);
+    }
+    free(inner);
+  }
+  delay_updates_context_destroy(context);
+  remove_tree(root);
+}
+
 /* The reserved staging name must be recognizable for validation, including
    with a trailing slash. */
 static void test_delay_updates_reserved_name_helper() {
@@ -288,6 +347,8 @@ static void test_delay_updates_reserved_name_helper() {
 
 void test_delay_updates() {
   test_delay_updates_reserved_name_helper();
+  test_delay_updates_unique_staging_name();
+  test_delay_updates_prepare_refuses_non_owned_collision();
   test_delay_updates_no_final_before_publish();
   test_delay_updates_publish_installs_files();
   test_delay_updates_cleanup_removes_staged();