Merge feat/p7-times: Phase 7 Wave D (real dir/symlink time preservation making -O/-J meaningful; secluded-args -> Impossible/Divergence; PROTOCOL 2.17.0)

src/client/client_send.c

@@ -127,6 +127,11 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann
   options->excluded_paths = NULL;
   options->excluded_mutex = NULL;
   options->hardlinks = NULL;
+  /* P7 Wave D: capture source directory times whenever metadata rides the
+     wire.  Whether they are APPLIED is decided receiver-side (-O skips). */
+  options->capture_dir_times = config->use_metadata;
+  options->dir_entries = NULL;
+  options->dir_entries_mutex = NULL;
   if (config->preserve_hard_links) {
     out->hardlinks = hardlink_table_create();
     if (!out->hardlinks) {
@@ -1115,14 +1120,51 @@ static bool send_file_direct(File* file, int fd, bool use_metadata, int compress
 }
 
 /* Transmit one explicit directory entry (--dirs): a STATUS_MKDIR frame whose
-   payload is only the destination path.  The receiver validates the path and
-   creates the directory under the receive root. */
-static bool send_directory_entry(Client* client, File* file) {
+   payload is the destination path and, when metadata is negotiated, the
+   directory's metadata frame.  The receiver validates the path, creates the
+   directory under the receive root, and (metadata case) defers applying its
+   times to the end of the transfer so -O/--omit-dir-times is honored. */
+static bool send_directory_entry(const Client* client, File* file, const Config* config) {
   if (!file || !file_wire_path(file))
     return false;
-  if (!send_status(client->file_descriptor, STATUS_MKDIR))
+  if (!send_status(client->file_descriptor, STATUS_MKDIR) ||
+      !send_wire_str(client->file_descriptor, file_wire_path(file)))
     return false;
-  return send_wire_str(client->file_descriptor, file_wire_path(file));
+  return !config->use_metadata || metadata_send(client->file_descriptor, file->metadata);
+}
+
+/* P7 Wave D: transmit every captured source directory's metadata in terminal
+   STATUS_DIR_TIMES frames (count, then (path, metadata) pairs) after all file
+   data and the optional delete manifest.  The receiver applies them at the END
+   of its own transfer (after deletion and --delay-updates publication) so a
+   directory's mtime is not clobbered by writing its children.  A non-metadata
+   transfer (or an empty set) sends nothing, keeping the stream byte-identical.
+
+   The receiver rejects a frame whose count exceeds MAX_MANIFEST_ENTRIES, so a
+   huge tree is CHUNKED into repeated frames of at most that many entries each
+   (the receiver's loop handles repeated STATUS_DIR_TIMES frames).  Every frame
+   stays within the receiver's bound, and a frame that would exceed it is never
+   emitted. */
+static bool send_dir_times(const Client* client, const Config* config, ArrayList* dir_entries) {
+  if (!client || !config || !config->use_metadata || !dir_entries || dir_entries->size == 0)
+    return true;
+  int fd = client->file_descriptor;
+  int index = 0;
+  while (index < dir_entries->size) {
+    int remaining = dir_entries->size - index;
+    int chunk = remaining > MAX_MANIFEST_ENTRIES ? MAX_MANIFEST_ENTRIES : remaining;
+    if (!send_status(fd, STATUS_DIR_TIMES) || !send_int(fd, chunk))
+      return false;
+    for (int i = 0; i < chunk; i++) {
+      File* file = (File*)dir_entries->items[index + i];
+      if (!file || !file_wire_path(file))
+        return false;
+      if (!send_wire_str(fd, file_wire_path(file)) || !metadata_send(fd, file->metadata))
+        return false;
+    }
+    index += chunk;
+  }
+  return true;
 }
 
 /* Transmit one symlink entry: a STATUS_SYMLINK frame carrying the destination
@@ -1313,10 +1355,10 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
     if (f == NULL)
       continue;
     if (f->is_dir) {
-      /* Explicit directory entry (--dirs): a MKDIR frame carrying only the
-         destination path.  Directories have no source to remove and no
-         incremental check. */
-      if (!send_directory_entry(client, f))
+      /* Explicit directory entry (--dirs): a MKDIR frame carrying the
+         destination path (and metadata when negotiated).  Directories have no
+         source to remove and no incremental check. */
+      if (!send_directory_entry(client, f, config))
         return -1;
       change_emit_dir_sent(config, f);
       continue;
@@ -1501,6 +1543,13 @@ static int send_chunks_multithreaded(void* pipeline_context) {
     if (send_delete_manifest(client->file_descriptor, NULL, NULL, context->missing_args) != 0)
       goto send_fail;
   }
+  /* P7 Wave D: transmit the captured directory times last.  The scanner thread
+     (and all parallel workers) has been joined before scanner_done was set, so
+     the list is complete and race-free; on an early stop the list may be
+     incomplete and is deliberately not sent. */
+  if (!context->scan_stopped_early &&
+      !send_dir_times(client, context->config, context->dir_entries))
+    goto send_fail;
   bool ok = finalize_transfer(client, context->config, context->remove_source_files);
   if (!ok && context->config->use_delete)
     log_message(LOG_LEVEL_ERROR,
@@ -1542,6 +1591,10 @@ static int scan_directory_multithreaded(void* pipeline_context) {
     return thrd_error;
   }
   prepared.options.stop_condition = &context->stop_condition;
+  /* P7 Wave D: the recursive scan feeds the shared directory-time list; the
+     parallel workers append under the context's dedicated mutex. */
+  prepared.options.dir_entries = context->dir_entries;
+  prepared.options.dir_entries_mutex = &context->dir_entries_mutex;
   /* The keep-set manifest for the late modes is built from this data pass, so
      the parallel scanner records the protected excluded prefixes here.  The
      early modes already transmitted the pre-scan keep-set and its protected
@@ -1843,6 +1896,9 @@ int send_files(Config* config) {
   DirectoryScanner* scanner = NULL;
   ArrayList* manifest = NULL;
   ArrayList* remove_sources = NULL;
+  /* P7 Wave D: captured source directory times, transmitted in trailing
+     STATUS_DIR_TIMES frame(s) (only when metadata rides the wire). */
+  ArrayList* dir_entries = NULL;
   /* Protected excluded prefixes (delete-excluded default protection). */
   ArrayList* excluded = NULL;
   bool delete_early = config->use_delete && config_delete_timing_early(config);
@@ -1855,6 +1911,11 @@ int send_files(Config* config) {
   receive_daemon_motd(client, config);
   if (!prepare_scanner(config, 0, &prepared))
     goto send_fail;
+  if (config->use_metadata) {
+    dir_entries = array_list_create(file_destroy);
+    if (!dir_entries)
+      goto send_fail;
+  }
   if (config->remove_source_files)
     remove_sources = array_list_create(source_file_destroy);
   if (config->remove_source_files && !remove_sources)
@@ -1919,6 +1980,10 @@ int send_files(Config* config) {
   StopCondition stop = stop_condition_make(config->stop_after_mins > 0, config->stop_after_mins,
                                            config->stop_at_set, config->stop_at, now_mono);
   prepared.options.stop_condition = &stop;
+  /* The early-delete pre-scan above already ran; only the data pass should feed
+     the directory-time list (otherwise every directory would be captured
+     twice). */
+  prepared.options.dir_entries = dir_entries;
   scanner = directory_scanner_create_with_options(config->send_directory, &prepared.options);
   if (!scanner)
     goto send_fail;
@@ -2037,6 +2102,11 @@ int send_files(Config* config) {
       }
     }
   }
+  /* P7 Wave D: every directory has now been traversed (or the scan stopped
+     early), so transmit the captured directory times last.  The receiver defers
+     applying them until after its own deletion/publication phase. */
+  if (!send_dir_times(client, config, dir_entries))
+    goto send_fail;
   bool ok = finalize_transfer(client, config, remove_sources);
   if (!ok && config->use_delete)
     log_message(LOG_LEVEL_ERROR,
@@ -2078,6 +2148,8 @@ send_fail:
     array_list_delete(missing_args);
   if (remove_sources)
     array_list_delete(remove_sources);
+  if (dir_entries)
+    array_list_delete(dir_entries);
   if (scanner)
     directory_scanner_destroy(scanner);
   prepared_scanner_destroy(&prepared);

src/client/scanner.c

@@ -488,6 +488,9 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
   scanner->hardlinks = options->hardlinks;
   scanner->prune_empty_dirs = options->prune_empty_dirs;
   scanner->stop_condition = options->stop_condition;
+  scanner->capture_dir_times = options->capture_dir_times;
+  scanner->dir_entries = options->dir_entries;
+  scanner->dir_entries_mutex = options->dir_entries_mutex;
   scanner->dirs_root_emitted = false;
   scanner->list_index = 0;
   scanner->dirs_batch = NULL;
@@ -595,6 +598,63 @@ static Chunk* chunk_data_to_chunk(ArrayList* chunk_data) {
   return chunk;
 }
 
+/* P7 Wave D: append one traversed source directory's captured metadata to the
+ * shared pending-directory-time list.  The File carries no payload; only the
+ * wire path (absolute fs path normally, the bare relative path under
+ * -R + --files-from) and its metadata are used, and the sender transmits them
+ * in trailing STATUS_DIR_TIMES frame(s).  `mutex` (optional) serializes the
+ * append for the parallel scanner's shared workers.  An unstattable or
+ * non-directory path is silently skipped (the transfer is unaffected); an
+ * allocation failure is fatal and reported to the caller. */
+static bool scanner_capture_dir_time(ArrayList* dir_entries, mtx_t* mutex, const char* root_path,
+                                     const char* fs_path, bool relative_mode, bool preserve_atimes,
+                                     bool preserve_crtimes) {
+  if (!dir_entries || !root_path || !fs_path)
+    return true;
+  struct stat st;
+  if (stat(fs_path, &st) != 0 || !S_ISDIR(st.st_mode))
+    return true;
+  char* rel = scanner_path_relative(root_path, fs_path);
+  if (!rel)
+    return true;
+  if (relative_mode && rel[0] == '\0') {
+    /* -R + --files-from: the transfer root itself has no bare relative wire
+       path (matches the -R scan, which never emits the root). */
+    free(rel);
+    return true;
+  }
+  File* file = file_create(fs_path);
+  if (!file) {
+    free(rel);
+    return false;
+  }
+  file->is_dir = true;
+  file->metadata = file_metadata_create(fs_path, &st, preserve_atimes, preserve_crtimes);
+  if (!file->metadata) {
+    free(rel);
+    file_destroy(file);
+    return false;
+  }
+  if (relative_mode) {
+    file->send_path = rel;
+    rel = NULL;
+  }
+  free(rel);
+  bool added;
+  if (mutex) {
+    mtx_lock(mutex);
+    added = array_list_add(dir_entries, file);
+    mtx_unlock(mutex);
+  } else {
+    added = array_list_add(dir_entries, file);
+  }
+  if (!added) {
+    file_destroy(file);
+    return false;
+  }
+  return true;
+}
+
 /* Open the next queued directory and set up its filter context.  Returns 1 when
    a directory is open, 0 when the queue is exhausted, and -1 on a fatal error.
    A directory that cannot be opened is an I/O error: it is recorded on the
@@ -661,6 +721,17 @@ static int open_next_directory(DirectoryScanner* scanner) {
       scanner->current_path = NULL;
       return -1;
     }
+    if (scanner->capture_dir_times &&
+        !scanner_capture_dir_time(scanner->dir_entries, scanner->dir_entries_mutex,
+                                  scanner->root_path, scanner->current_path, scanner->relative_mode,
+                                  scanner->preserve_atimes, scanner->preserve_crtimes)) {
+      closedir(scanner->current_dir);
+      scanner->current_dir = NULL;
+      free(scanner->current_path);
+      scanner->current_path = NULL;
+      scanner->failed = true;
+      return -1;
+    }
     return 1;
   }
   return 0;
@@ -1584,6 +1655,18 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
     parallel_scanner_destroy(ps);
     return NULL;
   }
+  /* P7 Wave D: the parallel scanner never runs a DirectoryScanner over the
+     transfer root itself (it hands the root's immediate subdirectories to
+     workers), so capture the root's directory time here. */
+  if (options->capture_dir_times &&
+      !scanner_capture_dir_time(options->dir_entries, options->dir_entries_mutex, root_directory,
+                                root_directory, options->relative && options->file_list != NULL,
+                                options->preserve_atimes, options->preserve_crtimes)) {
+    array_list_delete(root_files);
+    array_list_delete(subdirs);
+    parallel_scanner_destroy(ps);
+    return NULL;
+  }
 
   unsigned long long cs = options->chunk_size > 0 ? options->chunk_size : DESIRED_CHUNK_SIZE;
   ps->initial_chunk = batch_files(root_files, cs, ps->result_queue, &ps->failed);

src/client/scanner.h

@@ -98,6 +98,18 @@ typedef struct {
    * (without marking the scan as failed), so a busy scan itself stops early.
    * Client-only, never serialized to the wire. */
   const StopCondition* stop_condition;
+  /* P7 Wave D (protocol 2.17.0): directory-time capture sink.  When
+   * `capture_dir_times` is true the recursive scan appends one is_dir File
+   * (with metadata, no payload) per source directory it traverses to
+   * `dir_entries`, so the sender can transmit trailing STATUS_DIR_TIMES
+   * frame(s) and the receiver can apply directory mtimes AFTER all children
+   * are written.  `dir_entries_mutex` (optional) guards the list
+   * for the parallel scanner's shared worker threads; the caller owns both.
+   * The --dirs generator does not use this (its directory entries carry their
+   * metadata inline through STATUS_MKDIR). */
+  bool capture_dir_times;
+  ArrayList* dir_entries;
+  mtx_t* dir_entries_mutex;
 } ScannerOptions;
 
 /* Internal per-scanner filter state. FilterNode chains represent the ordered
@@ -173,6 +185,10 @@ typedef struct {
   HardLinkTable* hardlinks;
   /* Phase 6: sender stop deadline (from ScannerOptions). */
   const StopCondition* stop_condition;
+  /* P7 Wave D directory-time capture (see ScannerOptions). */
+  bool capture_dir_times;
+  ArrayList* dir_entries;
+  mtx_t* dir_entries_mutex;
 } DirectoryScanner;
 
 typedef struct {