Merge feat/p7-times: Phase 7 Wave D (real dir/symlink time preservation making -O/-J meaningful; secluded-args -> Impossible/Divergence; PROTOCOL 2.17.0)

This commit is contained in:
2026-09-12 11:22:23 +02:00
25 changed files with 1000 additions and 77 deletions
+81 -9
View File
@@ -127,6 +127,11 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann
options->excluded_paths = NULL;
options->excluded_mutex = NULL;
options->hardlinks = NULL;
/* P7 Wave D: capture source directory times whenever metadata rides the
wire. Whether they are APPLIED is decided receiver-side (-O skips). */
options->capture_dir_times = config->use_metadata;
options->dir_entries = NULL;
options->dir_entries_mutex = NULL;
if (config->preserve_hard_links) {
out->hardlinks = hardlink_table_create();
if (!out->hardlinks) {
@@ -1115,14 +1120,51 @@ static bool send_file_direct(File* file, int fd, bool use_metadata, int compress
}
/* Transmit one explicit directory entry (--dirs): a STATUS_MKDIR frame whose
payload is only the destination path. The receiver validates the path and
creates the directory under the receive root. */
static bool send_directory_entry(Client* client, File* file) {
payload is the destination path and, when metadata is negotiated, the
directory's metadata frame. The receiver validates the path, creates the
directory under the receive root, and (metadata case) defers applying its
times to the end of the transfer so -O/--omit-dir-times is honored. */
static bool send_directory_entry(const Client* client, File* file, const Config* config) {
if (!file || !file_wire_path(file))
return false;
if (!send_status(client->file_descriptor, STATUS_MKDIR))
if (!send_status(client->file_descriptor, STATUS_MKDIR) ||
!send_wire_str(client->file_descriptor, file_wire_path(file)))
return false;
return send_wire_str(client->file_descriptor, file_wire_path(file));
return !config->use_metadata || metadata_send(client->file_descriptor, file->metadata);
}
/* P7 Wave D: transmit every captured source directory's metadata in terminal
STATUS_DIR_TIMES frames (count, then (path, metadata) pairs) after all file
data and the optional delete manifest. The receiver applies them at the END
of its own transfer (after deletion and --delay-updates publication) so a
directory's mtime is not clobbered by writing its children. A non-metadata
transfer (or an empty set) sends nothing, keeping the stream byte-identical.
The receiver rejects a frame whose count exceeds MAX_MANIFEST_ENTRIES, so a
huge tree is CHUNKED into repeated frames of at most that many entries each
(the receiver's loop handles repeated STATUS_DIR_TIMES frames). Every frame
stays within the receiver's bound, and a frame that would exceed it is never
emitted. */
static bool send_dir_times(const Client* client, const Config* config, ArrayList* dir_entries) {
if (!client || !config || !config->use_metadata || !dir_entries || dir_entries->size == 0)
return true;
int fd = client->file_descriptor;
int index = 0;
while (index < dir_entries->size) {
int remaining = dir_entries->size - index;
int chunk = remaining > MAX_MANIFEST_ENTRIES ? MAX_MANIFEST_ENTRIES : remaining;
if (!send_status(fd, STATUS_DIR_TIMES) || !send_int(fd, chunk))
return false;
for (int i = 0; i < chunk; i++) {
File* file = (File*)dir_entries->items[index + i];
if (!file || !file_wire_path(file))
return false;
if (!send_wire_str(fd, file_wire_path(file)) || !metadata_send(fd, file->metadata))
return false;
}
index += chunk;
}
return true;
}
/* Transmit one symlink entry: a STATUS_SYMLINK frame carrying the destination
@@ -1313,10 +1355,10 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
if (f == NULL)
continue;
if (f->is_dir) {
/* Explicit directory entry (--dirs): a MKDIR frame carrying only the
destination path. Directories have no source to remove and no
incremental check. */
if (!send_directory_entry(client, f))
/* Explicit directory entry (--dirs): a MKDIR frame carrying the
destination path (and metadata when negotiated). Directories have no
source to remove and no incremental check. */
if (!send_directory_entry(client, f, config))
return -1;
change_emit_dir_sent(config, f);
continue;
@@ -1501,6 +1543,13 @@ static int send_chunks_multithreaded(void* pipeline_context) {
if (send_delete_manifest(client->file_descriptor, NULL, NULL, context->missing_args) != 0)
goto send_fail;
}
/* P7 Wave D: transmit the captured directory times last. The scanner thread
(and all parallel workers) has been joined before scanner_done was set, so
the list is complete and race-free; on an early stop the list may be
incomplete and is deliberately not sent. */
if (!context->scan_stopped_early &&
!send_dir_times(client, context->config, context->dir_entries))
goto send_fail;
bool ok = finalize_transfer(client, context->config, context->remove_source_files);
if (!ok && context->config->use_delete)
log_message(LOG_LEVEL_ERROR,
@@ -1542,6 +1591,10 @@ static int scan_directory_multithreaded(void* pipeline_context) {
return thrd_error;
}
prepared.options.stop_condition = &context->stop_condition;
/* P7 Wave D: the recursive scan feeds the shared directory-time list; the
parallel workers append under the context's dedicated mutex. */
prepared.options.dir_entries = context->dir_entries;
prepared.options.dir_entries_mutex = &context->dir_entries_mutex;
/* The keep-set manifest for the late modes is built from this data pass, so
the parallel scanner records the protected excluded prefixes here. The
early modes already transmitted the pre-scan keep-set and its protected
@@ -1843,6 +1896,9 @@ int send_files(Config* config) {
DirectoryScanner* scanner = NULL;
ArrayList* manifest = NULL;
ArrayList* remove_sources = NULL;
/* P7 Wave D: captured source directory times, transmitted in trailing
STATUS_DIR_TIMES frame(s) (only when metadata rides the wire). */
ArrayList* dir_entries = NULL;
/* Protected excluded prefixes (delete-excluded default protection). */
ArrayList* excluded = NULL;
bool delete_early = config->use_delete && config_delete_timing_early(config);
@@ -1855,6 +1911,11 @@ int send_files(Config* config) {
receive_daemon_motd(client, config);
if (!prepare_scanner(config, 0, &prepared))
goto send_fail;
if (config->use_metadata) {
dir_entries = array_list_create(file_destroy);
if (!dir_entries)
goto send_fail;
}
if (config->remove_source_files)
remove_sources = array_list_create(source_file_destroy);
if (config->remove_source_files && !remove_sources)
@@ -1919,6 +1980,10 @@ int send_files(Config* config) {
StopCondition stop = stop_condition_make(config->stop_after_mins > 0, config->stop_after_mins,
config->stop_at_set, config->stop_at, now_mono);
prepared.options.stop_condition = &stop;
/* The early-delete pre-scan above already ran; only the data pass should feed
the directory-time list (otherwise every directory would be captured
twice). */
prepared.options.dir_entries = dir_entries;
scanner = directory_scanner_create_with_options(config->send_directory, &prepared.options);
if (!scanner)
goto send_fail;
@@ -2037,6 +2102,11 @@ int send_files(Config* config) {
}
}
}
/* P7 Wave D: every directory has now been traversed (or the scan stopped
early), so transmit the captured directory times last. The receiver defers
applying them until after its own deletion/publication phase. */
if (!send_dir_times(client, config, dir_entries))
goto send_fail;
bool ok = finalize_transfer(client, config, remove_sources);
if (!ok && config->use_delete)
log_message(LOG_LEVEL_ERROR,
@@ -2078,6 +2148,8 @@ send_fail:
array_list_delete(missing_args);
if (remove_sources)
array_list_delete(remove_sources);
if (dir_entries)
array_list_delete(dir_entries);
if (scanner)
directory_scanner_destroy(scanner);
prepared_scanner_destroy(&prepared);
+83
View File
@@ -488,6 +488,9 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
scanner->hardlinks = options->hardlinks;
scanner->prune_empty_dirs = options->prune_empty_dirs;
scanner->stop_condition = options->stop_condition;
scanner->capture_dir_times = options->capture_dir_times;
scanner->dir_entries = options->dir_entries;
scanner->dir_entries_mutex = options->dir_entries_mutex;
scanner->dirs_root_emitted = false;
scanner->list_index = 0;
scanner->dirs_batch = NULL;
@@ -595,6 +598,63 @@ static Chunk* chunk_data_to_chunk(ArrayList* chunk_data) {
return chunk;
}
/* P7 Wave D: append one traversed source directory's captured metadata to the
* shared pending-directory-time list. The File carries no payload; only the
* wire path (absolute fs path normally, the bare relative path under
* -R + --files-from) and its metadata are used, and the sender transmits them
* in trailing STATUS_DIR_TIMES frame(s). `mutex` (optional) serializes the
* append for the parallel scanner's shared workers. An unstattable or
* non-directory path is silently skipped (the transfer is unaffected); an
* allocation failure is fatal and reported to the caller. */
static bool scanner_capture_dir_time(ArrayList* dir_entries, mtx_t* mutex, const char* root_path,
const char* fs_path, bool relative_mode, bool preserve_atimes,
bool preserve_crtimes) {
if (!dir_entries || !root_path || !fs_path)
return true;
struct stat st;
if (stat(fs_path, &st) != 0 || !S_ISDIR(st.st_mode))
return true;
char* rel = scanner_path_relative(root_path, fs_path);
if (!rel)
return true;
if (relative_mode && rel[0] == '\0') {
/* -R + --files-from: the transfer root itself has no bare relative wire
path (matches the -R scan, which never emits the root). */
free(rel);
return true;
}
File* file = file_create(fs_path);
if (!file) {
free(rel);
return false;
}
file->is_dir = true;
file->metadata = file_metadata_create(fs_path, &st, preserve_atimes, preserve_crtimes);
if (!file->metadata) {
free(rel);
file_destroy(file);
return false;
}
if (relative_mode) {
file->send_path = rel;
rel = NULL;
}
free(rel);
bool added;
if (mutex) {
mtx_lock(mutex);
added = array_list_add(dir_entries, file);
mtx_unlock(mutex);
} else {
added = array_list_add(dir_entries, file);
}
if (!added) {
file_destroy(file);
return false;
}
return true;
}
/* Open the next queued directory and set up its filter context. Returns 1 when
a directory is open, 0 when the queue is exhausted, and -1 on a fatal error.
A directory that cannot be opened is an I/O error: it is recorded on the
@@ -661,6 +721,17 @@ static int open_next_directory(DirectoryScanner* scanner) {
scanner->current_path = NULL;
return -1;
}
if (scanner->capture_dir_times &&
!scanner_capture_dir_time(scanner->dir_entries, scanner->dir_entries_mutex,
scanner->root_path, scanner->current_path, scanner->relative_mode,
scanner->preserve_atimes, scanner->preserve_crtimes)) {
closedir(scanner->current_dir);
scanner->current_dir = NULL;
free(scanner->current_path);
scanner->current_path = NULL;
scanner->failed = true;
return -1;
}
return 1;
}
return 0;
@@ -1584,6 +1655,18 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
parallel_scanner_destroy(ps);
return NULL;
}
/* P7 Wave D: the parallel scanner never runs a DirectoryScanner over the
transfer root itself (it hands the root's immediate subdirectories to
workers), so capture the root's directory time here. */
if (options->capture_dir_times &&
!scanner_capture_dir_time(options->dir_entries, options->dir_entries_mutex, root_directory,
root_directory, options->relative && options->file_list != NULL,
options->preserve_atimes, options->preserve_crtimes)) {
array_list_delete(root_files);
array_list_delete(subdirs);
parallel_scanner_destroy(ps);
return NULL;
}
unsigned long long cs = options->chunk_size > 0 ? options->chunk_size : DESIRED_CHUNK_SIZE;
ps->initial_chunk = batch_files(root_files, cs, ps->result_queue, &ps->failed);
+16
View File
@@ -98,6 +98,18 @@ typedef struct {
* (without marking the scan as failed), so a busy scan itself stops early.
* Client-only, never serialized to the wire. */
const StopCondition* stop_condition;
/* P7 Wave D (protocol 2.17.0): directory-time capture sink. When
* `capture_dir_times` is true the recursive scan appends one is_dir File
* (with metadata, no payload) per source directory it traverses to
* `dir_entries`, so the sender can transmit trailing STATUS_DIR_TIMES
* frame(s) and the receiver can apply directory mtimes AFTER all children
* are written. `dir_entries_mutex` (optional) guards the list
* for the parallel scanner's shared worker threads; the caller owns both.
* The --dirs generator does not use this (its directory entries carry their
* metadata inline through STATUS_MKDIR). */
bool capture_dir_times;
ArrayList* dir_entries;
mtx_t* dir_entries_mutex;
} ScannerOptions;
/* Internal per-scanner filter state. FilterNode chains represent the ordered
@@ -173,6 +185,10 @@ typedef struct {
HardLinkTable* hardlinks;
/* Phase 6: sender stop deadline (from ScannerOptions). */
const StopCondition* stop_condition;
/* P7 Wave D directory-time capture (see ScannerOptions). */
bool capture_dir_times;
ArrayList* dir_entries;
mtx_t* dir_entries_mutex;
} DirectoryScanner;
typedef struct {
+44 -2
View File
@@ -74,6 +74,25 @@ static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) {
return true;
}
/* P7 Wave D: read one STATUS_DIR_TIMES frame (a count followed by that many
* (path, metadata) directory entries) and route every entry through the regular
* store_file sink. A dir-time entry is RECORD-ONLY (file->dir_time_only): the
* sink accumulates its metadata for end-of-transfer application but creates
* nothing, so an empty/pruned source directory is never resurrected. A large
* tree arrives as repeated frames, each bounded by MAX_MANIFEST_ENTRIES; a
* malformed count or entry is a hard error. */
static bool receiver_process_dir_times(int fd, const Config* config, const ReceiverSink* sink) {
int count;
if (!receive_int(fd, &count) || count < 0 || count > MAX_MANIFEST_ENTRIES)
return false;
for (int i = 0; i < count; i++) {
File* dir = file_receive_dir_time(fd, config);
if (!dir || !sink->store_file(dir, sink->context))
return false;
}
return true;
}
static bool receiver_process_batch(Config* config, int file_descriptor) {
int count;
if (config->checksum || !receive_int(file_descriptor, &count) || count < 0 ||
@@ -161,7 +180,7 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK ||
status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH ||
status == STATUS_MKDIR || status == STATUS_MANIFEST || status == STATUS_HARDLINK ||
status == STATUS_SYMLINK || status == STATUS_SPECIAL) {
status == STATUS_SYMLINK || status == STATUS_SPECIAL || status == STATUS_DIR_TIMES) {
if (status == STATUS_KEEPALIVE) {
if (!send_status(file_descriptor, STATUS_KEEPALIVE))
goto fail;
@@ -185,9 +204,12 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
goto fail;
goto next_status;
} else if (status == STATUS_MKDIR) {
File* dir = file_receive_directory(file_descriptor);
File* dir = file_receive_directory(file_descriptor, config);
if (!dir || !sink->store_file(dir, sink->context))
goto receive_error;
} else if (status == STATUS_DIR_TIMES) {
if (!receiver_process_dir_times(file_descriptor, config, sink))
goto receive_error;
} else if (status == STATUS_HARDLINK) {
File* file = file_receive_hardlink(file_descriptor);
if (!file || !sink->store_file(file, sink->context))
@@ -311,6 +333,10 @@ receive_error:
typedef struct {
Config* config;
ReceiverOutcomes outcomes;
/* P7 Wave D: directory metadata accumulated during the stream, applied only
after the whole transfer (and its delete/publication phases) has run so a
child write never clobbers a directory mtime. */
DirTimeList dir_times;
} ReceiverSaveContext;
static bool receiver_save_file(File* file, void* context_pointer) {
@@ -323,6 +349,15 @@ static bool receiver_save_file(File* file, void* context_pointer) {
} else {
result = file_save_to_disk_full(context->config->receive_root_directory, file, context->config);
}
/* A directory's times are deferred, never applied inline: collect the
metadata now and apply it at the end. -O/--omit-dir-times is honored by
dir_time_list_apply's caller (see receiver_send_success_frame). */
if (result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
context->config->use_metadata && !context->config->omit_dir_times &&
!dir_time_list_add(&context->dir_times, file->path, file->metadata)) {
file_destroy(file);
return false;
}
if (result != FILE_SAVE_ERROR && context->config->remove_source_files && !file->is_dir &&
!file->is_special && !file->skip &&
!receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
@@ -346,15 +381,22 @@ static bool receiver_send_success_frame(int fd, void* context_pointer) {
return false;
}
}
/* P7 Wave D: every child is now written and the delete / --delay-updates
phases have committed, so it is finally safe to stamp directory times.
This runs after the deferred deletion because receiver_process commits it
before calling this success frame. */
dir_time_list_apply(&context->dir_times, context->config->receive_root_directory);
return receiver_send_final_success(fd, context->config, &context->outcomes);
}
int receiver_receive_files(Config* config, int file_descriptor) {
ReceiverSaveContext context = {.config = config, .outcomes = {0}};
dir_time_list_init(&context.dir_times);
ReceiverSink sink = {receiver_save_file, &context, true, true, receiver_send_success_frame};
int ret = receiver_process(config, file_descriptor, &sink);
if (ret != 0 && config->delay_updates && config->delay_context)
delay_updates_cleanup(config->delay_context);
receiver_outcomes_destroy(&context.outcomes);
dir_time_list_free(&context.dir_times);
return ret;
}
+6
View File
@@ -493,6 +493,12 @@ void handler(int file_descriptor) {
!delay_updates_publish(config->delay_context, config)) {
transfer_ok = false;
}
/* P7 Wave D: all writers have joined and the late deletion (and
--delay-updates publication) has committed above, so it is finally safe
to stamp directory times; a directory's mtime must not be clobbered by
its children or by an extra removal. */
if (transfer_ok)
dir_time_list_apply(&context->dir_times, config->receive_root_directory);
}
if (transfer_ok) {
if (!receiver_send_final_success(file_descriptor, config, &context->outcomes))
+22 -2
View File
@@ -543,8 +543,28 @@ typedef struct Config {
* new trailing bytes would desynchronize on the frame boundary, and the strict
* same-version handshake (config_receive rejects a mismatched version before
* parsing anything else) is what keeps a 2.16 client and a 2.15 server from
* ever reaching that state. */
#define PROTOCOL_VERSION "2.16.0"
* ever reaching that state.
*
* Times Wave (P7 Wave D): 2.16.0 -> 2.17.0.
*
* WHY the bump, grounded in the wire: this wave makes -O/--omit-dir-times and
* -J/--omit-link-times REAL by adding directory and symlink time preservation.
* The config-frame LAYOUT is unchanged (the omit flags already crossed the
* wire), but the FRAME STREAM gains a new terminal frame: after all file data
* and the optional delete manifest, the sender transmits STATUS_DIR_TIMES
* frame(s) (each a count followed by (path, metadata) pairs, chunked so no
* frame exceeds the receiver's MAX_MANIFEST_ENTRIES bound) carrying every
* source directory's captured times, so the receiver can apply them AFTER all of a
* directory's children have been written (writing a child bumps the parent's
* mtime). Symlink entries already carry their metadata on the STATUS_SYMLINK
* frame; the receiver now applies it (utimensat/lchown with
* AT_SYMLINK_NOFOLLOW) unless -J is set. Any change to the frame sequence must
* bump the protocol version: a 2.16 peer that does not know STATUS_DIR_TIMES
* would desynchronize on the unknown frame, and the strict same-version
* handshake (config_receive rejects a mismatched version before parsing
* anything else) is what keeps a 2.17 client and a 2.16 server from ever
* reaching that state. */
#define PROTOCOL_VERSION "2.17.0"
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
#define MAX_BASIS_DIRS 64
+1
View File
@@ -149,6 +149,7 @@ File* file_create(const char* path) {
file->metadata = NULL;
file->skip = false;
file->is_dir = false;
file->dir_time_only = false;
file->basis_link = NULL;
file->link_group = 0;
file->link_first = false;
+174 -5
View File
@@ -551,6 +551,18 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
return FILE_SAVE_ERROR;
}
/* P7 Wave D #1: a STATUS_DIR_TIMES entry is RECORD-ONLY. The scanner
captures every traversed directory -- including empty ones whose parents
were never created by a child write and directories pruned by
-m/--prune-empty-dirs. Creating them here would resurrect empty
directories (an -a behavior change) and could abort the whole transfer on a
pre-existing regular file/symlink at the mirror path. Short-circuit before
any device/write-devices/directory branch and report it as skipped so the
sink still accumulates its metadata for the deferred DirTimeList
application, but create nothing. */
if (file->dir_time_only)
return FILE_SAVE_SKIPPED;
/* Device/special node (--devices/--specials): recreate the node instead of
writing content (privilege-gated, confined, rdev-validated). */
if (file->is_special)
@@ -621,6 +633,12 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
}
ok = file_symlink_at_secure(link_path, target);
free(target);
/* P7 Wave D: apply the symlink's own metadata with no-follow primitives
(utimensat/lchown/fchmodat AT_SYMLINK_NOFOLLOW). -J/--omit-link-times
suppresses the timestamps; ownership stays gated by the identity policy.
A symlink has no children, so this can be applied immediately. */
if (ok && config && config->use_metadata)
file_restore_symlink_metadata(link_path, file->metadata, config->omit_link_times);
free(link_path);
return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR;
}
@@ -2137,12 +2155,119 @@ File* file_receive(const Config* config, int file_descriptor) {
return file;
}
/* ---- P7 Wave D: deferred directory times ---- */
void dir_time_list_init(DirTimeList* list) {
if (!list)
return;
list->paths = NULL;
list->entries = NULL;
list->count = 0;
list->capacity = 0;
}
void dir_time_list_free(DirTimeList* list) {
if (!list)
return;
for (size_t i = 0; i < list->count; i++)
free(list->paths[i]);
free(list->paths);
free(list->entries);
list->paths = NULL;
list->entries = NULL;
list->count = 0;
list->capacity = 0;
}
bool dir_time_list_add(DirTimeList* list, const char* wire_path, const FileMetadata* metadata) {
if (!list || !wire_path || !metadata)
return true; /* nothing to remember; never a hard error */
if (list->count == list->capacity) {
size_t new_capacity = list->capacity == 0 ? 16 : list->capacity * 2;
if (new_capacity < list->capacity)
return false;
/* Assign each grown array as soon as its realloc succeeds: the old block is
already freed by then, so discarding the pointer would dangle. capacity
is advanced only after BOTH reallocs succeed, so a partial failure leaves
capacity no larger than the entries allocation (the paths array may be
over-allocated, which is harmless) -- never a mismatched list the next
add could write past. */
char** grown_paths = realloc(list->paths, new_capacity * sizeof(char*));
if (!grown_paths)
return false;
list->paths = grown_paths;
FileMetadata* grown_entries = realloc(list->entries, new_capacity * sizeof(FileMetadata));
if (!grown_entries)
return false;
list->entries = grown_entries;
list->capacity = new_capacity;
}
char* copy = str_dup(wire_path);
if (!copy)
return false;
list->paths[list->count] = copy;
list->entries[list->count] = *metadata;
list->count++;
return true;
}
void dir_time_list_apply(const DirTimeList* list, const char* root_directory) {
if (!list || !root_directory)
return;
for (size_t i = 0; i < list->count; i++) {
char* dir_path = path_cat(root_directory, list->paths[i]);
if (!dir_path)
continue;
char* leaf = NULL;
/* The parent walk is fd-relative and O_NOFOLLOW, so a symlink planted in a
parent component can never redirect the utimensat outside the root. */
int parent_fd = file_open_secure_parent(dir_path, &leaf, false);
if (parent_fd < 0) {
free(dir_path);
continue;
}
/* A dir-time entry only records metadata: the directory is (deliberately)
not created from it, so an empty source directory (or one pruned by
-m/--prune-empty-dirs) may well not exist here. Skip absent paths
QUIETLY rather than warning for every one, and apply the times only to a
real directory that does exist. AT_SYMLINK_NOFOLLOW keeps a same-named
symlink from being followed; a pre-existing regular file/symlink is not a
directory, so it is left completely untouched. */
struct stat st;
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0 || !S_ISDIR(st.st_mode)) {
close(parent_fd);
free(leaf);
free(dir_path);
continue;
}
struct timespec times[2] = {
{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
{.tv_sec = list->entries[i].mtime_sec, .tv_nsec = list->entries[i].mtime_nsec}};
if (list->entries[i].atime_valid) {
times[0].tv_sec = list->entries[i].atime_sec;
times[0].tv_nsec = list->entries[i].atime_nsec;
}
if (utimensat(parent_fd, leaf, times, AT_SYMLINK_NOFOLLOW) != 0) {
char* escaped_path = output_escape(dir_path, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING, "Failed to set directory timestamps on %s: %s",
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
free(escaped_path);
}
close(parent_fd);
free(leaf);
free(dir_path);
}
}
/* Receive an explicit directory entry (--dirs): a STATUS_MKDIR frame carries
only the destination path; the entry carries no payload. The same path
validation as a regular file applies (non-empty, relative-or-mirrored, no
traversal), and the created File is routed through the regular store_file
sink so single-threaded and -m receivers handle directories identically. */
File* file_receive_directory(int file_descriptor) {
the destination path and, when metadata is negotiated, the directory's
metadata frame. The same path validation as a regular file applies
(non-empty, relative-or-mirrored, no traversal), and the created File is
routed through the regular store_file sink so single-threaded and -m
receivers handle directories identically. The metadata is NOT applied here:
the sink accumulates it into a DirTimeList that is applied only after the
whole transfer (children would otherwise clobber the directory mtime). */
File* file_receive_directory(int file_descriptor, const Config* config) {
char* path = receive_wire_str(file_descriptor);
if (path == NULL)
return NULL;
@@ -2159,6 +2284,50 @@ File* file_receive_directory(int file_descriptor) {
if (file == NULL)
return NULL;
file->is_dir = true;
if (config && config->use_metadata) {
int meta_ok = 1;
file->metadata = metadata_receive(file_descriptor, &meta_ok);
if (!meta_ok) {
file_destroy(file);
return NULL;
}
}
return file;
}
/* Receive one directory-time entry from a STATUS_DIR_TIMES frame: the
* destination-relative wire path and (when metadata is negotiated) the
* directory's metadata frame. The created File is an is_dir, dir_time_only
* entry routed through the regular store_file sink: the sink records its
* metadata into the deferred DirTimeList but never creates the directory (the
* scanner captures every traversed directory, including empty ones). Unlike a
* STATUS_MKDIR entry, this one must not create anything. */
File* file_receive_dir_time(int file_descriptor, const Config* config) {
char* path = receive_wire_str(file_descriptor);
if (path == NULL)
return NULL;
if (path[0] == '\0' || (!file_get_trust_sender() && has_path_traversal(path))) {
char* escaped_path = output_escape(path, log_get_8_bit_output());
log_message(LOG_LEVEL_ERROR, "Invalid received directory-time path: %s",
escaped_path ? escaped_path : "<allocation failed>");
free(escaped_path);
free(path);
return NULL;
}
File* file = file_create(path);
free(path);
if (!file)
return NULL;
file->is_dir = true;
file->dir_time_only = true;
if (config && config->use_metadata) {
int meta_ok = 1;
file->metadata = metadata_receive(file_descriptor, &meta_ok);
if (!meta_ok) {
file_destroy(file);
return NULL;
}
}
return file;
}
+28 -1
View File
@@ -8,13 +8,40 @@
/* Server-side file receive/save path. */
File* file_receive(const Config* config, int file_descriptor);
File* file_receive_directory(int file_descriptor);
File* file_receive_directory(int file_descriptor, const Config* config);
File* file_receive_dir_time(int file_descriptor, const Config* config);
File* file_receive_hardlink(int file_descriptor);
File* file_receive_symlink(int file_descriptor, const Config* config);
File* file_receive_special(int file_descriptor);
bool file_special_rdev_valid(int32_t major, int32_t minor, mode_t mode);
File* receive_incremental_check(int fd, const Config* config, bool* skipped);
/* P7 Wave D directory-time accumulator. The receiver collects the metadata of
* every directory it creates/receives (STATUS_MKDIR with metadata and/or the
* trailing STATUS_DIR_TIMES frame(s)) and applies the times only at the END of the
* transfer, after all children have been written and after the delete /
* --delay-updates phases have committed (writing or removing a child bumps the
* parent's mtime). -O/--omit-dir-times skips the application entirely. The
* list owns deep copies of the paths and metadata; freed on every path. */
typedef struct {
char** paths; /* owned, destination-relative wire paths */
FileMetadata* entries; /* owned, parallel to paths */
size_t count;
size_t capacity;
} DirTimeList;
void dir_time_list_init(DirTimeList* list);
void dir_time_list_free(DirTimeList* list);
/* Deep-copy one directory's path + metadata into the list. Returns false on
* allocation failure (the caller fails the transfer). */
bool dir_time_list_add(DirTimeList* list, const char* wire_path, const FileMetadata* metadata);
/* Apply every accumulated directory's mtime (and atime when captured) beneath
* `root_directory`, confined fd-relative. Best-effort per entry: an absent
* directory (an empty/pruned source dir that was deliberately not created) or a
* non-directory at the path is skipped QUIETLY, an unreachable one with a
* warning, and never fatal. */
void dir_time_list_apply(const DirTimeList* list, const char* root_directory);
/* A received delete-manifest frame: the keep-set (`keeps`, destination-relative
paths the sender transferred/keeps) plus `protected`, destination-relative
prefixes the sender asks the receiver never to delete (paths excluded on the
+8
View File
@@ -42,6 +42,14 @@ typedef struct {
/* True when this entry is an explicit directory entry (--dirs mode): the
* receiver creates the directory instead of writing a regular file. */
bool is_dir;
/* Receiver-only (P7 Wave D): this is a STATUS_DIR_TIMES entry. It carries a
* traversed source directory's metadata for DEFERRED application, but must
* NEVER create the directory: the scanner captures every traversed directory
* (including empty ones whose parents no child write created), so creation
* would resurrect the empty dirs that FastSync deliberately never transfers.
* file_save_to_disk_full short-circuits such an entry as FILE_SAVE_SKIPPED,
* and the sink still accumulates the metadata into its DirTimeList. */
bool dir_time_only;
/* Receiver-only, --link-dest: when set, install the destination entry as a
* hard link to this absolute (root-confined) path instead of writing
* `data`. The matching code has already verified the link target's content
+59 -30
View File
@@ -2,6 +2,7 @@
#include "log.h"
#include "utils.h"
#include <errno.h>
#include <fcntl.h>
#include <grp.h>
#include <limits.h>
#include <pwd.h>
@@ -370,16 +371,11 @@ static bool identity_map_lookup(const IdentityMap* map, int count, int32_t sourc
return false;
}
void identity_apply_ownership(int fd, int32_t source_uid, int32_t source_gid) {
/* Ownership application is OFF unless the client requested an identity flag.
* This is the controlled gate: a default (or plain -M) transfer never changes
* ownership, byte-for-byte preserving FastSync's existing behavior. */
if (!identity_active_enabled() || fd < 0)
return;
struct stat st;
if (fstat(fd, &st) != 0)
return;
/* Resolve the target ownership from the negotiated policy against the entry's
* current stat. Shared by the fd (regular file) and no-follow (symlink) apply
* paths. Returns false when no side is to be changed. */
static bool identity_resolve_targets(const struct stat* st, int32_t source_uid, int32_t source_gid,
uid_t* out_uid, gid_t* out_gid) {
bool set_uid = false;
bool set_gid = false;
uid_t uid = 0;
@@ -431,29 +427,62 @@ void identity_apply_ownership(int fd, int32_t source_uid, int32_t source_gid) {
}
if (!set_uid && !set_gid)
return;
return false;
/* An unset side keeps the file's current id so the other side can change. */
if (!set_uid)
uid = st.st_uid;
uid = st->st_uid;
if (!set_gid)
gid = st.st_gid;
gid = st->st_gid;
/* Only change ownership when the target differs (avoid needless syscalls and
* any chance of clearing setuid/setgid on an already-correct entry). */
if (st->st_uid == uid && st->st_gid == gid)
return false;
*out_uid = uid;
*out_gid = gid;
return true;
}
/* Only call fchown when the target differs (avoid needless syscalls and any
* chance of clearing setuid/setgid on an already-correct file). */
if (st.st_uid == uid && st.st_gid == gid)
static void identity_log_chown_failure(const char* what, uid_t uid, gid_t gid) {
/* EPERM/EACCES are expected when the receiver is not privileged (e.g. the CI
* `nobody` user): warn and continue, never abort the transfer. Any other
* error (EIO/EROFS/ENOSPC/...) is a real failure and must not be silently
* downgraded to a warning. */
if (errno == EPERM || errno == EACCES)
log_message(LOG_LEVEL_WARNING, "could not apply ownership (uid=%ld gid=%ld): %s; leaving as-is",
(long)uid, (long)gid, strerror(errno));
else
log_message(LOG_LEVEL_ERROR, "failed to apply ownership on %s (uid=%ld gid=%ld): %s", what,
(long)uid, (long)gid, strerror(errno));
}
void identity_apply_ownership(int fd, int32_t source_uid, int32_t source_gid) {
/* Ownership application is OFF unless the client requested an identity flag.
* This is the controlled gate: a default (or plain -M) transfer never changes
* ownership, byte-for-byte preserving FastSync's existing behavior. */
if (!identity_active_enabled() || fd < 0)
return;
struct stat st;
if (fstat(fd, &st) != 0)
return;
uid_t uid;
gid_t gid;
if (!identity_resolve_targets(&st, source_uid, source_gid, &uid, &gid))
return;
if (fchown(fd, uid, gid) != 0)
identity_log_chown_failure("file", uid, gid);
}
if (fchown(fd, uid, gid) != 0) {
/* EPERM/EACCES are expected when the receiver is not privileged (e.g. the
* CI `nobody` user): warn and continue, never abort the transfer. Any
* other error (EIO/EROFS/ENOSPC/...) is a real failure and must not be
* silently downgraded to a warning. */
if (errno == EPERM || errno == EACCES)
log_message(LOG_LEVEL_WARNING,
"could not apply ownership (uid=%ld gid=%ld): %s; leaving as-is", (long)uid,
(long)gid, strerror(errno));
else
log_message(LOG_LEVEL_ERROR, "failed to apply ownership (uid=%ld gid=%ld): %s", (long)uid,
(long)gid, strerror(errno));
}
}
void identity_apply_ownership_link(int parent_fd, const char* leaf, int32_t source_uid,
int32_t source_gid) {
if (!identity_active_enabled() || parent_fd < 0 || !leaf)
return;
struct stat st;
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0)
return;
uid_t uid;
gid_t gid;
if (!identity_resolve_targets(&st, source_uid, source_gid, &uid, &gid))
return;
if (fchownat(parent_fd, leaf, uid, gid, AT_SYMLINK_NOFOLLOW) != 0)
identity_log_chown_failure("symlink", uid, gid);
}
+7
View File
@@ -55,6 +55,13 @@ bool identity_active_enabled(void);
* never fatal (rsync parity: the transfer must not abort). */
void identity_apply_ownership(int fd, int32_t source_uid, int32_t source_gid);
/* P7 Wave D: the no-follow (symlink) counterpart. Resolves the same
* usermap/groupmap/chown/numeric-ids policy but applies it with
* fchownat(..., AT_SYMLINK_NOFOLLOW) so a symlink's own ownership is changed
* without ever dereferencing it. A no-op unless an identity flag is active. */
void identity_apply_ownership_link(int parent_fd, const char* leaf, int32_t source_uid,
int32_t source_gid);
/* Receiver-side wire validation of the resolved identity fields. */
bool identity_wire_valid(const Config* config);
+37
View File
@@ -357,6 +357,43 @@ void file_restore_metadata(const char* path, const FileMetadata* metadata,
}
}
void file_restore_symlink_metadata(const char* path, const FileMetadata* metadata,
bool omit_link_times) {
if (path == NULL || metadata == NULL)
return;
char* leaf = NULL;
int parent_fd = file_open_secure_parent(path, &leaf, false);
if (parent_fd < 0)
return;
/* Ownership (only when the identity policy is active) via lchown semantics:
fchownat with AT_SYMLINK_NOFOLLOW never dereferences the link. */
identity_apply_ownership_link(parent_fd, leaf, (int32_t)metadata->uid, (int32_t)metadata->gid);
/* Symlink mode: not settable on Linux (fchmodat AT_SYMLINK_NOFOLLOW returns
EOPNOTSUPP/ENOTSUP); attempt it for platforms that support it and quietly
ignore the unsupported case so the transfer never fails over it. */
mode_t link_mode = metadata->mode & 0777;
if (fchmodat(parent_fd, leaf, link_mode, AT_SYMLINK_NOFOLLOW) != 0 && errno != EOPNOTSUPP &&
errno != ENOTSUP && errno != ENOSYS) {
log_message(LOG_LEVEL_DEBUG, "Could not set symlink mode on %s: %s", path, strerror(errno));
}
if (!omit_link_times) {
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
{.tv_sec = metadata->mtime_sec, .tv_nsec = metadata->mtime_nsec}};
if (metadata->atime_valid) {
times[0].tv_sec = metadata->atime_sec;
times[0].tv_nsec = metadata->atime_nsec;
}
if (utimensat(parent_fd, leaf, times, AT_SYMLINK_NOFOLLOW) != 0) {
char* escaped_path = output_escape(path, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING, "Failed to set symlink timestamps on %s: %s",
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
free(escaped_path);
}
}
close(parent_fd);
free(leaf);
}
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserve_executability) {
if (fd < 0 || metadata == NULL)
return metadata == NULL;
+8
View File
@@ -39,6 +39,14 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok);
void file_restore_metadata(const char* path, const FileMetadata* metadata,
bool preserve_executability);
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserve_executability);
/* P7 Wave D: apply a SYMLINK's own metadata using no-follow primitives only
* (utimensat/lchown/fchmodat with AT_SYMLINK_NOFOLLOW), confined fd-relative
* under the authorized root. `omit_link_times` (-J/--omit-link-times)
* suppresses the timestamps; the link's mode/ownership are still attempted
* (ownership stays gated by the identity policy and by default is not applied).
* A null metadata or an unfollowable parent is a harmless no-op. */
void file_restore_symlink_metadata(const char* path, const FileMetadata* metadata,
bool omit_link_times);
/* Compare timestamps using rsync's whole-second modification window. */
bool metadata_mtime_matches(time_t left_sec, long left_nsec, time_t right_sec, long right_nsec,
+38
View File
@@ -39,7 +39,14 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
atomic_init(&context->cancelled, false);
protocol_session_init(&context->allocation_session, -1, -1);
protocol_session_set_max_alloc(&context->allocation_session, config->max_alloc);
context->dir_entries = NULL;
context->dir_entries_mutex_init = false;
int init = 0;
if (config->use_metadata) {
context->dir_entries = array_list_create(file_destroy);
if (!context->dir_entries)
goto fail;
}
if (mtx_init(&context->mutex_scanner, mtx_plain) != thrd_success)
goto fail;
init++;
@@ -62,10 +69,17 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
goto fail;
// cppcheck-suppress unreadVariable
init++;
if (mtx_init(&context->dir_entries_mutex, mtx_plain) != thrd_success)
goto fail;
context->dir_entries_mutex_init = true;
return context;
fail:
log_perror("Error initializing synchronization objects");
if (context->dir_entries_mutex_init)
mtx_destroy(&context->dir_entries_mutex);
if (context->dir_entries)
array_list_delete(context->dir_entries);
if (init >= 6)
cnd_destroy(&context->condition_not_empty_loader);
if (init >= 5)
@@ -92,6 +106,10 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) {
array_list_delete(context->missing_args);
if (context->remove_source_files)
array_list_delete(context->remove_source_files);
if (context->dir_entries)
array_list_delete(context->dir_entries);
if (context->dir_entries_mutex_init)
mtx_destroy(&context->dir_entries_mutex);
config_delete(context->config);
queue_destroy(context->queue_scanner);
queue_destroy(context->queue_loader);
@@ -117,6 +135,7 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
context->outcomes.entries = NULL;
context->outcomes.count = 0;
context->outcomes.capacity = 0;
dir_time_list_init(&context->dir_times);
protocol_session_init(&context->session, file_descriptor, file_descriptor);
protocol_session_set_ssl(&context->session, ssl);
context->receiver_done = false;
@@ -155,6 +174,7 @@ void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
delete_manifest_free(context->deferred_manifest);
queue_destroy(context->queue);
receiver_outcomes_destroy(&context->outcomes);
dir_time_list_free(&context->dir_times);
mtx_destroy(&context->mutex);
cnd_destroy(&context->condition_not_full);
cnd_destroy(&context->condition_not_empty);
@@ -307,6 +327,24 @@ int write_thread(void* pipeline_context) {
return thrd_error;
}
}
/* P7 Wave D: a directory's times are never applied inline (a later child
write would clobber them); accumulate the metadata here and let the
caller apply it once every writer has drained. */
if (result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
context->config->use_metadata && !context->config->omit_dir_times &&
!dir_time_list_add(&context->dir_times, file->path, file->metadata)) {
file_destroy(file);
pipeline_context_receiver_note_bytes_released(context, file_bytes);
mtx_lock(&context->mutex);
atomic_store(&context->cancelled, true);
context->receiver_done = true;
cnd_broadcast(&context->condition_not_full);
cnd_broadcast(&context->condition_not_empty);
mtx_unlock(&context->mutex);
free(root_directory);
protocol_session_unbind();
return thrd_error;
}
/* Record the per-file outcome so a --remove-source-files sender learns
which sources were actually written versus skipped on the receiver.
Explicit directory entries and recreated device/special nodes have no
+13 -2
View File
@@ -67,6 +67,13 @@ typedef struct {
* before it reads the manifest, so no additional synchronization is needed
* to suppress the manifest. */
bool scan_stopped_early;
/* P7 Wave D: captured source directory times, filled by the scanner thread
* (and its parallel workers, guarded by dir_entries_mutex) and drained by the
* sender thread in trailing STATUS_DIR_TIMES frame(s). Owned by the
* context; NULL for non-metadata transfers. */
ArrayList* dir_entries;
mtx_t dir_entries_mutex;
bool dir_entries_mutex_init;
} PipelineContextSender;
typedef struct PipelineContextReceiver {
@@ -94,9 +101,13 @@ typedef struct PipelineContextReceiver {
protocol stream but hands the manifest here instead of deleting while the
disk writer may still be draining; the caller (server.c) commits the
deletion after both threads have joined, so no extra is removed unless the
transfer truly succeeded. NULL in the early delete modes (which delete at
the manifest). */
transfer truly succeeded. NULL in the early delete modes (which delete at
the manifest). */
DeleteManifest* deferred_manifest;
/* P7 Wave D: directory metadata collected by write_thread from received
directory entries. Only write_thread mutates it (before it joins); the
caller (server.c) applies it after the delete/delay-updates phase. */
DirTimeList dir_times;
} PipelineContextReceiver;
PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* queue_scanner,
+6
View File
@@ -407,6 +407,12 @@ static const char* status_to_string(Status status) {
return "APPEND_DATA";
case STATUS_HARDLINK:
return "HARDLINK";
case STATUS_SYMLINK:
return "SYMLINK";
case STATUS_SPECIAL:
return "SPECIAL";
case STATUS_DIR_TIMES:
return "DIR_TIMES";
default:
return "UNKNOWN";
}
+11 -1
View File
@@ -103,7 +103,17 @@ enum NET_STATUS {
* int32 rdev major/minor fields. The receiver validates the kind and rdev,
* confines the node below the receive root, and recreates it (mknod/mkfifo),
* privilege-gating the mknod. Protocol 2.13.0. */
STATUS_SPECIAL
STATUS_SPECIAL,
/* Directory-time superstructure (P7 Wave D, protocol 2.17.0): one or more
* trailing frames sent after all file data (and after the optional delete
* manifest) carrying the source directories' captured metadata so the
* receiver can apply directory mtimes/atimes AFTER all of a directory's
* children have been written. Payload per frame: an int count, then count
* repetitions of (wire path string, metadata frame); an entry count larger
* than MAX_MANIFEST_ENTRIES is split across repeated frames. The receiver
* defers the actual utimensat until its own delete/publish phase has
* committed, then skips the whole set when -O/--omit-dir-times is set. */
STATUS_DIR_TIMES
};
void io_set_fds(int read_fd, int write_fd);