feat: protect filter-excluded destination mirrors by default (--delete-excluded opt-in), --ignore-errors scan continuation, --prune-empty-dirs

The scanners now record every entry pruned by user-selection rules
(--filter/-C/per-dir, --exclude/--include, --max-size/--min-size) as a
destination-relative protected path on a caller-supplied sink (thread-safe in
the parallel scanner); --files-from subset pruning and -R relative wire paths
are never recorded.  The sender transmits these as manifest protected prefixes,
giving rsync's default --delete behavior (excluded mirrors survive) with
--delete-excluded opting back into deleting them.  --ignore-errors makes an
unreadable source directory a recorded, non-fatal scan error: the run continues,
the deletion still runs, and the exit code reports the ignored error.
--prune-empty-dirs omits an empty source directory's explicit --dirs entry.
Empty directories were never transferred by recursive scans (rsync -m parity).
This commit is contained in:
2026-09-06 21:50:07 +02:00
parent c4e0de8f08
commit 0b25bacb18
3 changed files with 382 additions and 102 deletions
+108 -22
View File
@@ -102,6 +102,10 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann
options->per_dir_filters = config->per_dir_filter; options->per_dir_filters = config->per_dir_filter;
options->dirs = config->dirs; options->dirs = config->dirs;
options->relative = config->relative; options->relative = config->relative;
options->prune_empty_dirs = config->prune_empty_dirs;
options->ignore_io_errors = config->ignore_errors;
options->excluded_paths = NULL;
options->excluded_mutex = NULL;
return true; return true;
} }
@@ -255,7 +259,7 @@ static bool basis_oversize_preflight(const Config* config) {
if (!ok) if (!ok)
break; break;
} }
if (directory_scanner_failed(scanner)) if (directory_scanner_failed(scanner) || directory_scanner_had_io_error(scanner))
ok = false; ok = false;
directory_scanner_destroy(scanner); directory_scanner_destroy(scanner);
return ok; return ok;
@@ -596,7 +600,7 @@ static int send_list_only(const Config* config) {
if (oom) if (oom)
break; break;
} }
bool failed = oom || directory_scanner_failed(scanner); bool failed = oom || directory_scanner_failed(scanner) || directory_scanner_had_io_error(scanner);
directory_scanner_destroy(scanner); directory_scanner_destroy(scanner);
prepared_scanner_destroy(&prepared); prepared_scanner_destroy(&prepared);
if (failed) { if (failed) {
@@ -621,8 +625,11 @@ static int send_list_only(const Config* config) {
return 0; return 0;
} }
/* Send the delete manifest (list of files) to the server. Returns 0 on success, -1 on failure. */ /* Send the delete manifest (keep-set paths plus the protected excluded
static int send_delete_manifest(int fd, ArrayList* manifest) { prefixes) to the server. Returns 0 on success, -1 on failure. When
--delete-excluded is given `protected` is empty: excluded destination
mirrors are then ordinary extras and are removed. */
static int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes) {
if (!manifest) if (!manifest)
return -1; return -1;
if (!send_status(fd, STATUS_MANIFEST)) if (!send_status(fd, STATUS_MANIFEST))
@@ -633,6 +640,13 @@ static int send_delete_manifest(int fd, ArrayList* manifest) {
if (!send_str(fd, (char*)manifest->items[i])) if (!send_str(fd, (char*)manifest->items[i]))
return -1; return -1;
} }
int protected_count = protected_prefixes ? protected_prefixes->size : 0;
if (!send_int(fd, protected_count))
return -1;
for (int i = 0; i < protected_count; i++) {
if (!send_str(fd, (char*)protected_prefixes->items[i]))
return -1;
}
return 0; return 0;
} }
@@ -647,10 +661,11 @@ static int send_delete_manifest(int fd, ArrayList* manifest) {
instead of the default 60 s receive window. */ instead of the default 60 s receive window. */
#define DELETE_ACK_TIMEOUT_SEC 3600 #define DELETE_ACK_TIMEOUT_SEC 3600
static bool send_delete_manifest_early(Client* client, ArrayList* manifest) { static bool send_delete_manifest_early(Client* client, ArrayList* manifest,
ArrayList* protected_prefixes) {
if (!client || !manifest) if (!client || !manifest)
return false; return false;
if (send_delete_manifest(client->file_descriptor, manifest) != 0) if (send_delete_manifest(client->file_descriptor, manifest, protected_prefixes) != 0)
return false; return false;
Status ack; Status ack;
if (!receive_status_timed(client->file_descriptor, &ack, DELETE_ACK_TIMEOUT_SEC)) if (!receive_status_timed(client->file_descriptor, &ack, DELETE_ACK_TIMEOUT_SEC))
@@ -666,9 +681,14 @@ static bool send_delete_manifest_early(Client* client, ArrayList* manifest) {
paths, loading and sending nothing. --delete-before/--delete-during need the paths, loading and sending nothing. --delete-before/--delete-during need the
complete keep-set manifest before the first data byte, so it is built by a complete keep-set manifest before the first data byte, so it is built by a
dedicated pre-scan pass and transmitted early; the data pass then re-scans dedicated pre-scan pass and transmitted early; the data pass then re-scans
with a fresh scanner. */ with a fresh scanner. A source I/O error is fatal unless the options carry
--ignore-errors, in which case the scan continues past the unreadable
directory and *io_error_out reports it (the caller still performs the
deletion but reports the run as errored). */
static bool scan_paths_only(const Config* config, const ScannerOptions* options, static bool scan_paths_only(const Config* config, const ScannerOptions* options,
ArrayList* manifest) { ArrayList* manifest, bool* io_error_out) {
if (io_error_out)
*io_error_out = false;
DirectoryScanner* scanner = DirectoryScanner* scanner =
directory_scanner_create_with_options(config->send_directory, options); directory_scanner_create_with_options(config->send_directory, options);
if (!scanner) if (!scanner)
@@ -685,6 +705,8 @@ static bool scan_paths_only(const Config* config, const ScannerOptions* options,
} }
if (ok && directory_scanner_failed(scanner)) if (ok && directory_scanner_failed(scanner))
ok = false; ok = false;
if (io_error_out)
*io_error_out = directory_scanner_had_io_error(scanner);
directory_scanner_destroy(scanner); directory_scanner_destroy(scanner);
return ok; return ok;
} }
@@ -1004,7 +1026,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
if (context->early_delete) { if (context->early_delete) {
/* The keep-set manifest was prebuilt by a path-only pre-scan. Transmit it /* The keep-set manifest was prebuilt by a path-only pre-scan. Transmit it
and wait for the receiver to delete extras before streaming any data. */ and wait for the receiver to delete extras before streaming any data. */
if (!send_delete_manifest_early(client, context->manifest)) { if (!send_delete_manifest_early(client, context->manifest, context->excluded_paths)) {
pipeline_cancel(context); pipeline_cancel(context);
disconnect_transfer_client(client); disconnect_transfer_client(client);
mark_sender_done(context); mark_sender_done(context);
@@ -1026,10 +1048,15 @@ static int send_chunks_multithreaded(void* pipeline_context) {
return thrd_error; return thrd_error;
} }
if (context->config->use_delete && !context->early_delete) { if (context->config->use_delete && !context->early_delete) {
if (send_delete_manifest(client->file_descriptor, context->manifest) != 0) if (send_delete_manifest(client->file_descriptor, context->manifest,
context->excluded_paths) != 0)
goto send_fail; goto send_fail;
} }
bool ok = finalize_transfer(client, context->config, context->remove_source_files); bool ok = finalize_transfer(client, context->config, context->remove_source_files);
if (!ok && context->config->use_delete)
log_message(LOG_LEVEL_ERROR,
"server reported a deletion failure (--delete); see the server log for the "
"reason (a --max-delete limit that the run would exceed deletes nothing)");
if (ok) if (ok)
remove_transferred_sources(context->config, context->remove_source_files); remove_transferred_sources(context->config, context->remove_source_files);
mtx_lock(&context->mutex_progress); mtx_lock(&context->mutex_progress);
@@ -1091,6 +1118,12 @@ static int scan_directory_multithreaded(void* pipeline_context) {
protocol_session_unbind(); protocol_session_unbind();
return thrd_error; return thrd_error;
} }
/* The keep-set manifest for the late modes is built from this data pass, so
the parallel scanner records the protected excluded prefixes here. The
early modes already transmitted the pre-scan keep-set and its protected
list, so the data pass must not append to it again. */
if (!context->early_delete)
prepared.options.excluded_paths = context->excluded_paths;
bool dirs_mode = prepared.options.dirs; bool dirs_mode = prepared.options.dirs;
DirectoryScanner* dscanner = NULL; DirectoryScanner* dscanner = NULL;
ParallelScanner* scanner = NULL; ParallelScanner* scanner = NULL;
@@ -1153,6 +1186,15 @@ static int scan_directory_multithreaded(void* pipeline_context) {
protocol_session_unbind(); protocol_session_unbind();
return thrd_error; return thrd_error;
} }
/* --ignore-errors: an unreadable subdirectory was skipped (workers recorded
io_error, not failure); the deletion still runs but the run reports it. */
bool had_io =
dirs_mode ? directory_scanner_had_io_error(dscanner) : parallel_scanner_had_io_error(scanner);
if (had_io) {
mtx_lock(&context->mutex_scanner);
context->scan_had_io_error = true;
mtx_unlock(&context->mutex_scanner);
}
mtx_lock(&context->mutex_scanner); mtx_lock(&context->mutex_scanner);
context->scanner_done = true; context->scanner_done = true;
cnd_signal(&context->condition_not_empty_scanner); cnd_signal(&context->condition_not_empty_scanner);
@@ -1280,8 +1322,11 @@ int send_files(Config* config) {
DirectoryScanner* scanner = NULL; DirectoryScanner* scanner = NULL;
ArrayList* manifest = NULL; ArrayList* manifest = NULL;
ArrayList* remove_sources = NULL; ArrayList* remove_sources = NULL;
/* Protected excluded prefixes (delete-excluded default protection). */
ArrayList* excluded = NULL;
bool delete_early = config->use_delete && config_delete_timing_early(config); bool delete_early = config->use_delete && config_delete_timing_early(config);
bool send_failed = false; bool send_failed = false;
bool had_scan_io = false;
PreparedScanner prepared; PreparedScanner prepared;
memset(&prepared, 0, sizeof(prepared)); memset(&prepared, 0, sizeof(prepared));
if (!config_send(client->file_descriptor, config)) if (!config_send(client->file_descriptor, config))
@@ -1292,6 +1337,16 @@ int send_files(Config* config) {
remove_sources = array_list_create(source_file_destroy); remove_sources = array_list_create(source_file_destroy);
if (config->remove_source_files && !remove_sources) if (config->remove_source_files && !remove_sources)
goto send_fail; goto send_fail;
/* Unless --delete-excluded opts out, collect the paths the source scan prunes
by user-selection rules so the receiver protects their destination mirrors
from --delete (rsync's default). Only scans that build the keep-set get the
sink attached (prescan for early timing, the streaming data pass otherwise). */
if (config->use_delete && !config->delete_excluded) {
excluded = array_list_create(free);
if (!excluded)
goto send_fail;
prepared.options.excluded_paths = excluded;
}
/* The late-timing modes (plain --delete / --delete-after / --delete-delay) /* The late-timing modes (plain --delete / --delete-after / --delete-delay)
build the manifest while streaming and send it after the last data frame. build the manifest while streaming and send it after the last data frame.
The early modes (--delete-before/--delete-during) send it up front from a The early modes (--delete-before/--delete-during) send it up front from a
@@ -1303,13 +1358,15 @@ int send_files(Config* config) {
ArrayList* early_manifest = array_list_create(free); ArrayList* early_manifest = array_list_create(free);
if (!early_manifest) if (!early_manifest)
goto send_fail; goto send_fail;
if (!scan_paths_only(config, &prepared.options, early_manifest)) { bool prescan_ok = scan_paths_only(config, &prepared.options, early_manifest, &had_scan_io);
array_list_delete(early_manifest); bool early_ok = false;
goto send_fail; if (prescan_ok)
} early_ok = send_delete_manifest_early(client, early_manifest, excluded);
bool early_ok = send_delete_manifest_early(client, early_manifest);
array_list_delete(early_manifest); array_list_delete(early_manifest);
if (!early_ok) /* The keep-set (and its protected prefixes) are already on the wire; the
data pass must not append to the exclusion list again. */
prepared.options.excluded_paths = NULL;
if (!prescan_ok || !early_ok)
goto send_fail; goto send_fail;
} else if (config->use_delete) { } else if (config->use_delete) {
manifest = array_list_create(free); manifest = array_list_create(free);
@@ -1377,10 +1434,12 @@ int send_files(Config* config) {
} }
if (directory_scanner_failed(scanner)) if (directory_scanner_failed(scanner))
goto send_fail; goto send_fail;
if (directory_scanner_had_io_error(scanner))
had_scan_io = true;
if (manifest) { if (manifest) {
/* Late (commit) ordering: all file data is out; transmit the keep-set /* Late (commit) ordering: all file data is out; transmit the keep-set
manifest so the receiver deletes only after the transfer succeeds. */ manifest so the receiver deletes only after the transfer succeeds. */
if (send_delete_manifest(client->file_descriptor, manifest) != 0) { if (send_delete_manifest(client->file_descriptor, manifest, excluded) != 0) {
array_list_delete(manifest); array_list_delete(manifest);
manifest = NULL; manifest = NULL;
goto send_fail; goto send_fail;
@@ -1389,6 +1448,10 @@ int send_files(Config* config) {
manifest = NULL; manifest = NULL;
} }
bool ok = finalize_transfer(client, config, remove_sources); bool ok = finalize_transfer(client, config, remove_sources);
if (!ok && config->use_delete)
log_message(LOG_LEVEL_ERROR,
"server reported a deletion failure (--delete); see the server log for the "
"reason (a --max-delete limit that the run would exceed deletes nothing)");
if (ok) if (ok)
remove_transferred_sources(config, remove_sources); remove_transferred_sources(config, remove_sources);
if (config->show_progress && !config->quiet) if (config->show_progress && !config->quiet)
@@ -1410,13 +1473,17 @@ int send_files(Config* config) {
} }
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files, log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
total_bytes / 1048576.0); total_bytes / 1048576.0);
ret = ok ? 0 : 1; /* --ignore-errors: an unreadable source directory was skipped but the run
still completed (and deleted); report the run as errored like rsync does. */
ret = (ok && !had_scan_io) ? 0 : 1;
send_fail: send_fail:
/* Single cleanup path for all exits. The manifest is intentionally deleted /* Single cleanup path for all exits. The manifest is intentionally deleted
here even on success without --delete, fixing a pre-existing leak. */ here even on success without --delete, fixing a pre-existing leak. */
if (manifest) if (manifest)
array_list_delete(manifest); array_list_delete(manifest);
if (excluded)
array_list_delete(excluded);
if (remove_sources) if (remove_sources)
array_list_delete(remove_sources); array_list_delete(remove_sources);
if (scanner) if (scanner)
@@ -1468,20 +1535,32 @@ int send_files_multithreaded(Config** config_ptr) {
return 1; return 1;
} }
*config_ptr = NULL; /* context now owns config through all remaining paths */ *config_ptr = NULL; /* context now owns config through all remaining paths */
bool collect_excluded = config->use_delete && !config->delete_excluded;
if (config->use_delete) { if (config->use_delete) {
context->manifest = array_list_create(free); context->manifest = array_list_create(free);
if (!context->manifest) { if (!context->manifest) {
pipeline_context_sender_destroy(context); pipeline_context_sender_destroy(context);
return 1; return 1;
} }
if (collect_excluded) {
context->excluded_paths = array_list_create(free);
if (!context->excluded_paths) {
pipeline_context_sender_destroy(context);
return 1;
}
}
if (config_delete_timing_early(config)) { if (config_delete_timing_early(config)) {
/* --delete-before/--delete-during: build the complete keep-set manifest /* --delete-before/--delete-during: build the complete keep-set manifest
(paths only, nothing loaded or sent) up front so the sender thread can (paths only, nothing loaded or sent) up front so the sender thread can
transmit it before the first data byte. */ transmit it before the first data byte. The path-only pre-scan also
fills the protected excluded prefixes. */
PreparedScanner prepared; PreparedScanner prepared;
memset(&prepared, 0, sizeof(prepared)); memset(&prepared, 0, sizeof(prepared));
bool prebuilt = prepare_scanner(config, 4, &prepared) && bool prepared_ok = prepare_scanner(config, 4, &prepared);
scan_paths_only(config, &prepared.options, context->manifest); if (prepared_ok && context->excluded_paths)
prepared.options.excluded_paths = context->excluded_paths;
bool prebuilt = prepared_ok && scan_paths_only(config, &prepared.options, context->manifest,
&context->scan_had_io_error);
prepared_scanner_destroy(&prepared); prepared_scanner_destroy(&prepared);
if (!prebuilt) { if (!prebuilt) {
pipeline_context_sender_destroy(context); pipeline_context_sender_destroy(context);
@@ -1548,6 +1627,13 @@ int send_files_multithreaded(Config** config_ptr) {
thrd_join(progress, NULL); thrd_join(progress, NULL);
} }
bool scan_io;
mtx_lock(&context->mutex_scanner);
scan_io = context->scan_had_io_error;
mtx_unlock(&context->mutex_scanner);
bool sender_ok = sender_result == thrd_success;
/* --ignore-errors: the run completed (and deleted) past an unreadable source
directory; report it as errored like rsync does. */
pipeline_context_sender_destroy(context); pipeline_context_sender_destroy(context);
return sender_result == thrd_success ? 0 : 1; return sender_ok && !scan_io ? 0 : 1;
} }
+234 -80
View File
@@ -112,6 +112,10 @@ typedef struct {
char* path; char* path;
struct stat stats; struct stat stats;
bool is_directory; bool is_directory;
/* True when the entry was pruned by a user selection rule (--filter/-C/per-dir
rules, the --exclude/--include layer, or --max-size/--min-size) rather than
skipped for another reason (unreadable, symlink policy, not applicable). */
bool excluded;
} ScannerEntry; } ScannerEntry;
/* --one-file-system (-x) decision. Only directories can carry a different /* --one-file-system (-x) decision. Only directories can carry a different
@@ -161,6 +165,38 @@ static bool entry_passes_selection(const FileListSet* file_list, const FilterRul
return true; return true;
} }
/* Append `rel` to the caller's exclusion sink, taking `mtx` when shared across
parallel worker threads. Returns false on allocation failure (list left
unchanged). */
static bool excluded_sink_append(ArrayList* list, mtx_t* mtx, const char* rel) {
if (!list)
return true;
char* dup = str_dup(rel);
if (!dup)
return false;
if (mtx)
mtx_lock(mtx);
bool ok = array_list_add(list, dup);
if (mtx)
mtx_unlock(mtx);
if (!ok)
free(dup);
return ok;
}
/* Record one pruned-by-user-selection filesystem path in the scanner's
exclusion sink (see ScannerOptions.excluded_paths). The stored form is the
entry's wire/destination-relative path (a single leading '/' removed, exactly
how manifest keep entries are stored), so the receiver's walker prefixes
match the destination layout. An allocation failure is a fatal scan error. */
static void scanner_record_excluded(DirectoryScanner* scanner, const char* fs_path) {
if (!scanner->excluded_paths || !fs_path)
return;
const char* rel = *fs_path == '/' ? fs_path + 1 : fs_path;
if (!excluded_sink_append(scanner->excluded_paths, scanner->excluded_mutex, rel))
scanner->failed = true;
}
/* Merge the open directory's own .rsync-filter rules into the inherited /* Merge the open directory's own .rsync-filter rules into the inherited
* context, returning the context used for this directory's entries. On a parse * context, returning the context used for this directory's entries. On a parse
* error the scanner is marked failed. Returns 0 on success, -1 on failure. */ * error the scanner is marked failed. Returns 0 on success, -1 on failure. */
@@ -198,6 +234,7 @@ static int open_directory_filter_context(DirectoryScanner* scanner, const Filter
static int scanner_inspect_entry(const ScannerOptions* options, const char* source_root, static int scanner_inspect_entry(const ScannerOptions* options, const char* source_root,
const char* containing_dir, const char* name, const char* containing_dir, const char* name,
ScannerEntry* entry) { ScannerEntry* entry) {
entry->excluded = false;
entry->path = path_cat(containing_dir, name); entry->path = path_cat(containing_dir, name);
if (!entry->path) if (!entry->path)
return -1; return -1;
@@ -241,19 +278,25 @@ static int scanner_inspect_entry(const ScannerOptions* options, const char* sour
if (entry->is_directory) if (entry->is_directory)
return 1; return 1;
for (int i = 0; i < options->exclude_count; i++) for (int i = 0; i < options->exclude_count; i++)
if (glob_match(options->exclude_patterns[i], name)) if (glob_match(options->exclude_patterns[i], name)) {
entry->excluded = true;
goto skip; goto skip;
}
if (options->include_count > 0) { if (options->include_count > 0) {
bool included = false; bool included = false;
for (int i = 0; i < options->include_count; i++) for (int i = 0; i < options->include_count; i++)
if (glob_match(options->include_patterns[i], name)) if (glob_match(options->include_patterns[i], name))
included = true; included = true;
if (!included) if (!included) {
entry->excluded = true;
goto skip; goto skip;
}
} }
if ((options->max_size > 0 && (unsigned long long)entry->stats.st_size > options->max_size) || if ((options->max_size > 0 && (unsigned long long)entry->stats.st_size > options->max_size) ||
(options->min_size > 0 && (unsigned long long)entry->stats.st_size < options->min_size)) (options->min_size > 0 && (unsigned long long)entry->stats.st_size < options->min_size)) {
entry->excluded = true;
goto skip; goto skip;
}
return 1; return 1;
skip: skip:
@@ -306,8 +349,13 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
scanner->file_list = options->file_list; scanner->file_list = options->file_list;
scanner->base_filters = options->base_filters; scanner->base_filters = options->base_filters;
scanner->per_dir_filters = options->per_dir_filters; scanner->per_dir_filters = options->per_dir_filters;
scanner->excluded_paths = options->excluded_paths;
scanner->excluded_mutex = options->excluded_mutex;
scanner->ignore_io_errors = options->ignore_io_errors;
scanner->io_error = false;
scanner->dirs_mode = options->dirs; scanner->dirs_mode = options->dirs;
scanner->relative_mode = options->relative && options->file_list != NULL; scanner->relative_mode = options->relative && options->file_list != NULL;
scanner->prune_empty_dirs = options->prune_empty_dirs;
scanner->dirs_root_emitted = false; scanner->dirs_root_emitted = false;
scanner->list_index = 0; scanner->list_index = 0;
scanner->dirs_batch = NULL; scanner->dirs_batch = NULL;
@@ -360,27 +408,29 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
unsigned long long min_size, int max_depth, unsigned long long min_size, int max_depth,
bool follow_symlinks, bool copy_links, bool safe_links, bool follow_symlinks, bool copy_links, bool safe_links,
bool copy_unsafe_links, bool checksum) { bool copy_unsafe_links, bool checksum) {
ScannerOptions options = {use_metadata, ScannerOptions options = {
chunk_size, .use_metadata = use_metadata,
exclude_patterns, .chunk_size = chunk_size,
exclude_count, .exclude_patterns = exclude_patterns,
include_patterns, .exclude_count = exclude_count,
include_count, .include_patterns = include_patterns,
max_size, .include_count = include_count,
min_size, .max_size = max_size,
max_depth, .min_size = min_size,
0, .max_depth = max_depth,
follow_symlinks, .num_threads = 0,
copy_links, .follow_symlinks = follow_symlinks,
safe_links, .copy_links = copy_links,
copy_unsafe_links, .safe_links = safe_links,
checksum, .copy_unsafe_links = copy_unsafe_links,
false, .checksum = checksum,
NULL, .one_file_system = false,
NULL, .file_list = NULL,
false, .base_filters = NULL,
false, .per_dir_filters = false,
false}; .dirs = false,
.relative = false,
};
return directory_scanner_create_with_options(root_directory, &options); return directory_scanner_create_with_options(root_directory, &options);
} }
@@ -413,48 +463,66 @@ static Chunk* chunk_data_to_chunk(ArrayList* chunk_data) {
return chunk; return chunk;
} }
/* Open the next queued directory and set up its filter context. Returns 1 when
a directory is open, 0 when the queue is exhausted, and -1 on a fatal error.
A directory that cannot be opened is an I/O error: it is recorded on the
scanner and, when --ignore-errors is active, skipped so the rest of the tree
is still scanned (the caller decides whether to treat the recorded error as
fatal). */
static int open_next_directory(DirectoryScanner* scanner) { static int open_next_directory(DirectoryScanner* scanner) {
if (scanner->current_dir) { if (scanner->current_dir) {
closedir(scanner->current_dir); closedir(scanner->current_dir);
scanner->current_dir = NULL; scanner->current_dir = NULL;
} }
free(scanner->current_path); free(scanner->current_path);
scanner->current_path = NULL;
if (queue_is_empty(scanner->directories)) while (!queue_is_empty(scanner->directories)) {
return 0; DirEntry* de = (DirEntry*)queue_dequeue(scanner->directories);
scanner->current_path = de->path;
scanner->current_depth = de->depth;
/* The seed directory inherits the scanner's configured context (the root
* .rsync-filter context in parallel mode); other dirs inherit the context of
* the directory that enqueued them. */
const FilterNode* inherited = scanner->at_seed_dir ? scanner->seed_node : de->context;
scanner->at_seed_dir = false;
free(de);
DirEntry* de = (DirEntry*)queue_dequeue(scanner->directories); free(scanner->current_rel);
scanner->current_path = de->path; scanner->current_rel = scanner_path_relative(scanner->root_path, scanner->current_path);
scanner->current_depth = de->depth; if (!scanner->current_rel) {
/* The seed directory inherits the scanner's configured context (the root log_message(LOG_LEVEL_ERROR, "Could not compute relative path under %s", scanner->root_path);
* .rsync-filter context in parallel mode); other dirs inherit the context of scanner->failed = true;
* the directory that enqueued them. */ free(scanner->current_path);
const FilterNode* inherited = scanner->at_seed_dir ? scanner->seed_node : de->context; scanner->current_path = NULL;
scanner->at_seed_dir = false; return -1;
free(de); }
free(scanner->current_rel); scanner->current_dir = opendir(scanner->current_path);
scanner->current_rel = scanner_path_relative(scanner->root_path, scanner->current_path); if (scanner->current_dir == NULL) {
if (!scanner->current_rel) { scanner->io_error = true;
log_message(LOG_LEVEL_ERROR, "Could not compute relative path under %s", scanner->root_path); log_perror("Could not open directory");
scanner->failed = true; free(scanner->current_rel);
return -1; scanner->current_rel = NULL;
free(scanner->current_path);
scanner->current_path = NULL;
if (!scanner->ignore_io_errors) {
scanner->failed = true;
return -1;
}
/* --ignore-errors: record the I/O error and keep scanning the rest. */
continue;
}
if (open_directory_filter_context(scanner, inherited) != 0) {
closedir(scanner->current_dir);
scanner->current_dir = NULL;
free(scanner->current_path);
scanner->current_path = NULL;
return -1;
}
return 1;
} }
return 0;
scanner->current_dir = opendir(scanner->current_path);
if (scanner->current_dir == NULL) {
log_perror("Could not open directory");
free(scanner->current_path);
scanner->current_path = NULL;
scanner->failed = true;
return -1;
}
if (open_directory_filter_context(scanner, inherited) != 0) {
closedir(scanner->current_dir);
scanner->current_dir = NULL;
return -1;
}
return 1;
} }
/* ---- --dirs mode ---- /* ---- --dirs mode ----
@@ -563,12 +631,35 @@ static File* dirs_file_for_entry(DirectoryScanner* scanner, const char* entry) {
return file; return file;
} }
/* True when the directory contains no entries at all (ignoring "." and "..").
An unreadable directory is reported as non-empty so the regular (erroring)
root-entry path runs instead of silently transferring nothing. */
static bool dirs_source_dir_is_empty(const char* path) {
DIR* dir = opendir(path);
if (!dir)
return false;
bool empty = true;
const struct dirent* entry;
while ((entry = readdir(dir)) != NULL) {
if (strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0) {
empty = false;
break;
}
}
closedir(dir);
return empty;
}
/* The next File from the --dirs generator, or NULL when exhausted. */ /* The next File from the --dirs generator, or NULL when exhausted. */
static File* dirs_next_file(DirectoryScanner* scanner) { static File* dirs_next_file(DirectoryScanner* scanner) {
if (!scanner->file_list) { if (!scanner->file_list) {
if (scanner->dirs_root_emitted) if (scanner->dirs_root_emitted)
return NULL; return NULL;
scanner->dirs_root_emitted = true; scanner->dirs_root_emitted = true;
/* --prune-empty-dirs: a physically empty source directory's explicit entry
would only create an empty destination directory, so it is omitted. */
if (scanner->prune_empty_dirs && dirs_source_dir_is_empty(scanner->root_path))
return NULL;
return dirs_root_dir_file(scanner); return dirs_root_dir_file(scanner);
} }
while (scanner->list_index < scanner->file_list->count) { while (scanner->list_index < scanner->file_list->count) {
@@ -663,27 +754,29 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
continue; continue;
ScannerOptions options = {scanner->use_metadata, ScannerOptions options = {
scanner->chunk_size, .use_metadata = scanner->use_metadata,
scanner->exclude_patterns, .chunk_size = scanner->chunk_size,
scanner->exclude_count, .exclude_patterns = scanner->exclude_patterns,
scanner->include_patterns, .exclude_count = scanner->exclude_count,
scanner->include_count, .include_patterns = scanner->include_patterns,
scanner->max_size, .include_count = scanner->include_count,
scanner->min_size, .max_size = scanner->max_size,
scanner->max_depth, .min_size = scanner->min_size,
0, .max_depth = scanner->max_depth,
scanner->follow_symlinks, .num_threads = 0,
scanner->copy_links, .follow_symlinks = scanner->follow_symlinks,
scanner->safe_links, .copy_links = scanner->copy_links,
scanner->copy_unsafe_links, .safe_links = scanner->safe_links,
scanner->checksum, .copy_unsafe_links = scanner->copy_unsafe_links,
scanner->one_file_system, .checksum = scanner->checksum,
scanner->file_list, .one_file_system = scanner->one_file_system,
scanner->base_filters, .file_list = scanner->file_list,
scanner->per_dir_filters, .base_filters = scanner->base_filters,
false, .per_dir_filters = scanner->per_dir_filters,
false}; .dirs = false,
.relative = false,
};
ScannerEntry inspected; ScannerEntry inspected;
int inspection = scanner_inspect_entry(&options, scanner->current_path, scanner->current_path, int inspection = scanner_inspect_entry(&options, scanner->current_path, scanner->current_path,
entry->d_name, &inspected); entry->d_name, &inspected);
@@ -691,8 +784,21 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
scanner->failed = true; scanner->failed = true;
break; break;
} }
if (inspection == 0) if (inspection == 0) {
/* The entry was pruned by a user selection rule (exclude/include/size) or
skipped for another reason; only the user-selection prunes protect the
corresponding destination mirror from --delete. */
if (inspected.excluded) {
char* abs_path = path_cat(scanner->current_path, entry->d_name);
if (!abs_path) {
scanner->failed = true;
break;
}
scanner_record_excluded(scanner, abs_path);
free(abs_path);
}
continue; continue;
}
char* cur_path = inspected.path; char* cur_path = inspected.path;
struct stat stats = inspected.stats; struct stat stats = inspected.stats;
@@ -708,6 +814,16 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
bool passes_selection = bool passes_selection =
entry_passes_selection(scanner->file_list, scanner->base_filters, scanner->current_node, entry_passes_selection(scanner->file_list, scanner->base_filters, scanner->current_node,
rel, entry->d_name, is_dir, scanner->per_dir_filters); rel, entry->d_name, is_dir, scanner->per_dir_filters);
if (!passes_selection) {
/* --files-from subset pruning is not a filter exclusion: its delete
semantics stay keep-set-only (an unlisted source path is treated as
absent, so its destination mirror is a deletable extra). A rule-based
exclusion is recorded as a protected prefix. -R + --files-from bare
wire paths are never recorded (see ScannerOptions.excluded_paths). */
bool files_from_prune = scanner->file_list && !file_list_affects(scanner->file_list, rel);
if (!files_from_prune && !scanner->relative_mode)
scanner_record_excluded(scanner, cur_path);
}
/* With -R + --files-from the wire/destination path is the entry's bare /* With -R + --files-from the wire/destination path is the entry's bare
relative path; keep `rel` alive to attach it to a transferred file. */ relative path; keep `rel` alive to attach it to a transferred file. */
char* rel_copy = scanner->relative_mode ? str_dup(rel) : NULL; char* rel_copy = scanner->relative_mode ? str_dup(rel) : NULL;
@@ -796,6 +912,10 @@ bool directory_scanner_failed(const DirectoryScanner* scanner) {
return scanner == NULL || scanner->failed; return scanner == NULL || scanner->failed;
} }
bool directory_scanner_had_io_error(const DirectoryScanner* scanner) {
return scanner != NULL && scanner->io_error;
}
typedef struct { typedef struct {
ParallelScanner* ps; ParallelScanner* ps;
char** dirs; char** dirs;
@@ -826,10 +946,12 @@ static int parallel_worker_thread(void* arg) {
/* Root .rsync-filter rules (parsed by the parallel scanner) apply to the /* Root .rsync-filter rules (parsed by the parallel scanner) apply to the
* contents of every assigned subdirectory. Relative paths (used by the * contents of every assigned subdirectory. Relative paths (used by the
* allow-set and per-directory rules) are computed against the transfer * allow-set and per-directory rules) are computed against the transfer
* root, not the subdirectory the worker is seeded with. */ * root, not the subdirectory the worker is seeded with. Exclusion
* recording shares one caller-owned list across the workers. */
free(ds->root_path); free(ds->root_path);
ds->root_path = str_dup(wa->root_dir); ds->root_path = str_dup(wa->root_dir);
ds->seed_node = wa->ps->root_filter_node; ds->seed_node = wa->ps->root_filter_node;
ds->excluded_mutex = &wa->ps->result_mutex;
Chunk* chunk; Chunk* chunk;
while ((chunk = directory_scanner_next(ds)) != NULL) { while ((chunk = directory_scanner_next(ds)) != NULL) {
if (!queue_enqueue_multithreaded_cancel(wa->ps->result_queue, chunk, &wa->ps->result_mutex, if (!queue_enqueue_multithreaded_cancel(wa->ps->result_queue, chunk, &wa->ps->result_mutex,
@@ -846,6 +968,11 @@ static int parallel_worker_thread(void* arg) {
cnd_broadcast(&wa->ps->result_not_empty); cnd_broadcast(&wa->ps->result_not_empty);
cnd_broadcast(&wa->ps->result_not_full); cnd_broadcast(&wa->ps->result_not_full);
mtx_unlock(&wa->ps->result_mutex); mtx_unlock(&wa->ps->result_mutex);
} else if (directory_scanner_had_io_error(ds)) {
/* --ignore-errors path: an unreadable directory was skipped, not fatal. */
mtx_lock(&wa->ps->result_mutex);
wa->ps->io_error = true;
mtx_unlock(&wa->ps->result_mutex);
} }
directory_scanner_destroy(ds); directory_scanner_destroy(ds);
free(wa->dirs[i]); free(wa->dirs[i]);
@@ -993,8 +1120,23 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
ps->failed = true; ps->failed = true;
return; return;
} }
if (inspection == 0) if (inspection == 0) {
if (inspected.excluded && options->excluded_paths) {
/* A root-level user-selection prune protects the destination mirror of
the same-named wire path (at the root the bare name is the wire path in
every layout). */
char* abs_path = path_cat(root_directory, entry->d_name);
if (!abs_path) {
ps->failed = true;
} else {
const char* rel = *abs_path == '/' ? abs_path + 1 : abs_path;
if (!excluded_sink_append(options->excluded_paths, options->excluded_mutex, rel))
ps->failed = true;
free(abs_path);
}
}
return; return;
}
char* cur_path = inspected.path; char* cur_path = inspected.path;
struct stat st = inspected.stats; struct stat st = inspected.stats;
bool is_dir = inspected.is_directory; bool is_dir = inspected.is_directory;
@@ -1009,6 +1151,14 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
/* -R + --files-from: root-level files keep their bare relative send path. */ /* -R + --files-from: root-level files keep their bare relative send path. */
bool use_rel = options->relative && options->file_list != NULL; bool use_rel = options->relative && options->file_list != NULL;
if (!passes) { if (!passes) {
/* --files-from subset pruning is not a filter exclusion; -R bare-wire-path
exclusions are never recorded (see ScannerOptions.excluded_paths). */
bool files_from_prune = options->file_list && !file_list_affects(options->file_list, rel);
if (!files_from_prune && !use_rel && options->excluded_paths) {
const char* rel_path = *cur_path == '/' ? cur_path + 1 : cur_path;
if (!excluded_sink_append(options->excluded_paths, options->excluded_mutex, rel_path))
ps->failed = true;
}
free(rel); free(rel);
free(cur_path); free(cur_path);
return; return;
@@ -1258,6 +1408,10 @@ bool parallel_scanner_failed(const ParallelScanner* ps) {
return ps == NULL || ps->failed; return ps == NULL || ps->failed;
} }
bool parallel_scanner_had_io_error(const ParallelScanner* ps) {
return ps != NULL && ps->io_error;
}
void parallel_scanner_destroy(ParallelScanner* ps) { void parallel_scanner_destroy(ParallelScanner* ps) {
if (!ps) if (!ps)
return; return;
+40
View File
@@ -37,6 +37,28 @@ typedef struct {
bool per_dir_filters; /* -F: read .rsync-filter per directory */ bool per_dir_filters; /* -F: read .rsync-filter per directory */
bool dirs; /* -d/--dirs: transfer dir entries, no recursion */ bool dirs; /* -d/--dirs: transfer dir entries, no recursion */
bool relative; /* -R/--relative (dest rel paths, with --files-from) */ bool relative; /* -R/--relative (dest rel paths, with --files-from) */
/* --prune-empty-dirs (long only): in --dirs mode an empty source directory's
explicit entry is omitted from the transfer file list (so nothing is
created at the destination and it can be pruned by --delete); explicitly
--files-from-listed directories always pass through. Recursive transfers
never emit empty directories, so the flag has no additional effect there. */
bool prune_empty_dirs;
/* Delete-excluded protection sink (optional): when non-NULL the scanner
* appends the destination-relative path of every entry it prunes because a
* USER SELECTION rule excluded it (--filter/-C/per-dir rules, the legacy
* --exclude/--include layer, and --max-size/--min-size). The sender turns
* this list into the manifest's protected prefixes so `--delete` leaves the
* destination mirror of excluded source paths alone (rsync's default), and
* empties it when --delete-excluded opts back into deleting them. NOT
* recorded for --files-from subset pruning (whose delete semantics stay
* keep-set-only) or for -R/--files-from relative wire paths. When
* `excluded_mutex` is non-NULL it is taken around every append (the parallel
* scanner shares one list across its worker threads). */
ArrayList* excluded_paths;
mtx_t* excluded_mutex;
/* --ignore-errors: an unreadable directory during the scan is recorded as an
* I/O error and skipped instead of aborting the scan. Client-only. */
bool ignore_io_errors;
} ScannerOptions; } ScannerOptions;
/* Internal per-scanner filter state. FilterNode chains represent the ordered /* Internal per-scanner filter state. FilterNode chains represent the ordered
@@ -79,10 +101,21 @@ typedef struct {
the recursive scan). */ the recursive scan). */
bool dirs_mode; bool dirs_mode;
bool relative_mode; /* file_list && relative: send bare relative wire paths */ bool relative_mode; /* file_list && relative: send bare relative wire paths */
bool prune_empty_dirs;
bool dirs_root_emitted; bool dirs_root_emitted;
int list_index; int list_index;
ArrayList* dirs_batch; /* owned when non-NULL */ ArrayList* dirs_batch; /* owned when non-NULL */
unsigned long long dirs_batch_size; unsigned long long dirs_batch_size;
/* Excluded-path sink (see ScannerOptions). `excluded_mutex` is shared across
parallel worker threads. */
ArrayList* excluded_paths;
mtx_t* excluded_mutex;
/* --ignore-errors: continue past unreadable directories (records io_error). */
bool ignore_io_errors;
/* A directory could not be opened (I/O error, e.g. EACCES). With
--ignore-errors the scan continues past it and the caller decides what to
do; `failed` is reserved for fatal errors that always abort the scan. */
bool io_error;
} DirectoryScanner; } DirectoryScanner;
typedef struct { typedef struct {
@@ -96,6 +129,8 @@ typedef struct {
thrd_t* threads; thrd_t* threads;
bool done; bool done;
bool failed; bool failed;
/* A worker skipped an unreadable directory under --ignore-errors (non-fatal). */
bool io_error;
atomic_bool cancelled; atomic_bool cancelled;
int completed; int completed;
Chunk* initial_chunk; Chunk* initial_chunk;
@@ -131,6 +166,11 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
ProtocolSession* allocation_session); ProtocolSession* allocation_session);
Chunk* parallel_scanner_next(ParallelScanner* scanner); Chunk* parallel_scanner_next(ParallelScanner* scanner);
bool parallel_scanner_failed(const ParallelScanner* scanner); bool parallel_scanner_failed(const ParallelScanner* scanner);
bool parallel_scanner_had_io_error(const ParallelScanner* scanner);
void parallel_scanner_destroy(ParallelScanner* scanner); void parallel_scanner_destroy(ParallelScanner* scanner);
/* True when a directory could not be opened during the scan (an I/O error,
recorded even when --ignore-errors keeps the scan going past it). */
bool directory_scanner_had_io_error(const DirectoryScanner* scanner);
#endif #endif