feat(filter): per-directory merge rules at the receiver; implement e/n/w/- modifiers

This commit is contained in:
2026-09-23 22:46:14 +02:00
parent 29f8be161c
commit b414f197af
22 changed files with 1154 additions and 192 deletions
+21 -8
View File
@@ -330,9 +330,11 @@ int send_list_only(const Config* config) {
}
/* Send the delete manifest to the server. Returns 0 on success, -1 on
failure. It carries FOUR sections: the keep-set paths, the protected
excluded prefixes, the --delete-missing-args exact-delete paths, and the
destination-relative directories the sender synchronized this run.
failure. It carries FOUR path sections (keep-set paths, protected excluded
prefixes, --delete-missing-args exact-delete paths, and the destination-
relative directories the sender synchronized this run) followed by the
protocol-2.30.0 per-directory filter-rule block (`per_dir_rules`, the rules
the scan compiled from each directory's merge files).
When --delete-excluded is given `protected` is empty: excluded destination
mirrors are then ordinary extras and are removed. When
--delete-missing-args is active `missing_args` holds the destination mirrors
@@ -346,7 +348,8 @@ int send_list_only(const Config* config) {
frame. A heavily filtered source whose exclusion list is large therefore
fails the run cleanly on the receiver rather than being truncated. */
int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes,
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs) {
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs,
const FilterRuleList* per_dir_rules) {
if (!send_status(fd, STATUS_MANIFEST))
return -1;
int keep_count = manifest ? manifest->size : 0;
@@ -389,6 +392,11 @@ int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefi
if (!send_wire_str(fd, (char*)synced_dirs->items[i]))
return -1;
}
/* Protocol 2.30.0: the receiver-side per-directory filter rules discovered by
the sender's scan, so the whole-tree commit walker can shield a
destination-only entry that matches only a per-directory merge rule. */
if (!delete_filter_dir_rules_send(fd, per_dir_rules))
return -1;
return 0;
}
@@ -409,11 +417,11 @@ int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefi
bool send_delete_manifest_early(Client* client, ArrayList* manifest, ArrayList* protected_prefixes,
ArrayList* size_skipped, ArrayList* missing_args,
ArrayList* synced_dirs) {
ArrayList* synced_dirs, const FilterRuleList* per_dir_rules) {
if (!client || !manifest)
return false;
if (send_delete_manifest(client->file_descriptor, manifest, protected_prefixes, size_skipped,
missing_args, synced_dirs) != 0)
missing_args, synced_dirs, per_dir_rules) != 0)
return false;
Status ack;
/* The wait is long (up to an hour) and runs inline on this thread: a helper
@@ -497,6 +505,7 @@ int send_dry_run_remote(Config* config) {
ArrayList* dry_dirs = NULL;
ArrayList* dry_excluded = NULL;
ArrayList* dry_size_skipped = NULL;
FilterRuleList* dry_per_dir = NULL;
if (!config_send(client->file_descriptor, config))
goto dry_fail;
receive_daemon_motd(client, config);
@@ -509,8 +518,10 @@ int send_dry_run_remote(Config* config) {
dry_manifest = array_list_create(free);
dry_dirs = array_list_create(free);
dry_size_skipped = array_list_create(free);
if (!dry_manifest || !dry_dirs || !dry_size_skipped)
dry_per_dir = filter_rule_list_create();
if (!dry_manifest || !dry_dirs || !dry_size_skipped || !dry_per_dir)
goto dry_fail;
prepared.options.per_dir_rules = dry_per_dir;
if (!config->delete_excluded) {
dry_excluded = array_list_create(free);
if (!dry_excluded)
@@ -612,7 +623,7 @@ int send_dry_run_remote(Config* config) {
bool early_delete = config->use_delete && config_delete_timing_early(config);
if (dry_manifest) {
if (send_delete_manifest(client->file_descriptor, dry_manifest, dry_excluded, dry_size_skipped,
NULL, dry_dirs) != 0)
NULL, dry_dirs, dry_per_dir) != 0)
goto dry_fail;
if (early_delete) {
Status ack;
@@ -672,6 +683,8 @@ dry_fail:
array_list_delete(dry_excluded);
if (dry_size_skipped)
array_list_delete(dry_size_skipped);
if (dry_per_dir)
filter_rule_list_free(dry_per_dir);
if (scanner)
directory_scanner_destroy(scanner);
prepared_scanner_destroy(&prepared);
+25 -11
View File
@@ -1033,7 +1033,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
and wait for the receiver to delete extras before streaming any data. */
if (!send_delete_manifest_early(client, context->manifest, context->excluded_paths,
context->size_skipped_paths, context->missing_args,
context->synced_dirs)) {
context->synced_dirs, context->per_dir_rules)) {
pipeline_cancel(context);
disconnect_transfer_client(client);
mark_sender_done(context);
@@ -1163,7 +1163,8 @@ static int send_chunks_multithreaded(void* pipeline_context) {
log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion");
} else if (send_delete_manifest(client->file_descriptor, context->manifest,
context->excluded_paths, context->size_skipped_paths,
context->missing_args, context->synced_dirs) != 0) {
context->missing_args, context->synced_dirs,
context->per_dir_rules) != 0) {
goto send_fail;
}
} else if (context->config->delete_missing_args && !context->early_delete &&
@@ -1171,7 +1172,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
/* --delete-missing-args without --delete: no keep-set is built, but the
exact-delete paths still ride the same manifest frame (commit once the
transfer succeeded). */
if (send_delete_manifest(client->file_descriptor, NULL, NULL, NULL, context->missing_args,
if (send_delete_manifest(client->file_descriptor, NULL, NULL, NULL, context->missing_args, NULL,
NULL) != 0)
goto send_fail;
}
@@ -1285,6 +1286,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
its protected lists, so the data pass must not append to them again. */
if (!context->early_delete && !context->delete_plans) {
prepared.options.excluded_paths = context->excluded_paths;
prepared.options.per_dir_rules = context->per_dir_rules;
/* The root marker for a full recursive transfer is already in the list; do
not let the scanner append every directory to it. */
if (context->config->files_from_set != NULL)
@@ -1518,6 +1520,8 @@ typedef struct {
ArrayList* synced_dirs;
ArrayList* plan_dirs;
ArrayList* missing_args;
/* Per-directory filter rules compiled by the scan (protocol 2.30.0). */
FilterRuleList* per_dir_rules;
PreparedScanner prepared;
StopCondition stop;
TransferStats transfer_stats;
@@ -1562,9 +1566,11 @@ static bool send_files_prepare(Config* config, SendFilesState* state) {
}
state->size_skipped = array_list_create(free);
state->synced_dirs = array_list_create(free);
if (!state->size_skipped || !state->synced_dirs)
state->per_dir_rules = filter_rule_list_create();
if (!state->size_skipped || !state->synced_dirs || !state->per_dir_rules)
return false;
state->prepared.options.size_skipped_paths = state->size_skipped;
state->prepared.options.per_dir_rules = state->per_dir_rules;
/* Only a --files-from subset confines the extras walk to the directories
the scan synchronized; a full recursive transfer deletes throughout the
receive root, so mark the root itself (the "." sentinel) and let the
@@ -1629,9 +1635,9 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion");
skip_delete = true;
} else {
early_ok =
send_delete_manifest_early(client, early_manifest, state->excluded, state->size_skipped,
state->missing_args, state->synced_dirs);
early_ok = send_delete_manifest_early(client, early_manifest, state->excluded,
state->size_skipped, state->missing_args,
state->synced_dirs, state->per_dir_rules);
}
}
array_list_delete(early_manifest);
@@ -1640,6 +1646,7 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
state->prepared.options.excluded_paths = NULL;
state->prepared.options.size_skipped_paths = NULL;
state->prepared.options.synced_dirs = NULL;
state->prepared.options.per_dir_rules = NULL;
if (!prescan_ok || (!early_ok && !skip_delete)) {
array_list_delete(prescan_chunks);
return false;
@@ -1669,7 +1676,7 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
config->files_from_set ? state->synced_dirs : (walk_root ? state->synced_dirs : NULL);
delete_plan_sender_finalize(state->plan_sender, scope, walk_root);
delete_plan_sender_set_config(state->plan_sender, state->excluded, state->size_skipped,
state->missing_args);
state->missing_args, state->per_dir_rules);
if (state->had_scan_io && delete_plan_sender_empty(state->plan_sender)) {
log_message(LOG_LEVEL_ERROR,
"source scan hit an I/O error before finding any file; refusing to delete "
@@ -1693,6 +1700,7 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
state->prepared.options.size_skipped_paths = NULL;
state->prepared.options.synced_dirs = NULL;
state->prepared.options.plan_dirs = NULL;
state->prepared.options.per_dir_rules = NULL;
if (!prescan_ok || (!plans_ok && !skip_delete))
return false;
} else if (config->use_delete) {
@@ -1874,7 +1882,8 @@ static int send_files_finalize(const Config* config, SendFilesState* state) {
modes the deletion already went out with the data, so nothing is
re-sent here. */
if (send_delete_manifest(client->file_descriptor, state->manifest, state->excluded,
state->size_skipped, state->missing_args, state->synced_dirs) != 0) {
state->size_skipped, state->missing_args, state->synced_dirs,
state->per_dir_rules) != 0) {
if (state->manifest) {
array_list_delete(state->manifest);
state->manifest = NULL;
@@ -1949,6 +1958,8 @@ static void send_files_cleanup(SendFilesState* state) {
array_list_delete(state->plan_dirs);
if (state->missing_args)
array_list_delete(state->missing_args);
if (state->per_dir_rules)
filter_rule_list_free(state->per_dir_rules);
if (state->remove_sources)
array_list_delete(state->remove_sources);
if (state->dir_entries)
@@ -2130,7 +2141,8 @@ static int send_files_multithreaded_impl(Config* config) {
confined; only a --files-from subset records concrete directories. */
context->size_skipped_paths = array_list_create(free);
context->synced_dirs = array_list_create(free);
if (!context->size_skipped_paths || !context->synced_dirs) {
context->per_dir_rules = filter_rule_list_create();
if (!context->size_skipped_paths || !context->synced_dirs || !context->per_dir_rules) {
pipeline_context_sender_destroy(context);
return 1;
}
@@ -2159,6 +2171,7 @@ static int send_files_multithreaded_impl(Config* config) {
if (context->excluded_paths)
prepared.options.excluded_paths = context->excluded_paths;
prepared.options.size_skipped_paths = context->size_skipped_paths;
prepared.options.per_dir_rules = context->per_dir_rules;
/* The root marker for a full recursive transfer is already in the list;
only a --files-from subset needs the scanner to record directories. */
if (config->files_from_set != NULL)
@@ -2199,7 +2212,8 @@ static int send_files_multithreaded_impl(Config* config) {
: (walk_root ? context->synced_dirs : NULL);
delete_plan_sender_finalize(context->delete_plans, scope, walk_root);
delete_plan_sender_set_config(context->delete_plans, context->excluded_paths,
context->size_skipped_paths, context->missing_args);
context->size_skipped_paths, context->missing_args,
context->per_dir_rules);
}
bool empty = per_dir
? (context->delete_plans && delete_plan_sender_empty(context->delete_plans))
+3 -2
View File
@@ -102,9 +102,10 @@ int send_dry_run_manifest(const Config* config);
int send_list_only(const Config* config);
int send_dry_run_remote(Config* config);
int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes,
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs);
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs,
const FilterRuleList* per_dir_rules);
bool send_delete_manifest_early(Client* client, ArrayList* manifest, ArrayList* protected_prefixes,
ArrayList* size_skipped, ArrayList* missing_args,
ArrayList* synced_dirs);
ArrayList* synced_dirs, const FilterRuleList* per_dir_rules);
#endif
+7
View File
@@ -115,6 +115,13 @@ typedef struct {
* directories, exactly like rsync; the receive root is the "." sentinel.
* Guarded by `excluded_mutex`. */
ArrayList* synced_dirs;
/* Per-directory filter-rule sink (optional): when non-NULL the scanner appends
* a deep copy of every rule it reads from a per-directory merge file, each
* carrying its owner directory and no-inherit flag (see filter.h). The delete
* carriers transmit them so the receiver re-derives the per-directory
* protect/risk set for destination-only entries. Guarded by `excluded_mutex`
* like the other sinks. */
FilterRuleList* per_dir_rules;
/* Delete-plan directory sink (optional): when non-NULL the scanner appends
* the destination-relative path of every directory it traverses (except the
* receive root). The per-directory --delete-during/--delete-delay plan
+20 -2
View File
@@ -511,13 +511,31 @@ FilterRuleList* read_dir_filters(const ScannerOptions* options, const char* dir_
}
if (base) {
for (int i = 0; i < base->dir_merge_count; i++) {
if (!filter_file_append(own, dir_path, base->dir_merge_names[i], rel, &opts, &exists, err,
err_size))
if (!filter_dir_merge_append(own, dir_path, &base->dir_merges[i], rel, &opts, &exists, err,
err_size))
goto fail;
if (exists && any_exists)
*any_exists = true;
}
}
/* Mirror the directory's rules into the delete-carrier sink so the receiver
* can reconstruct its per-directory protect/risk set. */
if (options->per_dir_rules && own->count > 0) {
mtx_t* mtx = options->excluded_mutex;
if (mtx)
mtx_lock(mtx);
for (int i = 0; i < own->count; i++) {
FilterRule* copy = filter_rule_clone(own->items[i]);
if (!copy || !filter_rule_list_add(options->per_dir_rules, copy)) {
filter_rule_free(copy);
if (mtx)
mtx_unlock(mtx);
goto fail;
}
}
if (mtx)
mtx_unlock(mtx);
}
return own;
fail:
filter_rule_list_free(own);