feat(filter): per-directory merge rules at the receiver; implement e/n/w/- modifiers
This commit is contained in:
@@ -330,9 +330,11 @@ int send_list_only(const Config* config) {
|
||||
}
|
||||
|
||||
/* Send the delete manifest to the server. Returns 0 on success, -1 on
|
||||
failure. It carries FOUR sections: the keep-set paths, the protected
|
||||
excluded prefixes, the --delete-missing-args exact-delete paths, and the
|
||||
destination-relative directories the sender synchronized this run.
|
||||
failure. It carries FOUR path sections (keep-set paths, protected excluded
|
||||
prefixes, --delete-missing-args exact-delete paths, and the destination-
|
||||
relative directories the sender synchronized this run) followed by the
|
||||
protocol-2.30.0 per-directory filter-rule block (`per_dir_rules`, the rules
|
||||
the scan compiled from each directory's merge files).
|
||||
When --delete-excluded is given `protected` is empty: excluded destination
|
||||
mirrors are then ordinary extras and are removed. When
|
||||
--delete-missing-args is active `missing_args` holds the destination mirrors
|
||||
@@ -346,7 +348,8 @@ int send_list_only(const Config* config) {
|
||||
frame. A heavily filtered source whose exclusion list is large therefore
|
||||
fails the run cleanly on the receiver rather than being truncated. */
|
||||
int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes,
|
||||
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs) {
|
||||
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs,
|
||||
const FilterRuleList* per_dir_rules) {
|
||||
if (!send_status(fd, STATUS_MANIFEST))
|
||||
return -1;
|
||||
int keep_count = manifest ? manifest->size : 0;
|
||||
@@ -389,6 +392,11 @@ int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefi
|
||||
if (!send_wire_str(fd, (char*)synced_dirs->items[i]))
|
||||
return -1;
|
||||
}
|
||||
/* Protocol 2.30.0: the receiver-side per-directory filter rules discovered by
|
||||
the sender's scan, so the whole-tree commit walker can shield a
|
||||
destination-only entry that matches only a per-directory merge rule. */
|
||||
if (!delete_filter_dir_rules_send(fd, per_dir_rules))
|
||||
return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -409,11 +417,11 @@ int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefi
|
||||
|
||||
bool send_delete_manifest_early(Client* client, ArrayList* manifest, ArrayList* protected_prefixes,
|
||||
ArrayList* size_skipped, ArrayList* missing_args,
|
||||
ArrayList* synced_dirs) {
|
||||
ArrayList* synced_dirs, const FilterRuleList* per_dir_rules) {
|
||||
if (!client || !manifest)
|
||||
return false;
|
||||
if (send_delete_manifest(client->file_descriptor, manifest, protected_prefixes, size_skipped,
|
||||
missing_args, synced_dirs) != 0)
|
||||
missing_args, synced_dirs, per_dir_rules) != 0)
|
||||
return false;
|
||||
Status ack;
|
||||
/* The wait is long (up to an hour) and runs inline on this thread: a helper
|
||||
@@ -497,6 +505,7 @@ int send_dry_run_remote(Config* config) {
|
||||
ArrayList* dry_dirs = NULL;
|
||||
ArrayList* dry_excluded = NULL;
|
||||
ArrayList* dry_size_skipped = NULL;
|
||||
FilterRuleList* dry_per_dir = NULL;
|
||||
if (!config_send(client->file_descriptor, config))
|
||||
goto dry_fail;
|
||||
receive_daemon_motd(client, config);
|
||||
@@ -509,8 +518,10 @@ int send_dry_run_remote(Config* config) {
|
||||
dry_manifest = array_list_create(free);
|
||||
dry_dirs = array_list_create(free);
|
||||
dry_size_skipped = array_list_create(free);
|
||||
if (!dry_manifest || !dry_dirs || !dry_size_skipped)
|
||||
dry_per_dir = filter_rule_list_create();
|
||||
if (!dry_manifest || !dry_dirs || !dry_size_skipped || !dry_per_dir)
|
||||
goto dry_fail;
|
||||
prepared.options.per_dir_rules = dry_per_dir;
|
||||
if (!config->delete_excluded) {
|
||||
dry_excluded = array_list_create(free);
|
||||
if (!dry_excluded)
|
||||
@@ -612,7 +623,7 @@ int send_dry_run_remote(Config* config) {
|
||||
bool early_delete = config->use_delete && config_delete_timing_early(config);
|
||||
if (dry_manifest) {
|
||||
if (send_delete_manifest(client->file_descriptor, dry_manifest, dry_excluded, dry_size_skipped,
|
||||
NULL, dry_dirs) != 0)
|
||||
NULL, dry_dirs, dry_per_dir) != 0)
|
||||
goto dry_fail;
|
||||
if (early_delete) {
|
||||
Status ack;
|
||||
@@ -672,6 +683,8 @@ dry_fail:
|
||||
array_list_delete(dry_excluded);
|
||||
if (dry_size_skipped)
|
||||
array_list_delete(dry_size_skipped);
|
||||
if (dry_per_dir)
|
||||
filter_rule_list_free(dry_per_dir);
|
||||
if (scanner)
|
||||
directory_scanner_destroy(scanner);
|
||||
prepared_scanner_destroy(&prepared);
|
||||
|
||||
+25
-11
@@ -1033,7 +1033,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
and wait for the receiver to delete extras before streaming any data. */
|
||||
if (!send_delete_manifest_early(client, context->manifest, context->excluded_paths,
|
||||
context->size_skipped_paths, context->missing_args,
|
||||
context->synced_dirs)) {
|
||||
context->synced_dirs, context->per_dir_rules)) {
|
||||
pipeline_cancel(context);
|
||||
disconnect_transfer_client(client);
|
||||
mark_sender_done(context);
|
||||
@@ -1163,7 +1163,8 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion");
|
||||
} else if (send_delete_manifest(client->file_descriptor, context->manifest,
|
||||
context->excluded_paths, context->size_skipped_paths,
|
||||
context->missing_args, context->synced_dirs) != 0) {
|
||||
context->missing_args, context->synced_dirs,
|
||||
context->per_dir_rules) != 0) {
|
||||
goto send_fail;
|
||||
}
|
||||
} else if (context->config->delete_missing_args && !context->early_delete &&
|
||||
@@ -1171,7 +1172,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
/* --delete-missing-args without --delete: no keep-set is built, but the
|
||||
exact-delete paths still ride the same manifest frame (commit once the
|
||||
transfer succeeded). */
|
||||
if (send_delete_manifest(client->file_descriptor, NULL, NULL, NULL, context->missing_args,
|
||||
if (send_delete_manifest(client->file_descriptor, NULL, NULL, NULL, context->missing_args, NULL,
|
||||
NULL) != 0)
|
||||
goto send_fail;
|
||||
}
|
||||
@@ -1285,6 +1286,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
||||
its protected lists, so the data pass must not append to them again. */
|
||||
if (!context->early_delete && !context->delete_plans) {
|
||||
prepared.options.excluded_paths = context->excluded_paths;
|
||||
prepared.options.per_dir_rules = context->per_dir_rules;
|
||||
/* The root marker for a full recursive transfer is already in the list; do
|
||||
not let the scanner append every directory to it. */
|
||||
if (context->config->files_from_set != NULL)
|
||||
@@ -1518,6 +1520,8 @@ typedef struct {
|
||||
ArrayList* synced_dirs;
|
||||
ArrayList* plan_dirs;
|
||||
ArrayList* missing_args;
|
||||
/* Per-directory filter rules compiled by the scan (protocol 2.30.0). */
|
||||
FilterRuleList* per_dir_rules;
|
||||
PreparedScanner prepared;
|
||||
StopCondition stop;
|
||||
TransferStats transfer_stats;
|
||||
@@ -1562,9 +1566,11 @@ static bool send_files_prepare(Config* config, SendFilesState* state) {
|
||||
}
|
||||
state->size_skipped = array_list_create(free);
|
||||
state->synced_dirs = array_list_create(free);
|
||||
if (!state->size_skipped || !state->synced_dirs)
|
||||
state->per_dir_rules = filter_rule_list_create();
|
||||
if (!state->size_skipped || !state->synced_dirs || !state->per_dir_rules)
|
||||
return false;
|
||||
state->prepared.options.size_skipped_paths = state->size_skipped;
|
||||
state->prepared.options.per_dir_rules = state->per_dir_rules;
|
||||
/* Only a --files-from subset confines the extras walk to the directories
|
||||
the scan synchronized; a full recursive transfer deletes throughout the
|
||||
receive root, so mark the root itself (the "." sentinel) and let the
|
||||
@@ -1629,9 +1635,9 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
|
||||
log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion");
|
||||
skip_delete = true;
|
||||
} else {
|
||||
early_ok =
|
||||
send_delete_manifest_early(client, early_manifest, state->excluded, state->size_skipped,
|
||||
state->missing_args, state->synced_dirs);
|
||||
early_ok = send_delete_manifest_early(client, early_manifest, state->excluded,
|
||||
state->size_skipped, state->missing_args,
|
||||
state->synced_dirs, state->per_dir_rules);
|
||||
}
|
||||
}
|
||||
array_list_delete(early_manifest);
|
||||
@@ -1640,6 +1646,7 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
|
||||
state->prepared.options.excluded_paths = NULL;
|
||||
state->prepared.options.size_skipped_paths = NULL;
|
||||
state->prepared.options.synced_dirs = NULL;
|
||||
state->prepared.options.per_dir_rules = NULL;
|
||||
if (!prescan_ok || (!early_ok && !skip_delete)) {
|
||||
array_list_delete(prescan_chunks);
|
||||
return false;
|
||||
@@ -1669,7 +1676,7 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
|
||||
config->files_from_set ? state->synced_dirs : (walk_root ? state->synced_dirs : NULL);
|
||||
delete_plan_sender_finalize(state->plan_sender, scope, walk_root);
|
||||
delete_plan_sender_set_config(state->plan_sender, state->excluded, state->size_skipped,
|
||||
state->missing_args);
|
||||
state->missing_args, state->per_dir_rules);
|
||||
if (state->had_scan_io && delete_plan_sender_empty(state->plan_sender)) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"source scan hit an I/O error before finding any file; refusing to delete "
|
||||
@@ -1693,6 +1700,7 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
|
||||
state->prepared.options.size_skipped_paths = NULL;
|
||||
state->prepared.options.synced_dirs = NULL;
|
||||
state->prepared.options.plan_dirs = NULL;
|
||||
state->prepared.options.per_dir_rules = NULL;
|
||||
if (!prescan_ok || (!plans_ok && !skip_delete))
|
||||
return false;
|
||||
} else if (config->use_delete) {
|
||||
@@ -1874,7 +1882,8 @@ static int send_files_finalize(const Config* config, SendFilesState* state) {
|
||||
modes the deletion already went out with the data, so nothing is
|
||||
re-sent here. */
|
||||
if (send_delete_manifest(client->file_descriptor, state->manifest, state->excluded,
|
||||
state->size_skipped, state->missing_args, state->synced_dirs) != 0) {
|
||||
state->size_skipped, state->missing_args, state->synced_dirs,
|
||||
state->per_dir_rules) != 0) {
|
||||
if (state->manifest) {
|
||||
array_list_delete(state->manifest);
|
||||
state->manifest = NULL;
|
||||
@@ -1949,6 +1958,8 @@ static void send_files_cleanup(SendFilesState* state) {
|
||||
array_list_delete(state->plan_dirs);
|
||||
if (state->missing_args)
|
||||
array_list_delete(state->missing_args);
|
||||
if (state->per_dir_rules)
|
||||
filter_rule_list_free(state->per_dir_rules);
|
||||
if (state->remove_sources)
|
||||
array_list_delete(state->remove_sources);
|
||||
if (state->dir_entries)
|
||||
@@ -2130,7 +2141,8 @@ static int send_files_multithreaded_impl(Config* config) {
|
||||
confined; only a --files-from subset records concrete directories. */
|
||||
context->size_skipped_paths = array_list_create(free);
|
||||
context->synced_dirs = array_list_create(free);
|
||||
if (!context->size_skipped_paths || !context->synced_dirs) {
|
||||
context->per_dir_rules = filter_rule_list_create();
|
||||
if (!context->size_skipped_paths || !context->synced_dirs || !context->per_dir_rules) {
|
||||
pipeline_context_sender_destroy(context);
|
||||
return 1;
|
||||
}
|
||||
@@ -2159,6 +2171,7 @@ static int send_files_multithreaded_impl(Config* config) {
|
||||
if (context->excluded_paths)
|
||||
prepared.options.excluded_paths = context->excluded_paths;
|
||||
prepared.options.size_skipped_paths = context->size_skipped_paths;
|
||||
prepared.options.per_dir_rules = context->per_dir_rules;
|
||||
/* The root marker for a full recursive transfer is already in the list;
|
||||
only a --files-from subset needs the scanner to record directories. */
|
||||
if (config->files_from_set != NULL)
|
||||
@@ -2199,7 +2212,8 @@ static int send_files_multithreaded_impl(Config* config) {
|
||||
: (walk_root ? context->synced_dirs : NULL);
|
||||
delete_plan_sender_finalize(context->delete_plans, scope, walk_root);
|
||||
delete_plan_sender_set_config(context->delete_plans, context->excluded_paths,
|
||||
context->size_skipped_paths, context->missing_args);
|
||||
context->size_skipped_paths, context->missing_args,
|
||||
context->per_dir_rules);
|
||||
}
|
||||
bool empty = per_dir
|
||||
? (context->delete_plans && delete_plan_sender_empty(context->delete_plans))
|
||||
|
||||
@@ -102,9 +102,10 @@ int send_dry_run_manifest(const Config* config);
|
||||
int send_list_only(const Config* config);
|
||||
int send_dry_run_remote(Config* config);
|
||||
int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes,
|
||||
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs);
|
||||
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs,
|
||||
const FilterRuleList* per_dir_rules);
|
||||
bool send_delete_manifest_early(Client* client, ArrayList* manifest, ArrayList* protected_prefixes,
|
||||
ArrayList* size_skipped, ArrayList* missing_args,
|
||||
ArrayList* synced_dirs);
|
||||
ArrayList* synced_dirs, const FilterRuleList* per_dir_rules);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -115,6 +115,13 @@ typedef struct {
|
||||
* directories, exactly like rsync; the receive root is the "." sentinel.
|
||||
* Guarded by `excluded_mutex`. */
|
||||
ArrayList* synced_dirs;
|
||||
/* Per-directory filter-rule sink (optional): when non-NULL the scanner appends
|
||||
* a deep copy of every rule it reads from a per-directory merge file, each
|
||||
* carrying its owner directory and no-inherit flag (see filter.h). The delete
|
||||
* carriers transmit them so the receiver re-derives the per-directory
|
||||
* protect/risk set for destination-only entries. Guarded by `excluded_mutex`
|
||||
* like the other sinks. */
|
||||
FilterRuleList* per_dir_rules;
|
||||
/* Delete-plan directory sink (optional): when non-NULL the scanner appends
|
||||
* the destination-relative path of every directory it traverses (except the
|
||||
* receive root). The per-directory --delete-during/--delete-delay plan
|
||||
|
||||
@@ -511,13 +511,31 @@ FilterRuleList* read_dir_filters(const ScannerOptions* options, const char* dir_
|
||||
}
|
||||
if (base) {
|
||||
for (int i = 0; i < base->dir_merge_count; i++) {
|
||||
if (!filter_file_append(own, dir_path, base->dir_merge_names[i], rel, &opts, &exists, err,
|
||||
err_size))
|
||||
if (!filter_dir_merge_append(own, dir_path, &base->dir_merges[i], rel, &opts, &exists, err,
|
||||
err_size))
|
||||
goto fail;
|
||||
if (exists && any_exists)
|
||||
*any_exists = true;
|
||||
}
|
||||
}
|
||||
/* Mirror the directory's rules into the delete-carrier sink so the receiver
|
||||
* can reconstruct its per-directory protect/risk set. */
|
||||
if (options->per_dir_rules && own->count > 0) {
|
||||
mtx_t* mtx = options->excluded_mutex;
|
||||
if (mtx)
|
||||
mtx_lock(mtx);
|
||||
for (int i = 0; i < own->count; i++) {
|
||||
FilterRule* copy = filter_rule_clone(own->items[i]);
|
||||
if (!copy || !filter_rule_list_add(options->per_dir_rules, copy)) {
|
||||
filter_rule_free(copy);
|
||||
if (mtx)
|
||||
mtx_unlock(mtx);
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
if (mtx)
|
||||
mtx_unlock(mtx);
|
||||
}
|
||||
return own;
|
||||
fail:
|
||||
filter_rule_list_free(own);
|
||||
|
||||
Reference in New Issue
Block a user