rsync 3.4.1 drop-in parity (#285-#297) + parity completion (protocol 2.26.0) #298
@@ -89,6 +89,7 @@ set(SHARED_SRCS
|
||||
src/shared/daemon_limits.c
|
||||
src/shared/data.c
|
||||
src/shared/delay_updates.c
|
||||
src/shared/delete_plan.c
|
||||
src/shared/delta.c
|
||||
src/shared/file.c
|
||||
src/shared/file_list.c
|
||||
|
||||
+153
-31
@@ -7,6 +7,7 @@
|
||||
#include "compression.h"
|
||||
#include "config.h"
|
||||
#include "data.h"
|
||||
#include "delete_plan.h"
|
||||
#include "delta.h"
|
||||
#include "file.h"
|
||||
#include "file_list.h"
|
||||
@@ -1250,7 +1251,7 @@ static bool send_delete_manifest_early(Client* client, ArrayList* manifest,
|
||||
directory and *io_error_out reports it (the caller still performs the
|
||||
deletion but reports the run as errored). */
|
||||
static bool scan_paths_only(const Config* config, const ScannerOptions* options,
|
||||
ArrayList* manifest, bool* io_error_out) {
|
||||
ArrayList* manifest, DeletePlanSender* plans, bool* io_error_out) {
|
||||
if (io_error_out)
|
||||
*io_error_out = false;
|
||||
DirectoryScanner* scanner =
|
||||
@@ -1260,11 +1261,27 @@ static bool scan_paths_only(const Config* config, const ScannerOptions* options,
|
||||
bool ok = true;
|
||||
Chunk* chunk;
|
||||
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||
if (!add_chunk_to_manifest(manifest, chunk)) {
|
||||
if (manifest && !add_chunk_to_manifest(manifest, chunk)) {
|
||||
ok = false;
|
||||
chunk_destroy(chunk);
|
||||
break;
|
||||
}
|
||||
if (plans) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
File* f = chunk->items[i];
|
||||
if (!f)
|
||||
continue;
|
||||
const char* path = file_wire_path(f);
|
||||
if (!delete_plan_sender_add(plans, path, f->is_dir)) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!ok) {
|
||||
chunk_destroy(chunk);
|
||||
break;
|
||||
}
|
||||
}
|
||||
chunk_destroy(chunk);
|
||||
}
|
||||
if (ok && directory_scanner_failed(scanner))
|
||||
@@ -1275,6 +1292,25 @@ static bool scan_paths_only(const Config* config, const ScannerOptions* options,
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Transmit any not-yet-sent per-directory delete plan needed by the entries in
|
||||
* `chunk` (ancestors root-first, then the entry's own directory for --dirs
|
||||
* entries) before its data frames go out, so --delete-during/--delete-delay
|
||||
* clear a directory's extras (and any type conflict) before the directory's
|
||||
* first write. */
|
||||
static int send_chunk_delete_plans(Client* client, DeletePlanSender* plans, const Chunk* chunk) {
|
||||
if (!plans)
|
||||
return 0;
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
File* f = chunk->items[i];
|
||||
if (!f)
|
||||
continue;
|
||||
if (delete_plan_send_for_path(client->file_descriptor, plans, file_wire_path(f), f->is_dir) !=
|
||||
0)
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int incremental_check(Client* client, File* file, const Config* config,
|
||||
DeltaSignature** out_sig, unsigned long long* resume_offset) {
|
||||
*out_sig = NULL;
|
||||
@@ -2048,6 +2084,16 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
protocol_session_unbind();
|
||||
return thrd_error;
|
||||
}
|
||||
} else if (context->delete_plans) {
|
||||
/* --delete-during/--delete-delay: transmit the receive root's plan before
|
||||
any data, exactly like rsync's first generator directory. */
|
||||
if (delete_plan_send_root(client->file_descriptor, context->delete_plans) != 0) {
|
||||
pipeline_cancel(context);
|
||||
disconnect_transfer_client(client);
|
||||
mark_sender_done(context);
|
||||
protocol_session_unbind();
|
||||
return thrd_error;
|
||||
}
|
||||
}
|
||||
|
||||
while (true) {
|
||||
@@ -2087,6 +2133,15 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
}
|
||||
break;
|
||||
}
|
||||
if (send_chunk_delete_plans(client, context->delete_plans, current_chunk) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "unexpected error while sending delete plan");
|
||||
chunk_destroy(current_chunk);
|
||||
pipeline_cancel(context);
|
||||
disconnect_transfer_client(client);
|
||||
mark_sender_done(context);
|
||||
protocol_session_unbind();
|
||||
return thrd_error;
|
||||
}
|
||||
if (send_chunk_with_removal(client, current_chunk, context->config,
|
||||
context->remove_source_files) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "unexpected error while sending chunk");
|
||||
@@ -2134,7 +2189,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
"unscanned source mirrors are not deleted");
|
||||
else
|
||||
log_message(LOG_LEVEL_WARNING, "transfer stopped early (stop deadline)");
|
||||
} else if (context->config->use_delete && !context->early_delete) {
|
||||
} else if (context->config->use_delete && !context->early_delete && !context->delete_plans) {
|
||||
/* Empty keep-set + scan I/O error must not delete the whole destination
|
||||
(the source may not be genuinely empty -- see send_files). */
|
||||
bool empty_io;
|
||||
@@ -2151,7 +2206,8 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
context->size_skipped_paths, context->missing_args,
|
||||
context->synced_dirs) != 0)
|
||||
goto send_fail;
|
||||
} else if (context->config->delete_missing_args && !context->early_delete) {
|
||||
} else if (context->config->delete_missing_args && !context->early_delete &&
|
||||
!context->delete_plans) {
|
||||
/* --delete-missing-args without --delete: no keep-set is built, but the
|
||||
exact-delete paths still ride the same manifest frame (commit once the
|
||||
transfer succeeded). */
|
||||
@@ -2223,14 +2279,15 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
||||
synchronized directories here (the size-prune protection is collected in
|
||||
every mode). The early modes already transmitted the pre-scan keep-set and
|
||||
its protected lists, so the data pass must not append to them again. */
|
||||
if (!context->early_delete) {
|
||||
if (!context->early_delete && !context->delete_plans) {
|
||||
prepared.options.excluded_paths = context->excluded_paths;
|
||||
/* The root marker for a full recursive transfer is already in the list; do
|
||||
not let the scanner append every directory to it. */
|
||||
if (context->config->files_from_set != NULL)
|
||||
prepared.options.synced_dirs = context->synced_dirs;
|
||||
}
|
||||
prepared.options.size_skipped_paths = context->size_skipped_paths;
|
||||
if (!context->delete_plans)
|
||||
prepared.options.size_skipped_paths = context->size_skipped_paths;
|
||||
bool dirs_mode = prepared.options.dirs;
|
||||
/* -H also selects the sequential scanner (see the comment at the branch),
|
||||
* so the loop below must choose the scanner by which object exists, not by
|
||||
@@ -2268,7 +2325,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
||||
failed = use_dscanner ? directory_scanner_failed(dscanner) : parallel_scanner_failed(scanner);
|
||||
break;
|
||||
}
|
||||
if (context->config->use_delete && !context->early_delete) {
|
||||
if (context->config->use_delete && !context->early_delete && !context->delete_plans) {
|
||||
mtx_lock(&context->mutex_scanner);
|
||||
bool manifest_ok = add_chunk_to_manifest(context->manifest, current_chunk);
|
||||
mtx_unlock(&context->mutex_scanner);
|
||||
@@ -2531,6 +2588,7 @@ int send_files(Config* config) {
|
||||
int ret = 1;
|
||||
DirectoryScanner* scanner = NULL;
|
||||
ArrayList* manifest = NULL;
|
||||
DeletePlanSender* plan_sender = NULL;
|
||||
ArrayList* remove_sources = NULL;
|
||||
/* P7 Wave D: captured source directory times, transmitted in trailing
|
||||
STATUS_DIR_TIMES frame(s) (only when metadata rides the wire). */
|
||||
@@ -2541,6 +2599,10 @@ int send_files(Config* config) {
|
||||
ArrayList* size_skipped = NULL;
|
||||
ArrayList* synced_dirs = NULL;
|
||||
bool delete_early = config->use_delete && config_delete_timing_early(config);
|
||||
/* -d/--dirs does not recurse, so a per-directory plan would carry no child
|
||||
information and could delete the contents of an untraversed directory;
|
||||
fall back to the whole-tree end-of-transfer commit for that mode. */
|
||||
bool delete_per_dir = config->use_delete && config_delete_timing_per_dir(config) && !config->dirs;
|
||||
bool send_failed = false;
|
||||
bool had_scan_io = false;
|
||||
PreparedScanner prepared;
|
||||
@@ -2591,10 +2653,11 @@ int send_files(Config* config) {
|
||||
prepared.options.synced_dirs = synced_dirs;
|
||||
}
|
||||
}
|
||||
/* The late-timing modes (plain --delete / --delete-after / --delete-delay)
|
||||
build the manifest while streaming and send it after the last data frame.
|
||||
The early modes (--delete-before/--delete-during) send it up front from a
|
||||
dedicated path-only pre-scan, so no manifest is kept during the data pass. */
|
||||
/* The late-timing modes (plain --delete / --delete-after) build the manifest
|
||||
while streaming and send it after the last data frame. --delete-before
|
||||
sends a whole-tree keep-set up front; --delete-during/--delete-delay build a
|
||||
per-directory plan set up front (paths only) and stream the plans alongside
|
||||
the data, so no manifest is kept during the data pass. */
|
||||
if (delete_early) {
|
||||
/* Pass 1: collect the complete keep-set (paths only, no data loaded) and
|
||||
transmit it now, before any file data. The receiver removes extras and
|
||||
@@ -2602,7 +2665,8 @@ int send_files(Config* config) {
|
||||
ArrayList* early_manifest = array_list_create(free);
|
||||
if (!early_manifest)
|
||||
goto send_fail;
|
||||
bool prescan_ok = scan_paths_only(config, &prepared.options, early_manifest, &had_scan_io);
|
||||
bool prescan_ok =
|
||||
scan_paths_only(config, &prepared.options, early_manifest, NULL, &had_scan_io);
|
||||
bool early_ok = false;
|
||||
if (prescan_ok) {
|
||||
/* A scan that hit an I/O error and produced NO keep entries is ambiguous
|
||||
@@ -2628,6 +2692,33 @@ int send_files(Config* config) {
|
||||
prepared.options.synced_dirs = NULL;
|
||||
if (!prescan_ok || !early_ok)
|
||||
goto send_fail;
|
||||
} else if (delete_per_dir) {
|
||||
/* --delete-during/--delete-delay: build one plan per source directory from a
|
||||
path-only pre-scan and transmit the root plan now, before any data, so the
|
||||
receive root's extras are handled exactly like rsync's first generator
|
||||
directory. The remaining plans are streamed with the data below. */
|
||||
plan_sender = delete_plan_sender_create();
|
||||
if (!plan_sender)
|
||||
goto send_fail;
|
||||
bool prescan_ok = scan_paths_only(config, &prepared.options, NULL, plan_sender, &had_scan_io);
|
||||
bool plans_ok = false;
|
||||
if (prescan_ok) {
|
||||
delete_plan_sender_finalize(plan_sender, config->files_from_set ? synced_dirs : NULL);
|
||||
delete_plan_sender_set_config(plan_sender, excluded, size_skipped, missing_args);
|
||||
if (had_scan_io && delete_plan_sender_empty(plan_sender)) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"source scan hit an I/O error before finding any file; refusing to delete "
|
||||
"with an empty keep-set (--delete)");
|
||||
prescan_ok = false;
|
||||
} else {
|
||||
plans_ok = delete_plan_send_root(client->file_descriptor, plan_sender) == 0;
|
||||
}
|
||||
}
|
||||
prepared.options.excluded_paths = NULL;
|
||||
prepared.options.size_skipped_paths = NULL;
|
||||
prepared.options.synced_dirs = NULL;
|
||||
if (!prescan_ok || !plans_ok)
|
||||
goto send_fail;
|
||||
} else if (config->use_delete) {
|
||||
manifest = array_list_create(free);
|
||||
if (!manifest)
|
||||
@@ -2706,6 +2797,11 @@ int send_files(Config* config) {
|
||||
goto send_fail;
|
||||
}
|
||||
}
|
||||
if (send_chunk_delete_plans(client, plan_sender, current_chunk) != 0) {
|
||||
chunk_destroy(current_chunk);
|
||||
send_failed = true;
|
||||
break;
|
||||
}
|
||||
if (send_chunk_with_removal(client, current_chunk, config, remove_sources) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
|
||||
chunk_destroy(current_chunk);
|
||||
@@ -2763,12 +2859,13 @@ int send_files(Config* config) {
|
||||
"an empty keep-set (--delete)");
|
||||
goto send_fail;
|
||||
}
|
||||
if ((manifest || config->delete_missing_args) && !delete_early) {
|
||||
if ((manifest || config->delete_missing_args) && !delete_early && !delete_per_dir) {
|
||||
/* Late (commit) ordering: all file data is out; transmit the manifest so
|
||||
the receiver commits the extras walk (--delete) and/or the
|
||||
--delete-missing-args exact-path deletions only after the transfer
|
||||
succeeds. In the early modes (--delete-before/--delete-during) the
|
||||
manifest already went out up front, so nothing is re-sent here. */
|
||||
succeeds. In the early modes (--delete-before) and the per-directory
|
||||
modes the deletion already went out with the data, so nothing is
|
||||
re-sent here. */
|
||||
if (send_delete_manifest(client->file_descriptor, manifest, excluded, size_skipped,
|
||||
missing_args, synced_dirs) != 0) {
|
||||
if (manifest) {
|
||||
@@ -2817,6 +2914,8 @@ send_fail:
|
||||
here even on success without --delete, fixing a pre-existing leak. */
|
||||
if (manifest)
|
||||
array_list_delete(manifest);
|
||||
if (plan_sender)
|
||||
delete_plan_sender_destroy(plan_sender);
|
||||
if (excluded)
|
||||
array_list_delete(excluded);
|
||||
if (size_skipped)
|
||||
@@ -2916,11 +3015,6 @@ int send_files_multithreaded(Config** config_ptr) {
|
||||
config->stop_at, now_mono);
|
||||
bool collect_excluded = config->use_delete && !config->delete_excluded;
|
||||
if (config->use_delete) {
|
||||
context->manifest = array_list_create(free);
|
||||
if (!context->manifest) {
|
||||
pipeline_context_sender_destroy(context);
|
||||
return 1;
|
||||
}
|
||||
if (collect_excluded) {
|
||||
context->excluded_paths = array_list_create(free);
|
||||
if (!context->excluded_paths) {
|
||||
@@ -2946,11 +3040,15 @@ int send_files_multithreaded(Config** config_ptr) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
if (config_delete_timing_early(config)) {
|
||||
/* --delete-before/--delete-during: build the complete keep-set manifest
|
||||
/* -d/--dirs does not recurse, so a per-directory plan would carry no child
|
||||
information and could delete the contents of an untraversed directory;
|
||||
fall back to the whole-tree end-of-transfer commit for that mode. */
|
||||
bool per_dir = config_delete_timing_per_dir(config) && !config->dirs;
|
||||
if (config_delete_timing_early(config) || per_dir) {
|
||||
/* --delete-before / --delete-during / --delete-delay: build the keep-set
|
||||
(paths only, nothing loaded or sent) up front so the sender thread can
|
||||
transmit it before the first data byte. The path-only pre-scan also
|
||||
fills the protected excluded prefixes and synchronized directories. */
|
||||
transmit it before/with the data. The path-only pre-scan also fills the
|
||||
protected excluded prefixes and synchronized directories. */
|
||||
PreparedScanner prepared;
|
||||
memset(&prepared, 0, sizeof(prepared));
|
||||
bool prepared_ok = prepare_scanner(config, config->scanner_threads, &prepared);
|
||||
@@ -2963,12 +3061,29 @@ int send_files_multithreaded(Config** config_ptr) {
|
||||
if (config->files_from_set != NULL)
|
||||
prepared.options.synced_dirs = context->synced_dirs;
|
||||
}
|
||||
bool prebuilt = prepared_ok && scan_paths_only(config, &prepared.options, context->manifest,
|
||||
&context->scan_had_io_error);
|
||||
if (per_dir) {
|
||||
context->delete_plans = delete_plan_sender_create();
|
||||
prepared_ok = prepared_ok && context->delete_plans != NULL;
|
||||
} else {
|
||||
context->manifest = array_list_create(free);
|
||||
prepared_ok = prepared_ok && context->manifest != NULL;
|
||||
}
|
||||
bool prebuilt =
|
||||
prepared_ok && scan_paths_only(config, &prepared.options, context->manifest,
|
||||
context->delete_plans, &context->scan_had_io_error);
|
||||
prepared_scanner_destroy(&prepared);
|
||||
if (prebuilt && context->scan_had_io_error && context->manifest->size == 0) {
|
||||
/* Empty keep-set + scan I/O error: refusing an empty keep-set manifest
|
||||
would have deleted the whole destination (see send_files). */
|
||||
if (per_dir && prebuilt) {
|
||||
delete_plan_sender_finalize(context->delete_plans,
|
||||
config->files_from_set ? context->synced_dirs : NULL);
|
||||
delete_plan_sender_set_config(context->delete_plans, context->excluded_paths,
|
||||
context->size_skipped_paths, context->missing_args);
|
||||
}
|
||||
bool empty = per_dir
|
||||
? (context->delete_plans && delete_plan_sender_empty(context->delete_plans))
|
||||
: (context->manifest && context->manifest->size == 0);
|
||||
if (prebuilt && context->scan_had_io_error && empty) {
|
||||
/* Empty keep-set + scan I/O error: refusing an empty keep-set would
|
||||
have deleted the whole destination (see send_files). */
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"source scan hit an I/O error before finding any file; refusing to delete "
|
||||
"with an empty keep-set (--delete)");
|
||||
@@ -2978,12 +3093,19 @@ int send_files_multithreaded(Config** config_ptr) {
|
||||
pipeline_context_sender_destroy(context);
|
||||
return 1;
|
||||
}
|
||||
context->early_delete = true;
|
||||
if (!per_dir)
|
||||
context->early_delete = true;
|
||||
} else {
|
||||
context->manifest = array_list_create(free);
|
||||
if (!context->manifest) {
|
||||
pipeline_context_sender_destroy(context);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (config->remove_source_files)
|
||||
context->remove_source_files = array_list_create(source_file_destroy);
|
||||
if ((config->use_delete && !context->manifest) ||
|
||||
if ((config->use_delete && !context->manifest && !context->delete_plans) ||
|
||||
(config->remove_source_files && !context->remove_source_files)) {
|
||||
pipeline_context_sender_destroy(context);
|
||||
return 1;
|
||||
|
||||
+4
-4
@@ -66,11 +66,11 @@ void print_usage(void) {
|
||||
printf(" transfer has succeeded)\n");
|
||||
printf(" --delete-before Delete extras before the transfer starts\n");
|
||||
printf(" (implies --delete)\n");
|
||||
printf(" --delete-during Delete extras once the keep-set manifest is known,\n");
|
||||
printf(" before the data is applied (implies --delete)\n");
|
||||
printf(" --delete-during Delete a directory's extras as that directory is\n");
|
||||
printf(" processed (implies --delete)\n");
|
||||
printf(" --del Alias for --delete-during\n");
|
||||
printf(" --delete-delay Delete extras only after a successful transfer\n");
|
||||
printf(" (implies --delete)\n");
|
||||
printf(" --delete-delay Record the extras during the scan but remove them\n");
|
||||
printf(" only after a successful transfer (implies --delete)\n");
|
||||
printf(" --delete-after Delete only after the whole transfer succeeded\n");
|
||||
printf(" (the default --delete timing; implies --delete)\n");
|
||||
printf(" --delete-excluded Also delete destination files that were excluded on\n");
|
||||
|
||||
+64
-12
@@ -3,6 +3,7 @@
|
||||
#include "charset.h"
|
||||
#include "chunk.h"
|
||||
#include "config.h"
|
||||
#include "delete_plan.h"
|
||||
#include "delay_updates.h"
|
||||
#include "file.h"
|
||||
#include "file_receive.h"
|
||||
@@ -244,7 +245,7 @@ static bool receiver_note_status(const struct timespec* session_start,
|
||||
}
|
||||
|
||||
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink) {
|
||||
return receiver_process_pending(config, file_descriptor, sink, NULL);
|
||||
return receiver_process_pending(config, file_descriptor, sink, NULL, NULL);
|
||||
}
|
||||
|
||||
/* Runs the whole receive loop. The delete manifest may legitimately arrive
|
||||
@@ -258,7 +259,7 @@ int receiver_process(Config* config, int file_descriptor, const ReceiverSink* si
|
||||
the whole transfer succeeded. See receiver_process_pending() for how the -m
|
||||
receiver defers that commit until its disk writer has drained. */
|
||||
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
||||
DeleteManifest** pending_manifest) {
|
||||
DeleteManifest** pending_manifest, DeletePlanSession** pending_plans) {
|
||||
Status status;
|
||||
if (!receive_status(file_descriptor, &status))
|
||||
return -1;
|
||||
@@ -272,14 +273,22 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
if (!receiver_note_status(&session_start, &last_progress, status, file_descriptor, sink))
|
||||
return -1;
|
||||
bool early_delete = config_delete_timing_early(config);
|
||||
bool per_dir_delete = config_delete_timing_per_dir(config);
|
||||
/* Parked keep-set for the late/commit timing. Every exit path below frees it
|
||||
exactly once; the only exception is the successful FINISHED handoff, which
|
||||
transfers ownership to *pending_manifest (used by the -m receiver). */
|
||||
DeleteManifest* deferred_manifest = NULL;
|
||||
/* Per-directory delete session for --delete-during/--delete-delay. During the
|
||||
loop it applies plans inline (during) or snapshots their extras (delay); on
|
||||
a successful FINISHED it is either committed here or handed to
|
||||
*pending_plans so the -m caller commits after its disk writer drained. */
|
||||
DeletePlanSession* plan_session = NULL;
|
||||
bool delete_limit_noted = false;
|
||||
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK ||
|
||||
status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH ||
|
||||
status == STATUS_MKDIR || status == STATUS_MANIFEST || status == STATUS_HARDLINK ||
|
||||
status == STATUS_SYMLINK || status == STATUS_SPECIAL || status == STATUS_DIR_TIMES) {
|
||||
status == STATUS_SYMLINK || status == STATUS_SPECIAL || status == STATUS_DIR_TIMES ||
|
||||
status == STATUS_DELETE_PLAN) {
|
||||
if (status == STATUS_KEEPALIVE) {
|
||||
if (!send_status(file_descriptor, STATUS_KEEPALIVE))
|
||||
goto fail;
|
||||
@@ -344,11 +353,10 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
goto next_status;
|
||||
}
|
||||
if (early_delete) {
|
||||
/* --delete-before / --delete-during: the manifest is authoritative the
|
||||
moment it arrives, before any file data. Delete now and acknowledge
|
||||
so the sender only starts streaming once the deletion committed (or
|
||||
failed). This is the rsync delete-before/delete-during window: a
|
||||
later transfer failure does not restore these deletions. A
|
||||
/* --delete-before: the whole-tree manifest is authoritative the moment
|
||||
it arrives, before any file data. Delete now and acknowledge so the
|
||||
sender only starts streaming once the deletion committed (or failed).
|
||||
A later transfer failure does not restore these deletions. A
|
||||
--max-delete-capped commit still succeeds and the transfer proceeds;
|
||||
the terminal success frame reports the cap. */
|
||||
DeleteCommitResult deletion = (config->use_delete || config->delete_missing_args)
|
||||
@@ -364,9 +372,9 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
if (!send_status(file_descriptor, STATUS_OK))
|
||||
goto fail;
|
||||
} else if (config->use_delete || config->delete_missing_args) {
|
||||
/* Plain --delete / --delete-after / --delete-delay and the
|
||||
--delete-missing-args exact-path deletions: hold the manifest and
|
||||
commit it only after STATUS_FINISHED. */
|
||||
/* Plain --delete / --delete-after and the --delete-missing-args
|
||||
exact-path deletions: hold the manifest and commit it only after
|
||||
STATUS_FINISHED. The per-directory modes never send this frame. */
|
||||
if (deferred_manifest) {
|
||||
log_message(LOG_LEVEL_ERROR, "Received a second delete manifest");
|
||||
delete_manifest_free(deferred_manifest);
|
||||
@@ -380,6 +388,23 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
delete_manifest_free(manifest);
|
||||
}
|
||||
goto next_status;
|
||||
} else if (status == STATUS_DELETE_PLAN) {
|
||||
if (!per_dir_delete) {
|
||||
log_message(LOG_LEVEL_ERROR, "Received a per-directory delete plan without a per-dir "
|
||||
"delete timing");
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
goto fail;
|
||||
}
|
||||
if (!plan_session)
|
||||
plan_session = delete_plan_session_create(config);
|
||||
if (!plan_session || delete_plan_session_receive(plan_session, config, file_descriptor) != 0)
|
||||
goto fail;
|
||||
if (delete_plan_session_limit_reached(plan_session) && !delete_limit_noted &&
|
||||
sink->note_delete_limit) {
|
||||
sink->note_delete_limit(sink->context);
|
||||
delete_limit_noted = true;
|
||||
}
|
||||
goto next_status;
|
||||
} else {
|
||||
File* file = file_receive(config, file_descriptor);
|
||||
if (!file) {
|
||||
@@ -424,6 +449,28 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
sink->note_delete_limit(sink->context);
|
||||
}
|
||||
}
|
||||
/* Per-directory deletion: --delete-during already applied each plan inline, so
|
||||
this only finishes the missing-args deletions; --delete-delay committed
|
||||
nothing yet and applies its decompressed snapshot here. The -m receiver
|
||||
hands the session to its caller instead, which commits after the disk
|
||||
writer drained. */
|
||||
if (plan_session) {
|
||||
if (pending_plans) {
|
||||
*pending_plans = plan_session;
|
||||
plan_session = NULL;
|
||||
} else {
|
||||
DeleteCommitResult deletion = delete_plan_session_commit(plan_session, config);
|
||||
bool limit = delete_plan_session_limit_reached(plan_session);
|
||||
delete_plan_session_destroy(plan_session);
|
||||
plan_session = NULL;
|
||||
if (deletion == DELETE_COMMIT_ERROR) {
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
goto fail;
|
||||
}
|
||||
if (limit && !delete_limit_noted && sink->note_delete_limit)
|
||||
sink->note_delete_limit(sink->context);
|
||||
}
|
||||
}
|
||||
if (sink->send_success) {
|
||||
if (sink->send_success_frame) {
|
||||
if (!sink->send_success_frame(file_descriptor, sink->context))
|
||||
@@ -436,11 +483,14 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
|
||||
fail:
|
||||
/* Failure exits that must not (or already did) report a STATUS_ERROR. The
|
||||
parked keep-set is dropped: never commit a deletion for a failed stream. */
|
||||
parked keep-set/session is dropped: never commit a deletion for a failed
|
||||
stream. */
|
||||
if (deferred_manifest) {
|
||||
delete_manifest_free(deferred_manifest);
|
||||
deferred_manifest = NULL;
|
||||
}
|
||||
if (plan_session)
|
||||
delete_plan_session_destroy(plan_session);
|
||||
return -1;
|
||||
|
||||
receive_error:
|
||||
@@ -448,6 +498,8 @@ receive_error:
|
||||
delete_manifest_free(deferred_manifest);
|
||||
deferred_manifest = NULL;
|
||||
}
|
||||
if (plan_session)
|
||||
delete_plan_session_destroy(plan_session);
|
||||
if (sink->send_error)
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
return -1;
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
#define RECEIVER_H
|
||||
|
||||
#include "config.h"
|
||||
#include "delete_plan.h"
|
||||
#include "file.h"
|
||||
#include "file_receive.h"
|
||||
#include "protocol.h"
|
||||
@@ -53,10 +54,12 @@ int receiver_process(Config* config, int file_descriptor, const ReceiverSink* si
|
||||
when `pending_manifest` is non-NULL the receiver does NOT delete at
|
||||
STATUS_FINISHED itself; instead it stores the owned keep-set manifest there
|
||||
(leaving *pending_manifest untouched on early modes/errors) so the caller can
|
||||
commit the deletion only after its disk writer has fully drained. Pass NULL
|
||||
to keep the default behaviour (delete before the success frame). */
|
||||
commit the deletion only after its disk writer has fully drained. Likewise,
|
||||
when `pending_plans` is non-NULL the --delete-delay per-directory session is
|
||||
handed to the caller instead of being committed at STATUS_FINISHED. Pass NULL
|
||||
for either to keep the default behaviour (delete before the success frame). */
|
||||
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
||||
DeleteManifest** pending_manifest);
|
||||
DeleteManifest** pending_manifest, DeletePlanSession** pending_plans);
|
||||
int receiver_receive_files(Config* config, int file_descriptor);
|
||||
|
||||
/* ---- Connection time bounds (anti-slowloris) ----
|
||||
|
||||
@@ -27,6 +27,7 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
|
||||
context->queued_bytes = 0;
|
||||
context->max_queue_bytes = 0;
|
||||
context->deferred_manifest = NULL;
|
||||
context->deferred_plans = NULL;
|
||||
context->delete_limit_reached = false;
|
||||
atomic_init(&context->cancelled, false);
|
||||
int init = 0;
|
||||
@@ -58,6 +59,8 @@ void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
|
||||
config_delete(context->config);
|
||||
if (context->deferred_manifest)
|
||||
delete_manifest_free(context->deferred_manifest);
|
||||
if (context->deferred_plans)
|
||||
delete_plan_session_destroy(context->deferred_plans);
|
||||
queue_destroy(context->queue);
|
||||
receiver_outcomes_destroy(&context->outcomes);
|
||||
dir_time_list_free(&context->dir_times);
|
||||
@@ -164,8 +167,8 @@ int receive_thread(void* pipeline_context) {
|
||||
|
||||
ReceiverSink sink = {
|
||||
receiver_enqueue_file, context, false, false, NULL, receiver_pipeline_note_delete_limit};
|
||||
if (receiver_process_pending((Config*)config, file_descriptor, &sink,
|
||||
&context->deferred_manifest) != 0) {
|
||||
if (receiver_process_pending((Config*)config, file_descriptor, &sink, &context->deferred_manifest,
|
||||
&context->deferred_plans) != 0) {
|
||||
receiver_thread_fail(context);
|
||||
protocol_session_unbind();
|
||||
return thrd_error;
|
||||
|
||||
@@ -41,6 +41,11 @@ typedef struct PipelineContextReceiver {
|
||||
transfer truly succeeded. NULL in the early delete modes (which delete at
|
||||
the manifest). */
|
||||
DeleteManifest* deferred_manifest;
|
||||
/* Per-directory delete session for --delete-delay: receive_thread snapshots
|
||||
each plan's extras as it arrives and hands the session here instead of
|
||||
committing while the disk writer may still be draining; server.c commits it
|
||||
after both threads joined. NULL for every other timing. */
|
||||
DeletePlanSession* deferred_plans;
|
||||
/* Set by server.c when the deferred delete commit hit the --max-delete
|
||||
budget; the terminal success frame then carries STATUS_DELETE_LIMIT
|
||||
(rsync exit 25) while the transfer itself still succeeds. */
|
||||
|
||||
@@ -962,6 +962,19 @@ void handler(int file_descriptor) {
|
||||
delete_manifest_free(context->deferred_manifest);
|
||||
context->deferred_manifest = NULL;
|
||||
}
|
||||
/* --delete-delay: receive_thread snapshotted each plan's extras as it
|
||||
arrived; with the disk writer drained, commit the deferred removals.
|
||||
--delete-during already applied its plans on the receive thread. */
|
||||
if (context->deferred_plans) {
|
||||
DeleteCommitResult deletion = delete_plan_session_commit(context->deferred_plans, config);
|
||||
if (deletion == DELETE_COMMIT_ERROR) {
|
||||
transfer_ok = false;
|
||||
} else if (deletion == DELETE_COMMIT_LIMIT_REACHED) {
|
||||
context->delete_limit_reached = true;
|
||||
}
|
||||
delete_plan_session_destroy(context->deferred_plans);
|
||||
context->deferred_plans = NULL;
|
||||
}
|
||||
}
|
||||
if (transfer_ok && !config->dry_run) {
|
||||
/* --delay-updates: receive_thread has finished the whole protocol stream
|
||||
|
||||
+7
-1
@@ -228,7 +228,13 @@ Config* config_create(void) {
|
||||
bool config_delete_timing_early(const Config* config) {
|
||||
if (!config)
|
||||
return false;
|
||||
return config->delete_before || config->delete_during;
|
||||
return config->delete_before;
|
||||
}
|
||||
|
||||
bool config_delete_timing_per_dir(const Config* config) {
|
||||
if (!config)
|
||||
return false;
|
||||
return config->delete_during || config->delete_delay;
|
||||
}
|
||||
|
||||
/* A delete-timing flag is only meaningful together with --delete. At most one
|
||||
|
||||
+20
-12
@@ -551,13 +551,17 @@ typedef struct Config {
|
||||
/* rsync deletion-timing family (real from Phase 3). At most one of
|
||||
delete_before / delete_during / delete_delay / delete_after may be set, and
|
||||
only together with use_delete (the CLI implies --delete for each of them).
|
||||
delete_before and delete_during select the EARLY engine mode: the keep-set
|
||||
delete_before selects the EARLY engine mode: the whole-tree keep-set
|
||||
manifest is transmitted before any file data and extras are removed then,
|
||||
acknowledged, before the first data byte. delete_delay and delete_after
|
||||
select the LATE commit mode: extras are removed only after the whole
|
||||
transfer has succeeded (plain --delete keeps this mode). The exact
|
||||
semantics and the divergences from rsync are documented in RSYNC_COMPAT.md
|
||||
and in config_delete_timing_early() below. */
|
||||
acknowledged, before the first data byte. delete_during and delete_delay
|
||||
select the per-directory delete-plan mode (protocol 2.24.0): one plan per
|
||||
source directory is streamed in directory order, and the receiver removes
|
||||
each directory's extras when its plan arrives (during) or snapshots them
|
||||
and removes them only after a successful transfer (delay). delete_after
|
||||
(and plain --delete) keep the whole-tree commit mode: extras are removed
|
||||
from a fresh end-of-transfer destination scan only after the whole transfer
|
||||
succeeded. See config_delete_timing_early()/config_delete_timing_per_dir()
|
||||
below. */
|
||||
/* partial_dir */
|
||||
// PR #174: Partial transfer resumption
|
||||
/* suffix */
|
||||
@@ -916,7 +920,7 @@ typedef struct Config {
|
||||
* carries the new report_dest_info bool appended after the --copy-as block.
|
||||
* This is both a config-frame layout change (one trailing bool) and a frame
|
||||
* sequence change (the new status). */
|
||||
#define PROTOCOL_VERSION "2.23.0"
|
||||
#define PROTOCOL_VERSION "2.24.0"
|
||||
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
||||
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
||||
#define MAX_BASIS_DIRS 64
|
||||
@@ -1010,13 +1014,17 @@ int config_parse_daemon_dest(Config* config);
|
||||
* 0. */
|
||||
int config_parse_transport_dest(Config* config);
|
||||
|
||||
/* True when the negotiated delete timing performs the extra-file deletion
|
||||
* BEFORE the transfer data (--delete-before / --delete-during). The flag is
|
||||
* a pure function of the config and is used identically on the sender (to pick
|
||||
/* True for the whole-tree delete-before timing: a complete keep-set manifest is
|
||||
* transmitted before any data and committed (with an ack) before the first data
|
||||
* byte. Pure function of the config, used identically on the sender (to pick
|
||||
* the manifest-first frame order) and the receiver (to delete when the early
|
||||
* manifest arrives). When false the deletion is committed only after the whole
|
||||
* transfer succeeded (--delete / --delete-after / --delete-delay). */
|
||||
* manifest arrives). */
|
||||
bool config_delete_timing_early(const Config* config);
|
||||
/* True for the per-directory timings (--delete-during / --delete-delay). The
|
||||
* sender streams a delete plan per source directory in directory order; the
|
||||
* receiver applies each plan on arrival (during) or snapshots its extras and
|
||||
* commits them only after a fully-successful transfer (delay). */
|
||||
bool config_delete_timing_per_dir(const Config* config);
|
||||
/* Delete-timing sanity: with deletion enabled at most one timing flag may be
|
||||
* set (none = the default delete-after commit timing); without deletion no
|
||||
* timing flag may be set (each timing flag implies --delete). */
|
||||
|
||||
@@ -0,0 +1,844 @@
|
||||
#include "delete_plan.h"
|
||||
|
||||
#include "charset.h"
|
||||
#include "delay_updates.h"
|
||||
#include "file.h"
|
||||
#include "log.h"
|
||||
#include "utils.h"
|
||||
#include <dirent.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <limits.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* Mirrors MAX_SERVER_DELETE_COUNT in file_receive.c: the server's hard bound on
|
||||
* the number of entries one deletion commit may remove. A client
|
||||
* --max-delete=NUM smaller than this replaces it for the run. */
|
||||
#define DELETE_PLAN_SERVER_LIMIT 100000U
|
||||
/* Per-frame entry cap for the name sections (the dir/file child lists). */
|
||||
#define DELETE_PLAN_MAX_NAMES MAX_MANIFEST_ENTRIES
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Sender: plan builder */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
typedef struct PlanNode {
|
||||
char* dir;
|
||||
ArrayList* files; /* basenames kept directly in dir */
|
||||
ArrayList* dirs; /* basenames of kept child directories */
|
||||
bool sent;
|
||||
struct PlanNode* hash_next;
|
||||
} PlanNode;
|
||||
|
||||
struct DeletePlanSender {
|
||||
PlanNode** buckets;
|
||||
size_t capacity;
|
||||
size_t count;
|
||||
bool config_sent;
|
||||
bool all_synced;
|
||||
const ArrayList* synced_dirs;
|
||||
const ArrayList* protected_prefixes;
|
||||
const ArrayList* size_skipped;
|
||||
const ArrayList* missing_args;
|
||||
size_t entries;
|
||||
};
|
||||
|
||||
static size_t plan_hash(const char* key) {
|
||||
size_t h = 5381;
|
||||
for (const unsigned char* p = (const unsigned char*)key; *p; p++)
|
||||
h = ((h << 5) + h) + *p;
|
||||
return h;
|
||||
}
|
||||
|
||||
static bool list_contains_str(const ArrayList* list, const char* value) {
|
||||
if (!list)
|
||||
return false;
|
||||
for (int i = 0; i < list->size; i++) {
|
||||
if (strcmp((const char*)list->items[i], value) == 0)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool list_add_str_unique(ArrayList* list, const char* value) {
|
||||
if (!list || !value)
|
||||
return false;
|
||||
if (list_contains_str(list, value))
|
||||
return true;
|
||||
char* copy = str_dup(value);
|
||||
if (!copy)
|
||||
return false;
|
||||
if (!array_list_add(list, copy)) {
|
||||
free(copy);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
DeletePlanSender* delete_plan_sender_create(void) {
|
||||
DeletePlanSender* sender = calloc(1, sizeof(DeletePlanSender));
|
||||
if (!sender)
|
||||
return NULL;
|
||||
sender->capacity = 64;
|
||||
sender->buckets = calloc(sender->capacity, sizeof(PlanNode*));
|
||||
if (!sender->buckets) {
|
||||
free(sender);
|
||||
return NULL;
|
||||
}
|
||||
sender->all_synced = true;
|
||||
return sender;
|
||||
}
|
||||
|
||||
static void plan_node_destroy(PlanNode* node) {
|
||||
if (!node)
|
||||
return;
|
||||
free(node->dir);
|
||||
array_list_delete(node->files);
|
||||
array_list_delete(node->dirs);
|
||||
free(node);
|
||||
}
|
||||
|
||||
void delete_plan_sender_destroy(DeletePlanSender* sender) {
|
||||
if (!sender)
|
||||
return;
|
||||
for (size_t i = 0; i < sender->capacity; i++) {
|
||||
PlanNode* node = sender->buckets[i];
|
||||
while (node) {
|
||||
PlanNode* next = node->hash_next;
|
||||
plan_node_destroy(node);
|
||||
node = next;
|
||||
}
|
||||
}
|
||||
free(sender->buckets);
|
||||
free(sender);
|
||||
}
|
||||
|
||||
static PlanNode* plan_find(const DeletePlanSender* sender, const char* dir) {
|
||||
size_t index = plan_hash(dir) & (sender->capacity - 1);
|
||||
for (PlanNode* node = sender->buckets[index]; node; node = node->hash_next) {
|
||||
if (strcmp(node->dir, dir) == 0)
|
||||
return node;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static bool plan_grow(DeletePlanSender* sender) {
|
||||
size_t new_capacity = sender->capacity * 2;
|
||||
PlanNode** buckets = calloc(new_capacity, sizeof(PlanNode*));
|
||||
if (!buckets)
|
||||
return false;
|
||||
for (size_t i = 0; i < sender->capacity; i++) {
|
||||
PlanNode* node = sender->buckets[i];
|
||||
while (node) {
|
||||
PlanNode* next = node->hash_next;
|
||||
size_t index = plan_hash(node->dir) & (new_capacity - 1);
|
||||
node->hash_next = buckets[index];
|
||||
buckets[index] = node;
|
||||
node = next;
|
||||
}
|
||||
}
|
||||
free(sender->buckets);
|
||||
sender->buckets = buckets;
|
||||
sender->capacity = new_capacity;
|
||||
return true;
|
||||
}
|
||||
|
||||
static PlanNode* plan_ensure(DeletePlanSender* sender, const char* dir) {
|
||||
PlanNode* node = plan_find(sender, dir);
|
||||
if (node)
|
||||
return node;
|
||||
if (sender->count + 1 > sender->capacity * 3 / 4 && !plan_grow(sender))
|
||||
return NULL;
|
||||
node = calloc(1, sizeof(PlanNode));
|
||||
if (!node)
|
||||
return NULL;
|
||||
node->dir = str_dup(dir);
|
||||
node->files = array_list_create(free);
|
||||
node->dirs = array_list_create(free);
|
||||
if (!node->dir || !node->files || !node->dirs) {
|
||||
plan_node_destroy(node);
|
||||
return NULL;
|
||||
}
|
||||
size_t index = plan_hash(dir) & (sender->capacity - 1);
|
||||
node->hash_next = sender->buckets[index];
|
||||
sender->buckets[index] = node;
|
||||
sender->count++;
|
||||
return node;
|
||||
}
|
||||
|
||||
static char* path_parent_dir(const char* path) {
|
||||
const char* slash = strrchr(path, '/');
|
||||
if (!slash)
|
||||
return str_dup(".");
|
||||
if (slash == path)
|
||||
return str_dup(".");
|
||||
size_t len = (size_t)(slash - path);
|
||||
char* parent = malloc(len + 1);
|
||||
if (!parent)
|
||||
return NULL;
|
||||
memcpy(parent, path, len);
|
||||
parent[len] = '\0';
|
||||
return parent;
|
||||
}
|
||||
|
||||
static char* path_base_name(const char* path) {
|
||||
const char* slash = strrchr(path, '/');
|
||||
return str_dup(slash ? slash + 1 : path);
|
||||
}
|
||||
|
||||
/* Copy `path`, stripping a leading '/' and any trailing '/'. */
|
||||
static char* plan_clean_path(const char* path) {
|
||||
while (*path == '/')
|
||||
path++;
|
||||
size_t len = strlen(path);
|
||||
while (len > 0 && path[len - 1] == '/')
|
||||
len--;
|
||||
char* clean = malloc(len + 1);
|
||||
if (!clean)
|
||||
return NULL;
|
||||
memcpy(clean, path, len);
|
||||
clean[len] = '\0';
|
||||
return clean;
|
||||
}
|
||||
|
||||
static bool plan_ensure_ancestors(DeletePlanSender* sender, const char* dir) {
|
||||
char* current = str_dup(dir);
|
||||
if (!current)
|
||||
return false;
|
||||
bool ok = true;
|
||||
while (strcmp(current, ".") != 0) {
|
||||
char* parent = path_parent_dir(current);
|
||||
char* base = path_base_name(current);
|
||||
PlanNode* parent_node = parent ? plan_ensure(sender, parent) : NULL;
|
||||
if (!parent || !base || !parent_node || !list_add_str_unique(parent_node->dirs, base)) {
|
||||
ok = false;
|
||||
free(parent);
|
||||
free(base);
|
||||
break;
|
||||
}
|
||||
free(base);
|
||||
free(current);
|
||||
current = parent;
|
||||
}
|
||||
free(current);
|
||||
return ok;
|
||||
}
|
||||
|
||||
bool delete_plan_sender_add(DeletePlanSender* sender, const char* path, bool is_dir) {
|
||||
if (!sender || !path)
|
||||
return false;
|
||||
char* clean = plan_clean_path(path);
|
||||
if (!clean)
|
||||
return false;
|
||||
if (*clean == '\0') {
|
||||
free(clean);
|
||||
return true;
|
||||
}
|
||||
char* parent = path_parent_dir(clean);
|
||||
char* base = path_base_name(clean);
|
||||
PlanNode* parent_node = parent ? plan_ensure(sender, parent) : NULL;
|
||||
bool ok = parent && base && parent_node;
|
||||
if (ok) {
|
||||
if (is_dir) {
|
||||
ok = list_add_str_unique(parent_node->dirs, base) && plan_ensure(sender, clean) != NULL;
|
||||
} else {
|
||||
ok = list_add_str_unique(parent_node->files, base);
|
||||
}
|
||||
}
|
||||
if (ok)
|
||||
ok = plan_ensure_ancestors(sender, parent);
|
||||
if (ok)
|
||||
sender->entries++;
|
||||
free(clean);
|
||||
free(parent);
|
||||
free(base);
|
||||
return ok;
|
||||
}
|
||||
|
||||
void delete_plan_sender_finalize(DeletePlanSender* sender, const ArrayList* synced_dirs) {
|
||||
if (!sender)
|
||||
return;
|
||||
sender->synced_dirs = synced_dirs;
|
||||
sender->all_synced = synced_dirs == NULL;
|
||||
}
|
||||
|
||||
bool delete_plan_sender_empty(const DeletePlanSender* sender) {
|
||||
return !sender || sender->entries == 0;
|
||||
}
|
||||
|
||||
void delete_plan_sender_set_config(DeletePlanSender* sender, const ArrayList* protected_prefixes,
|
||||
const ArrayList* size_skipped, const ArrayList* missing_args) {
|
||||
if (!sender)
|
||||
return;
|
||||
sender->protected_prefixes = protected_prefixes;
|
||||
sender->size_skipped = size_skipped;
|
||||
sender->missing_args = missing_args;
|
||||
}
|
||||
|
||||
static bool plan_is_allowed(const DeletePlanSender* sender, const char* dir) {
|
||||
if (sender->all_synced)
|
||||
return true;
|
||||
return list_contains_str(sender->synced_dirs, dir);
|
||||
}
|
||||
|
||||
static int send_str_section(int fd, const ArrayList* list) {
|
||||
int count = list ? list->size : 0;
|
||||
if (!send_int(fd, count))
|
||||
return -1;
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (!send_wire_str(fd, (const char*)list->items[i]))
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int send_plan_node(int fd, DeletePlanSender* sender, PlanNode* node) {
|
||||
if (!send_status(fd, STATUS_DELETE_PLAN))
|
||||
return -1;
|
||||
if (!send_int(fd, sender->config_sent ? 0 : 1))
|
||||
return -1;
|
||||
if (!sender->config_sent) {
|
||||
if (send_str_section(fd, sender->protected_prefixes) != 0 ||
|
||||
send_str_section(fd, sender->size_skipped) != 0 ||
|
||||
send_str_section(fd, sender->missing_args) != 0)
|
||||
return -1;
|
||||
sender->config_sent = true;
|
||||
}
|
||||
if (!send_wire_str(fd, node->dir))
|
||||
return -1;
|
||||
if (send_str_section(fd, node->dirs) != 0 || send_str_section(fd, node->files) != 0)
|
||||
return -1;
|
||||
node->sent = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int send_prefix_plan(int fd, DeletePlanSender* sender, const char* dir) {
|
||||
PlanNode* node = plan_find(sender, dir);
|
||||
if (!node || node->sent)
|
||||
return 0;
|
||||
if (!plan_is_allowed(sender, dir))
|
||||
return 0;
|
||||
return send_plan_node(fd, sender, node);
|
||||
}
|
||||
|
||||
int delete_plan_send_root(int fd, DeletePlanSender* sender) {
|
||||
if (!sender)
|
||||
return -1;
|
||||
if (!plan_ensure(sender, "."))
|
||||
return -1;
|
||||
return send_prefix_plan(fd, sender, ".");
|
||||
}
|
||||
|
||||
int delete_plan_send_for_path(int fd, DeletePlanSender* sender, const char* path, bool is_dir) {
|
||||
if (!sender || !path)
|
||||
return -1;
|
||||
char* clean = plan_clean_path(path);
|
||||
if (!clean)
|
||||
return -1;
|
||||
int rc = send_prefix_plan(fd, sender, ".");
|
||||
if (rc == 0 && *clean != '\0') {
|
||||
size_t len = strlen(clean);
|
||||
size_t end = len;
|
||||
if (!is_dir) {
|
||||
const char* slash = strrchr(clean, '/');
|
||||
end = slash ? (size_t)(slash - clean) : 0;
|
||||
}
|
||||
for (size_t i = 1; i <= end && rc == 0; i++) {
|
||||
if (i == end || clean[i] == '/') {
|
||||
char* prefix = malloc(i + 1);
|
||||
if (!prefix) {
|
||||
rc = -1;
|
||||
break;
|
||||
}
|
||||
memcpy(prefix, clean, i);
|
||||
prefix[i] = '\0';
|
||||
rc = send_prefix_plan(fd, sender, prefix);
|
||||
free(prefix);
|
||||
}
|
||||
}
|
||||
}
|
||||
free(clean);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Receiver: delete session */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
struct DeletePlanSession {
|
||||
bool defer;
|
||||
bool dry_run;
|
||||
size_t max_delete;
|
||||
size_t deleted;
|
||||
size_t skipped;
|
||||
bool limit_hit;
|
||||
bool config_seen;
|
||||
bool missing_applied;
|
||||
ArrayList* protected_prefixes;
|
||||
ArrayList* size_skipped;
|
||||
ArrayList* missing;
|
||||
ArrayList* deferred;
|
||||
};
|
||||
|
||||
DeletePlanSession* delete_plan_session_create(const Config* config) {
|
||||
if (!config)
|
||||
return NULL;
|
||||
DeletePlanSession* session = calloc(1, sizeof(DeletePlanSession));
|
||||
if (!session)
|
||||
return NULL;
|
||||
session->defer = config->delete_delay;
|
||||
session->dry_run = config->dry_run;
|
||||
bool user_limited =
|
||||
config->max_delete >= 0 && (size_t)config->max_delete < DELETE_PLAN_SERVER_LIMIT;
|
||||
session->max_delete =
|
||||
user_limited ? (size_t)config->max_delete : (size_t)DELETE_PLAN_SERVER_LIMIT;
|
||||
session->protected_prefixes = array_list_create(free);
|
||||
session->size_skipped = array_list_create(free);
|
||||
session->missing = array_list_create(free);
|
||||
session->deferred = array_list_create(free);
|
||||
if (!session->protected_prefixes || !session->size_skipped || !session->missing ||
|
||||
!session->deferred) {
|
||||
delete_plan_session_destroy(session);
|
||||
return NULL;
|
||||
}
|
||||
return session;
|
||||
}
|
||||
|
||||
void delete_plan_session_destroy(DeletePlanSession* session) {
|
||||
if (!session)
|
||||
return;
|
||||
array_list_delete(session->protected_prefixes);
|
||||
array_list_delete(session->size_skipped);
|
||||
array_list_delete(session->missing);
|
||||
array_list_delete(session->deferred);
|
||||
free(session);
|
||||
}
|
||||
|
||||
bool delete_plan_session_limit_reached(const DeletePlanSession* session) {
|
||||
return session && session->limit_hit;
|
||||
}
|
||||
|
||||
/* True for a destination-relative path section entry (non-empty, relative,
|
||||
* traversal-free). */
|
||||
static bool valid_rel_path(const char* value) {
|
||||
return value && value[0] != '\0' && value[0] != '/' && !has_path_traversal(value);
|
||||
}
|
||||
|
||||
/* True for a single child name (non-empty, no slash, not "."/".."). */
|
||||
static bool valid_name(const char* value) {
|
||||
return value && value[0] != '\0' && strcmp(value, ".") != 0 && strcmp(value, "..") != 0 &&
|
||||
strchr(value, '/') == NULL;
|
||||
}
|
||||
|
||||
/* Read one count-prefixed section. `bytes` is the running per-frame budget,
|
||||
* shared across every section of the frame so a hostile peer cannot retain more
|
||||
* than MAX_MANIFEST_BYTES from one STATUS_DELETE_PLAN frame. */
|
||||
static bool read_section(int fd, ArrayList* list, bool rel_path, size_t* bytes) {
|
||||
int count;
|
||||
if (!receive_int(fd, &count) || count < 0 || count > MAX_MANIFEST_ENTRIES)
|
||||
return false;
|
||||
for (int i = 0; i < count; i++) {
|
||||
char* value = receive_wire_str(fd);
|
||||
bool ok = value && (rel_path ? valid_rel_path(value) : valid_name(value));
|
||||
if (ok) {
|
||||
size_t entry_size = strlen(value) + sizeof(char*) + 16;
|
||||
if (entry_size > MAX_MANIFEST_BYTES - *bytes) {
|
||||
ok = false;
|
||||
} else {
|
||||
*bytes += entry_size;
|
||||
ok = array_list_add(list, value);
|
||||
}
|
||||
}
|
||||
if (!ok) {
|
||||
free(value);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static int open_plan_dir(const Config* config, const char* dir) {
|
||||
char* full = (strcmp(dir, ".") == 0) ? str_dup(config->receive_root_directory)
|
||||
: path_cat(config->receive_root_directory, dir);
|
||||
if (!full)
|
||||
return -1;
|
||||
int root_fd = utils_get_authorized_root_fd();
|
||||
int fd = -1;
|
||||
if (root_fd >= 0) {
|
||||
if (utils_get_authorized_root_path())
|
||||
fd = utils_open_authorized_destination(full);
|
||||
else if (strcmp(dir, ".") == 0)
|
||||
fd = dup(root_fd);
|
||||
} else {
|
||||
fd = open(full, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
}
|
||||
free(full);
|
||||
return fd;
|
||||
}
|
||||
|
||||
typedef struct PlanSkips {
|
||||
DeleteSkipEntry* entries;
|
||||
int count;
|
||||
} PlanSkips;
|
||||
|
||||
static bool build_plan_skips(const Config* config, const DeletePlanSession* session,
|
||||
PlanSkips* out) {
|
||||
out->entries = NULL;
|
||||
out->count = 0;
|
||||
int count = (config->delay_updates ? 1 : 0) + config->basis_count +
|
||||
session->protected_prefixes->size + session->size_skipped->size;
|
||||
if (count == 0)
|
||||
return true;
|
||||
out->entries = calloc((size_t)count, sizeof(DeleteSkipEntry));
|
||||
if (!out->entries)
|
||||
return false;
|
||||
int idx = 0;
|
||||
if (config->delay_updates) {
|
||||
out->entries[idx].prefix = DELAY_UPDATES_STAGING_DIR;
|
||||
out->entries[idx].top_level_only = true;
|
||||
idx++;
|
||||
}
|
||||
for (int i = 0; i < config->basis_count; i++) {
|
||||
out->entries[idx].prefix = config->basis_dirs[i].path;
|
||||
out->entries[idx].top_level_only = false;
|
||||
idx++;
|
||||
}
|
||||
for (int i = 0; i < session->protected_prefixes->size; i++) {
|
||||
out->entries[idx].prefix = (const char*)session->protected_prefixes->items[i];
|
||||
out->entries[idx].top_level_only = false;
|
||||
idx++;
|
||||
}
|
||||
for (int i = 0; i < session->size_skipped->size; i++) {
|
||||
out->entries[idx].prefix = (const char*)session->size_skipped->items[i];
|
||||
out->entries[idx].top_level_only = false;
|
||||
idx++;
|
||||
}
|
||||
out->count = idx;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool budget_available(const DeletePlanSession* session) {
|
||||
return session->deleted < session->max_delete;
|
||||
}
|
||||
|
||||
static void note_skipped(DeletePlanSession* session) {
|
||||
session->limit_hit = true;
|
||||
session->skipped++;
|
||||
}
|
||||
|
||||
static void log_deleted(const char* rel) {
|
||||
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||
fprintf(stderr, " Deleted: %s\n", escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
}
|
||||
|
||||
/* Append a snapshot path for --delete-delay. */
|
||||
static bool defer_add(DeletePlanSession* session, const char* rel) {
|
||||
char* copy = str_dup(rel);
|
||||
if (!copy)
|
||||
return false;
|
||||
if (!array_list_add(session->deferred, copy)) {
|
||||
free(copy);
|
||||
return false;
|
||||
}
|
||||
session->deleted++;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Process the direct children of one directory. `keep_dirs`/`keep_files`
|
||||
* (basenames) are the source entries that must be kept; NULL means every child
|
||||
* is an extra (the forced path used inside a removed extra directory tree).
|
||||
* `survives` reports that at least one child remains (kept, protected, or
|
||||
* skipped by the budget). `force_now` removes even in --delete-delay mode
|
||||
* (type conflicts must clear before the incoming data). */
|
||||
static bool process_children(int dirfd, const char* dir_rel, const ArrayList* keep_dirs,
|
||||
const ArrayList* keep_files, bool at_root, bool force_now,
|
||||
const PlanSkips* skips, DeletePlanSession* session, bool* survives);
|
||||
|
||||
static bool process_extra_dir(int dirfd, const char* name, const char* child_rel, bool force_now,
|
||||
const PlanSkips* skips, DeletePlanSession* session, bool* removed) {
|
||||
*removed = false;
|
||||
int childfd = openat(dirfd, name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
if (childfd < 0) {
|
||||
if (errno == ENOENT) {
|
||||
*removed = true;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
bool survives = false;
|
||||
bool ok =
|
||||
process_children(childfd, child_rel, NULL, NULL, false, force_now, skips, session, &survives);
|
||||
close(childfd);
|
||||
if (!ok)
|
||||
return false;
|
||||
if (survives)
|
||||
return true;
|
||||
if (!budget_available(session)) {
|
||||
note_skipped(session);
|
||||
return true;
|
||||
}
|
||||
if (session->defer && !force_now) {
|
||||
if (!defer_add(session, child_rel))
|
||||
return false;
|
||||
*removed = true;
|
||||
return true;
|
||||
}
|
||||
if (unlinkat(dirfd, name, AT_REMOVEDIR) == 0) {
|
||||
session->deleted++;
|
||||
log_deleted(child_rel);
|
||||
*removed = true;
|
||||
return true;
|
||||
}
|
||||
if (errno == ENOENT) {
|
||||
*removed = true;
|
||||
return true;
|
||||
}
|
||||
/* ENOTEMPTY/EEXIST: a protected entry the walker leaves behind survived, so
|
||||
the directory stays; any other errno is a genuine failure. */
|
||||
return errno == ENOTEMPTY || errno == EEXIST;
|
||||
}
|
||||
|
||||
static bool process_extra_file(int dirfd, const char* name, const char* child_rel, bool force_now,
|
||||
DeletePlanSession* session) {
|
||||
if (!budget_available(session)) {
|
||||
note_skipped(session);
|
||||
return true;
|
||||
}
|
||||
if (session->defer && !force_now) {
|
||||
return defer_add(session, child_rel);
|
||||
}
|
||||
if (unlinkat(dirfd, name, 0) == 0) {
|
||||
session->deleted++;
|
||||
log_deleted(child_rel);
|
||||
} else if (errno != ENOENT) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool process_children(int dirfd, const char* dir_rel, const ArrayList* keep_dirs,
|
||||
const ArrayList* keep_files, bool at_root, bool force_now,
|
||||
const PlanSkips* skips, DeletePlanSession* session, bool* survives) {
|
||||
*survives = false;
|
||||
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
if (scanfd < 0)
|
||||
return false;
|
||||
DIR* dir = fdopendir(scanfd);
|
||||
if (!dir) {
|
||||
close(scanfd);
|
||||
return false;
|
||||
}
|
||||
bool operation_ok = true;
|
||||
bool local_survives = false;
|
||||
const struct dirent* entry;
|
||||
while ((entry = readdir(dir)) != NULL) {
|
||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||
continue;
|
||||
char* child_rel =
|
||||
(strcmp(dir_rel, ".") == 0) ? str_dup(entry->d_name) : path_cat(dir_rel, entry->d_name);
|
||||
if (!child_rel) {
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
if (path_under_skip_prefix(child_rel, at_root, skips->entries, skips->count)) {
|
||||
local_survives = true;
|
||||
free(child_rel);
|
||||
continue;
|
||||
}
|
||||
struct stat st;
|
||||
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||
if (errno != ENOENT)
|
||||
operation_ok = false;
|
||||
free(child_rel);
|
||||
continue;
|
||||
}
|
||||
bool is_dir = S_ISDIR(st.st_mode);
|
||||
bool in_keep_dirs = is_dir && list_contains_str(keep_dirs, entry->d_name);
|
||||
bool in_keep_files = !is_dir && list_contains_str(keep_files, entry->d_name);
|
||||
if (in_keep_dirs) {
|
||||
local_survives = true;
|
||||
} else if (keep_dirs && !is_dir && list_contains_str(keep_dirs, entry->d_name)) {
|
||||
/* Destination file blocks a source directory: clear it now, whatever the
|
||||
delete timing, so the directory can be created. */
|
||||
if (!process_extra_file(dirfd, entry->d_name, child_rel, true, session))
|
||||
operation_ok = false;
|
||||
} else if (in_keep_files) {
|
||||
local_survives = true;
|
||||
} else if (keep_files && is_dir && list_contains_str(keep_files, entry->d_name)) {
|
||||
/* Destination directory blocks a source file: remove it now. */
|
||||
bool removed = false;
|
||||
if (!process_extra_dir(dirfd, entry->d_name, child_rel, true, skips, session, &removed))
|
||||
operation_ok = false;
|
||||
else if (!removed)
|
||||
local_survives = true;
|
||||
} else if (is_dir) {
|
||||
bool removed = false;
|
||||
if (!process_extra_dir(dirfd, entry->d_name, child_rel, force_now, skips, session, &removed))
|
||||
operation_ok = false;
|
||||
else if (!removed)
|
||||
local_survives = true;
|
||||
} else {
|
||||
if (!process_extra_file(dirfd, entry->d_name, child_rel, force_now, session))
|
||||
operation_ok = false;
|
||||
}
|
||||
free(child_rel);
|
||||
}
|
||||
closedir(dir);
|
||||
*survives = local_survives;
|
||||
return operation_ok;
|
||||
}
|
||||
|
||||
static bool apply_plan_dir(DeletePlanSession* session, const Config* config, const char* dir,
|
||||
const ArrayList* dirs, const ArrayList* files) {
|
||||
int dirfd = open_plan_dir(config, dir);
|
||||
if (dirfd < 0) {
|
||||
/* An absent destination directory has nothing to delete. */
|
||||
return errno == ENOENT || errno == ENOTDIR;
|
||||
}
|
||||
PlanSkips skips;
|
||||
if (!build_plan_skips(config, session, &skips)) {
|
||||
close(dirfd);
|
||||
return false;
|
||||
}
|
||||
bool survives = false;
|
||||
bool ok = process_children(dirfd, dir, dirs, files, strcmp(dir, ".") == 0, false, &skips, session,
|
||||
&survives);
|
||||
free(skips.entries);
|
||||
close(dirfd);
|
||||
if (!ok)
|
||||
log_message(LOG_LEVEL_ERROR, "deletion failed while removing extraneous files");
|
||||
return ok;
|
||||
}
|
||||
|
||||
static bool apply_missing(DeletePlanSession* session, const Config* config) {
|
||||
if (session->missing_applied)
|
||||
return true;
|
||||
session->missing_applied = true;
|
||||
/* The server clears delete_missing_args when its --allow-delete policy is
|
||||
off; never honor the client's exact-path requests then. */
|
||||
if (!config->delete_missing_args || session->missing->size == 0)
|
||||
return true;
|
||||
DeleteManifest manifest = {
|
||||
.keeps = NULL, .protected = NULL, .missing = session->missing, .dirs = NULL};
|
||||
size_t remaining = budget_available(session) ? session->max_delete - session->deleted : 0;
|
||||
size_t deleted = 0;
|
||||
size_t skipped = 0;
|
||||
bool limit = false;
|
||||
bool ok = manifest_delete_missing_args_limited(config, &manifest, remaining, &deleted, &skipped,
|
||||
&limit);
|
||||
session->deleted += deleted;
|
||||
session->skipped += skipped;
|
||||
if (limit)
|
||||
session->limit_hit = true;
|
||||
return ok;
|
||||
}
|
||||
|
||||
int delete_plan_session_receive(DeletePlanSession* session, const Config* config, int fd) {
|
||||
if (!session || !config) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
int has_config;
|
||||
if (!receive_int(fd, &has_config) || (has_config != 0 && has_config != 1)) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
size_t bytes = 0;
|
||||
if (has_config) {
|
||||
if (session->config_seen || !read_section(fd, session->protected_prefixes, true, &bytes) ||
|
||||
!read_section(fd, session->size_skipped, true, &bytes) ||
|
||||
!read_section(fd, session->missing, true, &bytes)) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
session->config_seen = true;
|
||||
}
|
||||
char* dir = receive_wire_str(fd);
|
||||
ArrayList* dirs = array_list_create(free);
|
||||
ArrayList* files = array_list_create(free);
|
||||
bool parsed = dir && (strcmp(dir, ".") == 0 || valid_rel_path(dir)) && dirs && files &&
|
||||
read_section(fd, dirs, false, &bytes) && read_section(fd, files, false, &bytes);
|
||||
if (!parsed) {
|
||||
free(dir);
|
||||
array_list_delete(dirs);
|
||||
array_list_delete(files);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
bool enabled = config->use_delete || config->delete_missing_args;
|
||||
bool ok = true;
|
||||
if (!session->dry_run && enabled) {
|
||||
if (!session->defer && !apply_missing(session, config))
|
||||
ok = false;
|
||||
if (ok && !apply_plan_dir(session, config, dir, dirs, files))
|
||||
ok = false;
|
||||
}
|
||||
free(dir);
|
||||
array_list_delete(dirs);
|
||||
array_list_delete(files);
|
||||
if (!ok) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
if (session->limit_hit)
|
||||
log_message(LOG_LEVEL_WARNING, "Deletions stopped due to the delete limit (%zu skipped)",
|
||||
session->skipped);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Apply one snapshotted --delete-delay path (post-order: children precede their
|
||||
* parent directory). */
|
||||
static bool apply_deferred_path(DeletePlanSession* session, const Config* config, const char* rel) {
|
||||
(void)session;
|
||||
char* full = path_cat(config->receive_root_directory, rel);
|
||||
if (!full)
|
||||
return false;
|
||||
char* leaf = NULL;
|
||||
int parent_fd = file_open_secure_parent(full, &leaf, false);
|
||||
free(full);
|
||||
if (parent_fd < 0) {
|
||||
free(leaf);
|
||||
return errno == ENOENT || errno == ENOTDIR;
|
||||
}
|
||||
struct stat st;
|
||||
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||
bool absent = errno == ENOENT;
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
return absent;
|
||||
}
|
||||
int rc;
|
||||
if (S_ISDIR(st.st_mode))
|
||||
rc = unlinkat(parent_fd, leaf, AT_REMOVEDIR);
|
||||
else
|
||||
rc = unlinkat(parent_fd, leaf, 0);
|
||||
bool ok = rc == 0 || errno == ENOENT || errno == ENOTEMPTY || errno == EEXIST;
|
||||
if (rc == 0)
|
||||
log_deleted(rel);
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
return ok;
|
||||
}
|
||||
|
||||
DeleteCommitResult delete_plan_session_commit(DeletePlanSession* session, const Config* config) {
|
||||
if (!session || !config)
|
||||
return DELETE_COMMIT_ERROR;
|
||||
bool ok = true;
|
||||
if (session->defer) {
|
||||
for (int i = 0; i < session->deferred->size && ok; i++)
|
||||
ok = apply_deferred_path(session, config, (const char*)session->deferred->items[i]);
|
||||
}
|
||||
if (ok)
|
||||
ok = apply_missing(session, config);
|
||||
if (!ok)
|
||||
return DELETE_COMMIT_ERROR;
|
||||
if (session->limit_hit)
|
||||
return DELETE_COMMIT_LIMIT_REACHED;
|
||||
return DELETE_COMMIT_OK;
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
#ifndef DELETE_PLAN_H
|
||||
#define DELETE_PLAN_H
|
||||
|
||||
#include "array_list.h"
|
||||
#include "config.h"
|
||||
#include "file_receive.h"
|
||||
#include "protocol.h"
|
||||
#include <stdbool.h>
|
||||
|
||||
/* Per-directory delete plans (protocol 2.24.0).
|
||||
*
|
||||
* rsync's --delete-during removes a directory's extras while the generator
|
||||
* processes that directory, and --delete-delay records the deletion list during
|
||||
* the scan but applies it only after a fully-successful transfer. FastSync has
|
||||
* no per-directory generator pass; instead the sender streams one plan per
|
||||
* source directory, in directory order, and the receiver applies it when it
|
||||
* arrives (during) or snapshots its extras and commits them at the end (delay).
|
||||
*
|
||||
* The sender side builds a plan set from the path-only pre-scan (it needs every
|
||||
* directory's complete direct-child list before the first data byte of that
|
||||
* directory). The receiver side is a session that carries the global protected
|
||||
* prefixes (filter-excluded and size-skipped source mirrors), the
|
||||
* --delete-missing-args exact deletions, the shared --max-delete budget and,
|
||||
* for --delete-delay, the snapshotted extras. */
|
||||
|
||||
/* ---- Sender: plan builder ---- */
|
||||
|
||||
typedef struct DeletePlanSender DeletePlanSender;
|
||||
|
||||
DeletePlanSender* delete_plan_sender_create(void);
|
||||
void delete_plan_sender_destroy(DeletePlanSender* sender);
|
||||
/* Record one transmitted entry. `path` is the destination-relative wire path;
|
||||
* is_dir marks an explicit directory entry (--dirs, a -x mount point). */
|
||||
bool delete_plan_sender_add(DeletePlanSender* sender, const char* path, bool is_dir);
|
||||
/* Drop plans for directories outside `synced_dirs` (the --files-from
|
||||
* synchronization scope; pass NULL when a full recursive transfer synchronized
|
||||
* every directory). The receive root is the "." sentinel. */
|
||||
void delete_plan_sender_finalize(DeletePlanSender* sender, const ArrayList* synced_dirs);
|
||||
/* True when no transmitted entry was recorded (an ambiguous empty scan). */
|
||||
bool delete_plan_sender_empty(const DeletePlanSender* sender);
|
||||
/* Attach the global config sections advertised on the first plan frame. */
|
||||
void delete_plan_sender_set_config(DeletePlanSender* sender, const ArrayList* protected_prefixes,
|
||||
const ArrayList* size_skipped, const ArrayList* missing_args);
|
||||
/* Send the root plan (even before any data, so root extras are handled like
|
||||
* rsync's first generator directory). Returns -1 on I/O error. */
|
||||
int delete_plan_send_root(int fd, DeletePlanSender* sender);
|
||||
/* Send the plans for every ancestor of `path` (root-first) and, when is_dir,
|
||||
* for `path` itself; already-sent plans are skipped. */
|
||||
int delete_plan_send_for_path(int fd, DeletePlanSender* sender, const char* path, bool is_dir);
|
||||
|
||||
/* ---- Receiver: delete session ---- */
|
||||
|
||||
typedef struct DeletePlanSession DeletePlanSession;
|
||||
|
||||
DeletePlanSession* delete_plan_session_create(const Config* config);
|
||||
void delete_plan_session_destroy(DeletePlanSession* session);
|
||||
/* Read one STATUS_DELETE_PLAN frame (the leading status already consumed) and
|
||||
* act on it. Returns 0 on success (including a dry-run/disabled no-op) and -1
|
||||
* after signalling STATUS_ERROR on a malformed frame or a deletion failure. */
|
||||
int delete_plan_session_receive(DeletePlanSession* session, const Config* config, int fd);
|
||||
/* Apply the deferred snapshot (--delete-delay) and the missing-args deletions.
|
||||
* Safe to call once; returns the commit outcome. */
|
||||
DeleteCommitResult delete_plan_session_commit(DeletePlanSession* session, const Config* config);
|
||||
/* True once the shared --max-delete budget stopped part of a deletion. */
|
||||
bool delete_plan_session_limit_reached(const DeletePlanSession* session);
|
||||
|
||||
#endif
|
||||
@@ -3472,6 +3472,21 @@ bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest
|
||||
return delete_missing_args_budgeted(config, manifest, &budget);
|
||||
}
|
||||
|
||||
bool manifest_delete_missing_args_limited(const Config* config, DeleteManifest* manifest,
|
||||
size_t max_delete, size_t* deleted, size_t* skipped,
|
||||
bool* limit_hit) {
|
||||
DeleteBudgetState budget = {
|
||||
.max_delete = max_delete, .deleted = 0, .skipped = 0, .limit_hit = false};
|
||||
bool ok = delete_missing_args_budgeted(config, manifest, &budget);
|
||||
if (deleted)
|
||||
*deleted = budget.deleted;
|
||||
if (skipped)
|
||||
*skipped = budget.skipped;
|
||||
if (limit_hit)
|
||||
*limit_hit = budget.limit_hit;
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Commit every deletion family the manifest carries. The --delete-missing-args
|
||||
exact-path deletions run FIRST: they are explicit user requests and must not
|
||||
be blocked by the extras walker's filter-exclusion protection (a protected
|
||||
|
||||
@@ -118,6 +118,14 @@ bool manifest_delete_extras(const Config* config, DeleteManifest* manifest);
|
||||
confinement or I/O error (the run then fails); tolerated per-path cases are
|
||||
reported and skipped. */
|
||||
bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest);
|
||||
/* Budgeted form of manifest_delete_missing_args for the per-directory delete
|
||||
session: each removed mirror draws from `max_delete` (SIZE_MAX = unlimited)
|
||||
and the tallies are accumulated into `*deleted`/`*skipped`. `*limit_hit` is set
|
||||
when the budget stopped the pass with entries left over. Returns false only
|
||||
on a genuine deletion error. */
|
||||
bool manifest_delete_missing_args_limited(const Config* config, DeleteManifest* manifest,
|
||||
size_t max_delete, size_t* deleted, size_t* skipped,
|
||||
bool* limit_hit);
|
||||
/* Outcome of committing a delete manifest. LIMIT_REACHED reports rsync's
|
||||
partial --max-delete result: the budget allowed some deletions and the rest
|
||||
were skipped (the run still stores all file data but the client exits 25). */
|
||||
|
||||
@@ -36,6 +36,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
||||
context->scan_had_io_error = false;
|
||||
context->remove_source_files = NULL;
|
||||
context->early_delete = false;
|
||||
context->delete_plans = NULL;
|
||||
context->scan_stopped_early = false;
|
||||
context->total_files = 0;
|
||||
context->progress_bytes = 0;
|
||||
@@ -187,6 +188,8 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) {
|
||||
if (context->manifest) {
|
||||
array_list_delete(context->manifest);
|
||||
}
|
||||
if (context->delete_plans)
|
||||
delete_plan_sender_destroy(context->delete_plans);
|
||||
if (context->excluded_paths)
|
||||
array_list_delete(context->excluded_paths);
|
||||
if (context->size_skipped_paths)
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
#include "array_list.h"
|
||||
#include "chunk.h"
|
||||
#include "config.h"
|
||||
#include "delete_plan.h"
|
||||
#include "file.h"
|
||||
#include "protocol.h"
|
||||
#include "queue.h"
|
||||
@@ -66,11 +67,16 @@ typedef struct {
|
||||
--ignore-errors kept the run going. */
|
||||
bool scan_had_io_error;
|
||||
ArrayList* remove_source_files;
|
||||
/* True when --delete-before/--delete-during require the keep-set manifest to
|
||||
be transmitted before any file data: context->manifest is then prebuilt by
|
||||
a path-only pre-scan on the calling thread and the pipeline scanner must
|
||||
not append to it. Set once before the worker threads start. */
|
||||
/* True when --delete-before requires the whole-tree keep-set manifest to be
|
||||
transmitted before any file data: context->manifest is then prebuilt by a
|
||||
path-only pre-scan on the calling thread and the pipeline scanner must not
|
||||
append to it. Set once before the worker threads start. */
|
||||
bool early_delete;
|
||||
/* Non-NULL for --delete-during/--delete-delay: the per-directory plan set
|
||||
prebuilt by the path-only pre-scan on the calling thread. The sender
|
||||
thread transmits the root plan before any data and the remaining plans
|
||||
alongside the chunks. Set once before the worker threads start. */
|
||||
DeletePlanSender* delete_plans;
|
||||
mtx_t mutex_progress;
|
||||
int total_files;
|
||||
unsigned long long progress_bytes;
|
||||
|
||||
+15
-1
@@ -181,7 +181,21 @@ enum NET_STATUS {
|
||||
* (new vs modified, and which of size/time/perms/owner/group differ) without
|
||||
* changing the transfer decision itself. Appended after
|
||||
* STATUS_DELETE_LIMIT so no existing status is renumbered. */
|
||||
STATUS_DEST_INFO
|
||||
STATUS_DEST_INFO,
|
||||
/* Per-directory delete plan (protocol 2.24.0). The sender of a
|
||||
* --delete-during/--delete-delay transfer streams one frame per source
|
||||
* directory in directory order instead of a single whole-tree keep-set
|
||||
* manifest. The receiver applies the plan when it arrives
|
||||
* (--delete-during removes that directory's extras immediately) or records
|
||||
* the extras and applies them only after the whole transfer succeeded
|
||||
* (--delete-delay). Payload: an int32 has_config flag (1 on the first plan
|
||||
* of the run, 0 afterwards); when set, the three global config sections
|
||||
* (protected-prefix count+paths, size-skipped count+paths, missing-args
|
||||
* count+paths); then the destination-relative directory path wire string
|
||||
* ("." for the receive root); then the child-directory count + names and the
|
||||
* child-file count + names that must be kept. Appended after
|
||||
* STATUS_DEST_INFO so no existing status is renumbered. */
|
||||
STATUS_DELETE_PLAN
|
||||
};
|
||||
|
||||
void io_set_fds(int read_fd, int write_fd);
|
||||
|
||||
+2
-2
@@ -60,7 +60,7 @@ bool path_is_within_root(const char* root, const char* path) {
|
||||
* two differ in create-vs-no-create, in what path component they stop at, and
|
||||
* in the extra receiver policies they apply, so they are intentionally kept
|
||||
* separate. Both rely on the shared lexical path_is_within_root check. */
|
||||
static int open_authorized_destination(const char* dest_root) {
|
||||
int utils_open_authorized_destination(const char* dest_root) {
|
||||
int root_fd = utils_get_authorized_root_fd();
|
||||
const char* root_path = utils_get_authorized_root_path();
|
||||
if (root_fd < 0 || !root_path || !dest_root || !path_is_within_root(root_path, dest_root))
|
||||
@@ -752,7 +752,7 @@ DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* m
|
||||
int root_fd = utils_get_authorized_root_fd();
|
||||
if (root_fd >= 0) {
|
||||
if (utils_get_authorized_root_path())
|
||||
rootfd = open_authorized_destination(dest_root);
|
||||
rootfd = utils_open_authorized_destination(dest_root);
|
||||
else if (dest_root == NULL)
|
||||
rootfd = dup(root_fd);
|
||||
else
|
||||
|
||||
@@ -139,6 +139,11 @@ DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* m
|
||||
const DeleteSkipEntry* skips, int skip_count,
|
||||
size_t* deleted_out, size_t* skipped_out);
|
||||
bool delete_extras(const char* dest_root, const ArrayList* manifest);
|
||||
/* Open the existing destination directory at `dest_root`, confined to the
|
||||
authorized root with an O_NOFOLLOW component walk (the same confinement the
|
||||
deletion walker uses for its root). Returns a new fd the caller owns, or -1
|
||||
on error (including a destination that does not exist). */
|
||||
int utils_open_authorized_destination(const char* dest_root);
|
||||
bool utils_set_authorized_root(int fd, const char* canonical_path);
|
||||
/* The fd-only compatibility form is fail-closed for path-based operations;
|
||||
* callers should use utils_set_authorized_root with the canonical identity. */
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
"""Differential + regression coverage for rsync's delete timing.
|
||||
|
||||
``--delete-during``/``--delete-delay`` stream a per-directory delete plan instead
|
||||
of one whole-tree manifest, so the timing is observable:
|
||||
|
||||
* ``--delete-during`` removes a directory's extras as it processes that
|
||||
directory (so an interrupted transfer has already removed the extras of the
|
||||
directories it reached);
|
||||
* ``--delete-delay`` snapshots those extras while scanning and commits the
|
||||
removals only after a fully-successful transfer (so an extra created in the
|
||||
destination after its directory's plan survives, and a failed transfer
|
||||
removes nothing);
|
||||
* ``--delete-after`` re-scans the destination at the end (so that same
|
||||
late-created extra is removed).
|
||||
|
||||
The final-state tests compare against real ``rsync 3.4.1`` where a deterministic
|
||||
comparison exists; the timing tests use a byte-slicing proxy to force a
|
||||
mid-transfer failure or to create a destination entry while the transfer is in
|
||||
flight.
|
||||
"""
|
||||
import os
|
||||
import select
|
||||
import shutil
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import ( # noqa: E402
|
||||
BUILD_DIR,
|
||||
TEST_DATA_DIR,
|
||||
ServerManager,
|
||||
clean_dir,
|
||||
get_dest_received_dir,
|
||||
run_client,
|
||||
)
|
||||
|
||||
# Every test here is deterministic (the proxy throttles until the delete-plan
|
||||
# frames are processed), so the PR gate runs the whole module.
|
||||
pytestmark = pytest.mark.ci
|
||||
|
||||
RSYNC = shutil.which("rsync")
|
||||
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||
|
||||
BIG_BYTES = 8 * 1024 * 1024
|
||||
# Forward/cut this far into the stream: past the (small) delete-plan frames and
|
||||
# well into the big payload, so the receiver has already processed the plan.
|
||||
MID_TRANSFER_BYTES = 256 * 1024
|
||||
# Throttle the proxy so the receiver keeps up with the (fast) client and the
|
||||
# plan frames are provably processed before the hook/cut offset is reached.
|
||||
PROXY_THROTTLE = 0.001
|
||||
|
||||
|
||||
def _write(path, content):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(content)
|
||||
|
||||
|
||||
def _seed_pair(tag, big=False):
|
||||
"""Create a source tree and a destination mirror seeded with extras.
|
||||
|
||||
The tree is a single directory ``d`` containing the transferred files plus,
|
||||
in the destination, an extra ``d/old_extra``.
|
||||
"""
|
||||
source = os.path.join(TEST_DATA_DIR, f"dtp_{tag}_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"dtp_{tag}_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
_write(os.path.join(source, "d", "keep.txt"), b"kept payload\n")
|
||||
if big:
|
||||
_write(os.path.join(source, "d", "big.bin"), b"B" * BIG_BYTES)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
os.makedirs(os.path.join(received, "d"), exist_ok=True)
|
||||
_write(os.path.join(received, "d", "old_extra"), b"stale extra\n")
|
||||
return source, dest, received
|
||||
|
||||
|
||||
def _tree(root):
|
||||
"""Sorted relative paths of every entry below root (files and dirs)."""
|
||||
out = []
|
||||
for dirpath, dirs, files in os.walk(root):
|
||||
for name in dirs:
|
||||
out.append(os.path.relpath(os.path.join(dirpath, name), root))
|
||||
for name in files:
|
||||
out.append(os.path.relpath(os.path.join(dirpath, name), root))
|
||||
return sorted(out)
|
||||
|
||||
|
||||
def _rsync(args):
|
||||
env = dict(os.environ, LC_ALL="C")
|
||||
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120)
|
||||
|
||||
|
||||
class _SlicingProxy:
|
||||
"""Forward the client stream to a server, optionally cutting it or invoking a
|
||||
hook after a byte threshold. ``forward_limit`` mode resets both ends after
|
||||
that many client bytes (a mid-transfer failure). ``hook`` mode calls the
|
||||
hook once and keeps forwarding to completion."""
|
||||
|
||||
def __init__(self, target_port, forward_limit=None, hook=None, hook_after=0,
|
||||
throttle=0.0):
|
||||
self.target = ("127.0.0.1", target_port)
|
||||
self.forward_limit = forward_limit
|
||||
self.hook = hook
|
||||
self.hook_after = hook_after
|
||||
self.throttle = throttle
|
||||
self.hook_called = threading.Event()
|
||||
self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
self.listener.bind(("127.0.0.1", 0))
|
||||
self.listener.listen(1)
|
||||
self.listener.settimeout(20)
|
||||
self.port = self.listener.getsockname()[1]
|
||||
self._thread = threading.Thread(target=self._serve, daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def _serve(self):
|
||||
try:
|
||||
client, _ = self.listener.accept()
|
||||
except OSError:
|
||||
return
|
||||
try:
|
||||
backend = socket.create_connection(self.target, timeout=10)
|
||||
except OSError:
|
||||
client.close()
|
||||
return
|
||||
client.settimeout(20)
|
||||
backend.settimeout(20)
|
||||
forwarded = 0
|
||||
socks = [client, backend]
|
||||
try:
|
||||
while socks:
|
||||
ready, _, _ = select.select(socks, [], [], 20)
|
||||
if not ready:
|
||||
break
|
||||
for sock in ready:
|
||||
data = sock.recv(65536)
|
||||
if not data:
|
||||
socks.remove(sock)
|
||||
peer = backend if sock is client else client
|
||||
try:
|
||||
peer.shutdown(socket.SHUT_WR)
|
||||
except OSError:
|
||||
pass
|
||||
continue
|
||||
if sock is client:
|
||||
if self.forward_limit is not None:
|
||||
room = self.forward_limit - forwarded
|
||||
if room <= 0:
|
||||
socks = []
|
||||
break
|
||||
data = data[:room]
|
||||
backend.sendall(data)
|
||||
forwarded += len(data)
|
||||
if (self.hook is not None and not self.hook_called.is_set()
|
||||
and forwarded >= self.hook_after):
|
||||
# Give the receiver time to process the (tiny) plan
|
||||
# frames that precede this offset before the hook
|
||||
# mutates the destination.
|
||||
if self.throttle > 0:
|
||||
time.sleep(0.2)
|
||||
self.hook()
|
||||
self.hook_called.set()
|
||||
if self.forward_limit is not None and forwarded >= self.forward_limit:
|
||||
socks = []
|
||||
break
|
||||
if self.throttle > 0:
|
||||
time.sleep(self.throttle)
|
||||
else:
|
||||
client.sendall(data)
|
||||
except OSError:
|
||||
pass
|
||||
for sock in (client, backend):
|
||||
try:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0))
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
self.listener.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def finish(self):
|
||||
self._thread.join(30)
|
||||
try:
|
||||
self.listener.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
class TestDeleteTimingFinalStateParity:
|
||||
"""On a successful transfer the per-directory timings match rsync's result."""
|
||||
|
||||
def _run_fastsync(self, tag, timing):
|
||||
source, dest, received = _seed_pair(tag)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, flags=[timing], port=server.port)
|
||||
return result, received
|
||||
|
||||
@pytest.mark.parametrize("timing", ["--delete-during", "--delete-delay"])
|
||||
@requires_rsync
|
||||
def test_success_final_state_matches_rsync(self, timing):
|
||||
# Build the rsync fixture from the same seed so both sides start equal.
|
||||
source, dest, received = _seed_pair("parity_rsync")
|
||||
source2 = source
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, "dtp_parity_rsync_dst")
|
||||
clean_dir(rsync_dst)
|
||||
# rsync mirrors src/ into dst/; seed the same extra.
|
||||
_write(os.path.join(rsync_dst, "d", "old_extra"), b"stale extra\n")
|
||||
|
||||
rsync_result = _rsync(["-a", timing, source2 + "/", rsync_dst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
rsync_tree = _tree(rsync_dst)
|
||||
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, flags=[timing], port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
fastsync_tree = _tree(received)
|
||||
assert fastsync_tree == rsync_tree, (
|
||||
f"{timing}: fastsync tree {fastsync_tree} != rsync tree {rsync_tree}"
|
||||
)
|
||||
|
||||
|
||||
class TestDeleteTimingTypeConflictParity:
|
||||
"""A destination entry whose type differs from the source is replaced, in
|
||||
both per-directory timings and in both directions, exactly like rsync."""
|
||||
|
||||
@pytest.mark.parametrize("timing", ["--delete-during", "--delete-delay"])
|
||||
@requires_rsync
|
||||
def test_type_conflicts_match_rsync(self, timing):
|
||||
source = os.path.join(TEST_DATA_DIR, "dtc_src")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "foo"), b"now a file\n")
|
||||
_write(os.path.join(source, "bar", "inner.txt"), b"now a dir\n")
|
||||
|
||||
def seed_dest(root):
|
||||
clean_dir(root)
|
||||
_write(os.path.join(root, "foo", "inner.txt"), b"was a dir\n")
|
||||
_write(os.path.join(root, "bar"), b"was a file\n")
|
||||
|
||||
rsync_dst = os.path.join(TEST_DATA_DIR, "dtc_rsync_dst")
|
||||
seed_dest(rsync_dst)
|
||||
rsync_result = _rsync(["-a", timing, source + "/", rsync_dst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
rsync_tree = _tree(rsync_dst)
|
||||
|
||||
dest = os.path.join(TEST_DATA_DIR, "dtc_dst")
|
||||
clean_dir(dest)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
seed_dest(received)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, flags=[timing], port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert _tree(received) == rsync_tree, (
|
||||
f"{timing}: fastsync tree {_tree(received)} != rsync tree {rsync_tree}"
|
||||
)
|
||||
|
||||
|
||||
class TestDeleteTimingFailure:
|
||||
"""A mid-transfer failure distinguishes during from delay."""
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_during_removes_delay_preserves_on_failure(self, mt):
|
||||
source, dest, received = _seed_pair("failure", big=True)
|
||||
extra = os.path.join(received, "d", "old_extra")
|
||||
assert os.path.exists(extra)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
for timing, expect_removed in (("--delete-during", True),
|
||||
("--delete-delay", False)):
|
||||
# Re-seed the extra before each run.
|
||||
_write(extra, b"stale extra\n")
|
||||
proxy = _SlicingProxy(server.port, forward_limit=MID_TRANSFER_BYTES, throttle=PROXY_THROTTLE)
|
||||
flags = [timing] + (["--threads"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=proxy.port)
|
||||
proxy.finish()
|
||||
assert result.returncode != 0, f"{timing}: truncated transfer succeeded"
|
||||
present = os.path.exists(extra)
|
||||
assert present != expect_removed, (
|
||||
f"{timing} (mt={mt}): extra present={present}, expected "
|
||||
f"removed={expect_removed}"
|
||||
)
|
||||
|
||||
|
||||
class TestDeleteDelayVsAfterSnapshot:
|
||||
"""A destination entry created after its directory's scan survives under
|
||||
--delete-delay but is removed by --delete-after's fresh end scan."""
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_late_created_extra_survives_delay_not_after(self, mt):
|
||||
source, dest, received = _seed_pair("latecreate", big=True)
|
||||
old_extra = os.path.join(received, "d", "old_extra")
|
||||
new_extra = os.path.join(received, "d", "new_extra")
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
for timing, new_survives in (("--delete-delay", True),
|
||||
("--delete-after", False)):
|
||||
_write(old_extra, b"stale extra\n")
|
||||
if os.path.exists(new_extra):
|
||||
os.unlink(new_extra)
|
||||
|
||||
def hook():
|
||||
# Runs on the proxy thread while the big file is in flight,
|
||||
# after the directory's plan (delay) has been processed.
|
||||
_write(new_extra, b"created mid-transfer\n")
|
||||
|
||||
proxy = _SlicingProxy(server.port, hook=hook, hook_after=MID_TRANSFER_BYTES, throttle=PROXY_THROTTLE)
|
||||
flags = [timing] + (["--threads"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=proxy.port)
|
||||
proxy.finish()
|
||||
assert result.returncode == 0, (
|
||||
f"{timing}: {(result.stderr or result.stdout)[:300]}"
|
||||
)
|
||||
assert proxy.hook_called.is_set(), f"{timing}: hook never fired"
|
||||
assert not os.path.exists(old_extra), f"{timing}: old extra survived"
|
||||
assert os.path.exists(new_extra) == new_survives, (
|
||||
f"{timing} (mt={mt}): new_extra present="
|
||||
f"{os.path.exists(new_extra)}, expected survives={new_survives}"
|
||||
)
|
||||
@@ -36,7 +36,7 @@ from common import ( # noqa: E402
|
||||
verify_transfer,
|
||||
)
|
||||
|
||||
PROTOCOL_VERSION = b"2.23.0"
|
||||
PROTOCOL_VERSION = b"2.24.0"
|
||||
STATUS_MANIFEST = 5
|
||||
STATUS_OK = 0
|
||||
|
||||
|
||||
@@ -3755,15 +3755,15 @@ class TestDeleteTiming:
|
||||
assert _read_file(os.path.join(received, "sub", "deep.txt")) == b"deeply nested file\n", \
|
||||
f"{flag}: nested file was not written after the early deletion"
|
||||
|
||||
@pytest.mark.parametrize("flag", ["--delete", "--delete-after", "--delete-delay"])
|
||||
@pytest.mark.parametrize("flag", ["--delete", "--delete-after"])
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_late_flags_commit_only_after_success(self, flag, mt):
|
||||
"""Plain --delete/--delete-after/--delete-delay defer deletion until the
|
||||
whole transfer succeeds: a mid-transfer write failure must leave every
|
||||
extra in place (commit-style safety). The -m receiver must also keep
|
||||
the extras: the deferred keep-set is committed by the server only after
|
||||
the disk-writer thread has finished, and a failing writer means the
|
||||
manifest is freed, never applied."""
|
||||
"""Plain --delete/--delete-after defer deletion until the whole transfer
|
||||
succeeds: a mid-transfer write failure must leave every extra in place
|
||||
(commit-style safety). The -m receiver must also keep the extras: the
|
||||
deferred keep-set is committed by the server only after the disk-writer
|
||||
thread has finished, and a failing writer means the manifest is freed,
|
||||
never applied."""
|
||||
source = self._seed("late")
|
||||
dest = os.path.join(TEST_DATA_DIR, "deltiming_late_dst")
|
||||
clean_dir(dest)
|
||||
@@ -3789,10 +3789,42 @@ class TestDeleteTiming:
|
||||
assert os.path.isfile(blocker), \
|
||||
f"{flag} (mt={mt}) deleted the blocker although the transfer failed"
|
||||
|
||||
def test_early_flag_respected_when_server_refuses_delete(self, shared_server):
|
||||
"""With an --allow-delete-less server the client's early timing still
|
||||
completes (no deadlock on the pre-delete ack) and simply never deletes,
|
||||
exactly like the plain server policy."""
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_delete_delay_clears_type_conflict_like_rsync(self, mt):
|
||||
"""rsync clears a destination file that blocks a source directory even
|
||||
when the deletion itself is deferred (--delete-delay); the type conflict
|
||||
is resolved immediately so the nested write succeeds. The transfer must
|
||||
therefore succeed and the unrelated extra must still be removed."""
|
||||
source = self._seed("delayconflict")
|
||||
dest = os.path.join(TEST_DATA_DIR, "deltiming_delayconflict_dst")
|
||||
clean_dir(dest)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, port=server.port)
|
||||
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
extra = os.path.join(received, "extra.txt")
|
||||
with open(extra, "wb") as fh:
|
||||
fh.write(b"extra file")
|
||||
blocker = os.path.join(received, "sub")
|
||||
shutil.rmtree(blocker)
|
||||
with open(blocker, "wb") as fh:
|
||||
fh.write(b"blocks the nested destination directory")
|
||||
|
||||
flags = ["--delete-delay"] + (["--threads"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--delete-delay (mt={mt}) did not clear the type conflict: " \
|
||||
f"{(result.stderr or result.stdout)[:300]}"
|
||||
assert os.path.isdir(blocker), "blocker file was not replaced by the source directory"
|
||||
assert _read_file(os.path.join(received, "sub", "deep.txt")) == b"deeply nested file\n"
|
||||
assert not os.path.exists(extra), "--delete-delay did not remove the extra"
|
||||
|
||||
@pytest.mark.parametrize("flag", ["--delete-before", "--delete-during", "--delete-delay"])
|
||||
def test_early_flag_respected_when_server_refuses_delete(self, flag, shared_server):
|
||||
"""With an --allow-delete-less server the client's timing still completes
|
||||
(no deadlock on the pre-delete ack, no per-directory deletion) and simply
|
||||
never deletes, exactly like the plain server policy."""
|
||||
source = self._seed("refused")
|
||||
dest = os.path.join(TEST_DATA_DIR, "deltiming_refused_dst")
|
||||
clean_dir(dest)
|
||||
@@ -3802,9 +3834,9 @@ class TestDeleteTiming:
|
||||
extra = os.path.join(received, "extra.txt")
|
||||
with open(extra, "wb") as fh:
|
||||
fh.write(b"extra file")
|
||||
result, _ = run_client(source, dest, flags=["--delete-before"], port=shared_server.port)
|
||||
result, _ = run_client(source, dest, flags=[flag], port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--delete-before against a refuse-delete server failed: {result.stderr[:300]}"
|
||||
f"{flag} against a refuse-delete server failed: {result.stderr[:300]}"
|
||||
assert os.path.exists(extra), "unauthorized delete removed an extra file"
|
||||
|
||||
|
||||
@@ -3899,6 +3931,31 @@ class TestDeleteScope:
|
||||
finally:
|
||||
server.stop()
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
@pytest.mark.parametrize("timing", ["--delete-during", "--delete-delay"])
|
||||
@pytest.mark.ci
|
||||
def test_files_from_per_dir_timing_confined_to_listed_dirs(self, mt, timing):
|
||||
"""The per-directory timings honor the same --files-from scope: an extra
|
||||
inside a listed directory is removed, while unlisted siblings and the
|
||||
receive-root extra survive."""
|
||||
source, dest, received, server = self._seed(f"pd_{timing.strip('-')}_{mt}")
|
||||
try:
|
||||
listed = _write_rel_list(b"listed.txt\nsub/\n")
|
||||
flags = ["--files-from", listed, timing] + (["--threads"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, f"{timing} delete failed: {result.stderr[:300]}"
|
||||
assert not os.path.exists(os.path.join(received, "sub", "extra.txt")), \
|
||||
f"{timing} did not delete the in-scope extra"
|
||||
assert os.path.isfile(os.path.join(received, "sub", "x.txt"))
|
||||
assert os.path.exists(os.path.join(received, "unlisted.txt")), \
|
||||
f"{timing} deleted an unlisted path (data loss)"
|
||||
assert os.path.exists(os.path.join(received, "other", "c.txt")), \
|
||||
f"{timing} deleted an unlisted sibling directory (data loss)"
|
||||
assert os.path.exists(os.path.join(received, "rootextra.txt")), \
|
||||
f"{timing} deleted the receive-root extra (data loss)"
|
||||
finally:
|
||||
server.stop()
|
||||
|
||||
|
||||
class TestDeleteExtraneousSymlinks:
|
||||
"""#290 (3): --delete unlinks extraneous destination symlinks (never follows
|
||||
@@ -3986,7 +4043,8 @@ class TestDeletePolicy:
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
@pytest.mark.parametrize("timing",
|
||||
["--delete", "--delete-before", "--delete-after", "--delete-delay"])
|
||||
["--delete", "--delete-before", "--delete-after", "--delete-delay",
|
||||
"--delete-during"])
|
||||
def test_delete_protects_excluded_by_default_and_delete_excluded_removes(self, mt, timing):
|
||||
"""rsync parity: with a --delete timing the destination mirror path whose
|
||||
source was excluded survives (protected by default); --delete-excluded
|
||||
@@ -4070,7 +4128,7 @@ class TestDeletePolicy:
|
||||
"--delete-excluded did not remove the excluded dir subtree"
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
@pytest.mark.parametrize("timing", ["--delete", "--delete-before"])
|
||||
@pytest.mark.parametrize("timing", ["--delete", "--delete-before", "--delete-during"])
|
||||
def test_max_delete_exceeded_deletes_up_to_cap_and_exits_25(self, mt, timing):
|
||||
"""rsync parity: --max-delete=N deletes up to N extras, skips the rest and
|
||||
still succeeds as a transfer, exiting 25 with a diagnostic."""
|
||||
|
||||
@@ -94,14 +94,14 @@ def _seed_protocol_source(source):
|
||||
class TestProtocol:
|
||||
@pytest.mark.ci
|
||||
def test_protocol_current_version_accepted(self, shared_server):
|
||||
"""--protocol=2.23.0 (the current PROTOCOL_VERSION) is accepted and the
|
||||
"""--protocol=2.24.0 (the current PROTOCOL_VERSION) is accepted and the
|
||||
transfer completes normally."""
|
||||
source = os.path.join(TEST_DATA_DIR, "proto_ok_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst")
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
os.makedirs(dest)
|
||||
_seed_protocol_source(source)
|
||||
result, _ = run_client(source, dest, flags=["--protocol=2.23.0"],
|
||||
result, _ = run_client(source, dest, flags=["--protocol=2.24.0"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
|
||||
|
||||
@@ -317,7 +317,7 @@ static void test_parse_args_protocol_accept_current() {
|
||||
Config* cfg = valid_client_config();
|
||||
EXPECT_NOT_NULL(cfg);
|
||||
char* argv_equals[] = {"fastsync", "--source-dir", "/src",
|
||||
"--dest-dir", "/dst", "--protocol=2.23.0"};
|
||||
"--dest-dir", "/dst", "--protocol=2.24.0"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 6, argv_equals, positional_args, &positional_count), 0);
|
||||
@@ -327,7 +327,7 @@ static void test_parse_args_protocol_accept_current() {
|
||||
cfg = valid_client_config();
|
||||
EXPECT_NOT_NULL(cfg);
|
||||
char* argv_space[] = {"fastsync", "--source-dir", "/src", "--dest-dir",
|
||||
"/dst", "--protocol", "2.23.0"};
|
||||
"/dst", "--protocol", "2.24.0"};
|
||||
positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 7, argv_space, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_STR(cfg->version, PROTOCOL_VERSION);
|
||||
|
||||
+12
-8
@@ -852,13 +852,15 @@ static void test_config_delete_timing_early_helper() {
|
||||
cfg->use_delete = true;
|
||||
cfg->delete_before = true;
|
||||
EXPECT_TRUE(config_delete_timing_early(cfg));
|
||||
EXPECT_FALSE(config_delete_timing_per_dir(cfg));
|
||||
EXPECT_TRUE(config_has_valid_delete_timing(cfg));
|
||||
config_delete(cfg);
|
||||
|
||||
cfg = config_create();
|
||||
cfg->use_delete = true;
|
||||
cfg->delete_during = true;
|
||||
EXPECT_TRUE(config_delete_timing_early(cfg));
|
||||
EXPECT_FALSE(config_delete_timing_early(cfg));
|
||||
EXPECT_TRUE(config_delete_timing_per_dir(cfg));
|
||||
EXPECT_TRUE(config_has_valid_delete_timing(cfg));
|
||||
config_delete(cfg);
|
||||
|
||||
@@ -866,6 +868,7 @@ static void test_config_delete_timing_early_helper() {
|
||||
cfg->use_delete = true;
|
||||
cfg->delete_delay = true;
|
||||
EXPECT_FALSE(config_delete_timing_early(cfg));
|
||||
EXPECT_TRUE(config_delete_timing_per_dir(cfg));
|
||||
EXPECT_TRUE(config_has_valid_delete_timing(cfg));
|
||||
config_delete(cfg);
|
||||
|
||||
@@ -873,6 +876,7 @@ static void test_config_delete_timing_early_helper() {
|
||||
cfg->use_delete = true;
|
||||
cfg->delete_after = true;
|
||||
EXPECT_FALSE(config_delete_timing_early(cfg));
|
||||
EXPECT_FALSE(config_delete_timing_per_dir(cfg));
|
||||
EXPECT_TRUE(config_has_valid_delete_timing(cfg));
|
||||
config_delete(cfg);
|
||||
|
||||
@@ -2771,14 +2775,14 @@ static void golden_config_populate(Config* c) {
|
||||
c->copy_as_gid = 222;
|
||||
}
|
||||
|
||||
/* The pinned golden frame (protocol 2.23.0). The values below are the only
|
||||
/* The pinned golden frame (protocol 2.24.0). The values below are the only
|
||||
* thing that ties the generated table to the historical wire format; update
|
||||
* them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.23.0
|
||||
* rsync-parity wave changes the config-frame layout (map-entry range + TO name,
|
||||
* one report_dest_info bool, and other wire changes landing in this version);
|
||||
* the byte-exact values are recomputed for the merged layout. */
|
||||
* them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.24.0
|
||||
* per-directory delete-plan wave changes only the version string in the config
|
||||
* frame (the frame layout itself is unchanged from 2.23.0); the byte-exact hash
|
||||
* is recomputed for the new version bytes. */
|
||||
#define GOLDEN_WIRE_LEN 697
|
||||
#define GOLDEN_WIRE_HASH 7835017034643051109ULL
|
||||
#define GOLDEN_WIRE_HASH 13736055061412501670ULL
|
||||
|
||||
static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) {
|
||||
unsigned long long h = 1469598103934665603ULL;
|
||||
@@ -2860,7 +2864,7 @@ static unsigned long long capture_wire_hash(const Config* cfg, size_t* out_len)
|
||||
return h;
|
||||
}
|
||||
|
||||
/* Byte-for-byte wire compatibility guard (protocol 2.23.0). The expected hash
|
||||
/* Byte-for-byte wire compatibility guard (protocol 2.24.0). The expected hash
|
||||
* pins the pre-X-macro byte stream; the refactor MUST NOT change it. */
|
||||
static void test_config_wire_golden() {
|
||||
if (is_running_under_valgrind())
|
||||
|
||||
@@ -65,7 +65,7 @@ static void test_receiver_aborts_idle_keepalive() {
|
||||
ssize_t wrote = write(sv[0], &keepalive, sizeof(keepalive));
|
||||
int result = -2;
|
||||
if (wrote == (ssize_t)sizeof(keepalive))
|
||||
result = receiver_process_pending(config, sv[1], &sink, NULL);
|
||||
result = receiver_process_pending(config, sv[1], &sink, NULL, NULL);
|
||||
Status reply = STATUS_OK;
|
||||
ssize_t got = -1;
|
||||
if (result == -1)
|
||||
|
||||
+2
-2
@@ -566,7 +566,7 @@ static Config* make_late_delete_config(const char* root) {
|
||||
|
||||
static int run_pending_receiver(Config* cfg, int fd, DeleteManifest** pending) {
|
||||
ReceiverSink sink = {0};
|
||||
return receiver_process_pending(cfg, fd, &sink, pending);
|
||||
return receiver_process_pending(cfg, fd, &sink, pending, NULL);
|
||||
}
|
||||
|
||||
static void test_late_manifest_abort_frees_keepset() {
|
||||
@@ -797,7 +797,7 @@ static void test_receiver_pending_commits_missing_args() {
|
||||
/* NULL pending: the single-threaded commit path deletes at FINISHED. The
|
||||
sink sends the terminal STATUS_OK success frame. */
|
||||
ReceiverSink sink = {.send_success = true};
|
||||
EXPECT_EQ_INT(receiver_process_pending(cfg, p[0], &sink, NULL), 0);
|
||||
EXPECT_EQ_INT(receiver_process_pending(cfg, p[0], &sink, NULL, NULL), 0);
|
||||
Status ack;
|
||||
EXPECT_TRUE(receive_status(p[1], &ack));
|
||||
EXPECT_EQ_INT(ack, STATUS_OK);
|
||||
|
||||
Reference in New Issue
Block a user