TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: implement rsync delete timing (--delete-before/--delete-during/--delete-delay/--delete-after)

Browse Source 
Deletion timing is now real and selected by the four rsync flags plus the
plain --delete default. Wire protocol bumps to 2.8.0: two new config
booleans (delete_during, delete_delay) are serialized and validated, joining
the existing delete_before/delete_after.

- Early modes (--delete-before, --delete-during/--del): the sender pre-scans
  the whole tree (paths only), transmits the keep-set manifest BEFORE any
  file data, and the receiver removes extras and acks STATUS_OK; the sender
  only streams data after the deletion committed. Deletion is thus performed
  even if a later transfer phase fails (rsync delete-before/during are
  destructive by definition). FastSync streams in a single scan so it cannot
  interleave per-directory like rsync delete-during; --delete-during selects
  the same engine mode as --delete-before (documented divergence).
- Late/commit modes (plain --delete, --delete-after, --delete-delay): the
  manifest closes the data stream and deletion is committed only after
  STATUS_FINISHED proves the whole transfer succeeded, preserving FastSync's
  commit-style safety. --delete-delay converges with --delete-after because
  FastSync never snapshots the destination during data flow (documented).
- The STATUS_MANIFEST frame is now self-delimiting and position-independent.
  Single-threaded receivers delete before the success frame; the -m receiver
  hands the keep-set to server.c, which commits the deletion only after the
  disk writer thread has drained (fixes a delete-vs-in-flight-temp race).
- Every timing flag implies --delete; at most one timing flag is allowed.
- Each timing flag implies --delete, matching rsync; conflicts are rejected.
This commit is contained in:
TapTap
2026-09-06 18:53:20 +02:00
parent 4cf34026e7
commit 4e725517f0
14 changed files with 370 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/client_cli.c
+14 -14
View File
@@ -370,7 +370,6 @@ typedef enum {
OPT_POS_INT,
OPT_NONNEG_INT,
OPT_ULL,
OPT_UNSUPPORTED,
} OptKind;
typedef struct {
@@ -430,7 +429,10 @@ static const OptionEntry OPTION_TABLE[] = {
{"--old-d", NULL, OPT_FLAG, offsetof(Config, dirs)},
{"--relative", "-R", OPT_FLAG, offsetof(Config, relative)},
{"--mkpath", NULL, OPT_FLAG, offsetof(Config, mkpath)},
{"--delete-during", "--del", OPT_UNSUPPORTED, 0},
{"--delete-before", NULL, OPT_FLAG, offsetof(Config, delete_before)},
{"--delete-during", "--del", OPT_FLAG, offsetof(Config, delete_during)},
{"--delete-delay", NULL, OPT_FLAG, offsetof(Config, delete_delay)},
{"--delete-after", NULL, OPT_FLAG, offsetof(Config, delete_after)},
{"--source-dir", NULL, OPT_STRING, offsetof(Config, send_directory)},
{"--dest-dir", NULL, OPT_STRING, offsetof(Config, receive_root_directory)},
@@ -547,8 +549,7 @@ static int apply_negation(Config* config, const char* arg) {
return 0;
}
static int apply_table_option(Config* config, const OptionEntry* entry, const char* option_name,
const char* value) {
static int apply_table_option(Config* config, const OptionEntry* entry, const char* value) {
if (entry->kind == OPT_NOOP)
return 0;
void* field = (char*)config + entry->offset;
@@ -578,11 +579,6 @@ static int apply_table_option(Config* config, const OptionEntry* entry, const ch
*(unsigned long long*)field = v;
return 0;
}
case OPT_UNSUPPORTED: {
const char* reason = "delete-during is not implemented";
log_message(LOG_LEVEL_ERROR, "%s: %s; refusing to ignore option", option_name, reason);
return -1;
}
}
return -1;
}
@@ -665,10 +661,8 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
if (!entry)
entry = find_table_option_with_equals(argv[i], &inline_value);
if (entry) {
const char* option_name = argv[i];
const char* value = NULL;
if (entry->kind != OPT_FLAG) {
if (entry->kind != OPT_UNSUPPORTED) {
value = inline_value;
if (!value && i + 1 < argc)
value = argv[++i];
@@ -676,12 +670,11 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
log_message(LOG_LEVEL_ERROR, "missing argument for %s", entry->name);
return -1;
}
}
if (strcmp(entry->name, "--compress-choice") == 0) {
if (set_compression_choice(config, value) != 0)
return -1;
} else {
if (apply_table_option(config, entry, option_name, value) != 0)
if (apply_table_option(config, entry, value) != 0)
return -1;
if (strcmp(entry->name, "--compress-level") == 0 &&
(config->compression_level < 1 || config->compression_level > 22)) {
@@ -697,11 +690,18 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
config->use_metadata = true;
}
}
} else if (apply_table_option(config, entry, option_name, NULL) != 0) {
} else if (apply_table_option(config, entry, NULL) != 0) {
return -1;
}
if (entry->offset == offsetof(Config, eight_bit_output))
protocol_set_8_bit_output(true);
/* A delete-timing flag selects when --delete removes extras, so it
implies --delete exactly like the rsync options do. */
if (entry->offset == offsetof(Config, delete_before) ||
entry->offset == offsetof(Config, delete_during) ||
entry->offset == offsetof(Config, delete_delay) ||
entry->offset == offsetof(Config, delete_after))
config->use_delete = true;
continue;
}
src/client/client_send.c
+125 -17
View File
@@ -254,10 +254,6 @@ static void disconnect_transfer_client(Client* client) {
client_delete(client);
}
static ArrayList* create_transfer_manifest(const Config* config) {
return config->use_delete ? array_list_create(free) : NULL;
}
static bool add_chunk_to_manifest(ArrayList* manifest, const Chunk* chunk) {
if (!manifest)
return true;
@@ -597,6 +593,53 @@ static int send_delete_manifest(int fd, ArrayList* manifest) {
return 0;
}
/* Transmit the keep-set manifest and wait for the receiver's verdict. Used by
--delete-before/--delete-during, where the extras are removed on the receiver
BEFORE the first byte of file data is sent: the receiver acknowledges with
STATUS_OK once the bounded delete committed, or STATUS_ERROR if it could not
(in which case the sender aborts without streaming any data). */
static bool send_delete_manifest_early(Client* client, ArrayList* manifest) {
if (!client || !manifest)
return false;
if (send_delete_manifest(client->file_descriptor, manifest) != 0)
return false;
Status ack;
if (!receive_status(client->file_descriptor, &ack))
return false;
if (ack != STATUS_OK) {
log_message(LOG_LEVEL_ERROR, "Server failed to delete files before the transfer");
return false;
}
return true;
}
/* Walk the whole source tree once collecting only destination-relative wire
paths, loading and sending nothing. --delete-before/--delete-during need the
complete keep-set manifest before the first data byte, so it is built by a
dedicated pre-scan pass and transmitted early; the data pass then re-scans
with a fresh scanner. */
static bool scan_paths_only(const Config* config, const ScannerOptions* options,
ArrayList* manifest) {
DirectoryScanner* scanner =
directory_scanner_create_with_options(config->send_directory, options);
if (!scanner)
return false;
bool ok = true;
Chunk* chunk;
while ((chunk = directory_scanner_next(scanner)) != NULL) {
if (!add_chunk_to_manifest(manifest, chunk)) {
ok = false;
chunk_destroy(chunk);
break;
}
chunk_destroy(chunk);
}
if (ok && directory_scanner_failed(scanner))
ok = false;
directory_scanner_destroy(scanner);
return ok;
}
static int incremental_check(Client* client, File* file, const Config* config,
DeltaSignature** out_sig) {
*out_sig = NULL;
@@ -906,6 +949,17 @@ static int send_chunks_multithreaded(void* pipeline_context) {
protocol_session_unbind();
return thrd_error;
}
if (context->early_delete) {
/* The keep-set manifest was prebuilt by a path-only pre-scan. Transmit it
and wait for the receiver to delete extras before streaming any data. */
if (!send_delete_manifest_early(client, context->manifest)) {
pipeline_cancel(context);
disconnect_transfer_client(client);
mark_sender_done(context);
protocol_session_unbind();
return thrd_error;
}
}
while (true) {
Chunk* current_chunk = queue_dequeue_multithreaded(
@@ -919,7 +973,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
protocol_session_unbind();
return thrd_error;
}
if (context->config->use_delete) {
if (context->config->use_delete && !context->early_delete) {
if (send_delete_manifest(client->file_descriptor, context->manifest) != 0)
goto send_fail;
}
@@ -1013,7 +1067,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
failed = dirs_mode ? directory_scanner_failed(dscanner) : parallel_scanner_failed(scanner);
break;
}
if (context->config->use_delete) {
if (context->config->use_delete && !context->early_delete) {
mtx_lock(&context->mutex_scanner);
bool manifest_ok = add_chunk_to_manifest(context->manifest, current_chunk);
mtx_unlock(&context->mutex_scanner);
@@ -1172,19 +1226,46 @@ int send_files(Config* config) {
DirectoryScanner* scanner = NULL;
ArrayList* manifest = NULL;
ArrayList* remove_sources = NULL;
bool delete_early = config->use_delete && config_delete_timing_early(config);
bool send_failed = false;
PreparedScanner prepared;
memset(&prepared, 0, sizeof(prepared));
if (!config_send(client->file_descriptor, config))
goto send_fail;
if (!prepare_scanner(config, 0, &prepared))
goto send_fail;
scanner = directory_scanner_create_with_options(config->send_directory, &prepared.options);
manifest = create_transfer_manifest(config);
if (config->remove_source_files)
remove_sources = array_list_create(source_file_destroy);
if (!scanner || (config->use_delete && !manifest) ||
(config->remove_source_files && !remove_sources))
if (config->remove_source_files && !remove_sources)
goto send_fail;
/* The late-timing modes (plain --delete / --delete-after / --delete-delay)
build the manifest while streaming and send it after the last data frame.
The early modes (--delete-before/--delete-during) send it up front from a
dedicated path-only pre-scan, so no manifest is kept during the data pass. */
if (delete_early) {
/* Pass 1: collect the complete keep-set (paths only, no data loaded) and
transmit it now, before any file data. The receiver removes extras and
acks; the transfer aborts here if the deletion could not commit. */
ArrayList* early_manifest = array_list_create(free);
if (!early_manifest)
goto send_fail;
if (!scan_paths_only(config, &prepared.options, early_manifest)) {
array_list_delete(early_manifest);
goto send_fail;
}
bool early_ok = send_delete_manifest_early(client, early_manifest);
array_list_delete(early_manifest);
if (!early_ok)
goto send_fail;
} else if (config->use_delete) {
manifest = array_list_create(free);
if (!manifest)
goto send_fail;
}
scanner = directory_scanner_create_with_options(config->send_directory, &prepared.options);
if (!scanner)
goto send_fail;
Chunk* current_chunk;
unsigned long long total_bytes = 0;
int total_files = 0;
@@ -1196,7 +1277,7 @@ int send_files(Config* config) {
chunk_bytes += current_chunk->items[i]->data->size;
total_files++;
}
if (!add_chunk_to_manifest(manifest, current_chunk)) {
if (manifest && !add_chunk_to_manifest(manifest, current_chunk)) {
chunk_destroy(current_chunk);
goto send_fail;
}
@@ -1220,9 +1301,7 @@ int send_files(Config* config) {
if (send_chunk_with_removal(client, current_chunk, config, remove_sources) != 0) {
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
chunk_destroy(current_chunk);
if (manifest)
array_list_delete(manifest);
manifest = NULL;
send_failed = true;
break;
}
total_bytes += chunk_bytes;
@@ -1235,9 +1314,18 @@ int send_files(Config* config) {
}
chunk_destroy(current_chunk);
}
if (directory_scanner_failed(scanner) || (config->use_delete && manifest == NULL))
if (send_failed) {
if (manifest) {
array_list_delete(manifest);
manifest = NULL;
}
goto send_fail;
if (config->use_delete) {
}
if (directory_scanner_failed(scanner))
goto send_fail;
if (manifest) {
/* Late (commit) ordering: all file data is out; transmit the keep-set
manifest so the receiver deletes only after the transfer succeeds. */
if (send_delete_manifest(client->file_descriptor, manifest) != 0) {
array_list_delete(manifest);
manifest = NULL;
@@ -1324,8 +1412,28 @@ int send_files_multithreaded(Config** config_ptr) {
return 1;
}
*config_ptr = NULL; /* context now owns config through all remaining paths */
if (config->use_delete)
if (config->use_delete) {
context->manifest = array_list_create(free);
if (!context->manifest) {
pipeline_context_sender_destroy(context);
return 1;
}
if (config_delete_timing_early(config)) {
/* --delete-before/--delete-during: build the complete keep-set manifest
(paths only, nothing loaded or sent) up front so the sender thread can
transmit it before the first data byte. */
PreparedScanner prepared;
memset(&prepared, 0, sizeof(prepared));
bool prebuilt = prepare_scanner(config, 4, &prepared) &&
scan_paths_only(config, &prepared.options, context->manifest);
prepared_scanner_destroy(&prepared);
if (!prebuilt) {
pipeline_context_sender_destroy(context);
return 1;
}
context->early_delete = true;
}
}
if (config->remove_source_files)
context->remove_source_files = array_list_create(source_file_destroy);
if ((config->use_delete && !context->manifest) ||
src/client/client_validation.c
+6
View File
@@ -71,5 +71,11 @@ bool validate_config(const Config* config) {
"staging directory)");
return false;
}
if (!config_has_valid_delete_timing(config)) {
log_message(LOG_LEVEL_ERROR,
"--delete-before/--delete-during/--delete-delay/--delete-after select the delete "
"timing; at most one may be given and each implies --delete");
return false;
}
return true;
}
src/client/usage.c
+10 -1
View File
@@ -24,6 +24,16 @@ void print_usage(void) {
printf(" -P Partial mode with progress (retention incomplete)\n");
printf(" -8, --8-bit-output Leave high-bit characters unescaped in output\n");
printf(" --delete Delete files on receiver not in source\n");
printf(" (default timing: delete only after the whole\n");
printf(" transfer has succeeded)\n");
printf(" --delete-before Delete extras before the transfer starts\n");
printf(" (implies --delete)\n");
printf(" --delete-during Delete extras once the keep-set is known, before\n");
printf(" --del data is applied (alias --del; implies --delete)\n");
printf(" --delete-delay Delete extras only after a successful transfer\n");
printf(" (implies --delete)\n");
printf(" --delete-after Alias of the default --delete timing: delete only\n");
printf(" after the transfer succeeded (implies --delete)\n");
printf(" --ignore-existing Skip files that already exist on receiver\n");
printf(" --delay-updates Put updated files into place only at the end of transfer\n");
printf(" --dirs, -d, --old-dirs, --old-d Transfer the named directory entries without\n");
@@ -37,7 +47,6 @@ void print_usage(void) {
printf(" parent directory is not itself listed\n");
printf(" --mkpath Create the destination root directory on the server when it\n");
printf(" does not exist yet\n");
printf(" --del Alias for --delete-during (not implemented)\n");
printf(" --exclude <pattern> Exclude files matching pattern\n");
printf(" --include <pattern> Only include files matching pattern\n");
printf(" --exclude-from <file> Read exclude patterns from file\n");
src/server/receiver.c
+76 -6
View File
@@ -129,16 +129,33 @@ static bool receiver_process_batch(Config* config, int file_descriptor) {
}
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink) {
return receiver_process_pending(config, file_descriptor, sink, NULL);
}
/* Runs the whole receive loop. The delete manifest may legitimately arrive
either FIRST (--delete-before / --delete-during: the sender transmits the
validated keep-set before any file data) or LAST (plain --delete /
--delete-after / --delete-delay: the manifest closes the data stream). In
the early modes the receiver deletes as soon as the manifest has been read
and acknowledges with STATUS_OK so the sender only starts streaming once the
deletion has committed (or failed); in the late modes the manifest is held
and the deletion is committed only after the terminal STATUS_FINISHED proves
the whole transfer succeeded. See receiver_process_pending() for how the -m
receiver defers that commit until its disk writer has drained. */
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
ArrayList** pending_manifest) {
Status status;
if (!receive_status(file_descriptor, &status))
return -1;
bool early_delete = config_delete_timing_early(config);
ArrayList* deferred_manifest = NULL;
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK ||
status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH ||
status == STATUS_MKDIR) {
status == STATUS_MKDIR || status == STATUS_MANIFEST) {
if (status == STATUS_KEEPALIVE) {
if (!send_status(file_descriptor, STATUS_KEEPALIVE))
return -1;
goto next;
goto next_status;
}
if (status == STATUS_ABORT) {
log_message(LOG_LEVEL_INFO, "Received abort from client, cleaning up");
@@ -156,11 +173,45 @@ int receiver_process(Config* config, int file_descriptor, const ReceiverSink* si
} else if (status == STATUS_CHECK_BATCH) {
if (!receiver_process_batch(config, file_descriptor))
return -1;
goto next;
goto next_status;
} else if (status == STATUS_MKDIR) {
File* dir = file_receive_directory(file_descriptor);
if (!dir || !sink->store_file(dir, sink->context))
goto receive_error;
} else if (status == STATUS_MANIFEST) {
ArrayList* manifest = receive_manifest_entries(file_descriptor);
if (!manifest)
return -1; /* receive_manifest_entries already sent STATUS_ERROR */
if (early_delete) {
/* --delete-before / --delete-during: the manifest is authoritative the
moment it arrives, before any file data. Delete now and acknowledge
so the sender only starts streaming once the deletion committed (or
failed). This is the rsync delete-before/delete-during window: a
later transfer failure does not restore these deletions. */
bool deletion_ok = config->use_delete ? manifest_delete_extras(config, manifest) : true;
array_list_delete(manifest);
if (!deletion_ok) {
send_status(file_descriptor, STATUS_ERROR);
return -1;
}
if (!send_status(file_descriptor, STATUS_OK))
return -1;
} else {
/* Plain --delete / --delete-after / --delete-delay: hold the keep-set
and commit the deletion only after STATUS_FINISHED. */
if (config->use_delete) {
if (deferred_manifest) {
log_message(LOG_LEVEL_ERROR, "Received a second delete manifest");
array_list_delete(manifest);
send_status(file_descriptor, STATUS_ERROR);
return -1;
}
deferred_manifest = manifest;
} else {
array_list_delete(manifest);
}
}
goto next_status;
} else {
File* file = file_receive(config, file_descriptor);
if (!file) {
@@ -170,16 +221,35 @@ int receiver_process(Config* config, int file_descriptor, const ReceiverSink* si
if (!sink->store_file(file, sink->context))
goto receive_error;
}
next:
next_status:
if (!receive_status(file_descriptor, &status))
goto receive_error;
}
if (status == STATUS_MANIFEST && receive_manifest(file_descriptor, config, &status) != 0)
return -1;
if (status != STATUS_FINISHED) {
log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status");
goto receive_error;
}
/* Commit-style (late) deletion: every data frame has been received and the
sender proved the whole tree with STATUS_FINISHED. The single-threaded
receiver stores files synchronously, so everything is on disk here and the
deletion can be committed before the --delay-updates publication in
send_success (the walker skips the staging dir, so staged files are never
treated as extras). The -m receiver passes `pending_manifest` because its
disk writer may still be draining; the caller commits after the writer has
joined so no extra file is removed unless the transfer is known to have
succeeded. */
if (deferred_manifest) {
if (pending_manifest) {
*pending_manifest = deferred_manifest;
} else {
bool deletion_ok = manifest_delete_extras(config, deferred_manifest);
array_list_delete(deferred_manifest);
if (!deletion_ok) {
send_status(file_descriptor, STATUS_ERROR);
return -1;
}
}
}
if (sink->send_success) {
if (sink->send_success_frame) {
if (!sink->send_success_frame(file_descriptor, sink->context))
src/server/receiver.h
+8
View File
@@ -35,6 +35,14 @@ void receiver_outcomes_destroy(ReceiverOutcomes* outcomes);
bool receiver_send_final_success(int fd, const Config* config, const ReceiverOutcomes* outcomes);
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink);
/* receiver_process with an escape hatch for the commit-style (late) deletion:
when `pending_manifest` is non-NULL the receiver does NOT delete at
STATUS_FINISHED itself; instead it stores the owned keep-set manifest there
(leaving *pending_manifest untouched on early modes/errors) so the caller can
commit the deletion only after its disk writer has fully drained. Pass NULL
to keep the default behaviour (delete before the success frame). */
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
ArrayList** pending_manifest);
int receiver_receive_files(Config* config, int file_descriptor);
#endif
src/server/server.c
+14
View File
@@ -255,6 +255,20 @@ void handler(int file_descriptor) {
thrd_join(receiver, &receiver_result);
thrd_join(writer, &writer_result);
bool transfer_ok = receiver_result == thrd_success && writer_result == thrd_success;
if (transfer_ok) {
/* Commit-style (late) deletion: receive_thread handed the keep-set
manifest here instead of deleting while write_thread might still be
draining, so by now every file is on disk and the whole transfer is
known to have succeeded. Remove the extras before publishing a
--delay-updates run; the walker skips the staging directory. */
if (context->deferred_manifest) {
if (!manifest_delete_extras(config, context->deferred_manifest)) {
transfer_ok = false;
}
array_list_delete(context->deferred_manifest);
context->deferred_manifest = NULL;
}
}
if (transfer_ok) {
/* --delay-updates: receive_thread has finished the whole protocol stream
(including manifest/delete handling) and write_thread has drained its
src/shared/config.c
+29 -2
View File
@@ -115,6 +115,8 @@ static void config_set_defaults(Config* config) {
config->partial_dir = NULL;
config->suffix = NULL;
config->delete_before = false;
config->delete_during = false;
config->delete_delay = false;
config->address = NULL;
config->bind_address = NULL;
config->ipv6 = false;
@@ -161,12 +163,14 @@ static bool validate_received_config(const Config* config) {
valid_wire_bool(config->inplace) && valid_wire_bool(config->append) &&
valid_wire_bool(config->use_fsync) && valid_wire_bool(config->append_verify) &&
valid_wire_bool(config->delete_excluded) && valid_wire_bool(config->delete_after) &&
valid_wire_bool(config->delete_delay) && valid_wire_bool(config->delete_during) &&
valid_wire_bool(config->relative) && valid_wire_bool(config->prune_empty_dirs) &&
valid_wire_bool(config->delay_updates) && valid_wire_bool(config->mkpath) &&
!(config->delay_updates && config->inplace) &&
!(config->delay_updates && delay_updates_staging_name_conflict(config->backup_dir)) &&
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
config_has_valid_delete_timing(config) &&
!(config->skip_compress_set && config->use_chunk_serialization) &&
(!config->use_compression ||
(config->compression_level >= 1 && config->compression_level <= 22)) &&
@@ -188,6 +192,26 @@ Config* config_create(void) {
return config;
}
bool config_delete_timing_early(const Config* config) {
if (!config)
return false;
return config->delete_before || config->delete_during;
}
/* A delete-timing flag is only meaningful together with --delete. At most one
of the four flags may be set; several simultaneous timings are a client bug
and are rejected on both ends. */
bool config_has_valid_delete_timing(const Config* config) {
if (!config)
return false;
if (!config->use_delete)
return !config->delete_before && !config->delete_during && !config->delete_delay &&
!config->delete_after;
int timing_count = (config->delete_before ? 1 : 0) + (config->delete_during ? 1 : 0) +
(config->delete_delay ? 1 : 0) + (config->delete_after ? 1 : 0);
return timing_count <= 1;
}
bool config_is_remote_dest(const char* s) {
if (s == NULL)
return false;
@@ -311,7 +335,8 @@ static bool send_selection_options(int fd, const Config* c) {
send_int(fd, c->use_fsync) && send_int(fd, c->append_verify) &&
send_int(fd, c->delete_excluded) && send_int(fd, c->delete_after) &&
send_n_data(fd, &c->max_delete, sizeof(c->max_delete)) && send_int(fd, c->relative) &&
send_int(fd, c->prune_empty_dirs) && send_int(fd, c->mkpath);
send_int(fd, c->prune_empty_dirs) && send_int(fd, c->mkpath) &&
send_int(fd, c->delete_during) && send_int(fd, c->delete_delay);
}
static bool send_skip_compress_options(int fd, const Config* c) {
@@ -418,7 +443,9 @@ static bool receive_selection_options(int fd, Config* c) {
return false;
if (!receive_wire_bool(fd, &c->mkpath))
return false;
return true;
if (!receive_wire_bool(fd, &c->delete_during))
return false;
return receive_wire_bool(fd, &c->delete_delay);
}
static bool receive_resume_options(int fd, Config* c) {
src/shared/config.h
+26 -1
View File
@@ -147,6 +147,19 @@ typedef struct Config {
// PR #179: Delete policies
bool delete_before;
/* rsync deletion-timing family (real from Phase 3). At most one of
delete_before / delete_during / delete_delay / delete_after may be set, and
only together with use_delete (the CLI implies --delete for each of them).
delete_before and delete_during select the EARLY engine mode: the keep-set
manifest is transmitted before any file data and extras are removed then,
acknowledged, before the first data byte. delete_delay and delete_after
select the LATE commit mode: extras are removed only after the whole
transfer has succeeded (plain --delete keeps this mode). The exact
semantics and the divergences from rsync are documented in RSYNC_COMPAT.md
and in config_delete_timing_early() below. */
bool delete_during;
bool delete_delay;
// PR #181: IPv6 and bind address
char* address;
char* bind_address;
@@ -174,7 +187,7 @@ typedef struct Config {
DelayUpdatesContext* delay_context;
} Config;
#define PROTOCOL_VERSION "2.7.0"
#define PROTOCOL_VERSION "2.8.0"
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
Config* config_create(void);
@@ -184,4 +197,16 @@ Config* config_receive(int file_descriptor);
bool config_is_remote_dest(const char* s);
void config_parse_ssh_dest(Config* config);
/* True when the negotiated delete timing performs the extra-file deletion
* BEFORE the transfer data (--delete-before / --delete-during). The flag is
* a pure function of the config and is used identically on the sender (to pick
* the manifest-first frame order) and the receiver (to delete when the early
* manifest arrives). When false the deletion is committed only after the whole
* transfer succeeded (--delete / --delete-after / --delete-delay). */
bool config_delete_timing_early(const Config* config);
/* Delete-timing sanity: with deletion enabled at most one timing flag may be
* set (none = the default delete-after commit timing); without deletion no
* timing flag may be set (each timing flag implies --delete). */
bool config_has_valid_delete_timing(const Config* config);
#endif
src/shared/file_receive.c
+24 -33
View File
@@ -792,26 +792,27 @@ File* file_receive_directory(int file_descriptor) {
return file;
}
int receive_manifest(int fd, const Config* config, int* next_status) {
if (!config) {
send_status(fd, STATUS_ERROR);
return -1;
}
int received_status = STATUS_ERROR;
int* status_out = next_status ? next_status : &received_status;
/* Read a delete-manifest frame (the STATUS_MANIFEST leading code has already
been consumed): an entry count followed by that many destination-relative
paths. The frame is self-delimiting (the count is authoritative), so the
caller decides what to do next and continues reading the following STATUS_*
frame. Returns an owned ArrayList of validated path strings, or NULL after
sending STATUS_ERROR when the frame is malformed (bad count, empty/absolute
path, path traversal, or an aggregate size beyond MAX_MANIFEST_BYTES). */
ArrayList* receive_manifest_entries(int fd) {
int count;
if (!receive_int(fd, &count)) {
send_status(fd, STATUS_ERROR);
return -1;
return NULL;
}
if (count < 0 || count > MAX_MANIFEST_ENTRIES) {
send_status(fd, STATUS_ERROR);
return -1;
return NULL;
}
ArrayList* manifest = array_list_create(free);
if (!manifest) {
send_status(fd, STATUS_ERROR);
return -1;
return NULL;
}
size_t manifest_bytes = 0;
for (int i = 0; i < count; i++) {
@@ -823,32 +824,22 @@ int receive_manifest(int fd, const Config* config, int* next_status) {
free(s);
array_list_delete(manifest);
send_status(fd, STATUS_ERROR);
return -1;
return NULL;
}
}
if (!receive_status(fd, status_out)) {
array_list_delete(manifest);
send_status(fd, STATUS_ERROR);
return -1;
}
/* Deletion is a commit operation: never perform it until the sender has
completed the manifest frame successfully. */
if (*status_out != STATUS_FINISHED || !config->use_delete) {
array_list_delete(manifest);
if (*status_out != STATUS_FINISHED)
send_status(fd, STATUS_ERROR);
return *status_out == STATUS_FINISHED ? 0 : -1;
return manifest;
}
/* Remove every destination entry under the receive root that is not listed in
`manifest`, bounded by MAX_SERVER_DELETE_COUNT, using the symlink-safe
delete walker. With --delay-updates the not-yet-published staging directory
is a direct child of the receive root and must not be treated as a set of
extras. Prints a notice and returns true on success. */
bool manifest_delete_extras(const Config* config, ArrayList* manifest) {
if (!config || !manifest)
return false;
fprintf(stderr, "Deleting files not in manifest...\n");
/* With --delay-updates the staged (not yet published) files live directly
under the receive root in the staging directory; the delete walker must
not treat them as extras or it would remove every staged file before it
can be published. */
const char* skip_staging = config->delay_updates ? DELAY_UPDATES_STAGING_DIR : NULL;
bool deletion_ok = delete_extras_limited(config->receive_root_directory, manifest,
MAX_SERVER_DELETE_COUNT, skip_staging);
array_list_delete(manifest);
if (!deletion_ok)
send_status(fd, STATUS_ERROR);
return deletion_ok ? 0 : -1;
return delete_extras_limited(config->receive_root_directory, manifest, MAX_SERVER_DELETE_COUNT,
skip_staging);
}
src/shared/file_receive.h
+8 -1
View File
@@ -10,7 +10,14 @@
File* file_receive(const Config* config, int file_descriptor);
File* file_receive_directory(int file_descriptor);
File* receive_incremental_check(int fd, const Config* config, bool* skipped);
int receive_manifest(int fd, const Config* config, int* next_status);
/* Read a delete-manifest frame: entry count then paths (self-delimiting; the
leading STATUS_MANIFEST code has been consumed). Returns an owned path
ArrayList, or NULL after signalling STATUS_ERROR on a malformed frame. */
ArrayList* receive_manifest_entries(int fd);
/* Remove destination entries under config->receive_root_directory that are not
in `manifest` (bounded walk, staging-dir skip). The caller decides WHEN to
run it based on the negotiated delete timing. */
bool manifest_delete_extras(const Config* config, ArrayList* manifest);
/* Outcome of a single file_save_to_disk operation. The receiver needs to
distinguish "written" from "skipped" so --remove-source-files can be told
src/shared/multiprocessing.c
+6 -1
View File
@@ -27,6 +27,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
context->loader_done = false;
context->manifest = NULL;
context->remove_source_files = NULL;
context->early_delete = false;
context->total_files = 0;
context->progress_bytes = 0;
context->total_bytes = 0;
@@ -113,6 +114,7 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
context->receiver_done = false;
context->queued_bytes = 0;
context->max_queue_bytes = 0;
context->deferred_manifest = NULL;
atomic_init(&context->cancelled, false);
int init = 0;
if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
@@ -141,6 +143,8 @@ fail:
void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
config_delete(context->config);
if (context->deferred_manifest)
array_list_delete(context->deferred_manifest);
queue_destroy(context->queue);
receiver_outcomes_destroy(&context->outcomes);
mtx_destroy(&context->mutex);
@@ -236,7 +240,8 @@ int receive_thread(void* pipeline_context) {
mtx_unlock(&context->mutex);
ReceiverSink sink = {receiver_enqueue_file, context, false, false, NULL};
if (receiver_process((Config*)config, file_descriptor, &sink) != 0) {
if (receiver_process_pending((Config*)config, file_descriptor, &sink,
&context->deferred_manifest) != 0) {
receiver_thread_fail(context);
protocol_session_unbind();
return thrd_error;
src/shared/multiprocessing.h
+13
View File
@@ -26,6 +26,11 @@ typedef struct {
bool loader_done;
ArrayList* manifest;
ArrayList* remove_source_files;
/* True when --delete-before/--delete-during require the keep-set manifest to
be transmitted before any file data: context->manifest is then prebuilt by
a path-only pre-scan on the calling thread and the pipeline scanner must
not append to it. Set once before the worker threads start. */
bool early_delete;
mtx_t mutex_progress;
int total_files;
unsigned long long progress_bytes;
@@ -55,6 +60,14 @@ typedef struct PipelineContextReceiver {
budget instead of growing without bound. */
size_t queued_bytes;
size_t max_queue_bytes;
/* Keep-set manifest for the commit-style (late) deletion
(--delete/--delete-after/--delete-delay). receive_thread parses the whole
protocol stream but hands the manifest here instead of deleting while the
disk writer may still be draining; the caller (server.c) commits the
deletion after both threads have joined, so no extra is removed unless the
transfer truly succeeded. NULL in the early delete modes (which delete at
the manifest). */
ArrayList* deferred_manifest;
} PipelineContextReceiver;
PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* queue_scanner,
tests/test_server.c
+4 -1
View File
@@ -180,7 +180,10 @@ static void test_receive_manifest_rejects_traversal() {
io_set_fds(p[0], p[1]);
EXPECT_TRUE(send_int(p[1], 1));
EXPECT_TRUE(send_str(p[1], "../outside"));
EXPECT_EQ_INT(receive_manifest(p[0], cfg, NULL), -1);
EXPECT_NULL(receive_manifest_entries(p[0]));
Status status;
EXPECT_TRUE(receive_status(p[1], &status));
EXPECT_EQ_INT(status, STATUS_ERROR);
close(p[0]);
close(p[1]);
config_delete(cfg);