feat(protocol): destination-state itemize for dirs/symlinks; --stats deleted breakdown

This commit is contained in:
2026-09-23 21:52:40 +02:00
parent cb2979fdf1
commit 29f8be161c
20 changed files with 761 additions and 99 deletions
+16
View File
@@ -19,6 +19,22 @@ Wire backlog cycle (protocol 2.29.0 → 2.30.0; config-frame layout unchanged).
sends the terminal `STATUS_PARTIAL`; the client exits 23 like rsync and, under sends the terminal `STATUS_PARTIAL`; the client exits 23 like rsync and, under
`--remove-source-files`, still removes the sources it successfully `--remove-source-files`, still removes the sources it successfully
transferred. A clean run stays 0 and a fatal/connection error stays non-23. transferred. A clean run stays 0 and a fatal/connection error stays non-23.
- **Directory/symlink destination-state itemize (#314):** when `report_dest_info`
is negotiated (now also for `--progress`), the receiver answers `STATUS_MKDIR`
and `STATUS_SYMLINK` with the entry's pre-transfer destination snapshot
(existence, type, perms/owner/group/time, and whether an existing symlink's
target already matches), and the sender probes every ancestor directory before
the receiver creates it implicitly. A re-run over an unchanged tree no longer
emits per-directory `cd+++++++++` or unchanged-symlink lines, a changed
directory renders rsync's `.d..t......`, and a changed symlink renders
`cLc........` / `.L..t......`. Directory/symlink time comparison uses whole
seconds (rsync's `cmp_time`). The `STATUS_MKDIR` body gains a probe flag and
the `STATUS_DEST_INFO` record gains a symlink-target-match field; the
config-frame layout is unchanged. Differential-tested against rsync 3.4.1.
- **`--stats` deleted per-type breakdown (#316):** `STATUS_STATS` gains
`deleted_reg/dir/link/special`, tallied by the delete observers and rendered
as rsync's `Number of deleted files: X (reg: A, dir: B, link: C, special: D)`.
Differential-tested against rsync 3.4.1 for a mixed-type `--delete` tree.
## [2.29.0] - 2026-09-23 ## [2.29.0] - 2026-09-23
+5 -5
View File
File diff suppressed because one or more lines are too long
+67 -4
View File
@@ -123,8 +123,15 @@ static char itemize_type_char(const ChangeEvent* event) {
static bool times_match(const Config* config, const ChangeEvent* event) { static bool times_match(const Config* config, const ChangeEvent* event) {
if (!event->dest.known || !event->dest.existed) if (!event->dest.known || !event->dest.existed)
return false; return false;
if (event->mtime_sec == event->dest.mtime_sec) if (event->mtime_sec == event->dest.mtime_sec) {
/* A regular file's sub-second mtime IS preserved by the receiver, so an nsec
difference is a real change. A directory or symlink has no preserved
sub-second mtime (rsync's quick-check compares whole seconds there), so a
nanosecond-only difference must not render a spurious `.d..t` / `.L..t`. */
if (event->is_directory || event->is_symlink || event->is_special)
return true;
return event->mtime_nsec == event->dest.mtime_nsec; return event->mtime_nsec == event->dest.mtime_nsec;
}
long long delta = (long long)event->mtime_sec - (long long)event->dest.mtime_sec; long long delta = (long long)event->mtime_sec - (long long)event->dest.mtime_sec;
if (delta < 0) if (delta < 0)
delta = -delta; delta = -delta;
@@ -145,6 +152,10 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co
/* rsync: an existing directory that only has attribute changes carries no /* rsync: an existing directory that only has attribute changes carries no
transfer, so the update column is `.` rather than `>`. */ transfer, so the update column is `.` rather than `>`. */
update = '.'; update = '.';
else if (event->is_symlink)
/* rsync: an existing symlink whose target is unchanged is a `.` update
(attributes only); a changed target is `c` (the link value changed). */
update = event->dest.target_matches ? '.' : 'c';
else else
update = '>'; update = '>';
code[0] = update; code[0] = update;
@@ -155,12 +166,20 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co
code[11] = '\0'; code[11] = '\0';
return; return;
} }
bool size_diff = event->size != event->dest.size; /* rsync's value/checksum column: `c` for a symlink whose target changed (the
bool time_diff = !times_match(config, event); link value is the compared content); no destination digest is available for
a regular file. */
bool value_diff = event->is_symlink && !event->dest.target_matches;
/* rsync itemizes size only for regular files: a directory's st_size and a
symlink's target length are not compared. */
bool size_diff = !event->is_directory && !event->is_symlink && !event->is_special &&
event->size != event->dest.size;
/* rsync itemizes the time column only when -t/--times is in effect. */
bool time_diff = config->preserve_times && !times_match(config, event);
bool perms_diff = (event->mode & 07777) != (event->dest.mode & 07777); bool perms_diff = (event->mode & 07777) != (event->dest.mode & 07777);
bool owner_diff = event->uid != (uid_t)event->dest.uid; bool owner_diff = event->uid != (uid_t)event->dest.uid;
bool group_diff = event->gid != (gid_t)event->dest.gid; bool group_diff = event->gid != (gid_t)event->dest.gid;
code[2] = '.'; /* checksum: no destination digest available */ code[2] = value_diff ? 'c' : '.';
code[3] = size_diff ? 's' : '.'; code[3] = size_diff ? 's' : '.';
code[4] = time_diff ? 't' : '.'; code[4] = time_diff ? 't' : '.';
code[5] = (config->preserve_perms && perms_diff) ? 'p' : '.'; code[5] = (config->preserve_perms && perms_diff) ? 'p' : '.';
@@ -172,6 +191,24 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co
code[11] = '\0'; code[11] = '\0';
} }
/* True when the itemized destination entry is unchanged, i.e. rsync would print
* no line at all. Reuses itemize_code so suppression is exactly consistent
* with what would have been rendered: the update column must be `.` and every
* attribute column must be `.`. */
static bool itemize_is_unchanged(const Config* config, const ChangeEvent* event) {
if (!event->dest.known || !event->dest.existed)
return false;
char code[12];
itemize_code(config, event, code);
if (code[0] != '.')
return false;
for (int i = 2; i < 11; i++) {
if (code[i] != '.')
return false;
}
return true;
}
/* rsync %n: the transfer-relative name, with a trailing slash for directories. /* rsync %n: the transfer-relative name, with a trailing slash for directories.
* The transfer root is `.` (so `%n` renders `./`), matching rsync's root entry. */ * The transfer root is `.` (so `%n` renders `./`), matching rsync's root entry. */
static bool append_name(StrBuf* buf, const ChangeEvent* event) { static bool append_name(StrBuf* buf, const ChangeEvent* event) {
@@ -678,6 +715,19 @@ void change_emit_file_sent_bytes(const Config* config, const File* file,
char* name = NULL; char* name = NULL;
char* path = NULL; char* path = NULL;
fill_event_from_file(config, file, &event, &name, &path); fill_event_from_file(config, file, &event, &name, &path);
if (file->is_symlink) {
/* Output parity (protocol 2.30.0): an unchanged symlink is silent, like
rsync's quick check. The itemize/log stream suppresses it only when every
attribute matches; the name stream suppresses it whenever the link target
is unchanged (rsync names a symlink only when it relinks or creates it). */
bool itemize_output = config->itemize_changes || config->out_format != NULL ||
(config->log_file != NULL && config->log_file_format != NULL);
bool suppress = itemize_output
? itemize_is_unchanged(config, &event)
: (event.dest.known && event.dest.existed && event.dest.target_matches);
if (suppress)
event.decision = CHANGE_UP_TO_DATE;
}
if (name != NULL && path != NULL) { if (name != NULL && path != NULL) {
fill_event_checksum(config, file, &event); fill_event_checksum(config, file, &event);
change_emit(config, &event); change_emit(config, &event);
@@ -729,6 +779,19 @@ void change_emit_dir_sent(const Config* config, const File* file) {
char* name = NULL; char* name = NULL;
char* path = NULL; char* path = NULL;
fill_event_from_file(config, file, &event, &name, &path); fill_event_from_file(config, file, &event, &name, &path);
/* Output parity (protocol 2.30.0): suppress a directory rsync would leave
silent. The itemize/log stream suppresses it only when every attribute
matches (`.d.........`); the name stream suppresses any pre-existing
directory (rsync names a directory only when it is created). */
bool itemize_output = config->itemize_changes || config->out_format != NULL ||
(config->log_file != NULL && config->log_file_format != NULL);
bool suppress = itemize_output ? itemize_is_unchanged(config, &event)
: (event.dest.known && event.dest.existed);
/* The transfer root's line is an unconditional FastSync residual (rsync keys
it off the root's own attribute change); keep emitting it. */
bool is_root = event.name != NULL && event.name[0] == '\0';
if (suppress && !is_root)
event.decision = CHANGE_UP_TO_DATE;
if (name != NULL && path != NULL) if (name != NULL && path != NULL)
change_emit(config, &event); change_emit(config, &event);
free(name); free(name);
+4 -1
View File
@@ -2807,8 +2807,11 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool
* need the pre-transfer destination snapshot (new vs modified and which * need the pre-transfer destination snapshot (new vs modified and which
* attributes differ), so ask the receiver to report it on every per-file * attributes differ), so ask the receiver to report it on every per-file
* check. This is a wire field. */ * check. This is a wire field. */
bool progress_active =
!config->quiet && (config->show_progress || (config->info_level & LOG_INFO_PROGRESS) != 0);
config->report_dest_info = config->itemize_changes || config->out_format != NULL || config->report_dest_info = config->itemize_changes || config->out_format != NULL ||
(config->log_file != NULL && config->log_file_format != NULL); (config->log_file != NULL && config->log_file_format != NULL) ||
progress_active;
/* Wire-stats parity: --stats, --progress/-P, an --out-format token that needs /* Wire-stats parity: --stats, --progress/-P, an --out-format token that needs
* a wire counter (%b/%c), or a dry-run --delete need the receiver's * a wire counter (%b/%c), or a dry-run --delete need the receiver's
* end-of-transfer STATUS_STATS report. This is a wire field (protocol * end-of-transfer STATUS_STATS report. This is a wire field (protocol
+145 -4
View File
@@ -162,6 +162,9 @@ void report_transfer_stats(const Config* config, const TransferStats* stats, tim
created_breakdown, sizeof(created_breakdown)); created_breakdown, sizeof(created_breakdown));
unsigned long long created_total = unsigned long long created_total =
recv->created_reg + recv->created_dir + recv->created_link + recv->created_special; recv->created_reg + recv->created_dir + recv->created_link + recv->created_special;
char deleted_breakdown[128];
type_breakdown(recv->deleted_reg, recv->deleted_dir, recv->deleted_link, recv->deleted_special,
deleted_breakdown, sizeof(deleted_breakdown));
printf("\n"); printf("\n");
if (breakdown[0] != '\0') if (breakdown[0] != '\0')
printf("Number of files: %llu %s\n", flist_total, breakdown); printf("Number of files: %llu %s\n", flist_total, breakdown);
@@ -173,7 +176,13 @@ void report_transfer_stats(const Config* config, const TransferStats* stats, tim
printf("Number of created files: %llu %s\n", created_total, created_breakdown); printf("Number of created files: %llu %s\n", created_total, created_breakdown);
else else
printf("Number of created files: %llu\n", created_total); printf("Number of created files: %llu\n", created_total);
printf("Number of deleted files: %llu\n", recv->deleted_files); /* Protocol 2.30.0: the receiver reports the removed entries split by type, so
this line matches rsync's `Number of deleted files: X (reg: A, dir: B,
link: C, special: D)` (only the non-zero categories are listed). */
if (deleted_breakdown[0] != '\0')
printf("Number of deleted files: %llu %s\n", recv->deleted_files, deleted_breakdown);
else
printf("Number of deleted files: %llu\n", recv->deleted_files);
printf("Number of regular files transferred: %llu\n", stats->transferred_regular); printf("Number of regular files transferred: %llu\n", stats->transferred_regular);
printf("Total file size: %s bytes\n", total); printf("Total file size: %s bytes\n", total);
printf("Total transferred file size: %s bytes\n", transferred); printf("Total transferred file size: %s bytes\n", transferred);
@@ -526,9 +535,16 @@ static void client_progress_emit_ancestors(const Config* config, const char* rel
if (dir != NULL) if (dir != NULL)
change_emit_dir_sent(config, dir); change_emit_dir_sent(config, dir);
} else { } else {
char* escaped = output_escape(prefix, config->eight_bit_output); /* --progress/-P names a directory only when it is newly created;
printf("%s/\n", escaped ? escaped : prefix); rsync stays silent for a pre-existing directory even when one of
free(escaped); its children changed (protocol 2.30.0 dest-state report). */
const File* dir = progress_dir_lookup(prefix);
bool existed = dir != NULL && dir->dest_state.known && dir->dest_state.existed;
if (!existed) {
char* escaped = output_escape(prefix, config->eight_bit_output);
printf("%s/\n", escaped ? escaped : prefix);
free(escaped);
}
} }
g_progress_index++; g_progress_index++;
} else { } else {
@@ -553,6 +569,122 @@ void client_change_emit_ancestors(const Config* config, const File* file) {
client_progress_emit_ancestors(config, rel); client_progress_emit_ancestors(config, rel);
} }
/* Send one STATUS_MKDIR probe (probe=1) for a pre-count directory and cache the
* receiver's pre-transfer destination snapshot in `dir->dest_state`. Returns
* false on a protocol/transport error. */
static bool client_probe_dir_state(int fd, File* dir) {
if (dir == NULL || file_wire_path(dir) == NULL)
return true;
if (!send_status(fd, STATUS_MKDIR) || !send_int(fd, 1) || !send_wire_str(fd, file_wire_path(dir)))
return false;
Status status = STATUS_ERROR;
if (!receive_status(fd, &status) || status != STATUS_DEST_INFO ||
!format_dest_state_receive(fd, &dir->dest_state)) {
log_message(LOG_LEVEL_ERROR, "Directory destination-state probe failed");
return false;
}
return true;
}
/* Output parity (protocol 2.30.0): ask the receiver for each not-yet-probed
* ancestor directory's pre-transfer state BEFORE the entry that first triggers
* it is sent, so the ancestor's -i/--out-format line renders rsync's
* `.d..t......` (existing, attributes changed) versus `cd+++++++++` (created)
* and an unchanged directory is suppressed. The probe is a STATUS_MKDIR frame
* with probe=1 (the receiver reports and creates nothing), so it must run before
* the receiver implicitly creates the parent for the child. Each directory is
* probed at most once; the result is cached in the pre-count File's dest_state.
* Returns false on a protocol/transport error (the caller aborts the transfer). */
bool client_change_probe_ancestors(const Config* config, const File* file, int fd) {
if (config == NULL || file == NULL || fd < 0 || !config->report_dest_info)
return true;
if (!g_progress_dir_index_valid || !g_progress_precount.dir_refs)
return true;
const char* rel = delete_display_path(config, file_wire_path(file));
if (rel == NULL)
return true;
size_t rel_len = strlen(rel);
for (size_t i = 1; i < rel_len; i++) {
if (rel[i] != '/')
continue;
char* prefix = malloc(i + 1);
if (prefix == NULL)
return false;
memcpy(prefix, rel, i);
prefix[i] = '\0';
if (path_index_contains(&g_progress_dir_index, prefix)) {
File* dir = progress_dir_lookup(prefix);
/* The probe path is the same wire path the real STATUS_MKDIR would carry
(the pre-count File's send_path, or its absolute source path for a
plain recursive scan), not the display-relative prefix. */
if (dir != NULL && !dir->dest_state.known && !client_probe_dir_state(fd, dir)) {
free(prefix);
return false;
}
}
free(prefix);
}
return true;
}
/* Mark a directory the data pass already itemized/named so the end-of-transfer
* pending-directory flush does not report it a second time. `file` is a
* transferred directory entry (an empty-directory STATUS_MKDIR). */
void client_change_mark_dir(const Config* config, const File* file) {
if (config == NULL || file == NULL || !g_progress_emitted_valid ||
g_progress_emitted_keys == NULL)
return;
const char* rel = delete_display_path(config, file_wire_path(file));
if (rel == NULL || rel[0] == '\0' || str_hash_set_lookup(&g_progress_emitted, rel))
return;
char* key = str_dup(rel);
if (key == NULL)
return;
if (!array_list_add(g_progress_emitted_keys, key)) {
free(key);
return;
}
str_hash_set_insert_ref(&g_progress_emitted, key);
}
/* Emit the itemize lines for source directories that CHANGED but had no
* transferred child, so no ancestor emission reached them (rsync reports a
* directory whose attributes changed even when its contents did not). Runs at
* the end of the data pass, BEFORE the deferred STATUS_DIR_TIMES apply, so the
* probe still observes each untouched directory's pre-transfer state. Only the
* itemize/out-format/log streams report attribute-only directory changes;
* --progress/-P stays silent for them, matching rsync. Best-effort: a probe
* failure simply stops the flush (the transfer's verdict is unaffected). */
void client_change_emit_pending_dirs(const Config* config, int fd) {
if (config == NULL || fd < 0 || !config->report_dest_info)
return;
bool itemize_output = config->itemize_changes || config->out_format != NULL ||
(config->log_file != NULL && config->log_file_format != NULL);
if (!itemize_output)
return;
if (!g_progress_dir_index_valid || g_progress_precount.dir_refs == NULL ||
!g_progress_emitted_valid || g_progress_emitted_keys == NULL)
return;
for (int i = 0; i < g_progress_precount.dir_refs->size; i++) {
DirRef* ref = (DirRef*)g_progress_precount.dir_refs->items[i];
if (ref == NULL || ref->name == NULL || ref->name[0] == '\0')
continue;
if (str_hash_set_lookup(&g_progress_emitted, ref->name))
continue;
File* dir = ref->file;
if (dir == NULL)
continue;
if (!dir->dest_state.known && !client_probe_dir_state(fd, dir))
return;
change_emit_dir_sent(config, dir);
char* key = str_dup(ref->name);
if (key != NULL && array_list_add(g_progress_emitted_keys, key))
str_hash_set_insert_ref(&g_progress_emitted, key);
else
free(key);
}
}
/* rsync's --info=name/progress line for one entry: transfer-relative name (a /* rsync's --info=name/progress line for one entry: transfer-relative name (a
* trailing slash for directories) plus the ` -> target` symlink suffix. */ * trailing slash for directories) plus the ` -> target` symlink suffix. */
static char* progress_entry_line(const File* file, const char* rel) { static char* progress_entry_line(const File* file, const char* rel) {
@@ -645,6 +777,15 @@ void client_progress_file(const Config* config, const File* file) {
void client_progress_name(const Config* config, const File* file) { void client_progress_name(const Config* config, const File* file) {
if (!g_progress_active || file == NULL) if (!g_progress_active || file == NULL)
return; return;
/* rsync's --progress/-P name stream reports an entry only when it is created
or (for a symlink) actually relinked: a pre-existing directory or an
unchanged symlink is silent (protocol 2.30.0 dest-state report). */
if (file->dest_state.known && file->dest_state.existed) {
if (file->is_dir || (file->is_symlink && file->dest_state.target_matches)) {
g_progress_index++;
return;
}
}
const char* rel = delete_display_path(config, file_wire_path(file)); const char* rel = delete_display_path(config, file_wire_path(file));
if (!config->itemize_changes && config->out_format == NULL) { if (!config->itemize_changes && config->out_format == NULL) {
char* line = progress_entry_line(file, rel ? rel : ""); char* line = progress_entry_line(file, rel ? rel : "");
+54 -7
View File
@@ -615,14 +615,34 @@ static bool send_file_direct(File* file, int fd, bool use_metadata, int compress
static bool send_directory_entry(const Client* client, File* file, const Config* config) { static bool send_directory_entry(const Client* client, File* file, const Config* config) {
if (!file || !file_wire_path(file)) if (!file || !file_wire_path(file))
return false; return false;
if (!send_status(client->file_descriptor, STATUS_MKDIR) || int fd = client->file_descriptor;
!send_wire_str(client->file_descriptor, file_wire_path(file))) if (!send_status(fd, STATUS_MKDIR))
return false; return false;
if (config->use_metadata && !metadata_send(client->file_descriptor, file->metadata)) /* Output parity (protocol 2.30.0): when report_dest_info is negotiated every
STATUS_MKDIR body is prefixed with a probe flag (1 = probe only, 0 = a real
create), so the receiver knows whether to expect the metadata/xattr block. */
if (config->report_dest_info && !send_int(fd, 0))
return false;
if (!send_wire_str(fd, file_wire_path(file)))
return false;
if (config->use_metadata && !metadata_send(fd, file->metadata))
return false; return false;
/* Directory xattrs/ACLs (-X/-A) ride the same trailing block as regular files /* Directory xattrs/ACLs (-X/-A) ride the same trailing block as regular files
when the xattr transport was negotiated. */ when the xattr transport was negotiated. */
return !config->use_xattrs || xattr_send(client->file_descriptor, file->xattrs); if (config->use_xattrs && !xattr_send(fd, file->xattrs))
return false;
/* The receiver answers with the directory's pre-transfer destination state
BEFORE creating it, so the sender can render rsync's `.d..t......` versus
`cd+++++++++` and suppress an unchanged directory. */
if (config->report_dest_info) {
Status status;
if (!receive_status(fd, &status) || status != STATUS_DEST_INFO ||
!format_dest_state_receive(fd, &file->dest_state)) {
log_message(LOG_LEVEL_ERROR, "Unexpected reply to the directory destination-state report");
return false;
}
}
return true;
} }
/* P7 Wave D: transmit every captured source directory's metadata in terminal /* P7 Wave D: transmit every captured source directory's metadata in terminal
@@ -678,7 +698,21 @@ static bool send_symlink_entry(const Client* client, File* file, const Config* c
return false; return false;
/* Symlink xattrs/ACLs (-X/-A) ride the same trailing block as regular files /* Symlink xattrs/ACLs (-X/-A) ride the same trailing block as regular files
and directories when the xattr transport was negotiated. */ and directories when the xattr transport was negotiated. */
return !config->use_xattrs || xattr_send(fd, file->xattrs); if (config->use_xattrs && !xattr_send(fd, file->xattrs))
return false;
/* The receiver answers with the symlink's pre-transfer destination state
(including whether the on-disk link target already matches) BEFORE creating
it, so the sender can render rsync's `cLc........` / `.L..t......` and
suppress an unchanged symlink. */
if (config->report_dest_info) {
Status status;
if (!receive_status(fd, &status) || status != STATUS_DEST_INFO ||
!format_dest_state_receive(fd, &file->dest_state)) {
log_message(LOG_LEVEL_ERROR, "Unexpected reply to the symlink destination-state report");
return false;
}
}
return true;
} }
// Send a single file directly via sendfile (non-incremental path). // Send a single file directly via sendfile (non-incremental path).
@@ -869,10 +903,12 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
feed -i/--out-format its ancestor directory lines here. */ feed -i/--out-format its ancestor directory lines here. */
if (config->itemize_changes || config->out_format != NULL) if (config->itemize_changes || config->out_format != NULL)
client_change_emit_ancestors(config, chunk->items[i]); client_change_emit_ancestors(config, chunk->items[i]);
if (chunk->items[i]->is_dir) if (chunk->items[i]->is_dir) {
change_emit_dir_sent(config, chunk->items[i]); change_emit_dir_sent(config, chunk->items[i]);
else client_change_mark_dir(config, chunk->items[i]);
} else {
change_emit_file_sent(config, chunk->items[i]); change_emit_file_sent(config, chunk->items[i]);
}
if (!chunk->items[i]->is_dir) if (!chunk->items[i]->is_dir)
transfer_stats_note_transferred(stats, chunk->items[i]); transfer_stats_note_transferred(stats, chunk->items[i]);
} }
@@ -884,6 +920,11 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
if (f == NULL) if (f == NULL)
continue; continue;
transfer_stats_note_entry(stats, f); transfer_stats_note_entry(stats, f);
/* Output parity: probe this entry's ancestor directories' destination state
before the entry (or the first child below them) is sent, while the
receiver has not yet created them implicitly. */
if (!client_change_probe_ancestors(config, f, client->file_descriptor))
return -1;
if (f->is_dir) { if (f->is_dir) {
/* Explicit directory entry (--dirs): a MKDIR frame carrying the /* Explicit directory entry (--dirs): a MKDIR frame carrying the
destination path (and metadata when negotiated). Directories have no destination path (and metadata when negotiated). Directories have no
@@ -892,6 +933,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
return -1; return -1;
client_change_emit_ancestors(config, f); client_change_emit_ancestors(config, f);
change_emit_dir_sent(config, f); change_emit_dir_sent(config, f);
client_change_mark_dir(config, f);
client_progress_name(config, f); client_progress_name(config, f);
continue; continue;
} }
@@ -1137,6 +1179,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
(and all parallel workers) has been joined before scanner_done was set, so (and all parallel workers) has been joined before scanner_done was set, so
the list is complete and race-free; on an early stop the list may be the list is complete and race-free; on an early stop the list may be
incomplete and is deliberately not sent. */ incomplete and is deliberately not sent. */
client_change_emit_pending_dirs(context->config, client->file_descriptor);
if (!context->scan_stopped_early && if (!context->scan_stopped_early &&
!send_dir_times(client, context->config, context->dir_entries)) !send_dir_times(client, context->config, context->dir_entries))
goto send_fail; goto send_fail;
@@ -1844,6 +1887,10 @@ static int send_files_finalize(const Config* config, SendFilesState* state) {
} }
} }
} }
/* Output parity: report changed directories that had no transferred child
before the deferred directory times are applied (so the probe still sees
their pre-transfer state). */
client_change_emit_pending_dirs(config, client->file_descriptor);
/* P7 Wave D: every directory has now been traversed (or the scan stopped /* P7 Wave D: every directory has now been traversed (or the scan stopped
early), so transmit the captured directory times last. The receiver defers early), so transmit the captured directory times last. The receiver defers
applying them until after its own deletion/publication phase. */ applying them until after its own deletion/publication phase. */
+8
View File
@@ -68,6 +68,14 @@ void client_progress_name(const Config* config, const File* file);
/* Emit a transferred entry's ancestor directories (as -i/--out-format change /* Emit a transferred entry's ancestor directories (as -i/--out-format change
* lines or --progress name lines) before the entry's own line. */ * lines or --progress name lines) before the entry's own line. */
void client_change_emit_ancestors(const Config* config, const File* file); void client_change_emit_ancestors(const Config* config, const File* file);
/* Output parity (protocol 2.30.0): probe each not-yet-known ancestor directory's
* pre-transfer destination state before the entry that first triggers it is
* sent. Returns false on a protocol/transport error. */
bool client_change_probe_ancestors(const Config* config, const File* file, int fd);
/* Mark a transferred directory entry as already reported, and flush the
* itemize lines for changed directories that had no transferred child. */
void client_change_mark_dir(const Config* config, const File* file);
void client_change_emit_pending_dirs(const Config* config, int fd);
void client_progress_uptodate(const Config* config, const File* file); void client_progress_uptodate(const Config* config, const File* file);
void client_progress_prepare(const Config* config, const ArrayList* plan_dirs, void client_progress_prepare(const Config* config, const ArrayList* plan_dirs,
unsigned long long plan_non_dir_count); unsigned long long plan_non_dir_count);
+157 -23
View File
@@ -11,10 +11,13 @@
#include "metadata.h" #include "metadata.h"
#include "protocol.h" #include "protocol.h"
#include "utils.h" #include "utils.h"
#include <fcntl.h>
#include <limits.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <time.h> #include <time.h>
#include <unistd.h>
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code) { bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code) {
if (!outcomes) if (!outcomes)
@@ -101,14 +104,35 @@ static void receiver_tally_deleted(const ReceiverSink* sink, size_t deleted) {
sink->stats->deleted_files += deleted; sink->stats->deleted_files += deleted;
} }
/* Observer for --info=del: record each truly-removed destination-relative path /* Observer for --info=del/--stats: record each truly-removed destination-
in the ArrayList passed as the observer context, so the terminal STATUS_STATS relative path (when the context carries a path list) and tally it by type
frame can list it. A failed append is best-effort (the deletion already (when it carries a stats record), so the terminal STATUS_STATS frame can list
happened; output is cosmetic). Shared by the single-threaded receiver and the paths and render rsync's per-type `Number of deleted files` breakdown. A
the -m pipeline's deferred commit. */ failed append is best-effort (the deletion already happened; output is
void receiver_record_deleted_path(void* context, const char* rel_path) { cosmetic). Shared by the single-threaded receiver and the -m pipeline's
ArrayList* paths = context; deferred commit. */
if (!paths || !rel_path) void receiver_record_deleted_path(void* context, const char* rel_path, DeleteEntryType type) {
ReceiverDeleteContext* del = context;
if (!del || !rel_path)
return;
if (del->stats) {
switch (type) {
case DELETE_ENTRY_DIR:
del->stats->deleted_dir++;
break;
case DELETE_ENTRY_LINK:
del->stats->deleted_link++;
break;
case DELETE_ENTRY_SPECIAL:
del->stats->deleted_special++;
break;
default:
del->stats->deleted_reg++;
break;
}
}
ArrayList* paths = del->deleted_paths;
if (!paths)
return; return;
/* Bound the retained list like the keep-set manifest: only MAX_MANIFEST_ENTRIES /* Bound the retained list like the keep-set manifest: only MAX_MANIFEST_ENTRIES
paths are ever transmitted in the terminal STATUS_STATS frame, so recording paths are ever transmitted in the terminal STATUS_STATS frame, so recording
@@ -120,6 +144,16 @@ void receiver_record_deleted_path(void* context, const char* rel_path) {
free(copy); free(copy);
} }
/* Install the delete observer (and its context) for one commit when the sink
carries a stats record or a path list. Returns NULL when neither is needed,
so the delete engines skip the observer entirely. */
static DeletePathObserver receiver_delete_observer(const ReceiverSink* sink,
ReceiverDeleteContext* del) {
del->stats = sink ? sink->stats : NULL;
del->deleted_paths = sink ? sink->deleted_paths : NULL;
return (del->stats || del->deleted_paths) ? receiver_record_deleted_path : NULL;
}
static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) { static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) {
if (!chunk || !sink || !sink->store_file) if (!chunk || !sink || !sink->store_file)
return false; return false;
@@ -328,6 +362,9 @@ typedef struct {
a successful FINISHED it is either committed here or handed to a successful FINISHED it is either committed here or handed to
*pending_plans so the -m caller commits after its disk writer drained. */ *pending_plans so the -m caller commits after its disk writer drained. */
DeletePlanSession* plan_session; DeletePlanSession* plan_session;
/* Observer context for the per-directory delete session, which outlives the
frame handler; must stay alive until the session commits. */
ReceiverDeleteContext delete_ctx;
bool early_delete; bool early_delete;
bool per_dir_delete; bool per_dir_delete;
bool delete_limit_noted; bool delete_limit_noted;
@@ -403,9 +440,95 @@ static ReceiverStep receiver_handle_check_batch(ReceiverPendingState* state) {
return RECEIVER_STEP_NEXT; return RECEIVER_STEP_NEXT;
} }
/* Probe a destination entry's pre-transfer state for the output-parity
dest-info report (protocol 2.30.0). `wire_path` is the destination-relative
path; `incoming_target` is non-NULL only for a symlink probe, in which case
the on-disk link target is compared with the target the receiver is about to
store (after --munge-links). The final component is never followed and the
parent walk is confined below the receive root. Returns false only on an
allocation/secure-walk failure; a missing entry is reported as existed=false. */
static bool receiver_probe_dest_state(const Config* config, const char* wire_path,
const char* incoming_target, OutputDestState* out) {
memset(out, 0, sizeof(*out));
out->known = true;
if (!config || !wire_path || wire_path[0] == '\0')
return false;
char* full = path_cat(config->receive_root_directory, wire_path);
if (!full)
return false;
char* leaf = NULL;
int parent_fd = file_open_secure_parent(full, &leaf, false);
free(full);
if (parent_fd < 0) {
/* A missing/unreachable parent means the entry cannot exist yet. */
free(leaf);
return true;
}
struct stat st;
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) == 0) {
out->existed = true;
out->size = (unsigned long long)st.st_size;
out->mtime_sec = (long long)st.st_mtime;
#ifdef __linux__
out->mtime_nsec = st.st_mtim.tv_nsec;
#endif
out->mode = (uint32_t)st.st_mode;
out->uid = (int32_t)st.st_uid;
out->gid = (int32_t)st.st_gid;
if (incoming_target && S_ISLNK(st.st_mode)) {
char target_buf[PATH_MAX];
ssize_t n = readlinkat(parent_fd, leaf, target_buf, sizeof(target_buf) - 1);
if (n >= 0) {
target_buf[n] = '\0';
char* expected =
config->munge_links ? file_symlink_munge(incoming_target) : str_dup(incoming_target);
if (expected) {
out->target_matches = strcmp(target_buf, expected) == 0;
free(expected);
}
}
}
}
close(parent_fd);
free(leaf);
return true;
}
static ReceiverStep receiver_handle_mkdir(ReceiverPendingState* state) { static ReceiverStep receiver_handle_mkdir(ReceiverPendingState* state) {
File* dir = file_receive_directory(state->fd, state->config); const Config* config = state->config;
if (!dir || !state->sink->store_file(dir, state->sink->context)) int fd = state->fd;
if (config->report_dest_info) {
int probe = 0;
if (!receive_int(fd, &probe) || (probe != 0 && probe != 1))
return RECEIVER_STEP_FAIL;
if (probe) {
/* Probe-only frame: report the destination state and create nothing. */
char* path = receive_wire_str(fd);
if (!path || path[0] == '\0' || (!file_get_trust_sender() && has_path_traversal(path))) {
free(path);
send_status(fd, STATUS_ERROR);
return RECEIVER_STEP_FAIL;
}
OutputDestState info;
bool ok = receiver_probe_dest_state(config, path, NULL, &info);
free(path);
if (!ok || !send_status(fd, STATUS_DEST_INFO) || !format_dest_state_send(fd, &info))
return RECEIVER_STEP_FAIL;
return RECEIVER_STEP_NEXT;
}
}
File* dir = file_receive_directory(fd, config);
if (!dir)
return RECEIVER_STEP_ERROR;
if (config->report_dest_info) {
OutputDestState info;
bool ok = receiver_probe_dest_state(config, file_wire_path(dir), NULL, &info);
if (!ok || !send_status(fd, STATUS_DEST_INFO) || !format_dest_state_send(fd, &info)) {
file_destroy(dir);
return RECEIVER_STEP_FAIL;
}
}
if (!state->sink->store_file(dir, state->sink->context))
return RECEIVER_STEP_ERROR; return RECEIVER_STEP_ERROR;
return RECEIVER_STEP_NEXT; return RECEIVER_STEP_NEXT;
} }
@@ -424,8 +547,20 @@ static ReceiverStep receiver_handle_hardlink(ReceiverPendingState* state) {
} }
static ReceiverStep receiver_handle_symlink(ReceiverPendingState* state) { static ReceiverStep receiver_handle_symlink(ReceiverPendingState* state) {
File* sym = file_receive_symlink(state->fd, state->config); const Config* config = state->config;
if (!sym || !state->sink->store_file(sym, state->sink->context)) File* sym = file_receive_symlink(state->fd, config);
if (!sym)
return RECEIVER_STEP_ERROR;
if (config->report_dest_info) {
OutputDestState info;
bool ok = receiver_probe_dest_state(config, file_wire_path(sym), sym->symlink_target, &info);
if (!ok || !send_status(state->fd, STATUS_DEST_INFO) ||
!format_dest_state_send(state->fd, &info)) {
file_destroy(sym);
return RECEIVER_STEP_FAIL;
}
}
if (!state->sink->store_file(sym, state->sink->context))
return RECEIVER_STEP_ERROR; return RECEIVER_STEP_ERROR;
return RECEIVER_STEP_NEXT; return RECEIVER_STEP_NEXT;
} }
@@ -468,12 +603,11 @@ static ReceiverStep receiver_handle_manifest(ReceiverPendingState* state) {
--max-delete-capped commit still succeeds and the transfer proceeds; --max-delete-capped commit still succeeds and the transfer proceeds;
the terminal success frame reports the cap. */ the terminal success frame reports the cap. */
size_t deleted = 0; size_t deleted = 0;
DeletePathObserver observer = ReceiverDeleteContext delctx;
(config->report_deletes && sink->deleted_paths) ? receiver_record_deleted_path : NULL; DeletePathObserver observer = receiver_delete_observer(sink, &delctx);
DeleteCommitResult deletion = DeleteCommitResult deletion =
(config->use_delete || config->delete_missing_args) (config->use_delete || config->delete_missing_args)
? manifest_delete_all_observed(config, manifest, &deleted, observer, ? manifest_delete_all_observed(config, manifest, &deleted, observer, &delctx)
(void*)sink->deleted_paths)
: DELETE_COMMIT_OK; : DELETE_COMMIT_OK;
receiver_tally_deleted(sink, deleted); receiver_tally_deleted(sink, deleted);
delete_manifest_free(manifest); delete_manifest_free(manifest);
@@ -516,9 +650,9 @@ static ReceiverStep receiver_handle_delete_plan(ReceiverPendingState* state) {
} }
if (!state->plan_session) { if (!state->plan_session) {
state->plan_session = delete_plan_session_create(config); state->plan_session = delete_plan_session_create(config);
if (state->plan_session && config->report_deletes && sink->deleted_paths) if (state->plan_session && (sink->stats || sink->deleted_paths))
delete_plan_session_set_delete_observer(state->plan_session, receiver_record_deleted_path, delete_plan_session_set_delete_observer(state->plan_session, receiver_record_deleted_path,
(void*)sink->deleted_paths); &state->delete_ctx);
} }
if (!state->plan_session || delete_plan_session_receive(state->plan_session, config, fd) != 0) if (!state->plan_session || delete_plan_session_receive(state->plan_session, config, fd) != 0)
return RECEIVER_STEP_FAIL; return RECEIVER_STEP_FAIL;
@@ -623,6 +757,7 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
.pending_plans = pending_plans, .pending_plans = pending_plans,
.deferred_manifest = NULL, .deferred_manifest = NULL,
.plan_session = NULL, .plan_session = NULL,
.delete_ctx = {sink ? sink->stats : NULL, sink ? sink->deleted_paths : NULL},
.early_delete = config_delete_timing_early(config), .early_delete = config_delete_timing_early(config),
.per_dir_delete = config_delete_timing_per_dir(config), .per_dir_delete = config_delete_timing_per_dir(config),
.delete_limit_noted = false, .delete_limit_noted = false,
@@ -662,10 +797,9 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
state.deferred_manifest = NULL; state.deferred_manifest = NULL;
} else { } else {
size_t deleted = 0; size_t deleted = 0;
DeletePathObserver observer = DeletePathObserver observer = receiver_delete_observer(sink, &state.delete_ctx);
(config->report_deletes && sink->deleted_paths) ? receiver_record_deleted_path : NULL;
DeleteCommitResult deletion = manifest_delete_all_observed( DeleteCommitResult deletion = manifest_delete_all_observed(
config, state.deferred_manifest, &deleted, observer, (void*)sink->deleted_paths); config, state.deferred_manifest, &deleted, observer, &state.delete_ctx);
receiver_tally_deleted(sink, deleted); receiver_tally_deleted(sink, deleted);
delete_manifest_free(state.deferred_manifest); delete_manifest_free(state.deferred_manifest);
state.deferred_manifest = NULL; state.deferred_manifest = NULL;
@@ -683,9 +817,9 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
hands the session to its caller instead, which commits after the disk hands the session to its caller instead, which commits after the disk
writer drained. */ writer drained. */
if (state.plan_session) { if (state.plan_session) {
if (config->report_deletes && sink->deleted_paths) if (sink->stats || sink->deleted_paths)
delete_plan_session_set_delete_observer(state.plan_session, receiver_record_deleted_path, delete_plan_session_set_delete_observer(state.plan_session, receiver_record_deleted_path,
(void*)sink->deleted_paths); &state.delete_ctx);
if (state.pending_plans) { if (state.pending_plans) {
*state.pending_plans = state.plan_session; *state.pending_plans = state.plan_session;
state.plan_session = NULL; state.plan_session = NULL;
+14 -4
View File
@@ -55,10 +55,20 @@ typedef struct {
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code); bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code);
void receiver_outcomes_destroy(ReceiverOutcomes* outcomes); void receiver_outcomes_destroy(ReceiverOutcomes* outcomes);
/* DeletePathObserver implementation for --info=del: `context` is an ArrayList* /* Delete observer context: `deleted_paths` (optional) receives owned copies of
that receives owned copies of every truly-removed destination-relative path. every truly-removed destination-relative path for --info=del; `stats`
Shared by the single-threaded receiver and the -m pipeline's deferred commit. */ (optional) receives the per-type `Number of deleted files` tallies for
void receiver_record_deleted_path(void* context, const char* rel_path); --stats. Both may be NULL, in which case the observer is a no-op. */
typedef struct {
ReceiverStats* stats;
struct ArrayList* deleted_paths;
} ReceiverDeleteContext;
/* DeletePathObserver implementation: records each truly-removed path (when the
context carries a path list) and tallies it by type (when it carries a stats
record). Shared by the single-threaded receiver and the -m pipeline's
deferred commit. */
void receiver_record_deleted_path(void* context, const char* rel_path, DeleteEntryType type);
/* Send the terminal success frame. `final_status` is usually STATUS_OK, or /* Send the terminal success frame. `final_status` is usually STATUS_OK, or
STATUS_DELETE_LIMIT when a --max-delete commit was capped. */ STATUS_DELETE_LIMIT when a --max-delete commit was capped. */
+9 -6
View File
@@ -990,9 +990,11 @@ static void server_run_mt_receiver(ServerSession* state) {
server-contacting --dry-run deletes nothing (no manifest is sent). */ server-contacting --dry-run deletes nothing (no manifest is sent). */
if (context->deferred_manifest) { if (context->deferred_manifest) {
size_t deleted = 0; size_t deleted = 0;
DeletePathObserver observer = config->report_deletes ? receiver_record_deleted_path : NULL; ReceiverDeleteContext delctx = {&context->stats, context->deleted_paths};
DeleteCommitResult deletion = manifest_delete_all_observed( DeletePathObserver observer =
config, context->deferred_manifest, &deleted, observer, (void*)context->deleted_paths); (delctx.stats || delctx.deleted_paths) ? receiver_record_deleted_path : NULL;
DeleteCommitResult deletion = manifest_delete_all_observed(config, context->deferred_manifest,
&deleted, observer, &delctx);
context->stats.deleted_files += deleted; context->stats.deleted_files += deleted;
if (deletion == DELETE_COMMIT_ERROR) { if (deletion == DELETE_COMMIT_ERROR) {
transfer_ok = false; transfer_ok = false;
@@ -1010,9 +1012,10 @@ static void server_run_mt_receiver(ServerSession* state) {
if (context->deferred_plans) { if (context->deferred_plans) {
/* Defence in depth (the enclosing block already excludes dry-run): a /* Defence in depth (the enclosing block already excludes dry-run): a
-n run never commits a deletion. */ -n run never commits a deletion. */
if (config->report_deletes) ReceiverDeleteContext delctx = {&context->stats, context->deleted_paths};
delete_plan_session_set_delete_observer( if (delctx.stats || delctx.deleted_paths)
context->deferred_plans, receiver_record_deleted_path, (void*)context->deleted_paths); delete_plan_session_set_delete_observer(context->deferred_plans,
receiver_record_deleted_path, &delctx);
DeleteCommitResult deletion = DeleteCommitResult deletion =
config->dry_run ? DELETE_COMMIT_OK config->dry_run ? DELETE_COMMIT_OK
: delete_plan_session_commit(context->deferred_plans, config); : delete_plan_session_commit(context->deferred_plans, config);
+16 -3
View File
@@ -32,6 +32,17 @@ static bool keep_is_file(const PathIndex* index, const char* rel_path) {
return path_index_contains(index, rel_path); return path_index_contains(index, rel_path);
} }
/* Classify a removed entry from its st_mode for the per-type delete counters. */
DeleteEntryType delete_entry_type_of_mode(mode_t mode) {
if (S_ISDIR(mode))
return DELETE_ENTRY_DIR;
if (S_ISLNK(mode))
return DELETE_ENTRY_LINK;
if (S_ISREG(mode))
return DELETE_ENTRY_REG;
return DELETE_ENTRY_SPECIAL;
}
/* True when child_rel is, or lies below, a protected entry. A prefix "a" /* True when child_rel is, or lies below, a protected entry. A prefix "a"
therefore protects "a" and "a/b/c" but not "ab". Entries with top_level_only therefore protects "a" and "a/b/c" but not "ab". Entries with top_level_only
set only protect DIRECT children of the receive root (at_root); nested set only protect DIRECT children of the receive root (at_root); nested
@@ -126,6 +137,7 @@ bool delete_dir_entries_collect(int dirfd, DeleteDirEntry** out, size_t* count,
break; break;
} }
entries[used].is_dir = S_ISDIR(st.st_mode); entries[used].is_dir = S_ISDIR(st.st_mode);
entries[used].mode = st.st_mode;
used++; used++;
} }
closedir(dir); closedir(dir);
@@ -329,10 +341,10 @@ static bool delete_walk_fd(int dirfd, const char* rel_path, const PathIndex* kee
memcpy(with_slash, child_rel, len); memcpy(with_slash, child_rel, len);
with_slash[len] = '/'; with_slash[len] = '/';
with_slash[len + 1] = '\0'; with_slash[len + 1] = '\0';
state->observer(state->observer_context, with_slash); state->observer(state->observer_context, with_slash, DELETE_ENTRY_DIR);
free(with_slash); free(with_slash);
} else { } else {
state->observer(state->observer_context, child_rel); state->observer(state->observer_context, child_rel, DELETE_ENTRY_DIR);
} }
} }
} }
@@ -373,7 +385,8 @@ static bool delete_walk_fd(int dirfd, const char* rel_path, const PathIndex* kee
char* child_rel = path_cat((char*)rel_path, entries[i].name); char* child_rel = path_cat((char*)rel_path, entries[i].name);
if (child_rel) { if (child_rel) {
if (state->observer) if (state->observer)
state->observer(state->observer_context, child_rel); state->observer(state->observer_context, child_rel,
delete_entry_type_of_mode(entries[i].mode));
char* escaped_path = output_escape(child_rel, log_get_8_bit_output()); char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>"); fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
free(escaped_path); free(escaped_path);
+21 -3
View File
@@ -5,6 +5,7 @@
#include "config.h" #include "config.h"
#include <stdbool.h> #include <stdbool.h>
#include <stddef.h> #include <stddef.h>
#include <sys/stat.h>
/* Delete engine. /* Delete engine.
* *
@@ -66,6 +67,9 @@ bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSki
typedef struct { typedef struct {
char* name; char* name;
bool is_dir; bool is_dir;
/* The entry's full st_mode from the AT_SYMLINK_NOFOLLOW stat, so a delete
observer can classify a removed non-directory as reg/link/special. */
mode_t mode;
} DeleteDirEntry; } DeleteDirEntry;
/* Collect the entries of the directory open on `dirfd` (excluding "." and ".."), /* Collect the entries of the directory open on `dirfd` (excluding "." and ".."),
stat'ing each with AT_SYMLINK_NOFOLLOW. On success *out is a malloc'd array of stat'ing each with AT_SYMLINK_NOFOLLOW. On success *out is a malloc'd array of
@@ -99,10 +103,24 @@ DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* m
const FilterRuleList* protect_rules, size_t* deleted_out, const FilterRuleList* protect_rules, size_t* deleted_out,
size_t* skipped_out); size_t* skipped_out);
/* Entry kind of a removed path, reported to the delete observer so the receiver
can build rsync's `--stats` `Number of deleted files` per-type breakdown. The
four categories are a strict partition of every removed entry. */
typedef enum {
DELETE_ENTRY_REG = 0,
DELETE_ENTRY_DIR,
DELETE_ENTRY_LINK,
DELETE_ENTRY_SPECIAL
} DeleteEntryType;
/* Optional per-deletion observer: called for each destination-relative path /* Optional per-deletion observer: called for each destination-relative path
actually removed (a file, symlink, or directory), in removal order, so the actually removed (a file, symlink, or directory) with its entry kind, in
receiver can stream rsync's `--info=del`/`--info=remove` lines. */ removal order, so the receiver can stream rsync's `--info=del`/`--info=remove`
typedef void (*DeletePathObserver)(void* context, const char* rel_path); lines and tally the per-type `--stats` counters. */
typedef void (*DeletePathObserver)(void* context, const char* rel_path, DeleteEntryType type);
/* Classify a removed entry from its st_mode for the per-type delete counters. */
DeleteEntryType delete_entry_type_of_mode(mode_t mode);
/* `delete_extras_limited_observed` is delete_extras_limited with an optional /* `delete_extras_limited_observed` is delete_extras_limited with an optional
* observer; the observer is invoked only for entries truly removed. When * observer; the observer is invoked only for entries truly removed. When
+3 -3
View File
@@ -191,13 +191,13 @@ typedef struct {
const char* prefix; const char* prefix;
} PrefixedDeleteObserver; } PrefixedDeleteObserver;
static void prefixed_delete_observer(void* context, const char* rel) { static void prefixed_delete_observer(void* context, const char* rel, DeleteEntryType type) {
PrefixedDeleteObserver* prefixed = context; PrefixedDeleteObserver* prefixed = context;
if (!prefixed->inner || !rel) if (!prefixed->inner || !rel)
return; return;
char* joined = path_cat((char*)prefixed->prefix, rel); char* joined = path_cat((char*)prefixed->prefix, rel);
if (joined) { if (joined) {
prefixed->inner(prefixed->inner_context, joined); prefixed->inner(prefixed->inner_context, joined, type);
free(joined); free(joined);
} }
} }
@@ -359,7 +359,7 @@ static bool delete_missing_args_budgeted_observed(const Config* config,
if (removed) { if (removed) {
budget->deleted++; budget->deleted++;
if (observer) if (observer)
observer(observer_context, rel); observer(observer_context, rel, delete_entry_type_of_mode(st.st_mode));
char* escaped = output_escape(rel, log_get_8_bit_output()); char* escaped = output_escape(rel, log_get_8_bit_output());
fprintf(stderr, " Deleted: %s\n", escaped ? escaped : "<allocation failed>"); fprintf(stderr, " Deleted: %s\n", escaped ? escaped : "<allocation failed>");
free(escaped); free(escaped);
+10 -9
View File
@@ -478,9 +478,9 @@ struct DeletePlanSession {
}; };
/* Report one path the session truly removed (no-op without an observer). */ /* Report one path the session truly removed (no-op without an observer). */
static void notify_deleted(DeletePlanSession* session, const char* rel) { static void notify_deleted(DeletePlanSession* session, const char* rel, DeleteEntryType type) {
if (session && session->observer && rel) if (session && session->observer && rel)
session->observer(session->observer_context, rel); session->observer(session->observer_context, rel, type);
} }
/* A removed directory is reported with rsync's trailing slash (`deleting dir/`) /* A removed directory is reported with rsync's trailing slash (`deleting dir/`)
@@ -491,13 +491,13 @@ static void notify_deleted_dir(DeletePlanSession* session, const char* rel) {
size_t len = strlen(rel); size_t len = strlen(rel);
char* with_slash = malloc(len + 2); char* with_slash = malloc(len + 2);
if (!with_slash) { if (!with_slash) {
session->observer(session->observer_context, rel); session->observer(session->observer_context, rel, DELETE_ENTRY_DIR);
return; return;
} }
memcpy(with_slash, rel, len); memcpy(with_slash, rel, len);
with_slash[len] = '/'; with_slash[len] = '/';
with_slash[len + 1] = '\0'; with_slash[len + 1] = '\0';
session->observer(session->observer_context, with_slash); session->observer(session->observer_context, with_slash, DELETE_ENTRY_DIR);
free(with_slash); free(with_slash);
} }
@@ -711,8 +711,8 @@ static bool process_extra_dir(int dirfd, const char* name, const char* child_rel
return errno == ENOTEMPTY || errno == EEXIST; return errno == ENOTEMPTY || errno == EEXIST;
} }
static bool process_extra_file(int dirfd, const char* name, const char* child_rel, bool force_now, static bool process_extra_file(int dirfd, const char* name, const char* child_rel, mode_t mode,
DeletePlanSession* session) { bool force_now, DeletePlanSession* session) {
if (session->defer && !force_now) { if (session->defer && !force_now) {
return defer_add(session, child_rel); return defer_add(session, child_rel);
} }
@@ -724,7 +724,7 @@ static bool process_extra_file(int dirfd, const char* name, const char* child_re
session->deleted++; session->deleted++;
session->planned++; session->planned++;
log_deleted(child_rel); log_deleted(child_rel);
notify_deleted(session, child_rel); notify_deleted(session, child_rel, delete_entry_type_of_mode(mode));
} else if (errno != ENOENT) { } else if (errno != ENOENT) {
return false; return false;
} }
@@ -828,7 +828,8 @@ static bool process_children(int dirfd, const char* dir_rel, const ArrayList* ke
operation_ok = false; operation_ok = false;
continue; continue;
} }
if (!process_extra_file(dirfd, entries[i].name, child_rel, force[i] || force_now, session)) if (!process_extra_file(dirfd, entries[i].name, child_rel, entries[i].mode,
force[i] || force_now, session))
operation_ok = false; operation_ok = false;
free(child_rel); free(child_rel);
} }
@@ -1032,7 +1033,7 @@ static bool apply_deferred_path(DeletePlanSession* session, const Config* config
session->deleted++; session->deleted++;
session->planned++; session->planned++;
log_deleted(rel); log_deleted(rel);
notify_deleted(session, rel); notify_deleted(session, rel, delete_entry_type_of_mode(st.st_mode));
} else if (errno != ENOENT) { } else if (errno != ENOENT) {
close(parent_fd); close(parent_fd);
free(leaf); free(leaf);
+16 -6
View File
@@ -62,14 +62,16 @@ bool format_dest_state_send(int fd, const OutputDestState* state) {
if (!state) if (!state)
return false; return false;
int32_t has_old = state->existed ? 1 : 0; int32_t has_old = state->existed ? 1 : 0;
int32_t target_matches = state->target_matches ? 1 : 0;
uint64_t size = (uint64_t)state->size; uint64_t size = (uint64_t)state->size;
int64_t mtime = (int64_t)state->mtime_sec; int64_t mtime = (int64_t)state->mtime_sec;
int64_t mtime_nsec = state->mtime_nsec; int64_t mtime_nsec = state->mtime_nsec;
uint32_t mode = state->mode; uint32_t mode = state->mode;
int32_t uid = state->uid; int32_t uid = state->uid;
int32_t gid = state->gid; int32_t gid = state->gid;
return send_n_data(fd, &has_old, sizeof(has_old)) && send_n_data(fd, &size, sizeof(size)) && return send_n_data(fd, &has_old, sizeof(has_old)) &&
send_n_data(fd, &mtime, sizeof(mtime)) && send_n_data(fd, &target_matches, sizeof(target_matches)) &&
send_n_data(fd, &size, sizeof(size)) && send_n_data(fd, &mtime, sizeof(mtime)) &&
send_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) && send_n_data(fd, &mode, sizeof(mode)) && send_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) && send_n_data(fd, &mode, sizeof(mode)) &&
send_n_data(fd, &uid, sizeof(uid)) && send_n_data(fd, &gid, sizeof(gid)); send_n_data(fd, &uid, sizeof(uid)) && send_n_data(fd, &gid, sizeof(gid));
} }
@@ -78,14 +80,16 @@ bool format_dest_state_receive(int fd, OutputDestState* state) {
if (!state) if (!state)
return false; return false;
int32_t has_old = 0; int32_t has_old = 0;
int32_t target_matches = 0;
uint64_t size = 0; uint64_t size = 0;
int64_t mtime = 0; int64_t mtime = 0;
int64_t mtime_nsec = 0; int64_t mtime_nsec = 0;
uint32_t mode = 0; uint32_t mode = 0;
int32_t uid = 0; int32_t uid = 0;
int32_t gid = 0; int32_t gid = 0;
if (!receive_n_data(fd, &has_old, sizeof(has_old)) || !receive_n_data(fd, &size, sizeof(size)) || if (!receive_n_data(fd, &has_old, sizeof(has_old)) ||
!receive_n_data(fd, &mtime, sizeof(mtime)) || !receive_n_data(fd, &target_matches, sizeof(target_matches)) ||
!receive_n_data(fd, &size, sizeof(size)) || !receive_n_data(fd, &mtime, sizeof(mtime)) ||
!receive_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) || !receive_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) ||
!receive_n_data(fd, &mode, sizeof(mode)) || !receive_n_data(fd, &uid, sizeof(uid)) || !receive_n_data(fd, &mode, sizeof(mode)) || !receive_n_data(fd, &uid, sizeof(uid)) ||
!receive_n_data(fd, &gid, sizeof(gid))) !receive_n_data(fd, &gid, sizeof(gid)))
@@ -93,6 +97,7 @@ bool format_dest_state_receive(int fd, OutputDestState* state) {
memset(state, 0, sizeof(*state)); memset(state, 0, sizeof(*state));
state->known = true; state->known = true;
state->existed = has_old != 0; state->existed = has_old != 0;
state->target_matches = target_matches != 0;
state->size = size; state->size = size;
state->mtime_sec = mtime; state->mtime_sec = mtime;
state->mtime_nsec = mtime_nsec; state->mtime_nsec = mtime_nsec;
@@ -105,9 +110,10 @@ bool format_dest_state_receive(int fd, OutputDestState* state) {
bool format_stats_send(int fd, const ReceiverStats* stats) { bool format_stats_send(int fd, const ReceiverStats* stats) {
if (!stats) if (!stats)
return false; return false;
unsigned long long fields[8] = { unsigned long long fields[12] = {
stats->matched_data, stats->deleted_files, stats->would_delete_count, stats->literal_bytes, stats->matched_data, stats->deleted_files, stats->would_delete_count, stats->literal_bytes,
stats->created_reg, stats->created_dir, stats->created_link, stats->created_special, stats->created_reg, stats->created_dir, stats->created_link, stats->created_special,
stats->deleted_reg, stats->deleted_dir, stats->deleted_link, stats->deleted_special,
}; };
return send_n_data(fd, fields, sizeof(fields)); return send_n_data(fd, fields, sizeof(fields));
} }
@@ -115,7 +121,7 @@ bool format_stats_send(int fd, const ReceiverStats* stats) {
bool format_stats_receive(int fd, ReceiverStats* stats) { bool format_stats_receive(int fd, ReceiverStats* stats) {
if (!stats) if (!stats)
return false; return false;
unsigned long long fields[8] = {0}; unsigned long long fields[12] = {0};
if (!receive_n_data(fd, fields, sizeof(fields))) if (!receive_n_data(fd, fields, sizeof(fields)))
return false; return false;
memset(stats, 0, sizeof(*stats)); memset(stats, 0, sizeof(*stats));
@@ -127,5 +133,9 @@ bool format_stats_receive(int fd, ReceiverStats* stats) {
stats->created_dir = fields[5]; stats->created_dir = fields[5];
stats->created_link = fields[6]; stats->created_link = fields[6];
stats->created_special = fields[7]; stats->created_special = fields[7];
stats->deleted_reg = fields[8];
stats->deleted_dir = fields[9];
stats->deleted_link = fields[10];
stats->deleted_special = fields[11];
return true; return true;
} }
+25 -6
View File
@@ -17,10 +17,18 @@
/* Pre-transfer destination snapshot, reported by the receiver when the wire /* Pre-transfer destination snapshot, reported by the receiver when the wire
* config carries report_dest_info. `known` distinguishes "no report was * config carries report_dest_info. `known` distinguishes "no report was
* requested/received" from "the destination did not exist" (`existed == false` * requested/received" from "the destination did not exist" (`existed == false`
* with `known == true`). */ * with `known == true`).
*
* `target_matches` is meaningful only for a symlink destination (protocol
* 2.30.0): the receiver compares its on-disk link target with the incoming
* target and reports whether they are equal, so the sender can render rsync's
* `cLc........` (target changed) versus `.L..t......` (attributes only) and
* suppress an unchanged symlink's line entirely. It is always false for every
* other entry kind. */
typedef struct { typedef struct {
bool known; bool known;
bool existed; bool existed;
bool target_matches;
unsigned long long size; unsigned long long size;
long long mtime_sec; long long mtime_sec;
long long mtime_nsec; long long mtime_nsec;
@@ -57,17 +65,24 @@ bool format_dest_state_send(int fd, const OutputDestState* state);
bool format_dest_state_receive(int fd, OutputDestState* state); bool format_dest_state_receive(int fd, OutputDestState* state);
/* End-of-transfer receiver counters reported through STATUS_STATS (protocol /* End-of-transfer receiver counters reported through STATUS_STATS (protocol
* 2.25.0, extended in 2.28.0) when the wire config carries report_stats. * 2.25.0, extended in 2.28.0 and 2.30.0) when the wire config carries
* `would_delete_count` is the number of destination-relative paths the receiver * report_stats. `would_delete_count` is the number of destination-relative
* would have deleted in a -n/--dry-run --delete run; that many wire strings * paths the receiver would have deleted in a -n/--dry-run --delete run; that
* immediately follow the fixed record (sent/read by the caller). * many wire strings immediately follow the fixed record (sent/read by the
* caller).
* *
* Protocol 2.28.0 adds the receiver-observed counters the sender cannot see: * Protocol 2.28.0 adds the receiver-observed counters the sender cannot see:
* `literal_bytes` is the file data the receiver actually stored literally * `literal_bytes` is the file data the receiver actually stored literally
* (whole files plus the literal fragments of a delta) and the four `created_*` * (whole files plus the literal fragments of a delta) and the four `created_*`
* counters split the destination entries the receiver newly created by type, * counters split the destination entries the receiver newly created by type,
* reproducing rsync's `Number of created files` breakdown and an exact * reproducing rsync's `Number of created files` breakdown and an exact
* `Literal data` for a delta run. */ * `Literal data` for a delta run.
*
* Protocol 2.30.0 appends the four `deleted_*` counters: the same reg/dir/link/
* special split for the entries the receiver ACTUALLY removed, so `--stats` can
* render rsync's `Number of deleted files: X (reg: A, dir: B, link: C,
* special: D)` parenthetical. The scalar `deleted_files` stays the authoritative
* total (the breakdown is a strict partition of it). */
typedef struct { typedef struct {
unsigned long long matched_data; unsigned long long matched_data;
unsigned long long deleted_files; unsigned long long deleted_files;
@@ -77,6 +92,10 @@ typedef struct {
unsigned long long created_dir; unsigned long long created_dir;
unsigned long long created_link; unsigned long long created_link;
unsigned long long created_special; unsigned long long created_special;
unsigned long long deleted_reg;
unsigned long long deleted_dir;
unsigned long long deleted_link;
unsigned long long deleted_special;
} ReceiverStats; } ReceiverStats;
/* Fixed-width STATUS_STATS counter record. The status frame and the optional /* Fixed-width STATUS_STATS counter record. The status frame and the optional
+27 -14
View File
@@ -129,8 +129,13 @@ enum NET_STATUS {
STATUS_KEEPALIVE, STATUS_KEEPALIVE,
STATUS_ABORT, STATUS_ABORT,
STATUS_CHECK_BATCH, STATUS_CHECK_BATCH,
/* An explicit directory entry (--dirs): the sender transmits only the path; /* An explicit directory entry (--dirs / an empty source directory): the sender
* the receiver creates the directory below the receive root. */ * transmits the path and, when metadata/xattrs are negotiated, their blocks;
* the receiver creates the directory below the receive root. Protocol 2.30.0
* inserts an int32 probe flag right after the status when report_dest_info is
* negotiated: probe=1 is a report-only frame (path only; the receiver answers
* STATUS_DEST_INFO and creates nothing), probe=0 is a real create that is
* answered with the directory's pre-transfer state before it is created. */
STATUS_MKDIR, STATUS_MKDIR,
/* --append / --append-verify tail resume. STATUS_APPEND is sent by the /* --append / --append-verify tail resume. STATUS_APPEND is sent by the
* receiver after a per-file STATUS_CHECK when the existing destination file * receiver after a per-file STATUS_CHECK when the existing destination file
@@ -213,16 +218,22 @@ enum NET_STATUS {
* limit stopped deletions"). Appended after STATUS_DRY_RUN_TRANSFER so no * limit stopped deletions"). Appended after STATUS_DRY_RUN_TRANSFER so no
* existing status is renumbered. */ * existing status is renumbered. */
STATUS_DELETE_LIMIT, STATUS_DELETE_LIMIT,
/* Destination-state report for output parity (protocol 2.23.0). When the /* Destination-state report for output parity (protocol 2.23.0; extended to
* wire config carries report_dest_info=true, the receiver answers every * directories/symlinks in 2.30.0). When the wire config carries
* per-file STATUS_CHECK request with STATUS_DEST_INFO FIRST, followed by a * report_dest_info=true, the receiver answers every per-file STATUS_CHECK
* fixed record describing the pre-transfer destination entry * request with STATUS_DEST_INFO FIRST, followed by a fixed record describing
* (int32 has_old; uint64 size; int64 mtime; int64 mtime_nsec; uint32 mode; * the pre-transfer destination entry (int32 has_old; int32 target_matches;
* int32 uid; int32 gid). The ordinary STATUS_OK/STATUS_NEXT/... verdict * uint64 size; int64 mtime; int64 mtime_nsec; uint32 mode; int32 uid;
* follows, so the sender can render rsync-accurate -i/--out-format columns * int32 gid). The ordinary STATUS_OK/STATUS_NEXT/... verdict follows, so the
* (new vs modified, and which of size/time/perms/owner/group differ) without * sender can render rsync-accurate -i/--out-format columns (new vs modified,
* changing the transfer decision itself. Appended after * and which of size/time/perms/owner/group differ) without changing the
* STATUS_DELETE_LIMIT so no existing status is renumbered. */ * transfer decision itself. Protocol 2.30.0 also uses this record for
* STATUS_MKDIR and STATUS_SYMLINK: the sender consumes it into the entry's
* dest_state before emitting its change line, and target_matches reports
* whether an existing symlink's on-disk target already equals the incoming
* one (so the sender can render `cLc........` vs `.L..t......` and suppress
* an unchanged symlink). Appended after STATUS_DELETE_LIMIT so no existing
* status is renumbered. */
STATUS_DEST_INFO, STATUS_DEST_INFO,
/* Per-directory delete plan (protocol 2.24.0). The sender of a /* Per-directory delete plan (protocol 2.24.0). The sender of a
* --delete-during/--delete-delay transfer streams one frame per source * --delete-during/--delete-delay transfer streams one frame per source
@@ -244,8 +255,10 @@ enum NET_STATUS {
* config carries report_stats=true, the receiver sends this status once, * config carries report_stats=true, the receiver sends this status once,
* immediately before its terminal success status, followed by a fixed stats * immediately before its terminal success status, followed by a fixed stats
* record (see format_stats_send/receive in format.h) and, when the run is a * record (see format_stats_send/receive in format.h) and, when the run is a
* --dry-run with --delete, the would-delete path list. Appended after * --dry-run with --delete, the would-delete path list. Protocol 2.30.0
* STATUS_DELETE_PLAN so no existing status is renumbered. */ * appends the four deleted_reg/dir/link/special counters to that record, so
* --stats can render rsync's `Number of deleted files` per-type breakdown.
* Appended after STATUS_DELETE_PLAN so no existing status is renumbered. */
STATUS_STATS, STATUS_STATS,
/* Client diagnostic channel (protocol 2.30.0, rsync's --stderr=client / /* Client diagnostic channel (protocol 2.30.0, rsync's --stderr=client /
* --no-msgs2stderr). When the client's --stderr mode is `client`, the * --no-msgs2stderr). When the client's --stderr mode is `client`, the
@@ -17,6 +17,7 @@ Run locally::
python3 -m pytest tests/integration/test_differential_parity.py -n 4 --dist=load -m parity python3 -m pytest tests/integration/test_differential_parity.py -n 4 --dist=load -m parity
""" """
import os import os
import re
import shutil import shutil
import sys import sys
import warnings import warnings
@@ -666,6 +667,157 @@ def test_added_and_deleted_between_runs(parity_server_factory):
_run_and_check(case_id, result) _run_and_check(case_id, result)
def _seed_dest_tree(src, root):
"""Copy `src`'s tree into `root` (the transfer mirror), preserving symlinks
and directory mtimes, so a second differential run starts from an existing
destination exactly like a seeded rsync run."""
os.makedirs(root, exist_ok=True)
for dirpath, dirnames, filenames in os.walk(src):
rel = os.path.relpath(dirpath, src)
for name in dirnames:
s = os.path.join(dirpath, name)
d = os.path.join(root, rel, name) if rel != "." else os.path.join(root, name)
if os.path.islink(s):
continue
os.makedirs(d, exist_ok=True)
for name in filenames:
s = os.path.join(dirpath, name)
d = os.path.join(root, rel, name) if rel != "." else os.path.join(root, name)
os.makedirs(os.path.dirname(d), exist_ok=True)
if os.path.islink(s):
if os.path.lexists(d):
os.remove(d)
os.symlink(os.readlink(s), d)
else:
shutil.copy2(s, d)
if rel != ".":
os.utime(os.path.join(root, rel), None)
os.utime(root, None)
# A full rsync itemize code (11 columns) followed by the name. H._ITEMIZE_RE
# only matches created (`+`) entries, so the changed-attribute codes this test
# asserts need their own matcher.
_ITEMIZE_LINE_RE = re.compile(r"^[<>ch.*][fdLDS].{9} ")
def _itemize_dir_link_lines(text):
"""The itemize lines for directory and symlink entries, excluding the
transfer-root `./` line (FastSync emits it unconditionally; a documented
residual)."""
out = []
for line in (text or "").splitlines():
line = line.rstrip()
if not line or not _ITEMIZE_LINE_RE.match(line):
continue
name = line.rsplit(" ", 1)[-1]
if name == "./":
continue
if name.endswith("/") or " -> " in line:
out.append(line)
return sorted(out)
@requires_rsync
@parity
def test_itemize_rerun_dirs_symlinks_matches_rsync(parity_server_factory):
"""#314: a re-run reports directory/symlink destination state like rsync.
On an unchanged tree FastSync emits no per-directory `cd+++++++++` (or
symlink) lines, and after a changed directory mtime / symlink target it
renders rsync's `.d..t......` / `cLc........` instead of `cd`/`cL`."""
case_id = "itemize_rerun_dirs_symlinks"
src = os.path.join(TEST_DATA_DIR, "parity_itemds_src")
rdst = os.path.join(TEST_DATA_DIR, "parity_itemds_rdst")
fdst = os.path.join(TEST_DATA_DIR, "parity_itemds_fdst")
clean_dir(src)
_mk(os.path.join(src, "sub", "b.txt"), b"nested\n")
os.makedirs(os.path.join(src, "emptydir"), exist_ok=True)
os.symlink("a.txt", os.path.join(src, "link"))
_mk(os.path.join(src, "a.txt"), b"top\n")
clean_dir(rdst)
clean_dir(fdst)
server = parity_server_factory(SUPER)
rroot = rdst
froot = get_dest_received_dir(fdst, src)
_seed_dest_tree(src, rroot)
_seed_dest_tree(src, froot)
# Unchanged re-run: no directory or symlink itemize lines from either tool.
rs = H.run_rsync(src, rdst, ["-a", "-i"])
fs, _ = H.run_fastsync(src, fdst, ["-a", "-i", "--incremental"], server.port)
assert rs.returncode == 0, rs.stderr
assert fs.returncode == 0, fs.stderr
assert _itemize_dir_link_lines(rs.stdout) == []
fast_unchanged = _itemize_dir_link_lines(fs.stdout)
assert fast_unchanged == [], f"unchanged re-run itemized dirs/links: {fast_unchanged}"
# Change the directory mtime and the symlink target, then re-run.
_pin(os.path.join(src, "sub"), _OLD_MTIME)
os.remove(os.path.join(src, "link"))
os.symlink("b.txt", os.path.join(src, "link"))
rs = H.run_rsync(src, rdst, ["-a", "-i"])
fs, _ = H.run_fastsync(src, fdst, ["-a", "-i", "--incremental"], server.port)
assert rs.returncode == 0, rs.stderr
assert fs.returncode == 0, fs.stderr
expected = _itemize_dir_link_lines(rs.stdout)
actual = _itemize_dir_link_lines(fs.stdout)
assert actual == expected, f"rsync={rs.stdout!r} fastsync={fs.stdout!r}"
assert any(line.endswith(" sub/") and line.startswith(".d..t") for line in actual), actual
assert any(line.startswith("cLc") and " -> b.txt" in line for line in actual), actual
def _deleted_breakdown_line(text):
for line in (text or "").splitlines():
if line.startswith("Number of deleted files:"):
return " ".join(line.split())
return ""
@requires_rsync
@parity
def test_stats_deleted_breakdown_matches_rsync(parity_server_factory):
"""#316: `--stats` renders rsync's per-type `Number of deleted files`
breakdown for removed regular files, directories, symlinks and a special."""
case_id = "stats_deleted_breakdown"
src = os.path.join(TEST_DATA_DIR, "parity_delbd_src")
rdst = os.path.join(TEST_DATA_DIR, "parity_delbd_rdst")
fdst = os.path.join(TEST_DATA_DIR, "parity_delbd_fdst")
clean_dir(src)
_mk(os.path.join(src, "keep.txt"), b"keep\n")
server = parity_server_factory(DELETE)
def seed(_src, rroot, froot):
for root in (rroot, froot):
_mk(os.path.join(root, "extra1.txt"), b"e1\n", _OLD_MTIME)
_mk(os.path.join(root, "extradir", "inside.txt"), b"e2\n", _OLD_MTIME)
os.makedirs(os.path.join(root, "extradir"), exist_ok=True)
link = os.path.join(root, "extralink")
if not os.path.lexists(link):
os.symlink("keep.txt", link)
fifo = os.path.join(root, "extrafifo")
if not os.path.exists(fifo):
os.mkfifo(fifo)
def extra(_src, _rroot, _froot, rs, fs):
rs_line = _deleted_breakdown_line(rs.stdout)
fs_line = _deleted_breakdown_line(fs.stdout)
if not rs_line:
return ["rsync printed no deleted-files line"]
if rs_line != fs_line:
return [f"deleted breakdown rsync={rs_line!r} fastsync={fs_line!r}"]
if "reg:" not in rs_line or "dir:" not in rs_line or \
"link:" not in rs_line or "special:" not in rs_line:
return [f"breakdown missing a category: {rs_line!r}"]
return []
result = H.run_differential(
src, rdst, fdst, ["-a", "--delete", "--stats"],
["-a", "--delete", "--stats", "--incremental"], server,
seed=seed, extra_check=extra)
_run_and_check(case_id, result, ref="--stats deleted per-type breakdown")
@requires_rsync @requires_rsync
@parity @parity
def test_one_file_system(parity_server_factory): def test_one_file_system(parity_server_factory):
+10
View File
@@ -64,6 +64,7 @@ static void test_dest_state_roundtrip() {
memset(&out, 0, sizeof(out)); memset(&out, 0, sizeof(out));
out.known = true; out.known = true;
out.existed = true; out.existed = true;
out.target_matches = true;
out.size = 123456789ULL; out.size = 123456789ULL;
out.mtime_sec = 1700000000; out.mtime_sec = 1700000000;
out.mtime_nsec = 123456789; out.mtime_nsec = 123456789;
@@ -76,6 +77,7 @@ static void test_dest_state_roundtrip() {
EXPECT_TRUE(format_dest_state_receive(fds[1], &in)); EXPECT_TRUE(format_dest_state_receive(fds[1], &in));
EXPECT_TRUE(in.known); EXPECT_TRUE(in.known);
EXPECT_TRUE(in.existed); EXPECT_TRUE(in.existed);
EXPECT_TRUE(in.target_matches);
EXPECT_TRUE(in.size == out.size); EXPECT_TRUE(in.size == out.size);
EXPECT_TRUE(in.mtime_sec == out.mtime_sec); EXPECT_TRUE(in.mtime_sec == out.mtime_sec);
EXPECT_TRUE(in.mtime_nsec == out.mtime_nsec); EXPECT_TRUE(in.mtime_nsec == out.mtime_nsec);
@@ -103,6 +105,10 @@ static void test_stats_roundtrip() {
out.created_dir = 4; out.created_dir = 4;
out.created_link = 2; out.created_link = 2;
out.created_special = 1; out.created_special = 1;
out.deleted_reg = 9;
out.deleted_dir = 6;
out.deleted_link = 3;
out.deleted_special = 2;
ReceiverStats in; ReceiverStats in;
memset(&in, 0, sizeof(in)); memset(&in, 0, sizeof(in));
EXPECT_TRUE(format_stats_send(fds[0], &out)); EXPECT_TRUE(format_stats_send(fds[0], &out));
@@ -115,6 +121,10 @@ static void test_stats_roundtrip() {
EXPECT_TRUE(in.created_dir == out.created_dir); EXPECT_TRUE(in.created_dir == out.created_dir);
EXPECT_TRUE(in.created_link == out.created_link); EXPECT_TRUE(in.created_link == out.created_link);
EXPECT_TRUE(in.created_special == out.created_special); EXPECT_TRUE(in.created_special == out.created_special);
EXPECT_TRUE(in.deleted_reg == out.deleted_reg);
EXPECT_TRUE(in.deleted_dir == out.deleted_dir);
EXPECT_TRUE(in.deleted_link == out.deleted_link);
EXPECT_TRUE(in.deleted_special == out.deleted_special);
close(fds[0]); close(fds[0]);
close(fds[1]); close(fds[1]);
} }
+2 -1
View File
@@ -153,7 +153,8 @@ static void test_receiver_deleted_paths_gated_by_report_deletes() {
PipelineContextReceiver* ctx_info = pipeline_context_receiver_create(info, q_info, -1, NULL); PipelineContextReceiver* ctx_info = pipeline_context_receiver_create(info, q_info, -1, NULL);
EXPECT_NOT_NULL(ctx_info); EXPECT_NOT_NULL(ctx_info);
EXPECT_NOT_NULL(ctx_info->deleted_paths); EXPECT_NOT_NULL(ctx_info->deleted_paths);
receiver_record_deleted_path(ctx_info->deleted_paths, "d/old_extra"); ReceiverDeleteContext delctx = {NULL, ctx_info->deleted_paths};
receiver_record_deleted_path(&delctx, "d/old_extra", DELETE_ENTRY_REG);
EXPECT_EQ_INT(ctx_info->deleted_paths->size, 1); EXPECT_EQ_INT(ctx_info->deleted_paths->size, 1);
EXPECT_EQ_STR((const char*)ctx_info->deleted_paths->items[0], "d/old_extra"); EXPECT_EQ_STR((const char*)ctx_info->deleted_paths->items[0], "d/old_extra");
pipeline_context_receiver_destroy(ctx_info); pipeline_context_receiver_destroy(ctx_info);