diff --git a/src/client/change_list.c b/src/client/change_list.c index a2c5096..892cef7 100644 --- a/src/client/change_list.c +++ b/src/client/change_list.c @@ -141,6 +141,10 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co update = 'h'; else if (created) update = (event->is_directory || event->is_symlink || event->is_special) ? 'c' : '>'; + else if (event->is_directory) + /* rsync: an existing directory that only has attribute changes carries no + transfer, so the update column is `.` rather than `>`. */ + update = '.'; else update = '>'; code[0] = update; @@ -168,12 +172,15 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co code[11] = '\0'; } -/* rsync %n: the transfer-relative name, with a trailing slash for directories. */ +/* rsync %n: the transfer-relative name, with a trailing slash for directories. + * The transfer root is `.` (so `%n` renders `./`), matching rsync's root entry. */ static bool append_name(StrBuf* buf, const ChangeEvent* event) { - if (!strbuf_append(buf, event->name != NULL ? event->name : "")) + const char* name = event->name != NULL ? event->name : ""; + if (event->is_directory && name[0] == '\0') + return strbuf_append(buf, "./"); + if (!strbuf_append(buf, name)) return false; - if (event->is_directory && (event->name == NULL || event->name[0] == '\0' || - event->name[strlen(event->name) - 1] != '/')) + if (event->is_directory && name[strlen(name) - 1] != '/') return strbuf_append_char(buf, '/'); return true; } diff --git a/src/client/client_report.c b/src/client/client_report.c index 08050df..2cbe5df 100644 --- a/src/client/client_report.c +++ b/src/client/client_report.c @@ -243,17 +243,31 @@ void transfer_stats_note_transferred(TransferStats* stats, const File* file) { #define RSYNC_PROGRESS_IO_WINDOW (32ULL * 1024ULL) /* Paths-only pre-count of the source file list, built once at transfer start - * when progress output is requested. rsync's `to-chk` denominator is the whole - * file list -- every regular file, directory, symlink and special plus the - * transfer root -- while the streaming scan never emits directories. A - * metadata-only walk (no file reads, no hashing) supplies that total and the - * directory names, so the opt-in pass leaves non-progress runs untouched. */ + * when progress output or -i/--out-format needs it. rsync's `to-chk` + * denominator is the whole file list -- every regular file, directory, symlink + * and special plus the transfer root -- while the streaming scan only emits + * empty directories. A metadata-only walk (no file reads, no hashing) supplies + * that total and a metadata-bearing File for every directory, so --progress can + * name them and -i/--out-format can itemize them without a second full scan. */ +/* One directory in the pre-count, keyed by its transfer-relative display name + * ("" is the transfer root). `file` is owned by ProgressPrecount.dir_files and + * carries the source metadata needed by -i/--out-format (%M/%B/%U/%G). */ +typedef struct { + char* name; /* owned */ + File* file; +} DirRef; + typedef struct { unsigned long long total; ArrayList* dir_paths; /* owned char* in transfer-relative display form */ + ArrayList* dir_files; /* owned File* captured during the metadata walk */ + ArrayList* dir_refs; /* owned DirRef*, sorted by name for prefix lookup */ } ProgressPrecount; static bool g_progress_active; +/* True when -i/--out-format need the pre-counted directory entries fed into the + * change-event stream (independent of --progress). */ +static bool g_change_dirs_active; static unsigned long long g_progress_xferred; static unsigned long long g_progress_index; static unsigned long long g_progress_total; @@ -270,14 +284,101 @@ bool progress_requested(const Config* config) { (config->show_progress || (config->info_level & LOG_INFO_PROGRESS) != 0); } +static void dir_ref_destroy(void* item) { + DirRef* ref = (DirRef*)item; + if (ref == NULL) + return; + free(ref->name); + free(ref); +} + +/* Sort DirRef pointers by their transfer-relative name for binary search. */ +static int dir_ref_compare(const void* left, const void* right) { + const DirRef* a = *(const DirRef* const*)left; + const DirRef* b = *(const DirRef* const*)right; + return strcmp(a->name, b->name); +} + +/* Look up the pre-counted directory File for a transfer-relative name ("" is + * the transfer root). Returns NULL when no pre-count was built or the name is + * not a known directory. */ +static File* progress_dir_lookup(const char* name) { + if (name == NULL || g_progress_precount.dir_refs == NULL) + return NULL; + ArrayList* refs = g_progress_precount.dir_refs; + size_t lo = 0; + size_t hi = (size_t)refs->size; + while (lo < hi) { + size_t mid = lo + (hi - lo) / 2; + DirRef* ref = (DirRef*)refs->items[mid]; + int cmp = strcmp(ref->name, name); + if (cmp < 0) + lo = mid + 1; + else if (cmp > 0) + hi = mid; + else + return ref->file; + } + return NULL; +} + static void progress_precount_dispose(ProgressPrecount* p) { if (p->dir_paths != NULL) { array_list_delete(p->dir_paths); p->dir_paths = NULL; } + if (p->dir_files != NULL) { + array_list_delete(p->dir_files); + p->dir_files = NULL; + } + if (p->dir_refs != NULL) { + array_list_delete(p->dir_refs); + p->dir_refs = NULL; + } p->total = 0; } +/* Record the transfer root's pre-transfer state for -i/--out-format. The + * receive root always exists, so rsync never marks it `cd`; its only observable + * change is its timestamp, which FastSync cannot observe remotely. Force a time + * mismatch so the root renders rsync's `.d..t...... ./` rather than the `cd` + * a zeroed destination state would produce. */ +static void progress_precount_mark_root(File* root) { + if (root == NULL) + return; + root->dest_state.known = true; + root->dest_state.existed = true; + root->dest_state.mode = root->metadata != NULL ? root->metadata->mode : 0; + root->dest_state.uid = root->metadata != NULL ? root->metadata->uid : 0; + root->dest_state.gid = root->metadata != NULL ? root->metadata->gid : 0; + root->dest_state.size = 0; + root->dest_state.mtime_sec = (root->metadata != NULL ? root->metadata->mtime_sec : 0) - 3600; + root->dest_state.mtime_nsec = root->metadata != NULL ? root->metadata->mtime_nsec : 0; +} + +/* Append one DirRef (name -> file) to the pre-count, marking the transfer + * root's destination state. Returns false on allocation failure. */ +static bool progress_precount_add_ref(ProgressPrecount* p, const Config* config, File* file) { + const char* rel = delete_display_path(config, file_wire_path(file)); + char* name = rel != NULL ? str_dup(rel) : NULL; + if (name == NULL) + return false; + DirRef* ref = malloc(sizeof(*ref)); + if (ref == NULL) { + free(name); + return false; + } + ref->name = name; + ref->file = file; + if (name[0] == '\0') + progress_precount_mark_root(file); + if (!array_list_add(p->dir_refs, ref)) { + dir_ref_destroy(ref); + return false; + } + return true; +} + void client_progress_cleanup(void) { if (g_progress_dir_index_valid) { path_index_free(&g_progress_dir_index); @@ -293,6 +394,7 @@ void client_progress_cleanup(void) { } progress_precount_dispose(&g_progress_precount); g_progress_active = false; + g_change_dirs_active = false; g_progress_total = 0; g_progress_index = 0; g_progress_xferred = 0; @@ -395,6 +497,11 @@ void print_delete_reports(const Config* config, const ArrayList* paths) { fflush(stdout); } +/* Emit every not-yet-seen ancestor directory of `rel`, outermost first, in the + * order rsync's depth-first flist walk visits them. With -i/--out-format each + * ancestor becomes a real change line (`cd+++++++++ sub/`, `.d..t...... ./`) + * rendered by the shared itemize code; otherwise it is the `--info=name` / + * --progress directory name line. */ static void client_progress_emit_ancestors(const Config* config, const char* rel) { if (!g_progress_dir_index_valid || !g_progress_emitted_valid || g_progress_emitted_keys == NULL || rel == NULL) @@ -413,9 +520,15 @@ static void client_progress_emit_ancestors(const Config* config, const char* rel char* key = str_dup(prefix); if (key != NULL && array_list_add(g_progress_emitted_keys, key)) { str_hash_set_insert_ref(&g_progress_emitted, key); - char* escaped = output_escape(prefix, config->eight_bit_output); - printf("%s/\n", escaped ? escaped : prefix); - free(escaped); + if (g_change_dirs_active) { + File* dir = progress_dir_lookup(prefix); + if (dir != NULL) + change_emit_dir_sent(config, dir); + } else { + char* escaped = output_escape(prefix, config->eight_bit_output); + printf("%s/\n", escaped ? escaped : prefix); + free(escaped); + } g_progress_index++; } else { free(key); @@ -425,6 +538,20 @@ static void client_progress_emit_ancestors(const Config* config, const char* rel } } +/* Feed a transferred entry's ancestor directories into the change-event stream + * before the entry's own line, so -i/--out-format and --progress report + * directories in rsync's depth-first order. Every directory is an ancestor of + * some emitted entry (a file, symlink, special, hard link or the empty-directory + * entry the scanner emits for a leaf), so this covers the whole tree. */ +void client_change_emit_ancestors(const Config* config, const File* file) { + if (config == NULL || file == NULL) + return; + if (!g_progress_active && !g_change_dirs_active) + return; + const char* rel = delete_display_path(config, file_wire_path(file)); + client_progress_emit_ancestors(config, rel); +} + /* rsync's --info=name/progress line for one entry: transfer-relative name (a * trailing slash for directories) plus the ` -> target` symlink suffix. */ static char* progress_entry_line(const File* file, const char* rel) { @@ -462,7 +589,7 @@ void client_progress_begin(const Config* config) { g_progress_active = progress_requested(config); g_progress_xferred = 0; g_progress_index = 1; /* the transfer root is file-list entry #0 */ - if (!g_progress_active) { + if (!g_progress_active && !g_change_dirs_active) { /* `--info=flist` prints rsync's file-list header even without progress. */ if (!config->quiet && info_flag_enabled(config, LOG_INFO_FLIST)) { printf("sending incremental file list\n"); @@ -470,11 +597,18 @@ void client_progress_begin(const Config* config) { } return; } - printf("sending incremental file list\n"); - /* rsync prints the transfer-root directory's name before the first file when - that directory is created; FastSync mirrors the source root below the - receive root and creates it on a fresh destination, so emit it here. */ - printf("./\n"); + if (g_progress_active) + printf("sending incremental file list\n"); + /* rsync prints the transfer-root directory before the first entry. Under + -i/--out-format it is the root change line (`.d..t...... ./`); otherwise it + is the plain --info=name / --progress name line. */ + if (g_change_dirs_active) { + File* root = progress_dir_lookup(""); + if (root != NULL) + change_emit_dir_sent(config, root); + } else { + printf("./\n"); + } fflush(stdout); } @@ -487,7 +621,6 @@ void client_progress_file(const Config* config, const File* file) { unsigned long long size = file->data->size; if (!config->itemize_changes && config->out_format == NULL) { const char* rel = delete_display_path(config, file_wire_path(file)); - client_progress_emit_ancestors(config, rel); char* escaped = output_escape(rel, config->eight_bit_output); printf("%s\n", escaped ? escaped : (rel ? rel : "")); free(escaped); @@ -510,7 +643,6 @@ void client_progress_name(const Config* config, const File* file) { return; const char* rel = delete_display_path(config, file_wire_path(file)); if (!config->itemize_changes && config->out_format == NULL) { - client_progress_emit_ancestors(config, rel); char* line = progress_entry_line(file, rel ? rel : ""); if (line != NULL) { char* escaped = output_escape(line, config->eight_bit_output); @@ -550,8 +682,12 @@ static bool progress_precount_add_dir(ProgressPrecount* p, const char* path) { * so the data pass's link-group state is never perturbed. */ static bool progress_precount_scan(const Config* config, ProgressPrecount* out) { out->dir_paths = array_list_create(free); - if (out->dir_paths == NULL) + out->dir_files = array_list_create(file_destroy); + out->dir_refs = array_list_create(dir_ref_destroy); + if (out->dir_paths == NULL || out->dir_files == NULL || out->dir_refs == NULL) { + progress_precount_dispose(out); return false; + } out->total = 0; PreparedScanner prepared; memset(&prepared, 0, sizeof(prepared)); @@ -568,12 +704,14 @@ static bool progress_precount_scan(const Config* config, ProgressPrecount* out) local.preserve_xattrs = false; local.preserve_acls = false; local.checksum = false; - local.capture_dir_times = false; + /* Capture one metadata-bearing File per traversed directory (including the + transfer root) so -i/--out-format can render %M/%B/%U/%G for directories. */ + local.capture_dir_times = true; local.excluded_paths = NULL; local.size_skipped_paths = NULL; local.synced_dirs = NULL; local.plan_dirs = NULL; - local.dir_entries = NULL; + local.dir_entries = out->dir_files; local.dir_entries_mutex = NULL; local.hardlinks = NULL; DirectoryScanner* scanner = directory_scanner_create_with_options(config->send_directory, &local); @@ -598,6 +736,18 @@ static bool progress_precount_scan(const Config* config, ProgressPrecount* out) progress_precount_dispose(out); return false; } + /* Build the name -> File lookup from the captured directory Files. */ + for (int i = 0; i < out->dir_files->size; i++) { + File* f = (File*)out->dir_files->items[i]; + if (f == NULL) + continue; + if (!progress_precount_add_ref(out, config, f)) { + progress_precount_dispose(out); + return false; + } + } + if (out->dir_refs->size > 1) + qsort(out->dir_refs->items, (size_t)out->dir_refs->size, sizeof(DirRef*), dir_ref_compare); out->total += 1; /* the transfer root "." */ return true; } @@ -609,9 +759,26 @@ static bool progress_precount_from_plan_dirs(const Config* config, const ArrayLi unsigned long long non_dir_count, ProgressPrecount* out) { out->dir_paths = array_list_create(free); - if (out->dir_paths == NULL) + out->dir_files = array_list_create(file_destroy); + out->dir_refs = array_list_create(dir_ref_destroy); + if (out->dir_paths == NULL || out->dir_files == NULL || out->dir_refs == NULL) { + progress_precount_dispose(out); return false; + } out->total = non_dir_count + 1; + /* The delete pre-scan's plan list omits the transfer root, so synthesize its + entry here; it is only used for the root change line. */ + File* root = file_create(""); + if (root == NULL || !array_list_add(out->dir_files, root)) { + file_destroy(root); + progress_precount_dispose(out); + return false; + } + root->is_dir = true; + if (!progress_precount_add_ref(out, config, root)) { + progress_precount_dispose(out); + return false; + } for (int i = 0; i < plan_dirs->size; i++) { const char* path = (const char*)plan_dirs->items[i]; const char* rel = config->send_directory != NULL @@ -621,7 +788,25 @@ static bool progress_precount_from_plan_dirs(const Config* config, const ArrayLi progress_precount_dispose(out); return false; } + File* dir = file_create(""); + if (dir == NULL) { + progress_precount_dispose(out); + return false; + } + dir->is_dir = true; + dir->send_path = str_dup(rel != NULL ? rel : ""); + if (dir->send_path == NULL || !array_list_add(out->dir_files, dir)) { + file_destroy(dir); + progress_precount_dispose(out); + return false; + } + if (!progress_precount_add_ref(out, config, dir)) { + progress_precount_dispose(out); + return false; + } } + if (out->dir_refs->size > 1) + qsort(out->dir_refs->items, (size_t)out->dir_refs->size, sizeof(DirRef*), dir_ref_compare); out->total += (unsigned long long)out->dir_paths->size; return true; } @@ -633,7 +818,8 @@ void client_progress_prepare(const Config* config, const ArrayList* plan_dirs, unsigned long long plan_non_dir_count) { client_progress_cleanup(); g_progress_active = progress_requested(config); - if (!g_progress_active) + g_change_dirs_active = config->itemize_changes || config->out_format != NULL; + if (!g_progress_active && !g_change_dirs_active) return; bool ok = plan_dirs != NULL ? progress_precount_from_plan_dirs( config, plan_dirs, plan_non_dir_count, &g_progress_precount) diff --git a/src/client/client_send.c b/src/client/client_send.c index 3b9d623..9bb783f 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -838,6 +838,10 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config, if (chunk->items[i] == NULL) continue; transfer_stats_note_entry(stats, chunk->items[i]); + /* The chunk-serialization path emits no --progress name lines, so only + feed -i/--out-format its ancestor directory lines here. */ + if (config->itemize_changes || config->out_format != NULL) + client_change_emit_ancestors(config, chunk->items[i]); if (chunk->items[i]->is_dir) change_emit_dir_sent(config, chunk->items[i]); else @@ -859,6 +863,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config, source to remove and no incremental check. */ if (!send_directory_entry(client, f, config)) return -1; + client_change_emit_ancestors(config, f); change_emit_dir_sent(config, f); client_progress_name(config, f); continue; @@ -873,6 +878,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config, !send_int(client->file_descriptor, f->link_group) || !send_wire_str(client->file_descriptor, f->hardlink_target)) return -1; + client_change_emit_ancestors(config, f); change_emit_file_sent(config, f); client_progress_name(config, f); continue; @@ -881,6 +887,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config, if (f->is_symlink) { if (!send_symlink_entry(client, f, config)) return -1; + client_change_emit_ancestors(config, f); change_emit_file_sent(config, f); client_progress_name(config, f); continue; @@ -890,6 +897,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config, if (f->is_special) { if (!file_send_special(f, client->file_descriptor, config->use_metadata)) return -1; + client_change_emit_ancestors(config, f); change_emit_file_sent(config, f); client_progress_name(config, f); continue; @@ -913,6 +921,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config, return -1; } transfer_stats_note_transferred(stats, f); + client_change_emit_ancestors(config, f); change_emit_file_sent_bytes(config, f, protocol_bytes_written() - bytes_before, protocol_bytes_read() - read_before); client_progress_file(config, f); @@ -1567,7 +1576,10 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) { * runs the shared cleanup). */ static bool send_files_run(Config* config, SendFilesState* state) { Client* client = state->client; - if (progress_requested(config)) + /* --progress needs the file-list total; -i/--out-format needs the directory + entries. Either way one paths-only pre-count supplies both, and a + --delete-during/--delete-delay pre-scan is reused when present. */ + if (progress_requested(config) || config->itemize_changes || config->out_format != NULL) client_progress_prepare(config, state->plan_dirs, state->per_dir_non_dir_count); /* Phase 6: compute the client-only stop deadline once at transfer start. The early-delete pre-scan above deliberately ignores it so the keep-set (and @@ -2045,9 +2057,10 @@ int send_files_multithreaded(Config* config) { return 1; } /* --progress/--info=progress: pre-count the file list for rsync's to-chk - denominator, reusing a --delete-during/--delete-delay pre-scan when one + denominator; -i/--out-format: pre-count the directory entries. One pass + supplies both, reusing a --delete-during/--delete-delay pre-scan when one already ran. */ - if (progress_requested(config)) + if (progress_requested(config) || config->itemize_changes || config->out_format != NULL) client_progress_prepare(config, context->plan_dirs, pre_scan_non_dir); thrd_t scanner, loader, sender; diff --git a/src/client/client_send_internal.h b/src/client/client_send_internal.h index 6d042b5..d82d45d 100644 --- a/src/client/client_send_internal.h +++ b/src/client/client_send_internal.h @@ -64,6 +64,9 @@ void client_progress_cleanup(void); void client_progress_begin(const Config* config); void client_progress_file(const Config* config, const File* file); void client_progress_name(const Config* config, const File* file); +/* Emit a transferred entry's ancestor directories (as -i/--out-format change + * lines or --progress name lines) before the entry's own line. */ +void client_change_emit_ancestors(const Config* config, const File* file); void client_progress_uptodate(const Config* config, const File* file); void client_progress_prepare(const Config* config, const ArrayList* plan_dirs, unsigned long long plan_non_dir_count); diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index e60d8c7..03fcb9e 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -2786,7 +2786,12 @@ class TestItemizeChanges: flags=["--preserve", "-i", "--incremental"], port=shared_server.port) assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}" - itemized = [line for line in result.stdout.splitlines() if line and line[0] in ">.. fl, f"fastsync %b must include framing: {result.stdout!r}" @@ -421,7 +477,9 @@ class TestWireStatsParity: assert file_lines(result.stdout) == file_lines(rsync_result.stdout), ( f"rsync={rsync_result.stdout!r} fastsync={result.stdout!r}" ) - assert result.stdout.split()[0] == rsync_result.stdout.split()[0] == "16", ( + fs_c = _file_entry_line(result.stdout).split()[0] + rs_c = _file_entry_line(rsync_result.stdout).split()[0] + assert fs_c == rs_c == "16", ( f"%c must be rsync's 16-byte sum header: {result.stdout!r}" ) @@ -450,8 +508,8 @@ class TestWireStatsParity: "--out-format=" + fmt], port=shared_server.port) assert result.returncode == 0, result.stderr[:300] - rs_c = int(rsync_result.stdout.split()[0]) - fs_c = int(result.stdout.split()[0]) + rs_c = int(_file_entry_line(rsync_result.stdout).split()[0]) + fs_c = int(_file_entry_line(result.stdout).split()[0]) # No basis exists, so rsync still reports only its sum header. assert rs_c == 16, rsync_result.stdout # FastSync reports its own handshake bytes and is not aligned.