diff --git a/CMakeLists.txt b/CMakeLists.txt index e985d79..905daec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.22) -project(FastFileTransfer VERSION 2.27.0) +project(FastFileTransfer VERSION 2.28.0) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_C_STANDARD 11) diff --git a/RSYNC_COMPAT.md b/RSYNC_COMPAT.md index 2a6c809..31a8e0a 100644 --- a/RSYNC_COMPAT.md +++ b/RSYNC_COMPAT.md @@ -82,7 +82,7 @@ Every one of those has an entry below with its remaining caveats. | Flag | Rsync Description | FastSync Status | Notes | |------|-------------------|-----------------|-------| -| `--stats` | Give transfer stats | ⚠️ Caveat | Prints transfer statistics. Protocol 2.25.0 populates the receiver-only counters the sender cannot observe: `Matched data` (a delta basis's reused bytes) and `Number of deleted files` come from the receiver's `STATUS_STATS` report. The sender now tracks the scanned file list per type and only counts regular files the receiver actually stored, so `Number of files` carries rsync's `(reg: X, dir: Y, link: Z, special: W)` breakdown (directories come from the scanner's captured directory list, present for `-a`/`-t`/`-p`), `Number of regular files transferred` excludes symlinks/specials and up-to-date files, `Total file size` includes symlink target lengths, and `Total transferred file size`/`Literal data` count only transferred files — all differential-tested in the sequential and `--threads` paths. **Remaining divergences:** `Number of created files` is the transferred-regular count (FastSync cannot tell which entries the receiver newly created, so on an update where rsync reports 0 created FastSync can report the transferred file) and lacks the type breakdown; a recursive scan that preserves no directory attribute (`-r` without `-t`/`-p`) captures no directory entries, so the `dir:` category is then omitted; `Literal data` is exact for a whole-file transfer but an upper bound for a delta transfer (the sender counts each stored file's whole source size rather than only the literal fragments rsync ships, since it does not measure the delta payload it sends); rsync's per-type breakdown on `Number of deleted files` is not reproduced; and `Total bytes sent`/`received` are FastSync wire bytes framed differently from rsync's, so they are not numerically comparable | +| `--stats` | Give transfer stats | ⚠️ Caveat | Prints transfer statistics. Protocol 2.25.0 populates the receiver-only counters the sender cannot observe (`Matched data`, `Number of deleted files`) from the receiver's `STATUS_STATS` report; the sender tracks the scanned file list per type so `Number of files` carries rsync's `(reg: X, dir: Y, link: Z, special: W)` breakdown (directories come from the scanner's captured directory list, present for `-a`/`-t`/`-p`), `Number of regular files transferred` excludes symlinks/specials and up-to-date files, `Total file size` includes symlink target lengths, and `Total transferred file size` counts only transferred files. **Protocol 2.28.0 extends `STATUS_STATS`** with receiver-observed `literal_bytes` and the four `created_*` counters: `Number of created files` now carries rsync's `(reg/dir/link/special)` breakdown (the receiver reports which destination entries it newly created, including implicitly-created parent directories below the transfer root) and `Literal data` is exact for a delta transfer (the receiver counts the literal fragments it stored, not the whole source size) — all differential-tested in the sequential and `--threads` paths against rsync 3.4.1 for fresh-create, update and delta shapes. **Remaining divergences:** a recursive scan that preserves no directory attribute (`-r` without `-t`/`-p`) captures no directory entries, so the `dir:` category is then omitted from `Number of files`; rsync's per-type breakdown on `Number of deleted files` is not reproduced; and `Total bytes sent`/`received` are FastSync wire bytes framed differently from rsync's, so they are not numerically comparable | | `-h`, `--human-readable` | Human-readable numbers | ✅ Parity | Formats transfer byte and rate counts using rsync's **decimal** (base-1000) units, matching rsync `-h` (e.g. `1.23M`), not binary units. **A lone `-h` with no transfer arguments prints help instead** (protocol 2.26.0), matching the rsync idiom; `-h` alongside a transfer remains human-readable | | `-i`, `--itemize-changes` | Per-file change summary | ✅ Parity | Prints rsync-style `>f+++++++++` lines to stdout only for files actually sent (also under `-j`/`--threads`); unchanged files print nothing, matching single-`-i` behavior | | `--progress` | Show progress | ⚠️ Caveat | Protocol 2.25.0 prints rsync-style per-file progress blocks (percent, transferred/total bytes, rate, elapsed, `(xfr#N, to-chk=M/T)`) fed by the receiver's `STATUS_STATS`, in both the sequential and `--threads` send paths. FastSync now also prints rsync's leading `./` transfer-root line and counts that root entry in the `to-chk` denominator, so a **single-file transfer's name lines and deterministic frames are byte-identical to rsync** (differential test). **Remaining divergences:** for a multi-directory tree rsync prints a per-directory name line and its `to-chk` denominator includes every directory/symlink/special entry; FastSync's streaming scan emits only file-name lines and counts just the root plus transferred files (a full flist pre-count would be needed), and the rate/ETA are wall-clock dependent | diff --git a/src/client/client_send.c b/src/client/client_send.c index e984788..65e0ac7 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -93,12 +93,11 @@ static const char* stats_bytes(const Config* config, unsigned long long bytes, c return buffer; } -/* Build rsync's `Number of files` parenthetical: each non-zero category, in - reg/dir/link/special order. Empty when the flist counted nothing. */ -static void stats_type_breakdown(const TransferStats* stats, char* out, size_t out_size) { - unsigned long long total = - stats->flist_reg + stats->flist_dir + stats->flist_link + stats->flist_special; - if (total == 0) { +/* Build rsync's per-type parenthetical: each non-zero category, in + reg/dir/link/special order. Empty when every count is zero. */ +static void type_breakdown(unsigned long long reg, unsigned long long dir, unsigned long long link, + unsigned long long special, char* out, size_t out_size) { + if (reg + dir + link + special == 0) { out[0] = '\0'; return; } @@ -107,10 +106,7 @@ static void stats_type_breakdown(const TransferStats* stats, char* out, size_t o const struct { const char* name; unsigned long long count; - } parts[4] = {{"reg", stats->flist_reg}, - {"dir", stats->flist_dir}, - {"link", stats->flist_link}, - {"special", stats->flist_special}}; + } parts[4] = {{"reg", reg}, {"dir", dir}, {"link", link}, {"special", special}}; bool first = true; for (size_t i = 0; i < 4; i++) { if (parts[i].count == 0) @@ -127,6 +123,12 @@ static void stats_type_breakdown(const TransferStats* stats, char* out, size_t o out[used] = '\0'; } +/* Build rsync's `Number of files` parenthetical from the scan's flist counts. */ +static void stats_type_breakdown(const TransferStats* stats, char* out, size_t out_size) { + type_breakdown(stats->flist_reg, stats->flist_dir, stats->flist_link, stats->flist_special, out, + out_size); +} + /* Print the rsync `--stats` block on stdout. The source-side flist and transferred counters come from `stats` (filled while scanning/sending), the receiver-only counters from the STATUS_STATS frame, and the wire byte totals @@ -151,6 +153,7 @@ static void report_transfer_stats(const Config* config, const TransferStats* sta char total_buffer[32]; char transferred_buffer[32]; char literal_buffer[32]; + char matched_buffer[32]; char sent_buffer[32]; char recv_buffer[32]; char rate_buffer[32] = {0}; @@ -159,8 +162,15 @@ static void report_transfer_stats(const Config* config, const TransferStats* sta stats_bytes(config, stats->total_file_size, total_buffer, sizeof(total_buffer)); const char* transferred = stats_bytes(config, stats->transferred_file_size, transferred_buffer, sizeof(transferred_buffer)); - const char* literal = - stats_bytes(config, stats->literal_data, literal_buffer, sizeof(literal_buffer)); + /* Protocol 2.28.0: the receiver reports the bytes it literally stored, which + is exact for a delta transfer (the sender's own literal_data counts each + stored file's whole source size and is only an upper bound). Fall back to + the sender total when the receiver reported no delta/literal accounting + (e.g. a local no-server path). */ + unsigned long long literal_bytes = (recv->literal_bytes != 0 || recv->matched_data != 0) + ? recv->literal_bytes + : stats->literal_data; + const char* literal = stats_bytes(config, literal_bytes, literal_buffer, sizeof(literal_buffer)); const char* sent_s = stats_bytes(config, sent, sent_buffer, sizeof(sent_buffer)); const char* recv_s = stats_bytes(config, received, recv_buffer, sizeof(recv_buffer)); const char* rate_str = rate_buffer; @@ -177,21 +187,30 @@ static void report_transfer_stats(const Config* config, const TransferStats* sta stats_type_breakdown(stats, breakdown, sizeof(breakdown)); unsigned long long flist_total = stats->flist_reg + stats->flist_dir + stats->flist_link + stats->flist_special; + char created_breakdown[128]; + type_breakdown(recv->created_reg, recv->created_dir, recv->created_link, recv->created_special, + created_breakdown, sizeof(created_breakdown)); + unsigned long long created_total = + recv->created_reg + recv->created_dir + recv->created_link + recv->created_special; printf("\n"); if (breakdown[0] != '\0') printf("Number of files: %llu %s\n", flist_total, breakdown); else printf("Number of files: %llu\n", flist_total); - /* FastSync cannot tell which entries the receiver newly created, so it - reports the transferred regular files (which are created on a fresh - destination). See RSYNC_COMPAT.md for the documented residual. */ - printf("Number of created files: %llu\n", stats->transferred_regular); + /* Protocol 2.28.0: the receiver reports which destination entries it newly + created, split by type, so this line matches rsync exactly. */ + if (created_breakdown[0] != '\0') + printf("Number of created files: %llu %s\n", created_total, created_breakdown); + else + printf("Number of created files: %llu\n", created_total); printf("Number of deleted files: %llu\n", recv->deleted_files); printf("Number of regular files transferred: %llu\n", stats->transferred_regular); printf("Total file size: %s bytes\n", total); printf("Total transferred file size: %s bytes\n", transferred); printf("Literal data: %s bytes\n", literal); - printf("Matched data: %llu bytes\n", recv->matched_data); + const char* matched = + stats_bytes(config, recv->matched_data, matched_buffer, sizeof(matched_buffer)); + printf("Matched data: %s bytes\n", matched); printf("File list size: 0\n"); printf("File list generation time: 0.000 seconds\n"); printf("File list transfer time: 0.000 seconds\n"); diff --git a/src/server/receiver.c b/src/server/receiver.c index a6cd5a1..41678f0 100644 --- a/src/server/receiver.c +++ b/src/server/receiver.c @@ -613,6 +613,8 @@ typedef struct { static bool receiver_save_file(File* file, void* context_pointer) { ReceiverSaveContext* context = context_pointer; FileSaveResult result = FILE_SAVE_ERROR; + bool created = false; + unsigned created_dirs = 0; if (context->config->dry_run) { /* Defense in depth: a dry-run receiver mutates nothing even if a data frame reaches the sink (the sender is not supposed to send one). */ @@ -622,12 +624,17 @@ static bool receiver_save_file(File* file, void* context_pointer) { --remove-source-files sender keeps its source. */ result = FILE_SAVE_SKIPPED; } else { - result = file_save_to_disk_full(context->config->receive_root_directory, file, context->config); + result = file_save_to_disk_full_ex(context->config->receive_root_directory, file, + context->config, &created, &created_dirs); } /* Wire-stats tally: bytes reconstructed from the basis file (delta matches) count as matched data in the end-of-transfer report. */ if (result != FILE_SAVE_ERROR && file->matched_bytes > 0) context->stats.matched_data += file->matched_bytes; + /* Protocol 2.28.0: receiver-observed literal bytes and the created-entry + breakdown (regular/dir/link/special) for the `--stats` report. */ + if (result == FILE_SAVE_WRITTEN) + receiver_stats_note_saved(&context->stats, file, created, created_dirs); /* A directory's metadata is deferred, never applied inline: collect it now and apply it at the end. -O/--omit-dir-times and --preserve_perms/-times are honored by dir_metadata_list_apply's caller (see diff --git a/src/server/receiver_pipeline.c b/src/server/receiver_pipeline.c index 5fb2c68..5d1066e 100644 --- a/src/server/receiver_pipeline.c +++ b/src/server/receiver_pipeline.c @@ -247,12 +247,23 @@ int write_thread(void* pipeline_context) { } size_t file_bytes = file->data ? file->data->size : 0; FileSaveResult result = FILE_SAVE_SKIPPED; + bool created = false; + unsigned created_dirs = 0; /* Server-contacting --dry-run: never write. The receiver thread does not enqueue anything on the dry-run path, but this keeps the writer thread provably mutation-free if a data frame ever reached it. */ bool dry_run = context->config->dry_run; if (save_to_disk && !dry_run) { - result = file_save_to_disk_full(root_directory, file, context->config); + result = + file_save_to_disk_full_ex(root_directory, file, context->config, &created, &created_dirs); + if (result == FILE_SAVE_WRITTEN) { + /* Protocol 2.28.0: fold the receiver-observed literal bytes and the + created-entry type into the shared stats block under its mutex (the + receive thread also writes stats.matched_data). */ + mtx_lock(&context->mutex); + receiver_stats_note_saved(&context->stats, file, created, created_dirs); + mtx_unlock(&context->mutex); + } if (result == FILE_SAVE_ERROR) { file_destroy(file); pipeline_context_receiver_note_bytes_released(context, file_bytes); diff --git a/src/shared/config.h b/src/shared/config.h index 6cbbfe8..6cb9efe 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -82,7 +82,7 @@ typedef struct { typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode; /* =========================================================================== - * Config wire-field table (single source of truth for protocol 2.27.0). + * Config wire-field table (single source of truth for protocol 2.28.0). * * Every field below crosses the wire. The table is the ONLY place a * serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare @@ -1006,7 +1006,20 @@ typedef struct Config { * the sender can print rsync's `deleting PATH` lines for a real deletion. No * change to the fixed STATUS_STATS record itself; only a new trailing config * bool, which still requires the version bump for the strict lockstep. */ -#define PROTOCOL_VERSION "2.27.0" +/* (8) --stats receiver-observed counters (protocol 2.28.0): the fixed + * STATUS_STATS record grows from three counters to eight. The receiver now + * reports the bytes it literally stored (`literal_data`) and the count of + * destination entries it newly CREATED, split by type + * (reg/dir/link/special), so the sender can print rsync's exact + * `Number of created files: N (reg: X, dir: Y, link: Z, special: W)` line and + * an exact `Literal data` total even for delta transfers. The config-frame + * LAYOUT is unchanged (no new config field), but the STATUS_STATS body grows, + * so a 2.27 peer that does not consume the five new fixed-width counters would + * desynchronize on the trailing would-delete path list; the strict + * same-version handshake (config_receive rejects a mismatched version before + * parsing anything else) keeps mixed deployments from ever reaching that + * state. */ +#define PROTOCOL_VERSION "2.28.0" #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) /* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */ #define MAX_BASIS_DIRS 64 diff --git a/src/shared/file.c b/src/shared/file.c index 55c2f7f..96f9a9e 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -187,6 +187,7 @@ File* file_create(const char* path) { file->xattrs = NULL; file->dest_state = (OutputDestState){0}; file->matched_bytes = 0; + file->literal_bytes = 0; return file; } @@ -614,6 +615,103 @@ static int open_dir_beneath_root(const char* resolved, const char* root) { } int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs) { + return file_open_secure_parent_counted(path, leaf_out, create_dirs, NULL, NULL); +} + +/* The logical transfer root expressed in the same coordinate as the secure + * parent walk's `rel_buf` (relative to the authorized root, with a leading + * '/'), used as the floor at or below which a created directory is a real + * file-list entry. The on-disk transfer root is the receive root joined to the + * wire path; the mirror scaffolding above it (the absolute source path below + * the destination root) is not an rsync entry. Returns an allocated string or + * NULL (count every created component). */ +static char* transfer_root_floor(const Config* config) { + if (!config || !config->send_directory || config->send_directory[0] == '\0') + return NULL; + const char* spec = config->send_directory; + const char* after = spec; + if (spec[0] == '.' && spec[1] == '/') { + after = spec + 2; + } else { + const char* cut = strstr(spec, "/./"); + if (cut) + after = cut + 3; + } + while (*after == '/') + after++; + char* wire_root = str_dup(after); + if (!wire_root) + return NULL; + size_t wlen = strlen(wire_root); + while (wlen > 0 && wire_root[wlen - 1] == '/') + wire_root[--wlen] = '\0'; + if (wlen == 0) { + free(wire_root); + return NULL; + } + char* disk_root = config->receive_root_directory + ? path_cat(config->receive_root_directory, wire_root) + : str_dup(wire_root); + free(wire_root); + if (!disk_root) + return NULL; + const char* root_path = utils_get_authorized_root_path(); + const char* floor = disk_root; + if (root_path && root_path[0] == '/') { + size_t rl = strlen(root_path); + while (rl > 0 && root_path[rl - 1] == '/') + rl--; + if (strncmp(disk_root, root_path, rl) == 0 && (disk_root[rl] == '/' || disk_root[rl] == '\0')) + floor = disk_root + rl; + } + while (*floor == '/') + floor++; + char* out = str_dup(floor); + free(disk_root); + if (!out) + return NULL; + if (out[0] == '\0') { + free(out); + return NULL; + } + return out; +} + +/* A created parent component counts toward `Number of created files` only when + * its receive-root-relative path is at or below the logical transfer root + * (`count_floor`). The transfer root itself corresponds to rsync's `.` entry + * (created on a fresh destination, pre-existing otherwise); the mirror + * scaffolding above it is FastSync's absolute-path layout, not an rsync entry. */ +static bool created_dir_counts(const char* count_floor, const char* rel_buf, + const char* component) { + if (!count_floor) + return true; + char candidate[PATH_MAX]; + int n = snprintf(candidate, sizeof(candidate), "%s/%s", rel_buf, component); + if (n < 0 || (size_t)n >= sizeof(candidate)) + return false; + const char* cand = candidate; + while (*cand == '/') + cand++; + size_t fl = strlen(count_floor); + if (strncmp(cand, count_floor, fl) != 0) + return false; + return cand[fl] == '\0' || cand[fl] == '/'; +} + +/* Public wrapper for the receiver's created-directory accounting: the logical + * transfer root expressed receive-root-relative, or NULL when the wire paths + * carry no mirror scaffolding above it (--relative and --files-from, whose + * paths are already relative to the transfer root). The caller frees a + * non-NULL result. */ +char* file_transfer_root_floor(const Config* config) { + if (!config || config->relative || config->files_from_set != NULL) + return NULL; + return transfer_root_floor(config); +} + +int file_open_secure_parent_counted(const char* path, char** leaf_out, bool create_dirs, + unsigned* dirs_created, const char* count_floor) { char* copy = str_dup(path); if (!copy) return -1; @@ -674,6 +772,14 @@ int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs) if (next < 0 && create_dirs && errno == ENOENT) { bool created = mkdirat(fd, component, (mode_t)(0777 & ~(mode_t)file_process_umask())) == 0; if (created || errno == EEXIST) { + /* Protocol 2.28.0: only directories the logical file list would + create count toward `Number of created files`; the mirror + scaffolding above the transfer root (e.g. the absolute source path + under the destination root) is not an rsync entry. `count_floor` + is a receive-root-relative prefix that must be reached before a + created component is counted. */ + if (created && dirs_created && created_dir_counts(count_floor, rel_buf, component)) + (*dirs_created)++; /* P7 Wave E: --copy-as owns EVERY entry, including the intermediate directories this walk creates implicitly. Its target ids are a global policy, so they are available here without per-entry source @@ -1026,15 +1132,14 @@ static void restore_extra_fd(int fd, const FileMetadata* metadata, const FileXat } } -static bool file_to_disk_secure_impl(const char* path, const void* data, - unsigned long long data_size, bool inplace, bool sparse, - bool preallocate, const FileMetadata* metadata, - FileAttrPolicy policy, bool update, bool no_replace, - bool use_fsync, const char* temp_dir, - const FileXattrList* xattrs, bool fake_super, - bool keep_partial) { +static bool +file_to_disk_secure_impl(const char* path, const void* data, unsigned long long data_size, + bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata, + FileAttrPolicy policy, bool update, bool no_replace, bool use_fsync, + const char* temp_dir, const FileXattrList* xattrs, bool fake_super, + bool keep_partial, unsigned* dirs_created, const char* count_floor) { char* leaf = NULL; - int dirfd = file_open_secure_parent(path, &leaf, true); + int dirfd = file_open_secure_parent_counted(path, &leaf, true, dirs_created, count_floor); if (dirfd < 0) return false; int fd = -1; @@ -1325,7 +1430,7 @@ static bool file_to_disk_secure_impl(const char* path, const void* data, "non-atomic copy into the destination directory"); return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata, policy, update, no_replace, use_fsync, NULL, xattrs, fake_super, - keep_partial); + keep_partial, dirs_created, count_floor); } return ok; } @@ -1334,7 +1439,8 @@ bool file_to_disk_secure(const char* path, const void* data, unsigned long long bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata, FileAttrPolicy policy, const char* temp_dir) { return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata, - policy, false, false, false, temp_dir, NULL, false, false); + policy, false, false, false, temp_dir, NULL, false, false, NULL, + NULL); } bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size, @@ -1342,7 +1448,8 @@ bool file_to_disk_secure_update(const char* path, const void* data, unsigned lon const FileMetadata* metadata, FileAttrPolicy policy, const char* temp_dir) { return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata, - policy, true, false, false, temp_dir, NULL, false, false); + policy, true, false, false, temp_dir, NULL, false, false, NULL, + NULL); } bool file_to_disk_secure_with_fsync(const char* path, const void* data, @@ -1350,7 +1457,8 @@ bool file_to_disk_secure_with_fsync(const char* path, const void* data, bool preallocate, const FileMetadata* metadata, FileAttrPolicy policy, bool use_fsync, const char* temp_dir) { return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata, - policy, false, false, use_fsync, temp_dir, NULL, false, false); + policy, false, false, use_fsync, temp_dir, NULL, false, false, + NULL, NULL); } bool file_to_disk_secure_no_replace(const char* path, const void* data, @@ -1358,7 +1466,8 @@ bool file_to_disk_secure_no_replace(const char* path, const void* data, const FileMetadata* metadata, FileAttrPolicy policy, const char* temp_dir) { return file_to_disk_secure_impl(path, data, data_size, false, sparse, preallocate, metadata, - policy, false, true, false, temp_dir, NULL, false, false); + policy, false, true, false, temp_dir, NULL, false, false, NULL, + NULL); } /* Receiver write-path variant that also applies the per-file xattrs (-X/-A) @@ -1371,9 +1480,21 @@ bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long const FileMetadata* metadata, FileAttrPolicy policy, bool update, bool no_replace, bool use_fsync, const FileXattrList* xattrs, bool fake_super, bool keep_partial, const char* temp_dir) { + return file_to_disk_secure_attrs_counted(path, data, data_size, inplace, sparse, preallocate, + metadata, policy, update, no_replace, use_fsync, xattrs, + fake_super, keep_partial, temp_dir, NULL, NULL); +} + +bool file_to_disk_secure_attrs_counted(const char* path, const void* data, + unsigned long long data_size, bool inplace, bool sparse, + bool preallocate, const FileMetadata* metadata, + FileAttrPolicy policy, bool update, bool no_replace, + bool use_fsync, const FileXattrList* xattrs, bool fake_super, + bool keep_partial, const char* temp_dir, + unsigned* dirs_created, const char* count_floor) { return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata, policy, update, no_replace, use_fsync, temp_dir, xattrs, - fake_super, keep_partial); + fake_super, keep_partial, dirs_created, count_floor); } /* Atomic --link-dest install. The destination is replaced (via a temporary @@ -1396,11 +1517,12 @@ static bool file_to_disk_secure_link_impl(const char* path, const char* basis_pa bool preallocate, const FileMetadata* metadata, FileAttrPolicy policy, bool use_fsync, const FileXattrList* xattrs, bool fake_super, - const char* temp_dir) { + const char* temp_dir, unsigned* dirs_created, + const char* count_floor) { if (!path || !basis_path) return false; char* leaf = NULL; - int dirfd = file_open_secure_parent(path, &leaf, true); + int dirfd = file_open_secure_parent_counted(path, &leaf, true, dirs_created, count_floor); if (dirfd < 0) return false; @@ -1483,9 +1605,9 @@ static bool file_to_disk_secure_link_impl(const char* path, const char* basis_pa free(leaf); /* The basis file could not be linked in (missing, cross-device, refused by the filesystem). Write a byte-identical local copy instead. */ - return file_to_disk_secure_attrs(path, data, data_size, false, false, preallocate, metadata, - policy, false, false, use_fsync, xattrs, fake_super, false, - temp_dir); + return file_to_disk_secure_attrs_counted( + path, data, data_size, false, false, preallocate, metadata, policy, false, false, use_fsync, + xattrs, fake_super, false, temp_dir, dirs_created, count_floor); } if (scratch_dirfd >= 0) @@ -1500,7 +1622,7 @@ bool file_to_disk_secure_link(const char* path, const char* basis_path, const vo const FileMetadata* metadata, FileAttrPolicy policy, bool use_fsync, const char* temp_dir) { return file_to_disk_secure_link_impl(path, basis_path, data, data_size, preallocate, metadata, - policy, use_fsync, NULL, false, temp_dir); + policy, use_fsync, NULL, false, temp_dir, NULL, NULL); } bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, const void* data, @@ -1508,8 +1630,21 @@ bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, co const FileMetadata* metadata, FileAttrPolicy policy, bool use_fsync, const FileXattrList* xattrs, bool fake_super, const char* temp_dir) { + return file_to_disk_secure_link_attrs_counted(path, basis_path, data, data_size, preallocate, + metadata, policy, use_fsync, xattrs, fake_super, + temp_dir, NULL, NULL); +} + +bool file_to_disk_secure_link_attrs_counted(const char* path, const char* basis_path, + const void* data, unsigned long long data_size, + bool preallocate, const FileMetadata* metadata, + FileAttrPolicy policy, bool use_fsync, + const FileXattrList* xattrs, bool fake_super, + const char* temp_dir, unsigned* dirs_created, + const char* count_floor) { return file_to_disk_secure_link_impl(path, basis_path, data, data_size, preallocate, metadata, - policy, use_fsync, xattrs, fake_super, temp_dir); + policy, use_fsync, xattrs, fake_super, temp_dir, + dirs_created, count_floor); } bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size, diff --git a/src/shared/file.h b/src/shared/file.h index 939d86d..e106996 100644 --- a/src/shared/file.h +++ b/src/shared/file.h @@ -87,6 +87,11 @@ bool file_path_exists_secure(const char* path); bool file_stat_secure(const char* path, struct stat* st); bool file_destination_is_newer_secure(const char* path, const FileMetadata* metadata); int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs); +/* Protocol 2.28.0 variant: also increments *dirs_created for every missing + * parent directory this walk creates that lies strictly below `count_floor` + * (a receive-root-relative path, or NULL to count all of them). */ +int file_open_secure_parent_counted(const char* path, char** leaf_out, bool create_dirs, + unsigned* dirs_created, const char* count_floor); bool file_ensure_directory_secure(const char* path); bool file_directory_exists_secure(const char* path); bool file_rename_secure(const char* old_path, const char* new_path); @@ -158,5 +163,27 @@ bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, co const FileMetadata* metadata, FileAttrPolicy policy, bool use_fsync, const FileXattrList* xattrs, bool fake_super, const char* temp_dir); +/* Protocol 2.28.0 receiver-stat variants: like the two above but additionally + * report through `dirs_created` (when non-NULL) how many parent directories the + * confined secure walk had to create that lie strictly below `count_floor` (a + * receive-root-relative prefix, or NULL for all). Used to reproduce rsync's + * `Number of created files` directory count on a fresh destination. */ +bool file_to_disk_secure_attrs_counted(const char* path, const void* data, + unsigned long long data_size, bool inplace, bool sparse, + bool preallocate, const FileMetadata* metadata, + FileAttrPolicy policy, bool update, bool no_replace, + bool use_fsync, const FileXattrList* xattrs, bool fake_super, + bool keep_partial, const char* temp_dir, + unsigned* dirs_created, const char* count_floor); +bool file_to_disk_secure_link_attrs_counted(const char* path, const char* basis_path, + const void* data, unsigned long long data_size, + bool preallocate, const FileMetadata* metadata, + FileAttrPolicy policy, bool use_fsync, + const FileXattrList* xattrs, bool fake_super, + const char* temp_dir, unsigned* dirs_created, + const char* count_floor); +/* The logical transfer root expressed receive-root-relative, or NULL when the + * wire paths carry no mirror scaffolding above it. Caller frees non-NULL. */ +char* file_transfer_root_floor(const Config* config); #endif diff --git a/src/shared/file_receive.c b/src/shared/file_receive.c index f582cfb..1a65f9a 100644 --- a/src/shared/file_receive.c +++ b/src/shared/file_receive.c @@ -37,7 +37,12 @@ #define MANIFEST_ENTRY_OVERHEAD (sizeof(char*) + 16) bool file_save_to_disk(const char* root_directory, const File* file, const Config* config) { - return file_save_to_disk_full(root_directory, file, config) != FILE_SAVE_ERROR; + return file_save_to_disk_full_ex(root_directory, file, config, NULL, NULL) != FILE_SAVE_ERROR; +} + +FileSaveResult file_save_to_disk_full(const char* root_directory, const File* file, + const Config* config) { + return file_save_to_disk_full_ex(root_directory, file, config, NULL, NULL); } /* --delay-updates receiver path: write the file into a private staging tree @@ -208,13 +213,14 @@ static FileSaveResult hardlink_sibling_absent_first(const char* destination_path --existing/--ignore-existing/--update policies are decided against the final destination like every normal write. */ static FileSaveResult file_save_hardlink_sibling(const char* root_directory, const File* file, - const Config* config) { + const Config* config, bool* created) { Config* cfg = (Config*)config; if (!root_directory || !file || !file->path || !file->hardlink_target) return FILE_SAVE_ERROR; char* destination_path = path_cat(root_directory, file->path); if (!destination_path) return FILE_SAVE_ERROR; + bool existed = file_path_exists_secure(destination_path); if (cfg->existing && !file_path_exists_secure(destination_path)) { free(destination_path); @@ -278,6 +284,8 @@ static FileSaveResult file_save_hardlink_sibling(const char* root_directory, con free(staged_first); free(staged_sibling); free(destination_path); + if (ok && created && !existed) + *created = true; return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR; } @@ -324,6 +332,8 @@ static FileSaveResult file_save_hardlink_sibling(const char* root_directory, con free(content); free(first_disk); free(destination_path); + if (ok && created && !existed) + *created = true; return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR; } @@ -360,7 +370,7 @@ bool file_special_rdev_valid(int32_t major, int32_t minor, mode_t mode) { * non-device entry must carry an empty rdev. */ static FileSaveResult file_save_special_to_disk(const char* root_directory, const File* file, - const Config* config) { + const Config* config, bool* created) { /* The empty-path and structural checks stay unconditional; the redundant ".." list-path re-check is skipped under --trust-sender exactly like the receive layer (confinement is deferred to the secure parent walk below, @@ -412,6 +422,7 @@ static FileSaveResult file_save_special_to_disk(const char* root_directory, cons char* destination = path_cat(root_directory, file->path); if (!destination) return FILE_SAVE_ERROR; + bool existed = file_path_exists_secure(destination); char* leaf = NULL; int parent_fd = file_open_secure_parent(destination, &leaf, true); if (parent_fd < 0) { @@ -532,6 +543,8 @@ static FileSaveResult file_save_special_to_disk(const char* root_directory, cons free(destination); /* A failed required --copy-as ownership marks the node as failed; every other * identity policy stays best-effort. */ + if (owner_ok && created && !existed) + *created = true; return owner_ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR; } @@ -610,8 +623,13 @@ static FileSaveResult file_save_write_device(const char* root_directory, const F return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_SKIPPED; } -FileSaveResult file_save_to_disk_full(const char* root_directory, const File* file, - const Config* config) { +FileSaveResult file_save_to_disk_full_ex(const char* root_directory, const File* file, + const Config* config, bool* created, + unsigned* created_dirs) { + if (created) + *created = false; + if (created_dirs) + *created_dirs = 0; /* Central no-mutation guard: a server-contacting --dry-run (or a local batch apply that somehow carries dry_run) must never touch the destination, no matter which caller reached this primitive. The per-caller guards remain, @@ -658,7 +676,7 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi /* Device/special node (--devices/--specials): recreate the node instead of writing content (privilege-gated, confined, rdev-validated). */ if (file->is_special) - return file_save_special_to_disk(root_directory, file, config); + return file_save_special_to_disk(root_directory, file, config, created); /* --write-devices: write straight into an existing device node. Writing into a device is a super-user activity, so --no-super must suppress it just like device-node creation; the default AUTO/--super attempt it (the wide @@ -689,6 +707,7 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi char* dir_path = path_cat(root_directory, file->path); if (!dir_path) return FILE_SAVE_ERROR; + bool dir_existed = file_path_exists_secure(dir_path); bool ok = file_ensure_directory_secure(dir_path); /* P7 Wave E: apply the negotiated ownership to the directory ITSELF (not just the files inside it). --copy-as and every explicit identity policy @@ -712,6 +731,8 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi free(leaf); } free(dir_path); + if (ok && created && !dir_existed) + *created = true; return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR; } @@ -728,6 +749,7 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi char* link_path = path_cat(root_directory, file->path); if (!link_path) return FILE_SAVE_ERROR; + bool link_existed = file_path_exists_secure(link_path); /* The link value is stored verbatim (rsync -l parity: absolute and ".."-bearing targets are preserved; the scanner's --safe-links / --copy-unsafe-links decide which links are sent at all). --munge-links @@ -768,6 +790,8 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi ok = file_restore_symlink_metadata(link_path, file->metadata, link_policy, config->omit_link_times); } + if (ok && created && !link_existed) + *created = true; free(link_path); return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR; } @@ -777,7 +801,7 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi byte-identical copy of) the group's first member. Handled entirely here, before the normal data-write paths (which would create an empty file). */ if (file->link_group != 0 && !file->link_first && file->hardlink_target != NULL) { - return file_save_hardlink_sibling(root_directory, file, config); + return file_save_hardlink_sibling(root_directory, file, config, created); } /* These options arrive from the client. --backup-dir, --partial-dir and @@ -808,12 +832,18 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi free(disk_path); return FILE_SAVE_ERROR; } + /* Snapshot the final destination's existence BEFORE any backup/force/partial + step can move or remove it, so the receiver can report rsync's + `Number of created files` (protocol 2.28.0). */ + bool dest_existed = file_path_exists_secure(destination_path); /* --delay-updates diverts the whole write into the staging tree; the rest of this function is the immediate-install path. */ if (config && config->delay_updates) { FileSaveResult result = file_stage_delayed_update(root_directory, destination_path, file, (Config*)config); + if (result == FILE_SAVE_WRITTEN && created && !dest_existed) + *created = true; free(confined_backup); free(confined_partial); free(destination_path); @@ -939,19 +969,23 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi existing/ignore-existing/update/backup preamble above has already made the policy decision. */ bool ok; + char* count_floor = file_transfer_root_floor(config); if (config && file->basis_link) { - ok = file_to_disk_secure_link_attrs( + ok = file_to_disk_secure_link_attrs_counted( disk_path, file->basis_link, file->data->data, file->data->size, config->preallocate, - metadata, policy, config->use_fsync, file->xattrs, config->fake_super, confined_temp); + metadata, policy, config->use_fsync, file->xattrs, config->fake_super, confined_temp, + created_dirs, count_floor); } else { /* The plain no-replace / update / with-fsync engines, plus per-file xattr (-X/-A) and --fake-super application on the written fd. */ - ok = file_to_disk_secure_attrs( + ok = file_to_disk_secure_attrs_counted( disk_path, file->data->data, file->data->size, inplace, sparse, config && config->preallocate, metadata, policy, config && config->update, config && config->ignore_existing, config && config->use_fsync, file->xattrs, - config ? config->fake_super : false, config ? config->partial : false, confined_temp); + config ? config->fake_super : false, config ? config->partial : false, confined_temp, + created_dirs, count_floor); } + free(count_floor); free(confined_temp); confined_temp = NULL; if (!ok) @@ -972,6 +1006,8 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi free(confined_partial); free(destination_path); free(disk_path); + if (created && !dest_existed) + *created = true; return FILE_SAVE_WRITTEN; fail: @@ -984,6 +1020,30 @@ fail: return FILE_SAVE_ERROR; } +void receiver_stats_note_saved(ReceiverStats* stats, const File* file, bool created, + unsigned created_dirs) { + if (!stats || !file) + return; + bool is_sibling = file->link_group != 0 && !file->link_first; + if (!file->is_dir && !file->is_symlink && !file->is_special && !is_sibling) { + unsigned long long literal = file->literal_bytes; + if (literal == 0 && file->matched_bytes == 0) + literal = file->data ? file->data->size : 0; + stats->literal_bytes += literal; + } + stats->created_dir += created_dirs; + if (!created) + return; + if (file->is_dir) + stats->created_dir++; + else if (file->is_symlink) + stats->created_link++; + else if (file->is_special) + stats->created_special++; + else + stats->created_reg++; +} + /* Receive a file's xattr block (when the config enables xattr transport) and * attach it to `file`. Returns false on a malformed/oversized frame. */ static bool receive_file_xattrs(File* file, int fd, const Config* config) { @@ -1094,11 +1154,15 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_ return NULL; } /* Wire-stats tally: bytes taken straight from the basis file (matched - delta blocks). Computed before the delta is destroyed. */ + delta blocks) and bytes shipped literally (protocol 2.28.0). Computed + before the delta is destroyed. */ unsigned long long matched = 0; + unsigned long long literal = 0; for (uint32_t k = 0; k < delta->instruction_count; k++) { if (delta->instructions[k].type == DELTA_INSTR_BLOCK_MATCH) matched += delta->instructions[k].match.length; + else if (delta->instructions[k].type == DELTA_INSTR_LITERAL) + literal += delta->instructions[k].literal.length; } void* new_data = delta_apply(old_data, old_size, delta, config->delta_block_size); delta_destroy(delta); @@ -1119,6 +1183,7 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_ return NULL; } file->matched_bytes = matched; + file->literal_bytes = literal; if (config->use_metadata) { int meta_ok = 1; diff --git a/src/shared/file_receive.h b/src/shared/file_receive.h index ce08a7e..7604ffb 100644 --- a/src/shared/file_receive.h +++ b/src/shared/file_receive.h @@ -179,6 +179,22 @@ typedef enum { FILE_SAVE_ERROR = 0, FILE_SAVE_WRITTEN = 1, FILE_SAVE_SKIPPED = 2 FileSaveResult file_save_to_disk_full(const char* root_directory, const File* file, const Config* config); +/* Protocol 2.28.0 variant: also reports through `created` (when non-NULL) + * whether the destination entry did not exist before this save, and through + * `created_dirs` how many parent directories the confined walk created, so the + * receiver can build rsync's `Number of created files` breakdown. The plain + * file_save_to_disk_full() is this with both out-params NULL. */ +FileSaveResult file_save_to_disk_full_ex(const char* root_directory, const File* file, + const Config* config, bool* created, + unsigned* created_dirs); bool file_save_to_disk(const char* root_directory, const File* file, const Config* config); +/* Protocol 2.28.0 receiver counter accumulator: fold one successfully saved + * entry into `stats`, adding its receiver-observed literal bytes and, when + * `created`, the matching created-by-type counter (regular file / symlink / + * special) plus `created_dirs` implicitly-created parent directories. + * Non-first hardlink siblings contribute no literal bytes. */ +void receiver_stats_note_saved(ReceiverStats* stats, const File* file, bool created, + unsigned created_dirs); + #endif diff --git a/src/shared/file_types.h b/src/shared/file_types.h index c2c127d..bf5dc8f 100644 --- a/src/shared/file_types.h +++ b/src/shared/file_types.h @@ -97,6 +97,11 @@ typedef struct { * basis file (matched delta blocks) for this entry. 0 when the file was sent * whole. Accumulated into ReceiverStats.matched_data by the receiver sink. */ unsigned long long matched_bytes; + /* Receiver-only (protocol 2.28.0) wire-stats tally: the literal delta fragment + * bytes this entry carried (DELTA_INSTR_LITERAL). 0 when the file was sent + * whole; the sink then falls back to the whole payload size. Accumulated + * into ReceiverStats.literal_bytes. */ + unsigned long long literal_bytes; } File; /* The path that should be sent on the wire and used for the receiver-side diff --git a/src/shared/format.c b/src/shared/format.c index 147d002..df2c463 100644 --- a/src/shared/format.c +++ b/src/shared/format.c @@ -105,25 +105,27 @@ bool format_dest_state_receive(int fd, OutputDestState* state) { bool format_stats_send(int fd, const ReceiverStats* stats) { if (!stats) return false; - unsigned long long matched = stats->matched_data; - unsigned long long deleted = stats->deleted_files; - unsigned long long would = stats->would_delete_count; - return send_n_data(fd, &matched, sizeof(matched)) && send_n_data(fd, &deleted, sizeof(deleted)) && - send_n_data(fd, &would, sizeof(would)); + unsigned long long fields[8] = { + stats->matched_data, stats->deleted_files, stats->would_delete_count, stats->literal_bytes, + stats->created_reg, stats->created_dir, stats->created_link, stats->created_special, + }; + return send_n_data(fd, fields, sizeof(fields)); } bool format_stats_receive(int fd, ReceiverStats* stats) { if (!stats) return false; - unsigned long long matched = 0; - unsigned long long deleted = 0; - unsigned long long would = 0; - if (!receive_n_data(fd, &matched, sizeof(matched)) || - !receive_n_data(fd, &deleted, sizeof(deleted)) || !receive_n_data(fd, &would, sizeof(would))) + unsigned long long fields[8] = {0}; + if (!receive_n_data(fd, fields, sizeof(fields))) return false; memset(stats, 0, sizeof(*stats)); - stats->matched_data = matched; - stats->deleted_files = deleted; - stats->would_delete_count = would; + stats->matched_data = fields[0]; + stats->deleted_files = fields[1]; + stats->would_delete_count = fields[2]; + stats->literal_bytes = fields[3]; + stats->created_reg = fields[4]; + stats->created_dir = fields[5]; + stats->created_link = fields[6]; + stats->created_special = fields[7]; return true; } diff --git a/src/shared/format.h b/src/shared/format.h index 729ba2a..9da9a93 100644 --- a/src/shared/format.h +++ b/src/shared/format.h @@ -57,14 +57,26 @@ bool format_dest_state_send(int fd, const OutputDestState* state); bool format_dest_state_receive(int fd, OutputDestState* state); /* End-of-transfer receiver counters reported through STATUS_STATS (protocol - * 2.25.0) when the wire config carries report_stats. `would_delete_count` is - * the number of destination-relative paths the receiver would have deleted in a - * -n/--dry-run --delete run; that many wire strings immediately follow the - * fixed record (sent/read by the caller). */ + * 2.25.0, extended in 2.28.0) when the wire config carries report_stats. + * `would_delete_count` is the number of destination-relative paths the receiver + * would have deleted in a -n/--dry-run --delete run; that many wire strings + * immediately follow the fixed record (sent/read by the caller). + * + * Protocol 2.28.0 adds the receiver-observed counters the sender cannot see: + * `literal_bytes` is the file data the receiver actually stored literally + * (whole files plus the literal fragments of a delta) and the four `created_*` + * counters split the destination entries the receiver newly created by type, + * reproducing rsync's `Number of created files` breakdown and an exact + * `Literal data` for a delta run. */ typedef struct { unsigned long long matched_data; unsigned long long deleted_files; unsigned long long would_delete_count; + unsigned long long literal_bytes; + unsigned long long created_reg; + unsigned long long created_dir; + unsigned long long created_link; + unsigned long long created_special; } ReceiverStats; /* Fixed-width STATUS_STATS counter record. The status frame and the optional diff --git a/tests/integration/parity_harness.py b/tests/integration/parity_harness.py index bb18130..f7395d8 100644 --- a/tests/integration/parity_harness.py +++ b/tests/integration/parity_harness.py @@ -56,10 +56,13 @@ STDOUT_OUTFMT = "outfmt" STDOUT_STATS = "stats" # rsync --stats lines that are protocol-independent and must match exactly. -# Deliberately excluded: the per-type "Number of files"/"Number of created -# files" breakdown and Total bytes sent/received (documented residual, see the -# `--stats` row in RSYNC_COMPAT.md). +# `Number of files` and `Number of created files` carry rsync's per-type +# breakdown; protocol 2.28.0 reports the receiver-created split over +# STATUS_STATS. Deliberately excluded: Total bytes sent/received (protocol +# framing differs, see the `--stats` row in RSYNC_COMPAT.md). STATS_KEYS = ( + "Number of files", + "Number of created files", "Number of deleted files", "Number of regular files transferred", "Total file size", @@ -443,6 +446,11 @@ def run_differential( # noqa: PLR0913 (explicit scenario parameters) rroot, froot = os.path.join(rdst, rel), get_dest_received_dir(fdst, src) else: rroot, froot = rdst, get_dest_received_dir(fdst, src) + # rsync's destination root always exists (clean_dir created it). FastSync's + # logical transfer root is the mirror path below the destination argument, + # so pre-create it too: `Number of created files` counts the root only when + # it is genuinely absent, and the two tools must start from the same state. + os.makedirs(froot, exist_ok=True) if seed: seed(src, rroot, froot) diff --git a/tests/integration/test_fault_injection.py b/tests/integration/test_fault_injection.py index 2694b31..7b8eb46 100644 --- a/tests/integration/test_fault_injection.py +++ b/tests/integration/test_fault_injection.py @@ -36,7 +36,7 @@ from common import ( # noqa: E402 verify_transfer, ) -PROTOCOL_VERSION = b"2.27.0" +PROTOCOL_VERSION = b"2.28.0" STATUS_MANIFEST = 5 STATUS_OK = 0 diff --git a/tests/integration/test_output_parity.py b/tests/integration/test_output_parity.py index 60df77a..c96867b 100644 --- a/tests/integration/test_output_parity.py +++ b/tests/integration/test_output_parity.py @@ -289,6 +289,15 @@ def _make_one_file(root, name="f.bin", size=100): fh.write(bytes((i * 7 + 3) & 0xFF for i in range(size))) +def _pick_stats(text, keys): + out = {} + for line in text.splitlines(): + for key in keys: + if line.startswith(key + ":"): + out[key] = line + return out + + class TestWireStatsParity: """Wire-counter output parity: --out-format %b/%c/%C, --progress and --stats versus real rsync 3.4.1.""" @@ -496,6 +505,9 @@ class TestWireStatsParity: _make_one_file(source, "f.bin", 6000) clean_dir(dest) clean_dir(rdst) + # Start both tools from the same state: rsync's destination root exists, + # so pre-create FastSync's mirrored logical root as well. + os.makedirs(get_dest_received_dir(dest, source), exist_ok=True) rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"]) assert rsync_result.returncode == 0, rsync_result.stderr flags = ["-a", "--stats"] + (["--threads"] if mt else []) @@ -526,17 +538,18 @@ class TestWireStatsParity: @requires_rsync @pytest.mark.ci def test_stats_file_count_breakdown_matches_rsync(self, shared_server): - """`Number of files` now carries rsync's per-type breakdown: the scanner - accounts directory entries (captured for -a/-t/-p) plus reg/link/special - from the transfer list. `Number of created files` still lacks the type - breakdown (FastSync cannot tell which entries the receiver newly - created), so that residual is pinned separately.""" + """`Number of files` and `Number of created files` both carry rsync's + per-type breakdown (protocol 2.28.0 reports the receiver-created + reg/dir/link/special split over STATUS_STATS).""" source = os.path.join(TEST_DATA_DIR, "wire_stc_src") dest = os.path.join(TEST_DATA_DIR, "wire_stc_dst") rdst = os.path.join(TEST_DATA_DIR, "wire_stc_rdst") _make_one_file(source, "f.bin", 6000) clean_dir(dest) clean_dir(rdst) + # Start both tools from the same state: rsync's destination root exists, + # so pre-create FastSync's mirrored logical root as well. + os.makedirs(get_dest_received_dir(dest, source), exist_ok=True) rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"]) assert rsync_result.returncode == 0, rsync_result.stderr result, _ = run_client(source, dest, flags=["-a", "--stats"], @@ -556,10 +569,71 @@ class TestWireStatsParity: assert re.match(r"Number of files: 2 \(reg: 1, dir: 1\)$", r_files), r_files assert r_files == f_files, (r_files, f_files) - # rsync always carries the created type breakdown; FastSync prints the - # bare transferred-regular count (documented residual). assert re.match(r"Number of created files: 1 \(reg: 1\)$", r_created), r_created - assert re.fullmatch(r"Number of created files: 1", f_created), f_created + assert f_created == r_created, (r_created, f_created) + + @requires_rsync + @pytest.mark.ci + @pytest.mark.parametrize("mt", [False, True]) + def test_stats_created_and_literal_fresh_update_delta(self, shared_server, mt): + """The receiver-observed counters must match rsync for the three + transfer shapes: a fresh create (created breakdown + whole-file literal), + an update (created == 0, whole-file literal), and a delta update (only + the literal delta fragments are counted, not the whole file).""" + source = os.path.join(TEST_DATA_DIR, "wire_stcd_src") + dest = os.path.join(TEST_DATA_DIR, "wire_stcd_dst") + rdst = os.path.join(TEST_DATA_DIR, "wire_stcd_rdst") + clean_dir(source) + clean_dir(dest) + clean_dir(rdst) + os.makedirs(source, exist_ok=True) + os.makedirs(get_dest_received_dir(dest, source), exist_ok=True) + with open(os.path.join(source, "big.bin"), "wb") as fh: + fh.write(bytes(range(256)) * 4096) # 1 MiB + mt_flag = ["--threads"] if mt else [] + + def compare(tag): + # Pin the delta block size on both ends: rsync's adaptive block size + # would otherwise make the literal/matched split non-comparable. + rsync_result = _rsync(["-a", "--stats", "--no-whole-file", "-B8192", + source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + result, _ = run_client( + source, dest, + flags=["-a", "--stats", "--incremental", "--delta", "-B8192"] + mt_flag, + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + keys = ("Number of created files", "Literal data", "Matched data", + "Total transferred file size") + r = _pick_stats(rsync_result.stdout, keys) + f = _pick_stats(result.stdout, keys) + assert r == f, f"{tag}: rsync={r} fastsync={f}" + return r + + fresh = compare("fresh") + assert re.match(r"Number of created files: 1 \(reg: 1\)$", + fresh["Number of created files"]), fresh + + # Update the source and re-run: the destination already exists. + sleep_mtime = os.path.getmtime(os.path.join(source, "big.bin")) + 2 + with open(os.path.join(source, "big.bin"), "r+b") as fh: + fh.seek(100) + fh.write(b"XXXXXXXXXX") + os.utime(os.path.join(source, "big.bin"), (sleep_mtime, sleep_mtime)) + update = compare("update") + assert update["Number of created files"] == "Number of created files: 0", update + + # Second delta update: change bytes far apart, so rsync ships only the + # literal fragments and FastSync must report the same Literal data. + sleep_mtime = os.path.getmtime(os.path.join(source, "big.bin")) + 2 + with open(os.path.join(source, "big.bin"), "r+b") as fh: + fh.seek(500000) + fh.write(b"YYYYYYYYYY") + os.utime(os.path.join(source, "big.bin"), (sleep_mtime, sleep_mtime)) + delta = compare("delta") + assert delta["Number of created files"] == "Number of created files: 0", delta + lit = int(delta["Literal data"].split(":", 1)[1].strip().split()[0].replace(",", "")) + assert 0 < lit < 1024 * 1024, delta @requires_rsync @pytest.mark.ci diff --git a/tests/integration/test_preflight.py b/tests/integration/test_preflight.py index db3c62c..1f4e9d1 100644 --- a/tests/integration/test_preflight.py +++ b/tests/integration/test_preflight.py @@ -94,14 +94,14 @@ def _seed_protocol_source(source): class TestProtocol: @pytest.mark.ci def test_protocol_current_version_accepted(self, shared_server): - """--protocol=2.27.0 (the current PROTOCOL_VERSION) is accepted and the + """--protocol=2.28.0 (the current PROTOCOL_VERSION) is accepted and the transfer completes normally.""" source = os.path.join(TEST_DATA_DIR, "proto_ok_src") dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst") shutil.rmtree(dest, ignore_errors=True) os.makedirs(dest) _seed_protocol_source(source) - result, _ = run_client(source, dest, flags=["--protocol=2.27.0"], + result, _ = run_client(source, dest, flags=["--protocol=2.28.0"], port=shared_server.port) assert result.returncode == 0, \ f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}" diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index c017df3..b0f1694 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -318,7 +318,7 @@ static void test_parse_args_protocol_accept_current() { Config* cfg = valid_client_config(); EXPECT_NOT_NULL(cfg); char* argv_equals[] = {"fastsync", "--source-dir", "/src", - "--dest-dir", "/dst", "--protocol=2.27.0"}; + "--dest-dir", "/dst", "--protocol=2.28.0"}; int positional_args[2]; int positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 6, argv_equals, positional_args, &positional_count), 0); @@ -328,7 +328,7 @@ static void test_parse_args_protocol_accept_current() { cfg = valid_client_config(); EXPECT_NOT_NULL(cfg); char* argv_space[] = {"fastsync", "--source-dir", "/src", "--dest-dir", - "/dst", "--protocol", "2.27.0"}; + "/dst", "--protocol", "2.28.0"}; positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 7, argv_space, positional_args, &positional_count), 0); EXPECT_EQ_STR(cfg->version, PROTOCOL_VERSION); diff --git a/tests/test_config.c b/tests/test_config.c index 936ed25..406af8d 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -2831,15 +2831,17 @@ static void golden_config_populate(Config* c) { c->copy_as_gid = 222; } -/* The pinned golden frame (protocol 2.27.0). The values below are the only +/* The pinned golden frame (protocol 2.28.0). The values below are the only * thing that ties the generated table to the historical wire format; update * them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.24.0 * delete-plan wave changed only the version string; 2.25.0 appended the - * report_stats bool, 2.26.0 appended the compression_algo int, and 2.27.0 - * appended the report_deletes bool. The byte-exact values are recomputed for - * the merged layout. */ + * report_stats bool, 2.26.0 appended the compression_algo int, 2.27.0 appended + * the report_deletes bool, and 2.28.0 changed only the version string (the + * STATUS_STATS body grew, but the config frame layout is unchanged, so the + * frame length is identical). The byte-exact values are recomputed for the + * merged layout. */ #define GOLDEN_WIRE_LEN 709 -#define GOLDEN_WIRE_HASH 14423869696887880000ULL +#define GOLDEN_WIRE_HASH 417335736347473203ULL static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) { unsigned long long h = 1469598103934665603ULL; @@ -2921,7 +2923,7 @@ static unsigned long long capture_wire_hash(const Config* cfg, size_t* out_len) return h; } -/* Byte-for-byte wire compatibility guard (protocol 2.27.0). The expected hash +/* Byte-for-byte wire compatibility guard (protocol 2.28.0). The expected hash * pins the pre-X-macro byte stream; the refactor MUST NOT change it. */ static void test_config_wire_golden() { if (is_running_under_valgrind()) diff --git a/tests/test_format.c b/tests/test_format.c index 9b8bd35..7343e26 100644 --- a/tests/test_format.c +++ b/tests/test_format.c @@ -86,9 +86,43 @@ static void test_dest_state_roundtrip() { close(fds[1]); } +static void test_stats_roundtrip() { + /* STATUS_STATS grew from three counters (2.25.0) to eight (2.28.0); the codec + * must carry every field, including the receiver-observed literal/created + * counters, across the wire in order. */ + int fds[2]; + if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) != 0) + return; + ReceiverStats out; + memset(&out, 0, sizeof(out)); + out.matched_data = 111111111ULL; + out.deleted_files = 7; + out.would_delete_count = 3; + out.literal_bytes = 222222222ULL; + out.created_reg = 5; + out.created_dir = 4; + out.created_link = 2; + out.created_special = 1; + ReceiverStats in; + memset(&in, 0, sizeof(in)); + EXPECT_TRUE(format_stats_send(fds[0], &out)); + EXPECT_TRUE(format_stats_receive(fds[1], &in)); + EXPECT_TRUE(in.matched_data == out.matched_data); + EXPECT_TRUE(in.deleted_files == out.deleted_files); + EXPECT_TRUE(in.would_delete_count == out.would_delete_count); + EXPECT_TRUE(in.literal_bytes == out.literal_bytes); + EXPECT_TRUE(in.created_reg == out.created_reg); + EXPECT_TRUE(in.created_dir == out.created_dir); + EXPECT_TRUE(in.created_link == out.created_link); + EXPECT_TRUE(in.created_special == out.created_special); + close(fds[0]); + close(fds[1]); +} + void test_format(void) { test_human_size_decimal(); test_big_num_grouping(); test_datetime_format(); test_dest_state_roundtrip(); + test_stats_roundtrip(); } diff --git a/tests/test_server.c b/tests/test_server.c index 501c9a3..1538f07 100644 --- a/tests/test_server.c +++ b/tests/test_server.c @@ -3,6 +3,8 @@ #include "config.h" #include "delta.h" #include "file.h" +#include "file_receive.h" +#include "format.h" #include "log.h" #include "protocol.h" #include "test_utils.h" @@ -134,6 +136,124 @@ static void test_receive_files_single_file() { } } +/* Protocol 2.28.0: a fresh single-file transfer over the wire reports the + * receiver-observed literal bytes and the created-regular counter through the + * terminal STATUS_STATS frame, and an update reports created_reg == 0. */ +static void test_receive_stats_frame_created_and_literal() { + const char* content = "stats frame content"; + size_t len = strlen(content); + char root_template[] = "/tmp/fastsync_stats_XXXXXX"; + char* root = mkdtemp(root_template); + EXPECT_NOT_NULL(root); + + int p[2]; + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + io_set_fds(p[0], p[1]); + io_set_bwlimit(0); + + Config* cfg = config_create(); + EXPECT_NOT_NULL(cfg); + free(cfg->version); + cfg->version = str_dup(PROTOCOL_VERSION); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup(root); + cfg->save_to_disk = true; + cfg->report_stats = true; + + pid_t pid = fork(); + if (pid == 0) { + close(p[1]); + io_set_fds(p[0], p[0]); + int ret = receiver_receive_files(cfg, p[0]); + close(p[0]); + config_delete(cfg); + _exit(ret == 0 ? 0 : 1); + } + close(p[0]); + io_set_fds(p[1], p[1]); + send_status(p[1], STATUS_NEXT); + File* file = file_create("created.bin"); + EXPECT_NOT_NULL(file); + file->data->data = malloc(len); + EXPECT_NOT_NULL(file->data->data); + memcpy(file->data->data, content, len); + file->data->size = len; + send_str(p[1], file->path); + send_data(p[1], file->data); + file_destroy(file); + send_status(p[1], STATUS_FINISHED); + + Status status; + EXPECT_TRUE(receive_status(p[1], &status)); + EXPECT_EQ_INT(status, STATUS_STATS); + ReceiverStats stats; + EXPECT_TRUE(format_stats_receive(p[1], &stats)); + int would = 0; + EXPECT_TRUE(receive_int(p[1], &would)); + EXPECT_EQ_INT(would, 0); + EXPECT_TRUE(stats.created_reg == 1); + EXPECT_TRUE(stats.created_dir == 0); + EXPECT_TRUE(stats.created_link == 0); + EXPECT_TRUE(stats.created_special == 0); + EXPECT_TRUE(stats.literal_bytes == (unsigned long long)len); + EXPECT_TRUE(stats.matched_data == 0); + + Status final; + EXPECT_TRUE(receive_status(p[1], &final)); + EXPECT_EQ_INT(final, STATUS_OK); + int wstatus; + waitpid(pid, &wstatus, 0); + close(p[1]); + config_delete(cfg); + EXPECT_TRUE(WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0); + + /* Second run against the now-existing destination: no created file. */ + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + io_set_fds(p[0], p[1]); + io_set_bwlimit(0); + cfg = config_create(); + EXPECT_NOT_NULL(cfg); + free(cfg->version); + cfg->version = str_dup(PROTOCOL_VERSION); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup(root); + cfg->save_to_disk = true; + cfg->report_stats = true; + pid = fork(); + if (pid == 0) { + close(p[1]); + io_set_fds(p[0], p[0]); + int ret = receiver_receive_files(cfg, p[0]); + close(p[0]); + config_delete(cfg); + _exit(ret == 0 ? 0 : 1); + } + close(p[0]); + io_set_fds(p[1], p[1]); + send_status(p[1], STATUS_NEXT); + file = file_create("created.bin"); + EXPECT_NOT_NULL(file); + file->data->data = malloc(len); + EXPECT_NOT_NULL(file->data->data); + memcpy(file->data->data, content, len); + file->data->size = len; + send_str(p[1], file->path); + send_data(p[1], file->data); + file_destroy(file); + send_status(p[1], STATUS_FINISHED); + EXPECT_TRUE(receive_status(p[1], &status)); + EXPECT_EQ_INT(status, STATUS_STATS); + EXPECT_TRUE(format_stats_receive(p[1], &stats)); + EXPECT_TRUE(receive_int(p[1], &would)); + EXPECT_TRUE(stats.created_reg == 0); + EXPECT_TRUE(stats.literal_bytes == (unsigned long long)len); + EXPECT_TRUE(receive_status(p[1], &final)); + waitpid(pid, &wstatus, 0); + close(p[1]); + config_delete(cfg); + EXPECT_TRUE(WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0); +} + /* Test receive_files with STATUS_ABORT */ static void test_receive_files_abort() { Config* cfg = config_create(); @@ -1183,6 +1303,7 @@ void test_server() { if (!is_running_under_valgrind()) { test_receive_files_finished(); test_receive_files_single_file(); + test_receive_stats_frame_created_and_literal(); test_receive_files_abort(); test_receive_manifest_rejects_traversal(); test_receive_incremental_check_rejects_invalid_nanoseconds();