Merge branch 'feat/parity-wirestats' into feat/parity-completion

# Conflicts:
#	src/server/receiver_pipeline.c
#	src/shared/config.h
#	src/shared/protocol.h
#	tests/integration/test_fault_injection.py
#	tests/integration/test_preflight.py
#	tests/test_client_cli.c
#	tests/test_config.c
This commit is contained in:
2026-09-16 23:46:28 +02:00
27 changed files with 1111 additions and 164 deletions
+98 -1
View File
@@ -1,10 +1,14 @@
#include "checksum.h"
#include <fcntl.h>
#include <openssl/evp.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
/* delta.c owns the single XXH_IMPLEMENTATION that provides the xxHash symbols
* for the whole binary; this TU only needs the declarations. */
* for the whole binary; this TU only needs the declarations. The streaming
* state structs and XXH3_update are exposed only with XXH_STATIC_LINKING_ONLY. */
#define XXH_STATIC_LINKING_ONLY
#include <xxhash.h>
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
@@ -54,6 +58,99 @@ bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t
return false;
}
bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, uint8_t* out,
size_t out_capacity, size_t* out_len) {
if (!path || !out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
return false;
int fd = open(path, O_RDONLY | O_CLOEXEC);
if (fd < 0)
return false;
uint8_t buffer[64 * 1024];
bool ok = false;
if (algo == CHECKSUM_ALGO_MD5) {
EVP_MD_CTX* ctx = EVP_MD_CTX_new();
if (!ctx) {
close(fd);
return false;
}
unsigned int digest_len = 0;
if (EVP_DigestInit_ex(ctx, EVP_md5(), NULL) == 1) {
ok = true;
ssize_t got;
while ((got = read(fd, buffer, sizeof(buffer))) > 0) {
if (EVP_DigestUpdate(ctx, buffer, (size_t)got) != 1) {
ok = false;
break;
}
}
if (got < 0)
ok = false;
if (ok && EVP_DigestFinal_ex(ctx, out, &digest_len) == 1 && digest_len <= out_capacity)
*out_len = digest_len;
else
ok = false;
}
EVP_MD_CTX_free(ctx);
close(fd);
return ok;
}
XXH64_state_t xxh64;
XXH3_state_t* xxh3 = NULL;
if (algo == CHECKSUM_ALGO_XXH64) {
XXH64_reset(&xxh64, seed);
} else if (algo == CHECKSUM_ALGO_XXH3 || algo == CHECKSUM_ALGO_XXH128) {
xxh3 = XXH3_createState();
if (!xxh3) {
close(fd);
return false;
}
if (algo == CHECKSUM_ALGO_XXH3)
XXH3_64bits_reset_withSeed(xxh3, seed);
else
XXH3_128bits_reset_withSeed(xxh3, seed);
} else {
close(fd);
return false;
}
ok = true;
ssize_t got;
while ((got = read(fd, buffer, sizeof(buffer))) > 0) {
if (algo == CHECKSUM_ALGO_XXH64)
XXH64_update(&xxh64, buffer, (size_t)got);
else if (XXH3_64bits_update(xxh3, buffer, (size_t)got) == XXH_ERROR) {
ok = false;
break;
}
}
if (got < 0)
ok = false;
if (ok) {
if (algo == CHECKSUM_ALGO_XXH64) {
uint64_t digest = XXH64_digest(&xxh64);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
} else if (algo == CHECKSUM_ALGO_XXH3) {
uint64_t digest = XXH3_64bits_digest(xxh3);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
} else {
XXH128_hash_t digest = XXH3_128bits_digest(xxh3);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
}
}
if (xxh3)
XXH3_freeState(xxh3);
close(fd);
return ok;
}
int checksum_algo_from_name(const char* name) {
if (!name)
return -1;
+10 -5
View File
@@ -35,11 +35,16 @@ typedef enum {
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
size_t out_capacity, size_t* out_len);
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id.
* Accepts "xxh64"/"xxhash", "xxh3", "xxh128" and "md5". "auto", rsync's
* default automatic choice, is resolved to the default by the caller (it is not
* a distinct algorithm here). Returns -1 for any name FastSync does not
* implement (md4/sha1/none included). */
/* Streaming whole-file digest: hash the contents of `path` without holding the
* whole file in memory. Same digest/capacity contract as checksum_digest.
* Returns false on open/read failure or an undersized buffer. */
bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, uint8_t* out,
size_t out_capacity, size_t* out_len);
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id. * Accepts
* "xxh64"/"xxhash", "xxh3", "xxh128" and "md5". "auto", rsync's default automatic choice, is
* resolved to the default by the caller (it is not a distinct algorithm here). Returns -1 for any
* name FastSync does not implement (md4/sha1/none included). */
int checksum_algo_from_name(const char* name);
/* Canonical name of an algorithm (used in CLI error messages). */
+27 -4
View File
@@ -250,8 +250,17 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
* answer every per-file STATUS_CHECK with a STATUS_DEST_INFO snapshot of the
* pre-transfer destination entry (see protocol.h). It is set by the client
* only when -i/--itemize-changes or --out-format asks for per-file change
* output; the transfer decision itself is unchanged. */
#define CONFIG_WIRE_OUTPUT_FIELDS(X) X(report_dest_info, bool, false, BOOL)
* output; the transfer decision itself is unchanged.
*
* Wire-stats wave (protocol 2.25.0). report_stats tells the receiver to send a
* STATUS_STATS frame immediately before its terminal success status carrying
* the receiver-only counters (matched data, deleted-file count) and,
* for -n/--dry-run --delete, the destination-relative paths it WOULD have
* deleted. It is set by the client only when --stats, --progress/-P, an
* --out-format token needs a wire counter (%b/%c), or a dry-run carries
* --delete; the transfer decision itself is unchanged. */
#define CONFIG_WIRE_OUTPUT_FIELDS(X) \
X(report_dest_info, bool, false, BOOL) X(report_stats, bool, false, BOOL)
/* All serialized fields, in exact wire order. Concatenating the per-segment
* lists here is what keeps the declaration order = the wire order. */
@@ -919,8 +928,22 @@ typedef struct Config {
* snapshot of the old entry) before its ordinary verdict when the config frame
* carries the new report_dest_info bool appended after the --copy-as block.
* This is both a config-frame layout change (one trailing bool) and a frame
* sequence change (the new status). */
#define PROTOCOL_VERSION "2.24.0"
* sequence change (the new status).
*
* (4) Delete timing (protocol 2.24.0): the sender streams one delete plan per
* source directory so --delete-during/--delete-delay reproduce rsync's deletion
* timing (the plan fields and STATUS_DELETE_PLAN are documented at the keep-set
* / delete-plan definitions below).
*
* (5) Wire-stats parity (protocol 2.25.0): --stats, --progress/-P and the
* --out-format %b/%c tokens need receiver-only and wire counters that the push
* sender cannot observe, and -n/--dry-run --delete must report the extras it
* would have removed without deleting anything. The config frame gains one
* trailing report_stats bool and the receiver emits a new STATUS_STATS frame
* (carrying matched data, the deleted-file count and the would-delete path
* list) immediately before its terminal success status. Both a config-frame
* layout change and a frame-sequence change, hence the bump. */
#define PROTOCOL_VERSION "2.25.0"
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
#define MAX_BASIS_DIRS 64
+36
View File
@@ -3460,6 +3460,42 @@ static bool delete_missing_args_budgeted(const Config* config, DeleteManifest* m
/* Public wrappers used outside the commit path (and by unit tests): no
--max-delete budget. */
bool manifest_would_delete_list(const Config* config, DeleteManifest* manifest, ArrayList* out,
size_t* count_out) {
if (count_out)
*count_out = 0;
if (!config || !manifest || !manifest->keeps || !out)
return false;
int skip_count = (config->delay_updates ? 1 : 0) + config->basis_count +
(manifest->protected ? manifest->protected->size : 0);
DeleteSkipEntry* skips = NULL;
if (skip_count > 0) {
skips = calloc((size_t)skip_count, sizeof(DeleteSkipEntry));
if (!skips)
return false;
int idx = 0;
if (config->delay_updates) {
skips[idx].prefix = DELAY_UPDATES_STAGING_DIR;
skips[idx].top_level_only = true;
idx++;
}
for (int i = 0; i < config->basis_count; i++) {
skips[idx].prefix = config->basis_dirs[i].path;
skips[idx].top_level_only = false;
idx++;
}
for (int i = 0; i < manifest->protected->size; i++) {
skips[idx].prefix = (const char*)manifest->protected->items[i];
skips[idx].top_level_only = false;
idx++;
}
}
bool ok = delete_extras_list(config->receive_root_directory, manifest->keeps, manifest->dirs,
skips, skip_count, out, count_out);
free(skips);
return ok;
}
bool manifest_delete_extras(const Config* config, DeleteManifest* manifest) {
DeleteBudgetState budget = {
.max_delete = SIZE_MAX, .deleted = 0, .skipped = 0, .limit_hit = false};
+8
View File
@@ -143,6 +143,14 @@ typedef enum {
stopped part of the work, or DELETE_COMMIT_ERROR on a genuine failure. */
DeleteCommitResult manifest_delete_all(const Config* config, DeleteManifest* manifest);
/* -n/--dry-run --delete would-delete reporting: walk the destination exactly as
the delete pass would and append (strdup'd) destination-relative paths that
WOULD be removed to `out`, without touching disk. Uses the same staging-dir,
basis-dir and protected-prefix skips as the real commit. Returns true on a
clean walk; `*count_out` receives the number of paths appended. */
bool manifest_would_delete_list(const Config* config, DeleteManifest* manifest, ArrayList* out,
size_t* count_out);
/* Outcome of a single file_save_to_disk operation. The receiver needs to
distinguish "written" from "skipped" so --remove-source-files can be told
which sources were actually stored. */
+1
View File
@@ -182,6 +182,7 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta
close(fd);
return false;
}
protocol_note_bytes_written((unsigned long long)sent);
}
close(fd);
+26
View File
@@ -101,3 +101,29 @@ bool format_dest_state_receive(int fd, OutputDestState* state) {
state->gid = gid;
return true;
}
bool format_stats_send(int fd, const ReceiverStats* stats) {
if (!stats)
return false;
unsigned long long matched = stats->matched_data;
unsigned long long deleted = stats->deleted_files;
unsigned long long would = stats->would_delete_count;
return send_n_data(fd, &matched, sizeof(matched)) && send_n_data(fd, &deleted, sizeof(deleted)) &&
send_n_data(fd, &would, sizeof(would));
}
bool format_stats_receive(int fd, ReceiverStats* stats) {
if (!stats)
return false;
unsigned long long matched = 0;
unsigned long long deleted = 0;
unsigned long long would = 0;
if (!receive_n_data(fd, &matched, sizeof(matched)) ||
!receive_n_data(fd, &deleted, sizeof(deleted)) || !receive_n_data(fd, &would, sizeof(would)))
return false;
memset(stats, 0, sizeof(*stats));
stats->matched_data = matched;
stats->deleted_files = deleted;
stats->would_delete_count = would;
return true;
}
+17
View File
@@ -56,4 +56,21 @@ bool format_rsync_datetime(time_t when, bool dash, char* buffer, size_t buffer_s
bool format_dest_state_send(int fd, const OutputDestState* state);
bool format_dest_state_receive(int fd, OutputDestState* state);
/* End-of-transfer receiver counters reported through STATUS_STATS (protocol
* 2.25.0) when the wire config carries report_stats. `would_delete_count` is
* the number of destination-relative paths the receiver would have deleted in a
* -n/--dry-run --delete run; that many wire strings immediately follow the
* fixed record (sent/read by the caller). */
typedef struct {
unsigned long long matched_data;
unsigned long long deleted_files;
unsigned long long would_delete_count;
} ReceiverStats;
/* Fixed-width STATUS_STATS counter record. The status frame and the optional
* would-delete path list are sent/received by the caller. Returns false on I/O
* failure. */
bool format_stats_send(int fd, const ReceiverStats* stats);
bool format_stats_receive(int fd, ReceiverStats* stats);
#endif
+21
View File
@@ -29,6 +29,13 @@ static unsigned long long io_bwlimit = 0;
static mtx_t bw_mutex;
static once_flag bw_mutex_once = ONCE_FLAG_INIT;
/* Process-wide wire byte counters, used by the client to render rsync's
* --stats/--progress totals and the --out-format %b/%c tokens. The zero-copy
* sendfile path bypasses protocol_send_n_data, so it reports its bytes through
* protocol_note_bytes_written. */
static atomic_ullong io_bytes_written = 0;
static atomic_ullong io_bytes_read = 0;
static unsigned long long global_bwlimit(void);
static bool protocol_reserve_memory(ProtocolSession* session, size_t charge) {
@@ -242,6 +249,18 @@ SSL* io_get_ssl(void) {
return io_ssl;
}
unsigned long long protocol_bytes_written(void) {
return atomic_load(&io_bytes_written);
}
unsigned long long protocol_bytes_read(void) {
return atomic_load(&io_bytes_read);
}
void protocol_note_bytes_written(unsigned long long bytes) {
atomic_fetch_add(&io_bytes_written, bytes);
}
static ProtocolSession* legacy_session(int read_fd, int write_fd) {
if (bound_session)
return bound_session;
@@ -343,6 +362,7 @@ bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t dat
wait_events = POLLOUT;
}
log_debug_message(LOG_DEBUG_IO, " Send n Data: %zu", total_bytes_send);
atomic_fetch_add(&io_bytes_written, (unsigned long long)total_bytes_send);
return true;
}
@@ -430,6 +450,7 @@ static bool protocol_receive_n_data_until(ProtocolSession* session, void* data,
wait_events = POLLIN;
}
log_debug_message(LOG_DEBUG_IO, " Received n Data: %zu", total_bytes_received);
atomic_fetch_add(&io_bytes_read, (unsigned long long)total_bytes_received);
return true;
}
+16 -1
View File
@@ -195,7 +195,14 @@ enum NET_STATUS {
* ("." for the receive root); then the child-directory count + names and the
* child-file count + names that must be kept. Appended after
* STATUS_DEST_INFO so no existing status is renumbered. */
STATUS_DELETE_PLAN
STATUS_DELETE_PLAN,
/* End-of-transfer receiver counter report (protocol 2.25.0). When the wire
* config carries report_stats=true, the receiver sends this status once,
* immediately before its terminal success status, followed by a fixed stats
* record (see format_stats_send/receive in format.h) and, when the run is a
* --dry-run with --delete, the would-delete path list. Appended after
* STATUS_DELETE_PLAN so no existing status is renumbered. */
STATUS_STATS
};
void io_set_fds(int read_fd, int write_fd);
@@ -203,6 +210,14 @@ void io_set_bwlimit(unsigned long long bytes_per_sec);
void io_set_ssl(SSL* ssl);
SSL* io_get_ssl(void);
/* Process-wide wire byte counters. protocol_send_n_data/protocol_receive_n_data
* update them; the zero-copy sendfile path reports through
* protocol_note_bytes_written. Used by the client to render rsync's
* --stats/--progress totals and the --out-format %b/%c tokens. */
unsigned long long protocol_bytes_written(void);
unsigned long long protocol_bytes_read(void);
void protocol_note_bytes_written(unsigned long long bytes);
void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd);
/* Transitional bridge for helpers whose signatures still carry only an fd. */
void protocol_session_bind(ProtocolSession* session);
+142
View File
@@ -725,6 +725,148 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
return operation_ok;
}
/* Read-only mirror of delete_extras_fd: records the paths that WOULD be removed
without unlinking anything. A child directory is reported after its own
reportable children (depth-first), matching the delete pass's ordering. */
static bool list_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep,
const PathIndex* dirs, ArrayList* out, size_t* recorded,
const DeleteSkipEntry* skips, int skip_count, bool parent_deletable,
bool* all_removed) {
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (scanfd < 0)
return false;
DIR* dir = fdopendir(scanfd);
if (!dir) {
close(scanfd);
return false;
}
bool operation_ok = true;
bool local_survives = false;
bool deletable = parent_deletable || is_synced_dir(dirs, rel_path);
const struct dirent* entry;
while ((entry = readdir(dir)) != NULL) {
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
continue;
char* child_rel = path_cat((char*)rel_path, entry->d_name);
if (!child_rel) {
operation_ok = false;
continue;
}
if (path_under_skip_prefix(child_rel, rel_path[0] == '\0', skips, skip_count)) {
local_survives = true;
free(child_rel);
continue;
}
struct stat st;
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
if (errno != ENOENT)
operation_ok = false;
free(child_rel);
continue;
}
if (S_ISDIR(st.st_mode)) {
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
bool child_all_removed = false;
if (childfd >= 0) {
if (!list_extras_fd(childfd, child_rel, keep, dirs, out, recorded, skips, skip_count,
deletable, &child_all_removed))
operation_ok = false;
close(childfd);
} else if (errno != ENOENT) {
operation_ok = false;
}
bool child_synced = dirs && path_index_contains(dirs, child_rel);
if (child_synced || keep_is_dir(keep, child_rel)) {
local_survives = true;
} else if (child_all_removed && deletable) {
size_t len = strlen(child_rel);
char* copy = malloc(len + 2);
if (!copy) {
operation_ok = false;
} else {
memcpy(copy, child_rel, len);
copy[len] = '/';
copy[len + 1] = '\0';
if (!array_list_add(out, copy)) {
free(copy);
operation_ok = false;
} else {
(*recorded)++;
}
}
} else {
local_survives = true;
}
} else {
bool found = keep_is_file(keep, child_rel);
if (found || !deletable) {
local_survives = true;
} else {
char* copy = str_dup(child_rel);
if (!copy || !array_list_add(out, copy)) {
free(copy);
operation_ok = false;
} else {
(*recorded)++;
}
}
}
free(child_rel);
}
closedir(dir);
*all_removed = !local_survives;
return operation_ok;
}
bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
const ArrayList* synced_dirs, const DeleteSkipEntry* skips, int skip_count,
ArrayList* out, size_t* count_out) {
if (count_out)
*count_out = 0;
if (!manifest || !out)
return false;
PathIndex keep;
if (!build_keep_index(manifest, &keep))
return false;
PathIndex dirs;
bool have_dirs = synced_dirs != NULL;
if (have_dirs &&
!path_index_build(&dirs, (const char* const*)synced_dirs->items, (size_t)synced_dirs->size)) {
path_index_free(&keep);
return false;
}
int rootfd;
int root_fd = utils_get_authorized_root_fd();
if (root_fd >= 0) {
if (utils_get_authorized_root_path())
rootfd = utils_open_authorized_destination(dest_root);
else if (dest_root == NULL)
rootfd = dup(root_fd);
else
rootfd = -1;
} else {
rootfd = open(dest_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
}
if (rootfd < 0) {
path_index_free(&keep);
if (have_dirs)
path_index_free(&dirs);
return false;
}
bool all_removed = false;
size_t recorded = 0;
bool ok = list_extras_fd(rootfd, "", &keep, have_dirs ? &dirs : NULL, out, &recorded, skips,
skip_count, false, &all_removed);
if (close(rootfd) != 0)
ok = false;
path_index_free(&keep);
if (have_dirs)
path_index_free(&dirs);
if (count_out)
*count_out = recorded;
return ok;
}
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
const ArrayList* synced_dirs, size_t max_delete,
const DeleteSkipEntry* skips, int skip_count,
+8
View File
@@ -138,6 +138,14 @@ DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* m
const ArrayList* synced_dirs, size_t max_delete,
const DeleteSkipEntry* skips, int skip_count,
size_t* deleted_out, size_t* skipped_out);
/* Read-only companion to delete_extras_limited: walk the destination exactly as
the delete pass would and APPEND (strdup'd) destination-relative paths that
WOULD be removed, without touching disk. Used for -n/--dry-run --delete
would-delete reporting. Returns true on a clean walk; the caller owns the
strings appended to `out` and receives their count in *count_out. */
bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
const ArrayList* synced_dirs, const DeleteSkipEntry* skips, int skip_count,
ArrayList* out, size_t* count_out);
bool delete_extras(const char* dest_root, const ArrayList* manifest);
/* Open the existing destination directory at `dest_root`, confined to the
authorized root with an O_NOFOLLOW component walk (the same confinement the