feat(basis): rsync quick-check default + FastSync-only --verify-basis

- default basis match is rsync's metadata quick-check (size + mtime; size-only
  drops mtime; -I disables), no mandatory content digest
- new long-only --verify-basis (wire bool, protocol stays 2.28.0) restores the
  strict whole-file content equality
- --copy-dest re-applies source attributes; basis-hit 256 MiB cap removed by
  streaming the copy/hash; basis miss keeps the normal payload bound
- compare/copy/link-dest rows -> caveat; tally 116/13/28
This commit is contained in:
2026-09-19 15:55:51 +02:00
parent 8ec8cb7203
commit 67076bf218
23 changed files with 940 additions and 255 deletions
+13 -11
View File
@@ -224,23 +224,31 @@ bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, ui
if (fd < 0)
return false;
bool ok = checksum_digest_fd(algo, seed, fd, out, out_capacity, out_len);
close(fd);
return ok;
}
bool checksum_digest_fd(ChecksumAlgo algo, uint64_t seed, int fd, uint8_t* out, size_t out_capacity,
size_t* out_len) {
if (fd < 0 || !out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
return false;
if (algo == CHECKSUM_ALGO_NONE) {
/* No checksum requested: nothing to read; an empty digest succeeds. */
close(fd);
*out_len = 0;
return true;
}
uint8_t buffer[64 * 1024];
bool ok = false;
lseek(fd, 0, SEEK_SET);
if (algo == CHECKSUM_ALGO_MD5 || algo == CHECKSUM_ALGO_SHA1) {
const EVP_MD* md = algo == CHECKSUM_ALGO_MD5 ? EVP_md5() : EVP_sha1();
EVP_MD_CTX* ctx = EVP_MD_CTX_new();
if (!ctx) {
close(fd);
if (!ctx)
return false;
}
unsigned int digest_len = 0;
if (EVP_DigestInit_ex(ctx, md, NULL) == 1) {
ok = true;
@@ -259,7 +267,6 @@ bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, ui
ok = false;
}
EVP_MD_CTX_free(ctx);
close(fd);
return ok;
}
@@ -276,7 +283,6 @@ bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, ui
md4_final(&ctx, out);
*out_len = 16;
}
close(fd);
return ok;
}
@@ -286,16 +292,13 @@ bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, ui
XXH64_reset(&xxh64, seed);
} else if (algo == CHECKSUM_ALGO_XXH3 || algo == CHECKSUM_ALGO_XXH128) {
xxh3 = XXH3_createState();
if (!xxh3) {
close(fd);
if (!xxh3)
return false;
}
if (algo == CHECKSUM_ALGO_XXH3)
XXH3_64bits_reset_withSeed(xxh3, seed);
else
XXH3_128bits_reset_withSeed(xxh3, seed);
} else {
close(fd);
return false;
}
@@ -329,7 +332,6 @@ bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, ui
}
if (xxh3)
XXH3_freeState(xxh3);
close(fd);
return ok;
}
+7
View File
@@ -51,6 +51,13 @@ bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t
bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, uint8_t* out,
size_t out_capacity, size_t* out_len);
/* Descriptor form of the streaming digest: rewinds `fd` to the start and hashes
* to EOF without closing it. Used by the --verify-basis path to hash an
* already-open, root-confined basis descriptor. Same contract as
* checksum_digest_file. */
bool checksum_digest_fd(ChecksumAlgo algo, uint64_t seed, int fd, uint8_t* out, size_t out_capacity,
size_t* out_len);
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id.
* Accepts "xxh64"/"xxhash", "xxh3", "xxh128", "md5", "md4", "sha1", "none".
* "auto" is not an algorithm here; the caller resolves it to the negotiated
+10 -1
View File
@@ -198,9 +198,18 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
X(skip_compress_count, int, 0, INT_SKIPCOUNT) \
X(skip_compress_suffixes, char**, NULL, BLOCK_SKIP_SUFFIXES)
/* FastSync-only --verify-basis (protocol 2.28.0, no version bump by project
* decision): restores the stricter content equality on a basis hit. By
* default a basis hit is accepted on rsync's metadata quick-check alone (equal
* size plus equal mtime, or size alone under --size-only); with this flag the
* receiver ALSO requires the basis bytes' whole-file digest (the negotiated
* --checksum-choice algorithm) to equal the sender's, exactly FastSync's
* historical behavior. It is a receiver policy and crosses the wire so the
* receiver knows whether to read and hash the basis content. */
#define CONFIG_WIRE_BASIS_FIELDS(X) \
X(basis_count, int, 0, INT_BASISCOUNT) \
X(basis_dirs, BasisDest*, NULL, BLOCK_BASIS)
X(basis_dirs, BasisDest*, NULL, BLOCK_BASIS) \
X(verify_basis, bool, false, BOOL)
#define CONFIG_WIRE_FUZZY_FIELDS(X) X(fuzzy, bool, false, BOOL)
+209 -1
View File
@@ -15,6 +15,7 @@
#include <unistd.h>
#include "data.h"
#include "checksum.h"
#include "delta.h"
#include "file.h"
#include "file_store.h"
@@ -24,6 +25,13 @@
#include "utils.h"
#include "protocol.h"
#include "xattr.h"
#include <fcntl.h>
#include <unistd.h>
/* Files larger than this are not loaded whole for transfer (the sender streams
* them); a whole-file digest is computed from the path instead. Kept in sync
* with the sender's streaming threshold. */
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
static bool write_all(int fd, const void* data, unsigned long long size) {
const unsigned char* p = data;
@@ -39,6 +47,31 @@ static bool write_all(int fd, const void* data, unsigned long long size) {
return true;
}
/* Streaming copy of an open source descriptor into the just-created destination
`fd` (already at offset 0). Used by the --copy-dest basis install so a basis
larger than any in-memory whole-file bound still materializes without
buffering the entire file. `expected_size` is the caller-verified basis
size; the copy must produce exactly that many bytes (a short source is a hard
error, never a silently truncated destination). The final ftruncate drops
any residual tail a raced-in longer source might have left. */
static bool copy_fd_all(int dst_fd, int src_fd, unsigned long long expected_size) {
unsigned char buf[1 << 20];
unsigned long long done = 0;
while (done < expected_size) {
unsigned long long remaining = expected_size - done;
size_t want = remaining < sizeof(buf) ? (size_t)remaining : sizeof(buf);
ssize_t n = read(src_fd, buf, want);
if (n < 0 && errno == EINTR)
continue;
if (n <= 0)
return false;
if (!write_all(dst_fd, buf, (unsigned long long)n))
return false;
done += (unsigned long long)n;
}
return ftruncate(dst_fd, (off_t)expected_size) == 0;
}
/* Preallocate `size` bytes on `fd` before any data is written (--preallocate).
* fallocate(2) reserves real disk blocks, so an out-of-space condition
* (ENOSPC/EDQUOT) surfaces up front instead of partway through a transfer;
@@ -140,6 +173,12 @@ bool file_checksum(File* file, ChecksumAlgo algo, uint64_t seed, uint8_t* out, s
if (file->data->size == 0) {
return checksum_digest(algo, seed, "", 0, out, out_capacity, out_len);
}
/* A streamed source (data not loaded) may exceed any in-memory whole-file
bound; hash it from the file path in bounded buffers instead of forcing a
full load. This is the same digest the receiver recomputes on the basis. */
if (!file->data->data && file->path && file->data->size > STREAM_THRESHOLD &&
checksum_digest_file(algo, seed, file->path, out, out_capacity, out_len))
return true;
if (!file->data->data && !file_load_data(file))
return false;
return checksum_digest(algo, seed, file->data->data, file->data->size, out, out_capacity,
@@ -176,6 +215,7 @@ File* file_create(const char* path) {
file->is_dir = false;
file->dir_time_only = false;
file->basis_link = NULL;
file->basis_copy = NULL;
file->link_group = 0;
file->link_first = false;
file->hardlink_target = NULL;
@@ -205,6 +245,8 @@ void file_destroy(void* item) {
file->send_path = NULL;
free(file->basis_link);
file->basis_link = NULL;
free(file->basis_copy);
file->basis_copy = NULL;
free(file->hardlink_target);
file->hardlink_target = NULL;
free(file->symlink_target);
@@ -1512,6 +1554,161 @@ bool file_to_disk_secure_attrs_counted(const char* path, const void* data,
* basis). Likewise `xattrs`/`fake_super` are applied only on the copy
* fallback, so a fallback copy preserves the per-file attributes instead of
* silently dropping them. */
/* Streaming --copy-dest basis install: atomically materialize `path` from the
* bytes of `basis_path` without holding the file in memory, so a basis larger
* than any whole-file bound still works. Mirrors the ordinary secure store
* path (confined parent walk, temp + rename, --update/--ignore-existing/
* --preallocate/--temp-dir) but sources the data from the basis descriptor
* rather than a caller buffer, and applies the SOURCE metadata (rsync copies
* then fixes attributes). A hard-link install that falls back to a byte copy
* also routes through here when the caller supplies the basis path. */
static bool file_copy_basis_stream_impl(const char* path, const char* basis_path,
unsigned long long expected_size, bool preallocate,
const FileMetadata* metadata, FileAttrPolicy policy,
bool update, bool no_replace, bool use_fsync,
const FileXattrList* xattrs, bool fake_super,
const char* temp_dir, unsigned* dirs_created,
const char* count_floor) {
if (!path || !basis_path)
return false;
char* leaf = NULL;
int dirfd = file_open_secure_parent_counted(path, &leaf, true, dirs_created, count_floor);
if (dirfd < 0)
return false;
char* basis_leaf = NULL;
int basis_dirfd = file_open_secure_parent(basis_path, &basis_leaf, false);
int src_fd = -1;
if (basis_dirfd >= 0 && basis_leaf != NULL) {
/* O_NONBLOCK rejects a raced-in FIFO without blocking; the S_ISREG gate
below is the real type check. */
src_fd = openat(basis_dirfd, basis_leaf, O_RDONLY | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK);
struct stat src_st;
if (src_fd >= 0 && (fstat(src_fd, &src_st) != 0 || !S_ISREG(src_st.st_mode))) {
close(src_fd);
src_fd = -1;
}
}
if (basis_dirfd >= 0)
close(basis_dirfd);
free(basis_leaf);
if (src_fd < 0) {
close(dirfd);
free(leaf);
return false;
}
struct stat destination_stat;
bool destination_is_regular = fstatat(dirfd, leaf, &destination_stat, AT_SYMLINK_NOFOLLOW) == 0 &&
S_ISREG(destination_stat.st_mode);
if (update && metadata && destination_is_regular && stat_is_newer(&destination_stat, metadata)) {
close(src_fd);
close(dirfd);
free(leaf);
return true;
}
if (no_replace && file_path_exists_secure(path)) {
close(src_fd);
close(dirfd);
free(leaf);
return true;
}
int scratch_dirfd = -1;
if (temp_dir) {
scratch_dirfd = file_open_temp_dir(temp_dir);
if (scratch_dirfd < 0) {
int saved_errno = errno;
log_message(LOG_LEVEL_ERROR,
"--temp-dir '%s' could not be opened (rsync requires it to already exist): %s",
temp_dir, strerror(saved_errno));
close(src_fd);
close(dirfd);
free(leaf);
return false;
}
}
int target_dirfd = scratch_dirfd >= 0 ? scratch_dirfd : dirfd;
int tmp_size = snprintf(NULL, 0, ".%s.tmp.%ld.%llu", leaf, (long)getpid(), ~0ULL);
char* tmp = NULL;
bool ok = false;
if (tmp_size >= 0)
tmp = malloc((size_t)tmp_size + 1);
if (tmp) {
for (unsigned int i = 0; i < 100 && !ok; ++i) {
if (scratch_dirfd >= 0)
snprintf(tmp, (size_t)tmp_size + 1, ".%s.tmp.%ld.%llu", leaf, (long)getpid(),
next_temp_sequence());
else
snprintf(tmp, (size_t)tmp_size + 1, ".%s.tmp.%ld.%u", leaf, (long)getpid(), i);
int fd =
openat(target_dirfd, tmp, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600);
if (fd < 0) {
if (errno != EEXIST)
break;
continue;
}
bool wrote = true;
if (preallocate && expected_size > 0 && preallocate_fd(fd, expected_size) != 0)
wrote = false;
if (wrote)
wrote = copy_fd_all(fd, src_fd, expected_size);
if (wrote && metadata) {
if (!policy.perms &&
fchmod(fd, file_mode_base(metadata, destination_is_regular,
destination_is_regular ? destination_stat.st_mode & 0777
: 0)) != 0)
wrote = false;
if (wrote)
wrote = file_restore_metadata_fd(fd, metadata, policy);
} else if (wrote && fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) != 0) {
wrote = false;
}
if (wrote)
restore_extra_fd(fd, metadata, xattrs, fake_super, policy);
if (wrote && use_fsync)
wrote = fsync(fd) == 0;
if (close(fd) != 0)
wrote = false;
if (wrote && renameat(target_dirfd, tmp, dirfd, leaf) != 0)
wrote = false;
if (!wrote)
unlinkat(target_dirfd, tmp, 0);
ok = wrote;
}
free(tmp);
}
if (!ok && scratch_dirfd >= 0) {
/* Retry once with no scratch dir (rsync's EXDEV fallback). */
close(scratch_dirfd);
close(src_fd);
close(dirfd);
free(leaf);
return file_copy_basis_stream_impl(path, basis_path, expected_size, preallocate, metadata,
policy, update, no_replace, use_fsync, xattrs, fake_super,
NULL, dirs_created, count_floor);
}
if (scratch_dirfd >= 0)
close(scratch_dirfd);
close(src_fd);
close(dirfd);
free(leaf);
return ok;
}
/* --copy-dest basis install (streaming). Applies the source metadata and the
per-file xattrs / --fake-super record. */
bool file_copy_basis_stream_attrs(const char* path, const char* basis_path,
unsigned long long expected_size, bool preallocate,
const FileMetadata* metadata, FileAttrPolicy policy, bool update,
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
const char* temp_dir) {
return file_copy_basis_stream_impl(path, basis_path, expected_size, preallocate, metadata, policy,
update, false, use_fsync, xattrs, fake_super, temp_dir, NULL,
NULL);
}
static bool file_to_disk_secure_link_impl(const char* path, const char* basis_path,
const void* data, unsigned long long data_size,
bool preallocate, const FileMetadata* metadata,
@@ -1521,6 +1718,10 @@ static bool file_to_disk_secure_link_impl(const char* path, const char* basis_pa
const char* count_floor) {
if (!path || !basis_path)
return false;
/* The caller-supplied buffer is no longer used: the copy fallback streams
from the basis path (which may hold an over-limit file). Kept in the
signature for the existing API. */
(void)data;
char* leaf = NULL;
int dirfd = file_open_secure_parent_counted(path, &leaf, true, dirs_created, count_floor);
if (dirfd < 0)
@@ -1604,7 +1805,14 @@ static bool file_to_disk_secure_link_impl(const char* path, const char* basis_pa
close(dirfd);
free(leaf);
/* The basis file could not be linked in (missing, cross-device, refused
by the filesystem). Write a byte-identical local copy instead. */
by the filesystem). Stream a byte-identical local copy from the basis
itself (never the possibly-absent caller buffer) so an over-limit basis
still materializes. When the basis path is not a readable regular file
(e.g. a directory raced in), fall back to the caller-supplied bytes. */
if (file_copy_basis_stream_impl(path, basis_path, data_size, preallocate, metadata, policy,
false, false, use_fsync, xattrs, fake_super, temp_dir,
dirs_created, count_floor))
return true;
return file_to_disk_secure_attrs_counted(
path, data, data_size, false, false, preallocate, metadata, policy, false, false, use_fsync,
xattrs, fake_super, false, temp_dir, dirs_created, count_floor);
+10
View File
@@ -163,6 +163,16 @@ bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, co
const FileMetadata* metadata, FileAttrPolicy policy,
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
const char* temp_dir);
/* Streaming --copy-dest install: atomically materialize `path` by copying the
* bytes of `basis_path` through a bounded buffer (no whole-file buffering, so
* an arbitrarily large basis works), applying the SOURCE metadata and the
* per-file xattrs / --fake-super record. `update` honors a newer destination;
* a --temp-dir scratch location falls back to a direct write on EXDEV. */
bool file_copy_basis_stream_attrs(const char* path, const char* basis_path,
unsigned long long expected_size, bool preallocate,
const FileMetadata* metadata, FileAttrPolicy policy, bool update,
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
const char* temp_dir);
/* Protocol 2.28.0 receiver-stat variants: like the two above but additionally
* report through `dirs_created` (when non-NULL) how many parent directories the
* confined secure walk had to create that lie strictly below `count_floor` (a
+146 -98
View File
@@ -99,6 +99,12 @@ static FileSaveResult file_stage_delayed_update(const char* root_directory,
if (file->basis_link) {
ok = file_to_disk_secure_link(staged_path, file->basis_link, file->data->data, file->data->size,
config->preallocate, metadata, policy, config->use_fsync, NULL);
} else if (file->basis_copy) {
/* --copy-dest basis hit: stream the basis into the staging tree (bounded
buffers, so an over-limit basis still stages). */
ok = file_copy_basis_stream_attrs(staged_path, file->basis_copy, file->data->size,
config->preallocate, metadata, policy, config->update,
config->use_fsync, file->xattrs, config->fake_super, NULL);
} else {
ok =
file_to_disk_secure_attrs(staged_path, file->data->data, file->data->size, false, sparse,
@@ -652,7 +658,8 @@ FileSaveResult file_save_to_disk_full_ex(const char* root_directory, const File*
char* destination_path = NULL;
char *backup_path = NULL, *parent_copy = NULL;
if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data) ||
if (!file || !file->path || !file->data ||
(file->data->size != 0 && !file->data->data && !file->basis_link && !file->basis_copy) ||
(!file_get_trust_sender() && has_path_traversal(file->path)) ||
(backup_enabled &&
(!backup_suffix || backup_suffix[0] == '\0' || strchr(backup_suffix, '/') != NULL ||
@@ -975,6 +982,13 @@ FileSaveResult file_save_to_disk_full_ex(const char* root_directory, const File*
disk_path, file->basis_link, file->data->data, file->data->size, config->preallocate,
metadata, policy, config->use_fsync, file->xattrs, config->fake_super, confined_temp,
created_dirs, count_floor);
} else if (config && file->basis_copy) {
/* --copy-dest: stream the basis bytes through a bounded buffer so a basis
larger than any whole-file bound still materializes. The source
metadata was transmitted with the check frame. */
ok = file_copy_basis_stream_attrs(
disk_path, file->basis_copy, file->data->size, config->preallocate, metadata, policy,
config->update, config->use_fsync, file->xattrs, config->fake_super, confined_temp);
} else {
/* The plain no-replace / update / with-fsync engines, plus per-file xattr
(-X/-A) and --fake-super application on the written fd. */
@@ -1298,16 +1312,17 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
/* ---- Alternate basis directories (--compare-dest / --copy-dest / --link-dest) ----
* The receiver consults the ordered basis-dir list only when the destination
* entry is NOT already up to date. An "exact match" requires an equal size,
* an equal mtime (unless --size-only), and an equal content xxHash64, so a
* hard link / local copy is only ever made from byte-identical content. */
* entry is NOT already up to date. By default an "exact match" is rsync's
* metadata quick-check: an equal size and an equal mtime (unless --size-only).
* The FastSync-only --verify-basis additionally requires an equal whole-file
* content digest, so a hard link / local copy is only then made from
* byte-verified content. */
typedef struct BasisMatch {
bool hit;
BasisDestType type;
char* basis_path; /* owned absolute path of the matched basis file */
struct stat st; /* fstat() of the matched basis file */
Data* content; /* owned basis bytes (or empty Data), NULL when not loaded */
} BasisMatch;
static void basis_match_free(BasisMatch* match) {
@@ -1315,8 +1330,6 @@ static void basis_match_free(BasisMatch* match) {
return;
free(match->basis_path);
match->basis_path = NULL;
data_destroy(match->content);
match->content = NULL;
match->hit = false;
match->type = BASIS_DEST_NONE;
}
@@ -1348,32 +1361,10 @@ static bool basis_open_regular(const char* path, unsigned long long expected_siz
return true;
}
/* Read the whole remaining content of an open descriptor. A zero-length file
yields an empty Data (data pointer NULL). */
static Data* basis_read_content(int fd, unsigned long long size) {
if (size == 0)
return data_create_reserve(0);
if (size > MAX_RECEIVE_WHOLE_FILE_SIZE || size > SIZE_MAX)
return NULL;
void* buf = protocol_alloc((size_t)size);
if (!buf)
return NULL;
size_t got = 0;
while (got < (size_t)size) {
ssize_t n = read(fd, (char*)buf + got, (size_t)size - got);
if (n <= 0) {
free(buf);
return NULL;
}
got += (size_t)n;
}
return data_create(buf, (size_t)size);
}
/* --ignore-times forces every file to be updated, so no basis hit is ever
declared (matching rsync, where -I prevents link-dest from linking). */
static bool basis_quick_matches(const Config* config, const struct stat* st, time_t check_mtime,
long check_mtime_nsec) {
bool file_basis_quick_match(const Config* config, const struct stat* st, time_t check_mtime,
long check_mtime_nsec) {
if (config->size_only)
return true;
long mtime_nsec = 0;
@@ -1384,28 +1375,39 @@ static bool basis_quick_matches(const Config* config, const struct stat* st, tim
config->modify_window);
}
/* Search the basis-dir list in command-line order and return the first exact
match. When load_content is true the matched bytes are kept in out->content
so the caller can materialize the file without re-reading it.
/* True when a basis hit must be confirmed by a whole-file content digest
(--verify-basis). False is the rsync-parity default: the metadata
quick-check alone decides a hit. */
bool file_basis_content_required(const Config* config) {
return config != NULL && config->verify_basis;
}
An exact match ALSO requires the basis bytes' digest to equal the source's,
so `hash_content` gates the content read/hash itself. A server-contacting
--dry-run passes hash_content=false: no basis file may be read or hashed
(that would be a 1-bit content oracle against a client-supplied digest), so a
metadata-only pass can never confirm a hit and declines it. The real path
always passes hash_content=true, keeping its behavior byte-for-byte. */
/* Search the basis-dir list in command-line order and return the first match.
By default (no --verify-basis) rsync's metadata quick-check is sufficient:
basis_open_regular has already required an equal size, and
file_basis_quick_match applies rsync's mtime (or --size-only) rule.
--verify-basis additionally requires the basis bytes' whole-file digest to
equal the sender's, restoring FastSync's historical content equality; that
digest is computed by streaming the open basis descriptor, so an arbitrarily
large basis is verified without buffering it. A copy/link install re-reads
the basis from its path in bounded buffers, so no content buffer is kept.
`hash_content` gates content READS under --verify-basis: a server-contacting
--dry-run passes false because hashing a basis against a client-supplied
digest would be a 1-bit content oracle. Without --verify-basis a dry-run can
still confirm the metadata-only hit without reading any basis bytes, matching
rsync's read-only quick-check. */
static bool basis_match_find(const Config* config, const char* check_path,
unsigned long long check_size, time_t check_mtime,
long check_mtime_nsec, const uint8_t* check_digest,
size_t check_digest_len, bool load_content, bool hash_content,
BasisMatch* out) {
size_t check_digest_len, bool hash_content, BasisMatch* out) {
memset(out, 0, sizeof(*out));
if (!config || !config_has_basis(config) || config->ignore_times)
return false;
/* Dry-run: never read/hash basis content. A hit cannot be decided from
metadata alone, so report no match (the caller treats it as would-transfer)
without touching the file's contents. */
if (!hash_content)
/* --verify-basis needs the basis content; a content-blind (dry-run) pass can
never confirm it and must not read the file, so decline without touching
the basis bytes. */
if (file_basis_content_required(config) && !hash_content)
return false;
for (int i = 0; i < config->basis_count; i++) {
const BasisDest* entry = &config->basis_dirs[i];
@@ -1424,29 +1426,26 @@ static bool basis_match_find(const Config* config, const char* check_path,
int fd;
struct stat st;
if (basis_open_regular(candidate, check_size, &fd, &st)) {
if (basis_quick_matches(config, &st, check_mtime, check_mtime_nsec)) {
Data* content = basis_read_content(fd, check_size);
if (content) {
if (file_basis_quick_match(config, &st, check_mtime, check_mtime_nsec)) {
bool hit = true;
if (file_basis_content_required(config)) {
uint8_t basis_digest[CHECKSUM_MAX_DIGEST_LEN];
size_t basis_len = 0;
bool hashed = checksum_digest((ChecksumAlgo)config->checksum_algo, config->checksum_seed,
content->data, content->size, basis_digest,
sizeof(basis_digest), &basis_len);
if (hashed && basis_len == check_digest_len && check_digest_len > 0 &&
memcmp(basis_digest, check_digest, check_digest_len) == 0) {
out->hit = true;
out->type = entry->type;
out->basis_path = candidate;
candidate = NULL; /* ownership transferred to out */
out->st = st;
out->content = load_content ? content : NULL;
if (!load_content)
data_destroy(content);
close(fd);
return true;
}
bool hashed =
checksum_digest_fd((ChecksumAlgo)config->checksum_algo, config->checksum_seed, fd,
basis_digest, sizeof(basis_digest), &basis_len);
hit = hashed && basis_len == check_digest_len && check_digest_len > 0 &&
memcmp(basis_digest, check_digest, check_digest_len) == 0;
}
if (hit) {
out->hit = true;
out->type = entry->type;
out->basis_path = candidate;
candidate = NULL; /* ownership transferred to out */
out->st = st;
close(fd);
return true;
}
data_destroy(content);
}
close(fd);
}
@@ -1871,6 +1870,12 @@ typedef struct {
long long check_mtime_nsec;
uint8_t check_digest[CHECKSUM_MAX_DIGEST_LEN];
size_t check_digest_len;
/* Source metadata carried alongside the check frame whenever a basis dir is
configured (rsync keeps the whole file list; FastSync's sender-driven
incremental path otherwise never transmits metadata for a SKIPPED file).
A basis materialization applies these SOURCE attributes instead of the
basis inode's, matching rsync's "copy then fix attributes". */
FileMetadata* source_metadata;
bool dest_exists; /* any destination entry exists (lstat succeeded) */
bool has_old_file;
int old_fd;
@@ -1903,6 +1908,8 @@ static void incremental_check_state_cleanup(IncrementalCheckState* state) {
if (state->old_fd >= 0)
close(state->old_fd);
state->old_fd = -1;
file_metadata_destroy(state->source_metadata);
state->source_metadata = NULL;
free(state->full_path);
state->full_path = NULL;
free(state->check_path);
@@ -1927,7 +1934,7 @@ static IncrementalCheckOutcome incremental_check_receive_request(IncrementalChec
send_error_detail(fd, "invalid check mtime nanoseconds");
return INCREMENTAL_ERROR;
}
if ((config->checksum || config_has_basis(config))) {
if ((config->checksum || config->verify_basis)) {
uint8_t wire_len;
if (!receive_n_data(fd, &wire_len, sizeof(wire_len)) || wire_len == 0 ||
wire_len > CHECKSUM_MAX_DIGEST_LEN ||
@@ -1939,8 +1946,24 @@ static IncrementalCheckOutcome incremental_check_receive_request(IncrementalChec
if (!receive_n_data(fd, state->check_digest, state->check_digest_len))
return INCREMENTAL_ERROR;
}
/* The sender transmits the source metadata with every basis-configured check
so a basis hit can be materialized with the SOURCE's attributes (rsync
copies/copies-then-fixes; the receiver would otherwise only have the basis
inode's stat). The block is symmetric and consumed unconditionally here,
whether or not this file ends up as a basis hit. */
if (config_has_basis(config) && config->use_metadata) {
int meta_ok = 1;
state->source_metadata = metadata_receive(fd, &meta_ok);
if (!meta_ok)
return INCREMENTAL_ERROR;
}
if (state->check_size > MAX_RECEIVE_WHOLE_FILE_SIZE) {
/* A basis-configured run may materialize a file larger than the whole-file
payload bound: a basis hit is streamed from the basis path (bounded
buffers), so the check size is not itself an allocation. Every other
path (delta/append/full) still applies MAX_RECEIVE_WHOLE_FILE_SIZE, and a
miss simply falls through to the normal transfer with its own bound. */
if (!config_has_basis(config) && state->check_size > MAX_RECEIVE_WHOLE_FILE_SIZE) {
send_error_detail(fd, "check size exceeds receiver limit");
return INCREMENTAL_ERROR;
}
@@ -2030,6 +2053,20 @@ incremental_check_ignore_existing(const IncrementalCheckState* state) {
return INCREMENTAL_SKIP;
}
/* Metadata for a materialized basis hit: prefer the SOURCE metadata the sender
transmitted with the check frame (rsync copies then fixes the destination to
the source's attributes); fall back to the basis inode's own stat when
metadata was not negotiated. Consumes state->source_metadata on success. */
static FileMetadata* basis_take_metadata(IncrementalCheckState* state,
const struct stat* basis_st) {
if (state->source_metadata) {
FileMetadata* meta = state->source_metadata;
state->source_metadata = NULL;
return meta;
}
return file_metadata_create(NULL, basis_st, false, false);
}
/* --link-dest relink of an already up-to-date destination. rsync hard-links a
destination entry to a matching basis even when the entry is already correct,
so a run over an existing tree still maximizes sharing with the basis. Only a
@@ -2047,7 +2084,7 @@ static IncrementalCheckOutcome incremental_check_link_dest_relink(IncrementalChe
BasisMatch basis;
basis_match_find(config, state->check_path, state->check_size, (time_t)state->check_mtime,
(long)state->check_mtime_nsec, state->check_digest, state->check_digest_len,
true, true, &basis);
true, &basis);
/* Only a link-dest hit relinks; a copy-dest/compare-dest hit (or a miss) lets
the up-to-date check below keep the existing destination. */
if (!basis.hit || basis.type != BASIS_DEST_LINK) {
@@ -2060,11 +2097,16 @@ static IncrementalCheckOutcome incremental_check_link_dest_relink(IncrementalChe
return INCREMENTAL_CONTINUE;
}
File* materialized = file_create(state->check_path);
if (materialized && basis.content) {
if (materialized) {
data_destroy(materialized->data);
materialized->data = basis.content;
basis.content = NULL;
materialized->metadata = file_metadata_create(NULL, &basis.st, false, false);
materialized->data = data_create_reserve((size_t)state->check_size);
if (!materialized->data) {
file_destroy(materialized);
materialized = NULL;
}
}
if (materialized) {
materialized->metadata = basis_take_metadata(state, &basis.st);
materialized->skip = true;
materialized->basis_link = basis.basis_path;
basis.basis_path = NULL;
@@ -2072,9 +2114,6 @@ static IncrementalCheckOutcome incremental_check_link_dest_relink(IncrementalChe
file_destroy(materialized);
materialized = NULL;
}
} else {
file_destroy(materialized);
materialized = NULL;
}
if (materialized) {
if (!send_status(state->fd, STATUS_OK)) {
@@ -2175,12 +2214,13 @@ static IncrementalCheckOutcome incremental_check_quick_skip(IncrementalCheckStat
materialize nothing (no basis link/copy, no append/delta/full transfer) and
the sender must send no data, so answer STATUS_DRY_RUN_TRANSFER and stop.
The basis lookup is deliberately content-blind: a real run would only accept
a --compare-dest exact hit after hashing the basis file and comparing it with
the client-supplied digest, which in a dry-run is a 1-bit content oracle.
Under dry_run no basis bytes may be read, so an otherwise-matching entry is
treated as would-transfer instead of a skip. Everything read here (the
destination file's metadata, basis candidates' metadata) is read-only. */
The basis lookup is content-blind: under the default metadata quick-check a
hit needs no basis bytes and is honored here just as in a real run; under
--verify-basis a real run hashes the basis against the client-supplied digest,
which in a dry-run is a 1-bit content oracle, so no basis bytes may be read
and an otherwise-matching entry is reported as would-transfer. Everything
read here (the destination file's metadata, basis candidates' metadata) is
read-only. */
static IncrementalCheckOutcome incremental_check_dry_run_shortcut(IncrementalCheckState* state,
bool* skipped,
bool* would_transfer) {
@@ -2191,12 +2231,14 @@ static IncrementalCheckOutcome incremental_check_dry_run_shortcut(IncrementalChe
bool skip_via_compare = false;
if (config_has_basis(config) && !config->ignore_times) {
BasisMatch basis;
/* hash_content=false: a dry-run must not read or hash the basis file. No
content comparison is possible, so no compare-dest hit can be confirmed
and an otherwise-matching file is reported as would-transfer. */
/* hash_content=false: a dry-run must not read or hash the basis file, so
under --verify-basis no compare-dest hit can be confirmed and an
otherwise-matching file is reported as would-transfer. Without
--verify-basis the metadata quick-check confirms it without touching any
basis bytes. */
basis_match_find(config, state->check_path, state->check_size, (time_t)state->check_mtime,
(long)state->check_mtime_nsec, state->check_digest, state->check_digest_len,
false, false, &basis);
false, &basis);
if (basis.hit && basis.type == BASIS_DEST_COMPARE && !state->has_old_file)
skip_via_compare = true;
basis_match_free(&basis);
@@ -2224,7 +2266,7 @@ static IncrementalCheckOutcome incremental_check_try_basis(IncrementalCheckState
BasisMatch basis;
basis_match_find(config, state->check_path, state->check_size, (time_t)state->check_mtime,
(long)state->check_mtime_nsec, state->check_digest, state->check_digest_len,
true, true, &basis);
true, &basis);
if (basis.hit) {
if (basis.type == BASIS_DEST_COMPARE) {
basis_match_free(&basis);
@@ -2234,24 +2276,30 @@ static IncrementalCheckOutcome incremental_check_try_basis(IncrementalCheckState
return INCREMENTAL_SKIP;
}
} else {
/* Copy/link installs source their bytes from the basis PATH at install
time (bounded buffers), so no whole-file content buffer is needed here
even for an over-limit basis. */
File* materialized = file_create(state->check_path);
if (materialized && basis.content) {
if (materialized) {
data_destroy(materialized->data);
materialized->data = basis.content;
basis.content = NULL;
materialized->metadata = file_metadata_create(NULL, &basis.st, false, false);
materialized->skip = true; /* receiver must not ack this as a data file */
if (basis.type == BASIS_DEST_LINK) {
materialized->basis_link = basis.basis_path;
basis.basis_path = NULL;
materialized->data = data_create_reserve((size_t)state->check_size);
if (!materialized->data) {
file_destroy(materialized);
materialized = NULL;
}
}
if (materialized) {
materialized->metadata = basis_take_metadata(state, &basis.st);
materialized->skip = true; /* receiver must not ack this as a data file */
if (basis.type == BASIS_DEST_LINK)
materialized->basis_link = basis.basis_path;
else
materialized->basis_copy = basis.basis_path;
basis.basis_path = NULL;
if (!materialized->metadata) {
file_destroy(materialized);
materialized = NULL;
}
} else {
file_destroy(materialized);
materialized = NULL;
}
if (materialized) {
if (!send_status(fd, STATUS_OK)) {
+10
View File
@@ -24,6 +24,16 @@ File* file_receive_hardlink(int file_descriptor);
File* file_receive_symlink(int file_descriptor, const Config* config);
File* file_receive_special(int file_descriptor);
bool file_special_rdev_valid(int32_t major, int32_t minor, mode_t mode);
/* Testable basis quick-check / verification policy. file_basis_quick_match is
* rsync's metadata quick-check for a basis candidate (equal size is required
* separately by the caller; this adds the --size-only / mtime / --modify-window
* leg). file_basis_content_required reports whether a hit must ALSO be
* confirmed by a whole-file content digest (--verify-basis; false is the
* default rsync-parity behavior). */
bool file_basis_quick_match(const Config* config, const struct stat* st, time_t check_mtime,
long check_mtime_nsec);
bool file_basis_content_required(const Config* config);
File* receive_incremental_check(int fd, const Config* config, bool* skipped);
/* Extended variant used by the receiver. `would_transfer` (may be NULL) is set
* true only on the server-contacting --dry-run path when the file is not up to
+5
View File
@@ -57,6 +57,11 @@ typedef struct {
* equals the incoming file, and `data` is kept as the cross-filesystem
* fallback (a local copy) if the hard link cannot be created. */
char* basis_link;
/* Receiver-only, --copy-dest: when set (and basis_link is NULL), stream the
* basis file's bytes into the destination instead of `data`/`data->size`.
* This lets a basis larger than any whole-file bound materialize without
* buffering it; the source metadata on `metadata` is applied afterwards. */
char* basis_copy;
/* --hard-links (-H), sender + receiver wire state. link_group is a run-local
* id shared by every member of one source inode (0 = not part of a group).
* The FIRST member (link_first == true) carries its data on the wire and is