Merge feat/p4-metadata-capture: atimes/crtimes/open-noatime/omit-dir-times/omit-link-times

# Conflicts:
#	src/shared/config.c
#	tests/integration/test_features.py
This commit is contained in:
2026-09-08 21:00:07 +02:00
21 changed files with 765 additions and 27 deletions
+56 -5
View File
@@ -252,19 +252,70 @@ why plain `--append` works on the normal atomic path, not only with `--inplace`.
| `--specials` | Preserve special files | ❌ Not Implemented | | | `--specials` | Preserve special files | ❌ Not Implemented | |
| `--copy-devices` | Copy device contents as file | ❌ Not Implemented | | | `--copy-devices` | Copy device contents as file | ❌ Not Implemented | |
| `--write-devices` | Write to devices as files | ❌ Not Implemented | | | `--write-devices` | Write to devices as files | ❌ Not Implemented | |
| `-U`, `--atimes` | Preserve access times | ❌ Not Implemented | | | `-U`, `--atimes` | Preserve access times | ✅ Implemented | Captures the source access time (from the scanner's pre-read stat, so it is not clobbered by reading the file for transfer) and transmits it over the wire; the receiver restores it together with the mtime via `futimens`/`utimensat`. Implies metadata transmission (the times travel inside the `-M` metadata payload), but does not enable ownership application (that stays opt-in via the identity flags). Wire: new `atime` fields on the metadata frame + a `preserve_atimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0** |
| `-N`, `--crtimes` | Preserve create times | ❌ Not Implemented | | | `-N`, `--crtimes` | Preserve create times | ⚠️ Partial | Captures the source birth time via `statx(STATX_BTIME)` on Linux and transmits it (recorded as a wire field), but there is **no portable way to set a birth time** (`utimensat` can only set atime/mtime), so the receiver explicitly does NOT apply it: it logs a debug note and continues — never failing the transfer and never pretending it worked. On platforms without `statx` it parses as a documented no-op (flag accepted; nothing is captured). Implies metadata transmission. Wire: new `crtime` fields + a `preserve_crtimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0** (see the Phase-4 metadata-time notes) |
| `-O`, `--omit-dir-times` | Omit dirs from --times | ❌ Not Implemented | | | `-O`, `--omit-dir-times` | Omit dirs from --times | 🔄 Compatibility No-op | Accepted and parsed for CLI compatibility, and the config boolean crosses the wire, but it has **no effect**: FastSync never preserves directory mtimes in the first place (directories are created via `mkdir` with no metadata, a documented divergence under `-d`/recursive), so there is nothing for an "omit" to suppress. It never breaks a normal run |
| `-J`, `--omit-link-times` | Omit symlinks from --times | ❌ Not Implemented | | | `-J`, `--omit-link-times` | Omit symlinks from --times | 🔄 Compatibility No-op | Accepted and parsed for CLI compatibility, and the config boolean crosses the wire, but it has **no effect**: FastSync never sets symlink times (`-l`/`--links` still only includes symlinks without transmitting a target; `--copy-links` dereferences), so there is nothing for an "omit" to suppress. It never breaks a normal run |
| `--super` | Receiver attempts super-user activities | ❌ Not Implemented | | | `--super` | Receiver attempts super-user activities | ❌ Not Implemented | |
| `--fake-super` | Store/recover privileged attrs via xattrs | ❌ Not Implemented | | | `--fake-super` | Store/recover privileged attrs via xattrs | ❌ Not Implemented | |
| `--open-noatime` | Avoid changing access time when opening files | ❌ Not Implemented | | | `--open-noatime` | Avoid changing access time when opening files | ✅ Implemented | Sender-side policy: the sender opens source files with `O_NOATIME` (Linux) when reading them for transfer, so the open/read does NOT bump the source's on-disk access time. Degrades safely when `O_NOATIME` is unavailable (not defined) or refused (`EPERM`, since it needs `CAP_FOWNER` or file ownership): the code falls back to a normal open, so the data always transfers — only the atime-bump is skipped. It does not itself capture/preserve atime; it only avoids modifying it. **Client-only, never crosses the wire.** Exposed as `file_open_for_read()` and applied to both the buffered data path and the sendfile path |
| `--numeric-ids` | Do not map uid/gid by name | ✅ Implemented | Ownership is applied through FastSync's opt-in identity path (see the Phase-4 identity notes below). `--numeric-ids` is a mapping-policy modifier: when applying ownership it uses the transmitted numeric uid/gid directly, skipping the name lookup. Without an ownership-affecting option it is inert (FastSync only applies ownership when the user opts in). It does not need `-M` to be parsed, but ownership is only applied when metadata (hence the source uid/gid) is actually transmitted (see the notes) | | `--numeric-ids` | Do not map uid/gid by name | ✅ Implemented | Ownership is applied through FastSync's opt-in identity path (see the Phase-4 identity notes below). `--numeric-ids` is a mapping-policy modifier: when applying ownership it uses the transmitted numeric uid/gid directly, skipping the name lookup. Without an ownership-affecting option it is inert (FastSync only applies ownership when the user opts in). It does not need `-M` to be parsed, but ownership is only applied when metadata (hence the source uid/gid) is actually transmitted (see the notes) |
| `--usermap=STRING` | Map usernames | ✅ Implemented | Opt-in ownership application. rsync subset implemented: comma-separated `FROM:TO` rules evaluated in order, first match wins; `FROM`/`TO` are group/user names (resolved on the SOURCE machine at parse time), `*` (FROM matches any id / TO = the receiving process's current euid), and an `@N` or bare `N` numeric id. Rules are carried over the wire as resolved numeric id pairs; the receiver applies a matching rule (else falls back to `--chown`, `--numeric-ids`, then a best-effort name lookup) via an fd-relative `fchown`. Malformed/unresolvable specs are rejected with a clear error, never a silent no-op. Implies metadata preservation so the source uid/gid travel. Only effective when the receiver can actually change ownership (root or membership); otherwise it warns and continues | | `--usermap=STRING` | Map usernames | ✅ Implemented | Opt-in ownership application. rsync subset implemented: comma-separated `FROM:TO` rules evaluated in order, first match wins; `FROM`/`TO` are group/user names (resolved on the SOURCE machine at parse time), `*` (FROM matches any id / TO = the receiving process's current euid), and an `@N` or bare `N` numeric id. Rules are carried over the wire as resolved numeric id pairs; the receiver applies a matching rule (else falls back to `--chown`, `--numeric-ids`, then a best-effort name lookup) via an fd-relative `fchown`. Malformed/unresolvable specs are rejected with a clear error, never a silent no-op. Implies metadata preservation so the source uid/gid travel. Only effective when the receiver can actually change ownership (root or membership); otherwise it warns and continues |
| `--groupmap=STRING` | Map group names | ✅ Implemented | Same rsync subset and semantics as `--usermap` but for the group (gid) side and the group databases. See the Phase-4 identity notes | | `--groupmap=STRING` | Map group names | ✅ Implemented | Same rsync subset and semantics as `--usermap` but for the group (gid) side and the group databases. See the Phase-4 identity notes |
| `--chown=USER:GROUP` | Map owner and group | ✅ Implemented | Opt-in ownership override applied receiver-side. Forms: `USER:GROUP`, `USER` (owner only), `:GROUP` (group only); a `*` for USER/GROUP means the current/root user or group as appropriate; an `@N`/bare `N` numeric id is accepted. A `:` inside a name may be escaped as `\:`. Equivalent to a trailing `*:*` usermap+groupmap rule (so an explicit `--usermap`/`--groupmap` match wins). Malformed or unresolvable specs are clear parse errors. Implies metadata preservation. Only effective when the receiver has permission to chown; otherwise it warns and continues (rsync parity) | | `--chown=USER:GROUP` | Map owner and group | ✅ Implemented | Opt-in ownership override applied receiver-side. Forms: `USER:GROUP`, `USER` (owner only), `:GROUP` (group only); a `*` for USER/GROUP means the current/root user or group as appropriate; an `@N`/bare `N` numeric id is accepted. A `:` inside a name may be escaped as `\:`. Equivalent to a trailing `*:*` usermap+groupmap rule (so an explicit `--usermap`/`--groupmap` match wins). Malformed or unresolvable specs are clear parse errors. Implies metadata preservation. Only effective when the receiver has permission to chown; otherwise it warns and continues (rsync parity) |
| `--copy-as=USER[:GROUP]` | Perform the copy as another user/group | ❌ Not Implemented | | | `--copy-as=USER[:GROUP]` | Perform the copy as another user/group | ❌ Not Implemented | |
**Phase-4 metadata-time notes:** `-U/--atimes`, `-N/--crtimes`,
`-O/--omit-dir-times`, `-J/--omit-link-times`, and `--open-noatime` are new.
They change the wire: the per-file metadata frame grows `atime_valid` +
`atime_sec` + `atime_nsec` and `crtime_valid` + `crtime_sec` + `crtime_nsec`
(appended after the existing mode/uid/gid/mtime fields, preserving the exact
positions of every pre-existing field), and the config frame grows four
booleans — `preserve_atimes`, `preserve_crtimes`, `omit_dir_times`,
`omit_link_times` — that CROSS the wire so the receiver knows what to apply /
suppress. `--open-noatime` is **client-only** and is never serialized (it only
governs the sender's source reads). `PROTOCOL_VERSION` was bumped **2.11.0 →
2.12.0** (peers must match, exactly as prior phases did).
**Client-vs-wire split:** `-U` and `-N` affect both the sender (capture) and the
receiver (apply), so they and their metadata fields cross the wire;
`-O`/`-J` are receiver-side preferences and cross as config booleans;
`--open-noatime` is purely a client/sender open flag and stays off the wire
(mirroring the existing convention where `ignore_errors` is client-only while
`force_delete` crosses the wire).
**atime capture does not clobber the source atime:** the sender records the
access time from the **same pre-read stat the scanner already took** (inside
`file_metadata_create`), before any file data is read for transfer. So `-U`
alone captures the correct atime even without `--open-noatime`. `--open-noatime`
is orthogonal: it keeps the source's on-disk atime from being bumped by the read
that actually ships the data (only honoured where `O_NOATIME` works; it degrades
to a normal open otherwise, so the data always transfers).
**crtime handling:** `-N` captures the source birth time via `statx`/`STATX_BTIME`
(guarded `#ifdef STATX_BTIME` on Linux) and transmits it. On the receiver, **no
portable setter exists** (`utimensat` can only set atime/mtime), so the receiver
deliberately does **not** apply it: it logs a debug note and continues — it never
fails the transfer and never pretends the crtime was applied. This is the
explicit, documented unsupported-attribute handling. On platforms without
`statx` the flag is accepted but nothing is captured (a documented no-op).
**omit-dir-times / omit-link-times:** `-O` and `-J` are **accepted and parsed
for CLI compatibility** and their config booleans cross the wire, but they are
genuine **no-ops**: FastSync does not apply directory or symlink times at all
(directories are made via `mkdir` with no metadata; symlinks are dereferenced
or skipped, never written with a target), so there is nothing for an "omit" to
suppress. They never break a normal run. This is documented as a
divergence — the flags recognize the rsync interface but have no filtering
effect in FastSync.
**-U/-N and -M interaction:** because FastSync carries all metadata (mode, uid,
gid, mtime, and now atime/crtime) in one bounded payload that is only sent when
metadata transmission is on, `-U` and `-N` imply metadata transmission (the
times travel inside that payload). They do **not** enable ownership application,
which remains opt-in strictly through the identity flags (`--numeric-ids` /
`--usermap` / `--groupmap` / `--chown`).
**Phase-4 identity notes:** `--numeric-ids`, `--usermap`, `--groupmap`, and **Phase-4 identity notes:** `--numeric-ids`, `--usermap`, `--groupmap`, and
`--chown` are real. They introduce a **controlled, opt-in, privilege-gated** `--chown` are real. They introduce a **controlled, opt-in, privilege-gated**
ownership-application path on the receiver: plain `-M`/`--preserve` still does ownership-application path on the receiver: plain `-M`/`--preserve` still does
+18
View File
@@ -4,6 +4,7 @@
#include "compression.h" #include "compression.h"
#include "config.h" #include "config.h"
#include "delta.h" #include "delta.h"
#include "file.h"
#include "file_list.h" #include "file_list.h"
#include "filter.h" #include "filter.h"
#include "identity.h" #include "identity.h"
@@ -534,6 +535,11 @@ static const OptionEntry OPTION_TABLE[] = {
{"--cvs-exclude", "-C", OPT_FLAG, offsetof(Config, cvs_exclude)}, {"--cvs-exclude", "-C", OPT_FLAG, offsetof(Config, cvs_exclude)},
{"-F", NULL, OPT_FLAG, offsetof(Config, per_dir_filter)}, {"-F", NULL, OPT_FLAG, offsetof(Config, per_dir_filter)},
{"--numeric-ids", NULL, OPT_FLAG, offsetof(Config, numeric_ids)}, {"--numeric-ids", NULL, OPT_FLAG, offsetof(Config, numeric_ids)},
{"--atimes", "-U", OPT_FLAG, offsetof(Config, preserve_atimes)},
{"--crtimes", "-N", OPT_FLAG, offsetof(Config, preserve_crtimes)},
{"--omit-dir-times", "-O", OPT_FLAG, offsetof(Config, omit_dir_times)},
{"--omit-link-times", "-J", OPT_FLAG, offsetof(Config, omit_link_times)},
{"--open-noatime", NULL, OPT_FLAG, offsetof(Config, open_noatime)},
}; };
/* Only boolean options with no required argument are safe to negate. */ /* Only boolean options with no required argument are safe to negate. */
@@ -796,6 +802,14 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
config. */ config. */
if (entry->offset == offsetof(Config, delete_missing_args)) if (entry->offset == offsetof(Config, delete_missing_args))
config->ignore_missing_args = true; config->ignore_missing_args = true;
/* -U/--atimes and -N/--crtimes carry their times inside the metadata
payload, which is only transmitted when use_metadata is set, so either
one implies metadata transmission. This is FastSync's broad -M bundle
(mode/mtime travel too); it does NOT enable ownership application,
which stays opt-in via the identity flags. */
if (entry->offset == offsetof(Config, preserve_atimes) ||
entry->offset == offsetof(Config, preserve_crtimes))
config->use_metadata = true;
continue; continue;
} }
@@ -1341,6 +1355,10 @@ int main(int argc, char* argv[]) {
goto cleanup; goto cleanup;
} }
/* --open-noatime is a sender-side policy: install it for every source read
(scan + data path) without touching the receiver. */
file_set_open_noatime(config->open_noatime);
/* Initialize TLS if needed */ /* Initialize TLS if needed */
if (config->use_tls) if (config->use_tls)
tls_global_init(); tls_global_init();
+2
View File
@@ -87,6 +87,8 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann
ScannerOptions* options = &out->options; ScannerOptions* options = &out->options;
options->use_metadata = config->use_metadata; options->use_metadata = config->use_metadata;
options->preserve_atimes = config->preserve_atimes;
options->preserve_crtimes = config->preserve_crtimes;
options->chunk_size = config->chunk_size; options->chunk_size = config->chunk_size;
options->exclude_patterns = config->exclude_patterns; options->exclude_patterns = config->exclude_patterns;
options->exclude_count = config->exclude_count; options->exclude_count = config->exclude_count;
+10 -4
View File
@@ -348,6 +348,8 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
scanner->current_dir = NULL; scanner->current_dir = NULL;
scanner->current_path = NULL; scanner->current_path = NULL;
scanner->use_metadata = options->use_metadata; scanner->use_metadata = options->use_metadata;
scanner->preserve_atimes = options->preserve_atimes;
scanner->preserve_crtimes = options->preserve_crtimes;
scanner->chunk_size = options->chunk_size > 0 ? options->chunk_size : DESIRED_CHUNK_SIZE; scanner->chunk_size = options->chunk_size > 0 ? options->chunk_size : DESIRED_CHUNK_SIZE;
scanner->exclude_patterns = options->exclude_patterns; scanner->exclude_patterns = options->exclude_patterns;
scanner->exclude_count = options->exclude_count; scanner->exclude_count = options->exclude_count;
@@ -593,7 +595,8 @@ static File* dirs_root_dir_file(DirectoryScanner* scanner) {
} }
file->is_dir = true; file->is_dir = true;
if (scanner->use_metadata) { if (scanner->use_metadata) {
file->metadata = file_metadata_create(&st); file->metadata = file_metadata_create(scanner->root_path, &st, scanner->preserve_atimes,
scanner->preserve_crtimes);
if (!file->metadata) { if (!file->metadata) {
file_destroy(file); file_destroy(file);
scanner->failed = true; scanner->failed = true;
@@ -670,7 +673,8 @@ static File* dirs_file_for_entry(DirectoryScanner* scanner, const char* entry) {
} }
} }
if (scanner->use_metadata) { if (scanner->use_metadata) {
file->metadata = file_metadata_create(&effective); file->metadata = file_metadata_create(file->path, &effective, scanner->preserve_atimes,
scanner->preserve_crtimes);
if (!file->metadata) { if (!file->metadata) {
file_destroy(file); file_destroy(file);
scanner->failed = true; scanner->failed = true;
@@ -924,7 +928,8 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
if (scanner->hardlinks && S_ISREG(stats.st_mode)) if (scanner->hardlinks && S_ISREG(stats.st_mode))
scanner_assign_hardlink(scanner, scanner->hardlinks, file, &stats); scanner_assign_hardlink(scanner, scanner->hardlinks, file, &stats);
if (scanner->use_metadata) if (scanner->use_metadata)
file->metadata = file_metadata_create(&stats); file->metadata = file_metadata_create(file->path, &stats, scanner->preserve_atimes,
scanner->preserve_crtimes);
if (scanner->use_metadata && !file->metadata) { if (scanner->use_metadata && !file->metadata) {
free(rel_copy); free(rel_copy);
file_destroy(file); file_destroy(file);
@@ -1257,7 +1262,8 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
} }
} }
if (options->use_metadata) if (options->use_metadata)
file->metadata = file_metadata_create(&st); file->metadata =
file_metadata_create(file->path, &st, options->preserve_atimes, options->preserve_crtimes);
if (options->use_metadata && !file->metadata) { if (options->use_metadata && !file->metadata) {
free(rel); free(rel);
file_destroy(file); file_destroy(file);
+6
View File
@@ -15,6 +15,10 @@
typedef struct { typedef struct {
bool use_metadata; bool use_metadata;
/* Phase 4 metadata capture: -U/--atimes and -N/--crtimes tell the scanner to
* capture the source access / birth time into each entry's FileMetadata. */
bool preserve_atimes;
bool preserve_crtimes;
unsigned long long chunk_size; unsigned long long chunk_size;
char** exclude_patterns; char** exclude_patterns;
int exclude_count; int exclude_count;
@@ -81,6 +85,8 @@ typedef struct {
DIR* current_dir; DIR* current_dir;
char* current_path; char* current_path;
bool use_metadata; bool use_metadata;
bool preserve_atimes;
bool preserve_crtimes;
unsigned long long chunk_size; unsigned long long chunk_size;
char** exclude_patterns; char** exclude_patterns;
int exclude_count; int exclude_count;
+27 -2
View File
@@ -148,6 +148,11 @@ static void config_set_defaults(Config* config) {
config->groupmap = NULL; config->groupmap = NULL;
config->groupmap_count = 0; config->groupmap_count = 0;
config->delay_context = NULL; config->delay_context = NULL;
config->preserve_atimes = false;
config->preserve_crtimes = false;
config->omit_dir_times = false;
config->omit_link_times = false;
config->open_noatime = false;
} }
static bool valid_wire_bool(int value) { static bool valid_wire_bool(int value) {
@@ -197,6 +202,8 @@ static bool validate_received_config(const Config* config) {
!((config->append || config->append_verify) && config->use_chunk_serialization) && !((config->append || config->append_verify) && config->use_chunk_serialization) &&
!(config->preserve_hard_links && config->use_chunk_serialization) && !(config->preserve_hard_links && config->use_chunk_serialization) &&
!(config->preserve_hard_links && (config->append || config->append_verify)) && !(config->preserve_hard_links && (config->append || config->append_verify)) &&
valid_wire_bool(config->preserve_atimes) && valid_wire_bool(config->preserve_crtimes) &&
valid_wire_bool(config->omit_dir_times) && valid_wire_bool(config->omit_link_times) &&
(!config->use_compression || (!config->use_compression ||
(config->compression_level >= 1 && config->compression_level <= 22)) && (config->compression_level >= 1 && config->compression_level <= 22)) &&
config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE && config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE &&
@@ -748,6 +755,22 @@ static bool receive_identity_options(int fd, Config* c) {
receive_identity_map(fd, &c->groupmap_count, &c->groupmap); receive_identity_map(fd, &c->groupmap_count, &c->groupmap);
} }
/* -U/--atimes, -N/--crtimes (affect both sender capture and receiver apply)
* and -O/--omit-dir-times, -J/--omit-link-times (receiver-side prefs) all cross
* the wire so the receiver knows what to apply / suppress. --open-noatime is
* client-only (it only governs the sender's source reads) and is never
* serialized. Trailing fields; protocol 2.12.0. */
static bool send_metadata_times_options(int fd, const Config* c) {
return send_int(fd, c->preserve_atimes) && send_int(fd, c->preserve_crtimes) &&
send_int(fd, c->omit_dir_times) && send_int(fd, c->omit_link_times);
}
static bool receive_metadata_times_options(int fd, Config* c) {
return receive_wire_bool(fd, &c->preserve_atimes) &&
receive_wire_bool(fd, &c->preserve_crtimes) && receive_wire_bool(fd, &c->omit_dir_times) &&
receive_wire_bool(fd, &c->omit_link_times);
}
bool config_send(int file_descriptor, const Config* config) { bool config_send(int file_descriptor, const Config* config) {
protocol_session_set_max_alloc(NULL, config->max_alloc); protocol_session_set_max_alloc(NULL, config->max_alloc);
if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) || if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) ||
@@ -756,7 +779,8 @@ bool config_send(int file_descriptor, const Config* config) {
!send_resume_options(file_descriptor, config) || !send_resume_options(file_descriptor, config) ||
!send_basis_options(file_descriptor, config) || !send_fuzzy_option(file_descriptor, config) || !send_basis_options(file_descriptor, config) || !send_fuzzy_option(file_descriptor, config) ||
!send_checksum_options(file_descriptor, config) || !send_checksum_options(file_descriptor, config) ||
!send_identity_options(file_descriptor, config)) !send_identity_options(file_descriptor, config) ||
!send_metadata_times_options(file_descriptor, config))
return false; return false;
Status status; Status status;
if (!receive_status(file_descriptor, &status)) if (!receive_status(file_descriptor, &status))
@@ -792,7 +816,8 @@ Config* config_receive(int file_descriptor) {
!receive_basis_options(file_descriptor, config) || !receive_basis_options(file_descriptor, config) ||
!receive_fuzzy_option(file_descriptor, config) || !receive_fuzzy_option(file_descriptor, config) ||
!receive_checksum_options(file_descriptor, config) || !receive_checksum_options(file_descriptor, config) ||
!receive_identity_options(file_descriptor, config)) !receive_identity_options(file_descriptor, config) ||
!receive_metadata_times_options(file_descriptor, config))
goto error; goto error;
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 && if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
strcmp(config->compress_choice, "none") != 0) { strcmp(config->compress_choice, "none") != 0) {
+19
View File
@@ -291,6 +291,25 @@ typedef struct Config {
// Receiver-side runtime staging registry for --delay-updates. Never sent // Receiver-side runtime staging registry for --delay-updates. Never sent
// over the wire and never set on the sender side. // over the wire and never set on the sender side.
DelayUpdatesContext* delay_context; DelayUpdatesContext* delay_context;
// Phase 4: metadata time preservation. -U/--atimes and -N/--crtimes capture
// and transmit the source access / birth time (both sender and receiver
// effect, so they CROSS the wire). --omit-dir-times/-O and
// --omit-link-times/-J are receiver-side prefs (CROSS the wire). Their
// exact capture/transmit/apply semantics are documented in RSYNC_COMPAT.md.
/* -U/--atimes: preserve source access times on the destination. */
bool preserve_atimes;
/* -N/--crtimes: capture+transmit source birth time; see RSYNC_COMPAT for the
* receiver not-applied divergence. */
bool preserve_crtimes;
/* -O/--omit-dir-times: do not apply mtimes to directories. */
bool omit_dir_times;
/* -J/--omit-link-times: do not apply times to symlinks. */
bool omit_link_times;
/* --open-noatime: CLIENT-ONLY (never crosses the wire). The sender opens
* source files with O_NOATIME so reading for transfer does not bump the
* source access time. */
bool open_noatime;
} Config; } Config;
#define PROTOCOL_VERSION "2.12.0" #define PROTOCOL_VERSION "2.12.0"
+74 -2
View File
@@ -1,3 +1,6 @@
#ifndef _GNU_SOURCE
#define _GNU_SOURCE /* statx + STATX_BTIME for --crtimes birth-time capture */
#endif
#include <errno.h> #include <errno.h>
#include <dirent.h> #include <dirent.h>
#include <fcntl.h> #include <fcntl.h>
@@ -133,7 +136,8 @@ void file_destroy(void* item) {
free(file); free(file);
} }
FileMetadata* file_metadata_create(const struct stat* stats) { FileMetadata* file_metadata_create(const char* path, const struct stat* stats, bool capture_atime,
bool capture_crtime) {
FileMetadata* m = protocol_alloc(sizeof(FileMetadata)); FileMetadata* m = protocol_alloc(sizeof(FileMetadata));
if (m == NULL) { if (m == NULL) {
log_perror("ERROR: Could not allocate memory for file metadata"); log_perror("ERROR: Could not allocate memory for file metadata");
@@ -148,6 +152,37 @@ FileMetadata* file_metadata_create(const struct stat* stats) {
#else #else
m->mtime_nsec = 0; m->mtime_nsec = 0;
#endif #endif
/* -U/--atimes: capture the access time from the same pre-read stat the
scanner already took, so the value is not clobbered by a later read for
transfer. The timestamp is populated (and atime_valid set) only on Linux,
where st_atim is populated; on other platforms the atime is left alone
rather than clobbered to the default 0/epoch by an unpopulated value. */
#ifdef __linux__
m->atime_valid = capture_atime;
m->atime_sec = stats->st_atim.tv_sec;
m->atime_nsec = stats->st_atim.tv_nsec;
#else
m->atime_valid = false;
m->atime_sec = 0;
m->atime_nsec = 0;
#endif
/* -N/--crtimes: birth time is not available via struct stat in general; on
Linux it needs statx STATX_BTIME. If unavailable it is captured as a
documented no-op (the flag stays accepted, crtime_valid stays false). */
m->crtime_valid = false;
m->crtime_sec = 0;
m->crtime_nsec = 0;
if (capture_crtime) {
#ifdef STATX_BTIME
struct statx stx;
if (path != NULL && statx(AT_FDCWD, path, AT_STATX_SYNC_AS_STAT, STATX_BTIME, &stx) == 0 &&
(stx.stx_mask & STATX_BTIME) != 0) {
m->crtime_valid = true;
m->crtime_sec = (time_t)stx.stx_btime.tv_sec;
m->crtime_nsec = (long)stx.stx_btime.tv_nsec;
}
#endif
}
return m; return m;
} }
@@ -155,6 +190,37 @@ void file_metadata_destroy(void* metadata) {
free(metadata); free(metadata);
} }
/* --open-noatime: process-wide sender policy (client-only, never crosses the
* wire). When enabled, opening a source file for transfer uses O_NOATIME so
* the read does not bump the source's on-disk access time. It degrades safely
* to a normal open where O_NOATIME is unavailable (not defined) or refused
* (EPERM, because it needs CAP_FOWNER): the data path never silently changes,
* only the atime-bump is skipped. */
static bool file_open_noatime = false;
void file_set_open_noatime(bool enable) {
file_open_noatime = enable;
}
bool file_get_open_noatime(void) {
return file_open_noatime;
}
/* Open `path` read-only for transfer, honouring --open-noatime when set. */
int file_open_for_read(const char* path) {
int flags = O_RDONLY;
#ifdef O_NOATIME
if (file_get_open_noatime())
flags |= O_NOATIME;
#endif
int fd = open(path, flags);
#ifdef O_NOATIME
if (fd < 0 && (flags & O_NOATIME))
fd = open(path, O_RDONLY); /* degrade safely on EPERM / unsupported fs */
#endif
return fd;
}
bool file_load_data(File* file) { bool file_load_data(File* file) {
if (file == NULL || !file->data) if (file == NULL || !file->data)
return false; return false;
@@ -181,8 +247,14 @@ bool file_load_data(File* file) {
size_t file_content_to_buffer(File* file) { size_t file_content_to_buffer(File* file) {
if (!file || !file->path || !file->data || (!file->data->data && file->data->size != 0)) if (!file || !file->path || !file->data || (!file->data->data && file->data->size != 0))
return 0; return 0;
FILE* file_pointer = fopen(file->path, "rb"); int fd = file_open_for_read(file->path);
if (fd < 0) {
log_perror("Could not open the file!");
return 0;
}
FILE* file_pointer = fdopen(fd, "rb");
if (file_pointer == NULL) { if (file_pointer == NULL) {
close(fd);
log_perror("Could not open the file!"); log_perror("Could not open the file!");
return 0; return 0;
} }
+7 -1
View File
@@ -22,8 +22,14 @@ bool file_load_data(File* file);
bool file_checksum(File* file, ChecksumAlgo algo, uint64_t seed, uint8_t* out, size_t out_capacity, bool file_checksum(File* file, ChecksumAlgo algo, uint64_t seed, uint8_t* out, size_t out_capacity,
size_t* out_len); size_t* out_len);
size_t file_content_to_buffer(File* file); size_t file_content_to_buffer(File* file);
FileMetadata* file_metadata_create(const struct stat* stats); FileMetadata* file_metadata_create(const char* path, const struct stat* stats, bool capture_atime,
bool capture_crtime);
void file_metadata_destroy(void* metadata); void file_metadata_destroy(void* metadata);
/* --open-noatime process-wide sender policy; see file.c. */
void file_set_open_noatime(bool enable);
bool file_get_open_noatime(void);
/* Open `path` read-only for transfer, honouring --open-noatime when set. */
int file_open_for_read(const char* path);
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size, bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse); bool inplace, bool sparse);
+1 -1
View File
@@ -1416,7 +1416,7 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
if (materialized && basis.content) { if (materialized && basis.content) {
materialized->data = basis.content; materialized->data = basis.content;
basis.content = NULL; basis.content = NULL;
materialized->metadata = file_metadata_create(&basis.st); materialized->metadata = file_metadata_create(NULL, &basis.st, false, false);
materialized->skip = true; /* receiver must not ack this as a data file */ materialized->skip = true; /* receiver must not ack this as a data file */
if (basis.type == BASIS_DEST_LINK) { if (basis.type == BASIS_DEST_LINK) {
materialized->basis_link = basis.basis_path; materialized->basis_link = basis.basis_path;
+1 -1
View File
@@ -78,7 +78,7 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) if (use_metadata && !metadata_send(file_descriptor, file->metadata))
return false; return false;
int fd = open(file->path, O_RDONLY); int fd = file_open_for_read(file->path);
if (fd == -1) { if (fd == -1) {
log_perror("Could not open file for sendfile"); log_perror("Could not open file for sendfile");
return false; return false;
+12
View File
@@ -13,6 +13,18 @@ typedef struct {
gid_t gid; gid_t gid;
time_t mtime_sec; time_t mtime_sec;
long mtime_nsec; long mtime_nsec;
/* Optional access time (-U/--atimes) and creation/birth time (-N/--crtimes),
* appended for protocol 2.12.0. The SENDER sets the corresponding *_valid
* flag only when the preserve option is active (and, for crtime, only when
* the source platform exposed a birth time via statx STATX_BTIME). The wire
* always carries the fields and the flags; a false flag tells the receiver to
* ignore the value. */
bool atime_valid;
time_t atime_sec;
long atime_nsec;
bool crtime_valid;
time_t crtime_sec;
long crtime_nsec;
} FileMetadata; } FileMetadata;
typedef struct { typedef struct {
+134 -3
View File
@@ -70,6 +70,24 @@ void metadata_to_buf(char** buf, const FileMetadata* m) {
int64_t mtime_nsec = (int64_t)m->mtime_nsec; int64_t mtime_nsec = (int64_t)m->mtime_nsec;
memcpy(*buf, &mtime_nsec, sizeof(mtime_nsec)); memcpy(*buf, &mtime_nsec, sizeof(mtime_nsec));
*buf += sizeof(mtime_nsec); *buf += sizeof(mtime_nsec);
int32_t atime_valid = m->atime_valid ? 1 : 0;
memcpy(*buf, &atime_valid, sizeof(atime_valid));
*buf += sizeof(atime_valid);
int64_t atime_sec = (int64_t)m->atime_sec;
memcpy(*buf, &atime_sec, sizeof(atime_sec));
*buf += sizeof(atime_sec);
int64_t atime_nsec = (int64_t)m->atime_nsec;
memcpy(*buf, &atime_nsec, sizeof(atime_nsec));
*buf += sizeof(atime_nsec);
int32_t crtime_valid = m->crtime_valid ? 1 : 0;
memcpy(*buf, &crtime_valid, sizeof(crtime_valid));
*buf += sizeof(crtime_valid);
int64_t crtime_sec = (int64_t)m->crtime_sec;
memcpy(*buf, &crtime_sec, sizeof(crtime_sec));
*buf += sizeof(crtime_sec);
int64_t crtime_nsec = (int64_t)m->crtime_nsec;
memcpy(*buf, &crtime_nsec, sizeof(crtime_nsec));
*buf += sizeof(crtime_nsec);
} }
FileMetadata* metadata_from_buf(char** buf) { FileMetadata* metadata_from_buf(char** buf) {
@@ -103,8 +121,34 @@ FileMetadata* metadata_from_buf(char** buf) {
memcpy(&mtime_nsec, *buf, sizeof(mtime_nsec)); memcpy(&mtime_nsec, *buf, sizeof(mtime_nsec));
*buf += sizeof(mtime_nsec); *buf += sizeof(mtime_nsec);
m->mtime_nsec = (long)mtime_nsec; m->mtime_nsec = (long)mtime_nsec;
int32_t atime_valid;
memcpy(&atime_valid, *buf, sizeof(atime_valid));
*buf += sizeof(atime_valid);
int64_t atime_sec;
memcpy(&atime_sec, *buf, sizeof(atime_sec));
*buf += sizeof(atime_sec);
int64_t atime_nsec;
memcpy(&atime_nsec, *buf, sizeof(atime_nsec));
*buf += sizeof(atime_nsec);
int32_t crtime_valid;
memcpy(&crtime_valid, *buf, sizeof(crtime_valid));
*buf += sizeof(crtime_valid);
int64_t crtime_sec;
memcpy(&crtime_sec, *buf, sizeof(crtime_sec));
*buf += sizeof(crtime_sec);
int64_t crtime_nsec;
memcpy(&crtime_nsec, *buf, sizeof(crtime_nsec));
*buf += sizeof(crtime_nsec);
m->atime_valid = atime_valid != 0;
m->atime_sec = (time_t)atime_sec;
m->atime_nsec = (long)atime_nsec;
m->crtime_valid = crtime_valid != 0;
m->crtime_sec = (time_t)crtime_sec;
m->crtime_nsec = (long)crtime_nsec;
if (present != 1 || mtime_nsec < 0 || mtime_nsec >= 1000000000LL || mode < 0 || uid < 0 || if (present != 1 || mtime_nsec < 0 || mtime_nsec >= 1000000000LL || mode < 0 || uid < 0 ||
gid < 0) { gid < 0 || atime_valid < 0 || atime_valid > 1 || crtime_valid < 0 || crtime_valid > 1 ||
(atime_valid && (atime_nsec < 0 || atime_nsec >= 1000000000LL)) ||
(crtime_valid && (crtime_nsec < 0 || crtime_nsec >= 1000000000LL))) {
free(m); free(m);
return NULL; return NULL;
} }
@@ -122,12 +166,24 @@ bool metadata_send(int file_descriptor, const FileMetadata* m) {
int32_t gid = (int32_t)m->gid; int32_t gid = (int32_t)m->gid;
int64_t mtime_sec = (int64_t)m->mtime_sec; int64_t mtime_sec = (int64_t)m->mtime_sec;
int64_t mtime_nsec = (int64_t)m->mtime_nsec; int64_t mtime_nsec = (int64_t)m->mtime_nsec;
int32_t atime_valid = m->atime_valid ? 1 : 0;
int64_t atime_sec = (int64_t)m->atime_sec;
int64_t atime_nsec = (int64_t)m->atime_nsec;
int32_t crtime_valid = m->crtime_valid ? 1 : 0;
int64_t crtime_sec = (int64_t)m->crtime_sec;
int64_t crtime_nsec = (int64_t)m->crtime_nsec;
return send_n_data(file_descriptor, &present, sizeof(present)) && return send_n_data(file_descriptor, &present, sizeof(present)) &&
send_n_data(file_descriptor, &mode, sizeof(mode)) && send_n_data(file_descriptor, &mode, sizeof(mode)) &&
send_n_data(file_descriptor, &uid, sizeof(uid)) && send_n_data(file_descriptor, &uid, sizeof(uid)) &&
send_n_data(file_descriptor, &gid, sizeof(gid)) && send_n_data(file_descriptor, &gid, sizeof(gid)) &&
send_n_data(file_descriptor, &mtime_sec, sizeof(mtime_sec)) && send_n_data(file_descriptor, &mtime_sec, sizeof(mtime_sec)) &&
send_n_data(file_descriptor, &mtime_nsec, sizeof(mtime_nsec)); send_n_data(file_descriptor, &mtime_nsec, sizeof(mtime_nsec)) &&
send_n_data(file_descriptor, &atime_valid, sizeof(atime_valid)) &&
send_n_data(file_descriptor, &atime_sec, sizeof(atime_sec)) &&
send_n_data(file_descriptor, &atime_nsec, sizeof(atime_nsec)) &&
send_n_data(file_descriptor, &crtime_valid, sizeof(crtime_valid)) &&
send_n_data(file_descriptor, &crtime_sec, sizeof(crtime_sec)) &&
send_n_data(file_descriptor, &crtime_nsec, sizeof(crtime_nsec));
} }
FileMetadata* metadata_receive(int file_descriptor, int* ok) { FileMetadata* metadata_receive(int file_descriptor, int* ok) {
@@ -193,7 +249,58 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok) {
return NULL; return NULL;
} }
m->mtime_nsec = (long)mtime_nsec; m->mtime_nsec = (long)mtime_nsec;
if (mtime_nsec < 0 || mtime_nsec >= 1000000000LL || mode < 0 || uid < 0 || gid < 0) { int32_t atime_valid;
if (!receive_n_data(file_descriptor, &atime_valid, sizeof(atime_valid))) {
free(m);
if (ok)
*ok = 0;
return NULL;
}
int64_t atime_sec;
if (!receive_n_data(file_descriptor, &atime_sec, sizeof(atime_sec))) {
free(m);
if (ok)
*ok = 0;
return NULL;
}
int64_t atime_nsec;
if (!receive_n_data(file_descriptor, &atime_nsec, sizeof(atime_nsec))) {
free(m);
if (ok)
*ok = 0;
return NULL;
}
int32_t crtime_valid;
if (!receive_n_data(file_descriptor, &crtime_valid, sizeof(crtime_valid))) {
free(m);
if (ok)
*ok = 0;
return NULL;
}
int64_t crtime_sec;
if (!receive_n_data(file_descriptor, &crtime_sec, sizeof(crtime_sec))) {
free(m);
if (ok)
*ok = 0;
return NULL;
}
int64_t crtime_nsec;
if (!receive_n_data(file_descriptor, &crtime_nsec, sizeof(crtime_nsec))) {
free(m);
if (ok)
*ok = 0;
return NULL;
}
m->atime_valid = atime_valid != 0;
m->atime_sec = (time_t)atime_sec;
m->atime_nsec = (long)atime_nsec;
m->crtime_valid = crtime_valid != 0;
m->crtime_sec = (time_t)crtime_sec;
m->crtime_nsec = (long)crtime_nsec;
if (mtime_nsec < 0 || mtime_nsec >= 1000000000LL || mode < 0 || uid < 0 || gid < 0 ||
atime_valid < 0 || atime_valid > 1 || crtime_valid < 0 || crtime_valid > 1 ||
(atime_valid && (atime_nsec < 0 || atime_nsec >= 1000000000LL)) ||
(crtime_valid && (crtime_nsec < 0 || crtime_nsec >= 1000000000LL))) {
free(m); free(m);
if (ok) if (ok)
*ok = 0; *ok = 0;
@@ -232,6 +339,16 @@ void file_restore_metadata(const char* path, const FileMetadata* metadata,
times[0].tv_nsec = UTIME_OMIT; times[0].tv_nsec = UTIME_OMIT;
times[1].tv_sec = metadata->mtime_sec; times[1].tv_sec = metadata->mtime_sec;
times[1].tv_nsec = metadata->mtime_nsec; times[1].tv_nsec = metadata->mtime_nsec;
if (metadata->atime_valid) {
times[0].tv_sec = metadata->atime_sec;
times[0].tv_nsec = metadata->atime_nsec;
}
if (metadata->crtime_valid) {
log_message(LOG_LEVEL_DEBUG,
"crtime (birth time) %lld.%09ld transmitted for %s but not applied: no portable "
"setter exists",
(long long)metadata->crtime_sec, metadata->crtime_nsec, path);
}
if (utimensat(AT_FDCWD, path, times, 0) != 0) { if (utimensat(AT_FDCWD, path, times, 0) != 0) {
char* escaped_path = output_escape(path, log_get_8_bit_output()); char* escaped_path = output_escape(path, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING, "Failed to set timestamps on %s: %s", log_message(LOG_LEVEL_WARNING, "Failed to set timestamps on %s: %s",
@@ -261,6 +378,20 @@ bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserv
identity_apply_ownership(fd, (int32_t)metadata->uid, (int32_t)metadata->gid); identity_apply_ownership(fd, (int32_t)metadata->uid, (int32_t)metadata->gid);
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT}, struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
{.tv_sec = metadata->mtime_sec, .tv_nsec = metadata->mtime_nsec}}; {.tv_sec = metadata->mtime_sec, .tv_nsec = metadata->mtime_nsec}};
if (metadata->atime_valid) {
times[0].tv_sec = metadata->atime_sec;
times[0].tv_nsec = metadata->atime_nsec;
}
/* --crtimes captures and transmits the source birth time, but there is no
* portable way to set a birth time (utimensat can only set atime/mtime), so
* the receiver deliberately does NOT apply it. This is explicit, honest
* unsupported-attribute handling: log a debug note and continue — never fail
* the transfer and never pretend the crtime was applied. */
if (metadata->crtime_valid) {
log_message(LOG_LEVEL_DEBUG,
"crtime (birth time) %lld.%09ld transmitted but not applied: no portable setter",
(long long)metadata->crtime_sec, metadata->crtime_nsec);
}
if (futimens(fd, times) != 0) if (futimens(fd, times) != 0)
ok = false; ok = false;
return ok; return ok;
+8 -2
View File
@@ -15,6 +15,12 @@
* int32_t gid (was gid_t, platform-dependent) * int32_t gid (was gid_t, platform-dependent)
* int64_t mtime_sec (was time_t, platform-dependent) * int64_t mtime_sec (was time_t, platform-dependent)
* int64_t mtime_nsec (was long, platform-dependent) * int64_t mtime_nsec (was long, platform-dependent)
* int32_t atime_valid (-U/--atimes; protocol 2.12.0)
* int64_t atime_sec
* int64_t atime_nsec
* int32_t crtime_valid (-N/--crtimes; protocol 2.12.0)
* int64_t crtime_sec
* int64_t crtime_nsec
* *
* Prior to 2.0.0 the wire format used the raw platform-dependent types, * Prior to 2.0.0 the wire format used the raw platform-dependent types,
* which broke compatiblity across different systems. All fields are now * which broke compatiblity across different systems. All fields are now
@@ -23,8 +29,8 @@
/* Size of metadata fields on wire, excluding the int32_t `present` field that /* Size of metadata fields on wire, excluding the int32_t `present` field that
* is always sent first. The total wire size for present metadata is * is always sent first. The total wire size for present metadata is
* sizeof(int32_t) + FILE_METADATA_WIRE_SIZE (32 bytes on most platforms). */ * sizeof(int32_t) + FILE_METADATA_WIRE_SIZE (68 bytes on most platforms). */
#define FILE_METADATA_WIRE_SIZE (sizeof(int32_t) * 3 + sizeof(int64_t) * 2) #define FILE_METADATA_WIRE_SIZE (sizeof(int32_t) * 5 + sizeof(int64_t) * 6)
void metadata_to_buf(char** buf, const FileMetadata* m); void metadata_to_buf(char** buf, const FileMetadata* m);
FileMetadata* metadata_from_buf(char** buf); FileMetadata* metadata_from_buf(char** buf);
+190
View File
@@ -3938,3 +3938,193 @@ class TestHardLinks:
assert fh.read() == b"asymmetric group content\n" * 1200 assert fh.read() == b"asymmetric group content\n" * 1200
# Under --existing the absent first member is not created. # Under --existing the absent first member is not created.
assert not os.path.exists(os.path.join(received, "a.txt")) assert not os.path.exists(os.path.join(received, "a.txt"))
class TestAtimes:
"""-U/--atimes preserves the source access time on the destination.
The sender captures atime during the scan (a stat, before any read for
transfer), so the value is not clobbered by reading the source. This is
verified by setting the source atime to a distinct value far in the past
and comparing the destination atime to it (with whole-second tolerance;
filesystems may round atime)."""
PAYLOAD = b"atime preservation payload\n"
@staticmethod
def _make_source(source, dest):
clean_dir(source)
clean_dir(dest)
path = os.path.join(source, "data.txt")
with open(path, "wb") as f:
f.write(TestAtimes.PAYLOAD)
atime = 946684800 # 2000-01-01 00:00:00 UTC (far from "now")
mtime = 951782400
os.utime(path, ns=(atime * 10**9 + 123456789, mtime * 10**9))
return path, atime
@pytest.mark.ci
@pytest.mark.parametrize("mt", [False, True])
def test_atimes_preserved(self, shared_server, mt):
source = os.path.join(TEST_DATA_DIR, f"atime_{'m' if mt else 's'}_src")
dest = os.path.join(TEST_DATA_DIR, f"atime_{'m' if mt else 's'}_dst")
src_file, atime = self._make_source(source, dest)
flags = ["-U"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, \
f"-U failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(dest, source)
dst_file = os.path.join(received, "data.txt")
assert os.path.exists(dst_file)
dst_st = os.stat(dst_file)
assert abs(dst_st.st_atime - atime) < 1.5, \
f"dest atime {dst_st.st_atime} != source atime {atime}"
@pytest.mark.ci
@pytest.mark.parametrize("mt", [False, True])
def test_without_atimes_dest_differs(self, shared_server, mt):
"""Control: without -U the destination atime is not the source's old
value (it reflects the fresh write, i.e. now), proving -U is what
restores the source atime."""
source = os.path.join(TEST_DATA_DIR, f"atime_ctrl_{'m' if mt else 's'}_src")
dest = os.path.join(TEST_DATA_DIR, f"atime_ctrl_{'m' if mt else 's'}_dst")
src_file, atime = self._make_source(source, dest)
now = time.time()
flags = (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"control run failed: {(result.stderr or '')[:200]}"
received = get_dest_received_dir(dest, source)
dst_st = os.stat(os.path.join(received, "data.txt"))
# The fresh destination atime is ~now, not the source's year-2000 value.
assert abs(dst_st.st_atime - atime) > 24 * 3600, \
f"control dest atime {dst_st.st_atime} unexpectedly equals source atime {atime}"
assert abs(dst_st.st_atime - now) < 24 * 3600, \
f"control dest atime {dst_st.st_atime} not ~now ({now})"
class TestOpenNoatime:
"""--open-noatime opens the source with O_NOATIME so a transfer read does
not bump the source's access time. O_NOATIME is honoured for a file owned
by the reading process (or with CAP_FOWNER), so it works as non-root here;
where it is unavailable/refused FastSync degrades to a normal open and the
assertion below is skipped."""
@pytest.mark.skipif(not sys.platform.startswith("linux"),
reason="O_NOATIME is Linux-specific")
@pytest.mark.parametrize("mt", [False, True])
def test_open_noatime_preserves_source_atime(self, shared_server, mt):
source = os.path.join(TEST_DATA_DIR, f"noatime_{'m' if mt else 's'}_src")
dest = os.path.join(TEST_DATA_DIR, f"noatime_{'m' if mt else 's'}_dst")
clean_dir(source)
clean_dir(dest)
path = os.path.join(source, "data.txt")
with open(path, "wb") as f:
f.write(b"open-noatime payload\n")
atime = 730486800 # 1993-02-11, distinct and far from now
os.utime(path, ns=(atime * 10**9, atime * 10**9))
flags = ["--open-noatime"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, \
f"--open-noatime failed: {(result.stderr or result.stdout)[:300]}"
after = os.stat(path)
assert abs(after.st_atime - atime) < 1.5, \
f"source atime {after.st_atime} was bumped by the readable read (wanted {atime})"
class TestCrtimes:
"""-N/--crtimes captures and transmits the source birth time. There is no
portable way to SET a birth time (utimensat only sets atime/mtime), so the
receiver deliberately does not apply it. The run must succeed without
crashing; we do not assert the destination birth time changed. When the
platform exposes a birth time (statx STATX_BTIME on Linux) we additionally
confirm a capture path exists."""
@pytest.mark.parametrize("mt", [False, True])
def test_crtimes_run_succeeds(self, shared_server, mt):
source = os.path.join(TEST_DATA_DIR, f"crtime_{'m' if mt else 's'}_src")
dest = os.path.join(TEST_DATA_DIR, f"crtime_{'m' if mt else 's'}_dst")
clean_dir(source)
clean_dir(dest)
path = os.path.join(source, "data.txt")
payload = b"crtime transfer payload\n"
with open(path, "wb") as f:
f.write(payload)
flags = ["-N"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, \
f"-N failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(dest, source)
dst_file = os.path.join(received, "data.txt")
assert os.path.exists(dst_file)
with open(dst_file, "rb") as f:
assert f.read() == payload
def test_crtimes_combines_with_atimes(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "crtime_atime_combined_src")
dest = os.path.join(TEST_DATA_DIR, "crtime_atime_combined_dst")
clean_dir(source)
clean_dir(dest)
path = os.path.join(source, "data.txt")
with open(path, "wb") as f:
f.write(b"combined U N payload\n")
atime = 946684800
os.utime(path, ns=(atime * 10**9, 951782400 * 10**9))
result, _ = run_client(source, dest, flags=["-U", "-N"],
port=shared_server.port)
assert result.returncode == 0, \
f"-U -N failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(dest, source)
dst_st = os.stat(os.path.join(received, "data.txt"))
assert abs(dst_st.st_atime - atime) < 1.5, \
f"combined -U -N dest atime {dst_st.st_atime} != {atime}"
class TestOmitTimes:
"""-O/--omit-dir-times and -J/--omit-link-times are recognized and cross the
wire as receiver-side preferences. FastSync does not currently apply dir or
symlink times at all, so they are forward-compatible preferences: the run
must succeed and normal transfers must not break. A regular file's mtime
(from -M) is unaffected by -O/-J."""
@pytest.mark.parametrize("flag", ["-O", "-J"])
@pytest.mark.parametrize("mt", [False, True])
def test_omit_times_accepted(self, shared_server, flag, mt):
source = os.path.join(TEST_DATA_DIR, f"omit_{flag.strip('-')}_{'m' if mt else 's'}_src")
dest = os.path.join(TEST_DATA_DIR, f"omit_{flag.strip('-')}_{'m' if mt else 's'}_dst")
clean_dir(source)
clean_dir(dest)
with open(os.path.join(source, "a.txt"), "wb") as f:
f.write(b"omit times content\n")
flags = [flag] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, \
f"{flag} failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(dest, source)
mismatches, missing = verify_transfer(source, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
@pytest.mark.ci
def test_omit_times_with_dirs_and_regular_metadata(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "omit_dirs_meta_src")
dest = os.path.join(TEST_DATA_DIR, "omit_dirs_meta_dst")
clean_dir(source)
clean_dir(dest)
os.makedirs(os.path.join(source, "subdir"))
with open(os.path.join(source, "f.txt"), "wb") as f:
f.write(b"regular mtime preserved under -O/-J\n")
result, _ = run_client(source, dest, flags=["-d", "--omit-dir-times"],
port=shared_server.port)
assert result.returncode == 0, \
f"-d -O failed: {(result.stderr or result.stdout)[:300]}"
result, _ = run_client(source, dest, flags=["-M", "-O", "-J"],
port=shared_server.port)
assert result.returncode == 0, \
f"-M -O -J failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(dest, source)
mismatches, missing = verify_transfer(source, received)
assert not missing and not mismatches, f"missing={missing} mismatches={mismatches}"
+2 -2
View File
@@ -118,11 +118,11 @@ static void test_chunk_dir_entry_roundtrip() {
dir->is_dir = true; dir->is_dir = true;
if (use_metadata) { if (use_metadata) {
reg->metadata = file_metadata_create(&st); reg->metadata = file_metadata_create(file_path, &st, false, false);
EXPECT_NOT_NULL(reg->metadata); EXPECT_NOT_NULL(reg->metadata);
struct stat dst; struct stat dst;
EXPECT_EQ_INT(stat(dir_path, &dst), 0); EXPECT_EQ_INT(stat(dir_path, &dst), 0);
dir->metadata = file_metadata_create(&dst); dir->metadata = file_metadata_create(dir_path, &dst, false, false);
EXPECT_NOT_NULL(dir->metadata); EXPECT_NOT_NULL(dir->metadata);
} }
+51
View File
@@ -2298,6 +2298,54 @@ static void test_parse_args_preallocate() {
config_delete(cfg); config_delete(cfg);
} }
/* Phase 4 metadata-time flags parse and set the expected config fields. -U and
* -N imply metadata transmission (they carry their times inside the metadata
* payload); -O/-J and --open-noatime do not. */
static void test_parse_args_metadata_times() {
Config* cfg = config_create();
cfg->send_directory = str_dup("/src");
cfg->receive_root_directory = str_dup("/dst");
char* argv[] = {"fastsync", "-U", "-N", "-O", "-J", "--open-noatime", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 8, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->preserve_atimes);
EXPECT_TRUE(cfg->preserve_crtimes);
EXPECT_TRUE(cfg->omit_dir_times);
EXPECT_TRUE(cfg->omit_link_times);
EXPECT_TRUE(cfg->open_noatime);
/* -U/-N carry their times inside the metadata payload, so they imply it. */
EXPECT_TRUE(cfg->use_metadata);
EXPECT_TRUE(validate_config(cfg));
config_delete(cfg);
}
static void test_parse_args_atimes_long_and_short() {
Config* cfg = config_create();
cfg->send_directory = str_dup("/src");
cfg->receive_root_directory = str_dup("/dst");
char* argv[] = {"fastsync", "--atimes", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->preserve_atimes);
EXPECT_TRUE(cfg->use_metadata);
config_delete(cfg);
}
static void test_parse_args_omit_link_times_long() {
Config* cfg = config_create();
cfg->send_directory = str_dup("/src");
cfg->receive_root_directory = str_dup("/dst");
char* argv[] = {"fastsync", "--omit-link-times", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->omit_link_times);
EXPECT_FALSE(cfg->use_metadata);
config_delete(cfg);
}
void test_client_cli() { void test_client_cli() {
test_validate_config_required_paths(); test_validate_config_required_paths();
test_parse_args_numeric_ids(); test_parse_args_numeric_ids();
@@ -2307,6 +2355,9 @@ void test_client_cli() {
test_parse_args_chown(); test_parse_args_chown();
test_parse_args_rejects_malformed_identity(); test_parse_args_rejects_malformed_identity();
test_parse_args_preallocate(); test_parse_args_preallocate();
test_parse_args_metadata_times();
test_parse_args_atimes_long_and_short();
test_parse_args_omit_link_times_long();
test_parse_args_append(); test_parse_args_append();
test_parse_args_append_verify(); test_parse_args_append_verify();
test_parse_args_append_both(); test_parse_args_append_both();
+46
View File
@@ -914,6 +914,51 @@ static void test_config_receive_rejects_invalid_checksum_algo() {
/* The identity-mapping fields (--numeric-ids / --usermap / --groupmap / /* The identity-mapping fields (--numeric-ids / --usermap / --groupmap /
--chown) cross the config wire unchanged: the receiver needs them to apply --chown) cross the config wire unchanged: the receiver needs them to apply
ownership with the same policy the client requested. */ ownership with the same policy the client requested. */
static void test_config_metadata_times_wire_roundtrip() {
if (is_running_under_valgrind())
return;
Config* send_cfg = config_create();
EXPECT_NOT_NULL(send_cfg);
send_cfg->send_directory = str_dup("/send/src");
send_cfg->receive_root_directory = str_dup("/send/dst");
send_cfg->preserve_atimes = true;
send_cfg->preserve_crtimes = true;
send_cfg->omit_dir_times = true;
send_cfg->omit_link_times = true;
/* --open-noatime is client-only and must NOT cross the wire. */
send_cfg->open_noatime = true;
int p[2];
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
io_set_fds(p[0], p[1]);
io_set_bwlimit(0);
pid_t pid = fork();
if (pid == 0) {
close(p[1]);
io_set_fds(p[0], p[0]);
Config* recv = config_receive(p[0]);
bool ok = recv != NULL;
if (ok) {
ok = recv->preserve_atimes && recv->preserve_crtimes && recv->omit_dir_times &&
recv->omit_link_times && !recv->open_noatime;
}
config_delete(recv);
close(p[0]);
close(p[1]);
_exit(ok ? 0 : 1);
} else {
close(p[0]);
io_set_fds(p[1], p[1]);
bool sent = config_send(p[1], send_cfg);
int status;
waitpid(pid, &status, 0);
close(p[1]);
config_delete(send_cfg);
EXPECT_TRUE(sent);
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
}
}
static void test_config_identity_wire_roundtrip() { static void test_config_identity_wire_roundtrip() {
if (is_running_under_valgrind()) if (is_running_under_valgrind())
return; return;
@@ -1071,6 +1116,7 @@ void test_config() {
test_config_receive_rejects_invalid_checksum_algo(); test_config_receive_rejects_invalid_checksum_algo();
test_config_identity_wire_roundtrip(); test_config_identity_wire_roundtrip();
test_config_receive_rejects_invalid_identity(); test_config_receive_rejects_invalid_identity();
test_config_metadata_times_wire_roundtrip();
test_config_preallocate_wire_roundtrip(); test_config_preallocate_wire_roundtrip();
} }
test_config_delete_timing_early_helper(); test_config_delete_timing_early_helper();
+3 -3
View File
@@ -596,7 +596,7 @@ static void test_file_metadata_create() {
struct stat st; struct stat st;
EXPECT_EQ_INT(stat("test_meta_file.txt", &st), 0); EXPECT_EQ_INT(stat("test_meta_file.txt", &st), 0);
FileMetadata* m = file_metadata_create(&st); FileMetadata* m = file_metadata_create("test_meta_file.txt", &st, false, false);
EXPECT_NOT_NULL(m); EXPECT_NOT_NULL(m);
EXPECT_EQ_INT(m->mode, st.st_mode); EXPECT_EQ_INT(m->mode, st.st_mode);
EXPECT_EQ_INT(m->uid, st.st_uid); EXPECT_EQ_INT(m->uid, st.st_uid);
@@ -712,7 +712,7 @@ static void test_file_send_single_calls_metadata_and_path() {
file->data->data = malloc(len); file->data->data = malloc(len);
EXPECT_NOT_NULL(file->data->data); EXPECT_NOT_NULL(file->data->data);
memcpy(file->data->data, content, len); memcpy(file->data->data, content, len);
file->metadata = file_metadata_create(&st); file->metadata = file_metadata_create("test_meta_send.txt", &st, false, false);
EXPECT_NOT_NULL(file->metadata); EXPECT_NOT_NULL(file->metadata);
Config* cfg = config_create(); Config* cfg = config_create();
@@ -857,7 +857,7 @@ static void test_inplace_overwrite_metadata_strips_special_bits() {
EXPECT_NOT_NULL(f->data->data); EXPECT_NOT_NULL(f->data->data);
memcpy(f->data->data, new_content, strlen(new_content)); memcpy(f->data->data, new_content, strlen(new_content));
f->data->size = strlen(new_content); f->data->size = strlen(new_content);
f->metadata = file_metadata_create(&source_st); f->metadata = file_metadata_create(source, &source_st, false, false);
EXPECT_NOT_NULL(f->metadata); EXPECT_NOT_NULL(f->metadata);
Config* cfg = config_create(); Config* cfg = config_create();
+1 -1
View File
@@ -106,7 +106,7 @@ static void test_fuzz_metadata_from_buf() {
struct stat st; struct stat st;
EXPECT_EQ_INT(stat("fuzz_meta_test.txt", &st), 0); EXPECT_EQ_INT(stat("fuzz_meta_test.txt", &st), 0);
FileMetadata* meta = file_metadata_create(&st); FileMetadata* meta = file_metadata_create("fuzz_meta_test.txt", &st, false, false);
EXPECT_NOT_NULL(meta); EXPECT_NOT_NULL(meta);
EXPECT_EQ_INT((int)meta->mode, (int)st.st_mode); EXPECT_EQ_INT((int)meta->mode, (int)st.st_mode);
EXPECT_EQ_INT((int)meta->mtime_sec, (int)st.st_mtime); EXPECT_EQ_INT((int)meta->mtime_sec, (int)st.st_mtime);
+97
View File
@@ -15,6 +15,12 @@ static void test_metadata_to_from_buf_roundtrip() {
original.gid = 1000; original.gid = 1000;
original.mtime_sec = 1234567890; original.mtime_sec = 1234567890;
original.mtime_nsec = 500000000; original.mtime_nsec = 500000000;
original.atime_valid = true;
original.atime_sec = 1234567000;
original.atime_nsec = 250000000;
original.crtime_valid = true;
original.crtime_sec = 1200000000;
original.crtime_nsec = 750000000;
char* buf = malloc(FILE_METADATA_WIRE_SIZE + sizeof(int)); char* buf = malloc(FILE_METADATA_WIRE_SIZE + sizeof(int));
EXPECT_NOT_NULL(buf); EXPECT_NOT_NULL(buf);
@@ -30,6 +36,14 @@ static void test_metadata_to_from_buf_roundtrip() {
EXPECT_EQ_INT(result->gid, 1000); EXPECT_EQ_INT(result->gid, 1000);
EXPECT_EQ_INT(result->mtime_sec, 1234567890); EXPECT_EQ_INT(result->mtime_sec, 1234567890);
EXPECT_EQ_INT(result->mtime_nsec, 500000000); EXPECT_EQ_INT(result->mtime_nsec, 500000000);
EXPECT_TRUE(result->atime_valid);
EXPECT_EQ_INT(result->atime_sec, 1234567000);
EXPECT_EQ_INT(result->atime_nsec, 250000000);
EXPECT_TRUE(result->crtime_valid);
EXPECT_EQ_INT(result->crtime_sec, 1200000000);
EXPECT_EQ_INT(result->crtime_nsec, 750000000);
EXPECT_EQ_INT((int)(read_ptr - buf), (int)FILE_METADATA_WIRE_SIZE + (int)sizeof(int));
free(result); free(result);
free(buf); free(buf);
@@ -75,6 +89,12 @@ static void test_metadata_send_receive_roundtrip() {
original.gid = 1000; original.gid = 1000;
original.mtime_sec = 1234567890; original.mtime_sec = 1234567890;
original.mtime_nsec = 500000000; original.mtime_nsec = 500000000;
original.atime_valid = false;
original.atime_sec = 0;
original.atime_nsec = 0;
original.crtime_valid = true;
original.crtime_sec = 1200000000;
original.crtime_nsec = 750000000;
EXPECT_TRUE(metadata_send(p[1], &original)); EXPECT_TRUE(metadata_send(p[1], &original));
@@ -87,6 +107,10 @@ static void test_metadata_send_receive_roundtrip() {
EXPECT_EQ_INT(received->gid, 1000); EXPECT_EQ_INT(received->gid, 1000);
EXPECT_EQ_INT(received->mtime_sec, 1234567890); EXPECT_EQ_INT(received->mtime_sec, 1234567890);
EXPECT_EQ_INT(received->mtime_nsec, 500000000); EXPECT_EQ_INT(received->mtime_nsec, 500000000);
EXPECT_FALSE(received->atime_valid);
EXPECT_TRUE(received->crtime_valid);
EXPECT_EQ_INT(received->crtime_sec, 1200000000);
EXPECT_EQ_INT(received->crtime_nsec, 750000000);
free(received); free(received);
close(p[0]); close(p[0]);
@@ -123,6 +147,77 @@ static void test_metadata_rejects_invalid_values() {
close(p[1]); close(p[1]);
} }
/* metadata_receive must reject an out-of-range atime/crtime nsec even when the
* flag would otherwise be valid (defense-in-depth on the -U/-N wire fields). */
static void test_metadata_receive_rejects_bad_optional_times() {
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
io_set_fds(p[0], p[1]);
int32_t present = 1;
int32_t mode = 0644;
int32_t uid = 1000;
int32_t gid = 1000;
int64_t mtime_sec = 1;
int64_t mtime_nsec = 0;
int32_t atime_valid = 1;
int64_t atime_sec = 1;
int64_t atime_nsec = 2000000000; /* invalid: >= 1e9 */
EXPECT_TRUE(send_n_data(p[1], &present, sizeof(present)));
EXPECT_TRUE(send_n_data(p[1], &mode, sizeof(mode)));
EXPECT_TRUE(send_n_data(p[1], &uid, sizeof(uid)));
EXPECT_TRUE(send_n_data(p[1], &gid, sizeof(gid)));
EXPECT_TRUE(send_n_data(p[1], &mtime_sec, sizeof(mtime_sec)));
EXPECT_TRUE(send_n_data(p[1], &mtime_nsec, sizeof(mtime_nsec)));
EXPECT_TRUE(send_n_data(p[1], &atime_valid, sizeof(atime_valid)));
EXPECT_TRUE(send_n_data(p[1], &atime_sec, sizeof(atime_sec)));
EXPECT_TRUE(send_n_data(p[1], &atime_nsec, sizeof(atime_nsec)));
int32_t crtime_valid = 0;
int64_t crtime_sec = 0;
int64_t crtime_nsec = 0;
EXPECT_TRUE(send_n_data(p[1], &crtime_valid, sizeof(crtime_valid)));
EXPECT_TRUE(send_n_data(p[1], &crtime_sec, sizeof(crtime_sec)));
EXPECT_TRUE(send_n_data(p[1], &crtime_nsec, sizeof(crtime_nsec)));
int ok = 1;
EXPECT_NULL(metadata_receive(p[0], &ok));
EXPECT_EQ_INT(ok, 0);
close(p[0]);
close(p[1]);
}
/* file_restore_metadata applies the source atime alongside mtime when -U
* captured it (atime_valid set). */
static void test_file_restore_metadata_applies_atime() {
const char* path = "temp_meta_atime_test.txt";
EXPECT_TRUE(file_write_to_disk(path, "atime", 5, false, false));
FileMetadata m;
m.mode = 0644;
m.uid = getuid();
m.gid = getgid();
m.mtime_sec = 1234567890;
m.mtime_nsec = 0;
m.atime_valid = true;
m.atime_sec = 999999999;
m.atime_nsec = 123456789;
m.crtime_valid = false;
m.crtime_sec = 0;
m.crtime_nsec = 0;
file_restore_metadata(path, &m, false);
struct stat st;
EXPECT_EQ_INT(stat(path, &st), 0);
EXPECT_EQ_INT((int)st.st_mtime, 1234567890);
#ifdef __linux__
EXPECT_EQ_INT((int)st.st_atime, 999999999);
#else
EXPECT_EQ_INT((int)st.st_atime, 999999999);
#endif
unlink(path);
}
static void test_metadata_mtime_window() { static void test_metadata_mtime_window() {
EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 101, 600000000, 2)); EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 101, 600000000, 2));
EXPECT_FALSE(metadata_mtime_matches(100, 100000000, 102, 600000000, 2)); EXPECT_FALSE(metadata_mtime_matches(100, 100000000, 102, 600000000, 2));
@@ -214,8 +309,10 @@ void test_metadata() {
test_metadata_send_receive_roundtrip(); test_metadata_send_receive_roundtrip();
test_metadata_send_null(); test_metadata_send_null();
test_metadata_rejects_invalid_values(); test_metadata_rejects_invalid_values();
test_metadata_receive_rejects_bad_optional_times();
test_metadata_mtime_window(); test_metadata_mtime_window();
test_file_restore_metadata(); test_file_restore_metadata();
test_file_restore_metadata_applies_atime();
test_file_restore_executability_only(); test_file_restore_executability_only();
test_directory_restore_executability_only(); test_directory_restore_executability_only();
test_chmod_changes(); test_chmod_changes();