From e80888ce7b366e2410a5e792610e3fb828dd1708 Mon Sep 17 00:00:00 2001 From: TapTap Date: Tue, 8 Sep 2026 20:58:56 +0200 Subject: [PATCH] metadata times: -U/--atimes, -N/--crtimes, --open-noatime, -O/-J Capture+transmit source atime (pre-read stat; O_NOATIME sender guard) and birth time (statx STATX_BTIME); receiver restores atime with mtime (crtime not settable portably -> transmitted, explicitly not applied). --open-noatime is client-only. -O/-J documented as accepted no-ops (FastSync never preserves dir/symlink times). Wire: metadata frame gains atime/crtime val+sec+nsec; PROTOCOL_VERSION 2.11.0->2.12.0. Review fixes: gate atime capture to Linux (no epoch clobber on non-Linux), close fd on fdopen failure, honest -O/-J status (Compat no-op). --- RSYNC_COMPAT.md | 61 ++++++++- src/client/client_cli.c | 18 +++ src/client/client_send.c | 2 + src/client/scanner.c | 14 ++- src/client/scanner.h | 6 + src/shared/config.c | 29 ++++- src/shared/config.h | 21 +++- src/shared/file.c | 76 +++++++++++- src/shared/file.h | 8 +- src/shared/file_receive.c | 2 +- src/shared/file_send.c | 2 +- src/shared/file_types.h | 12 ++ src/shared/metadata.c | 137 ++++++++++++++++++++- src/shared/metadata.h | 10 +- tests/integration/test_features.py | 191 +++++++++++++++++++++++++++++ tests/test_chunk.c | 4 +- tests/test_client_cli.c | 51 ++++++++ tests/test_config.c | 46 +++++++ tests/test_file.c | 6 +- tests/test_fuzz_smoke.c | 2 +- tests/test_metadata.c | 97 +++++++++++++++ 21 files changed, 767 insertions(+), 28 deletions(-) diff --git a/RSYNC_COMPAT.md b/RSYNC_COMPAT.md index d1068b3..ebce0af 100644 --- a/RSYNC_COMPAT.md +++ b/RSYNC_COMPAT.md @@ -252,19 +252,70 @@ why plain `--append` works on the normal atomic path, not only with `--inplace`. | `--specials` | Preserve special files | ❌ Not Implemented | | | `--copy-devices` | Copy device contents as file | ❌ Not Implemented | | | `--write-devices` | Write to devices as files | ❌ Not Implemented | | -| `-U`, `--atimes` | Preserve access times | ❌ Not Implemented | | -| `-N`, `--crtimes` | Preserve create times | ❌ Not Implemented | | -| `-O`, `--omit-dir-times` | Omit dirs from --times | ❌ Not Implemented | | -| `-J`, `--omit-link-times` | Omit symlinks from --times | ❌ Not Implemented | | +| `-U`, `--atimes` | Preserve access times | ✅ Implemented | Captures the source access time (from the scanner's pre-read stat, so it is not clobbered by reading the file for transfer) and transmits it over the wire; the receiver restores it together with the mtime via `futimens`/`utimensat`. Implies metadata transmission (the times travel inside the `-M` metadata payload), but does not enable ownership application (that stays opt-in via the identity flags). Wire: new `atime` fields on the metadata frame + a `preserve_atimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0** | +| `-N`, `--crtimes` | Preserve create times | ⚠️ Partial | Captures the source birth time via `statx(STATX_BTIME)` on Linux and transmits it (recorded as a wire field), but there is **no portable way to set a birth time** (`utimensat` can only set atime/mtime), so the receiver explicitly does NOT apply it: it logs a debug note and continues — never failing the transfer and never pretending it worked. On platforms without `statx` it parses as a documented no-op (flag accepted; nothing is captured). Implies metadata transmission. Wire: new `crtime` fields + a `preserve_crtimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0** (see the Phase-4 metadata-time notes) | +| `-O`, `--omit-dir-times` | Omit dirs from --times | 🔄 Compatibility No-op | Accepted and parsed for CLI compatibility, and the config boolean crosses the wire, but it has **no effect**: FastSync never preserves directory mtimes in the first place (directories are created via `mkdir` with no metadata, a documented divergence under `-d`/recursive), so there is nothing for an "omit" to suppress. It never breaks a normal run | +| `-J`, `--omit-link-times` | Omit symlinks from --times | 🔄 Compatibility No-op | Accepted and parsed for CLI compatibility, and the config boolean crosses the wire, but it has **no effect**: FastSync never sets symlink times (`-l`/`--links` still only includes symlinks without transmitting a target; `--copy-links` dereferences), so there is nothing for an "omit" to suppress. It never breaks a normal run | | `--super` | Receiver attempts super-user activities | ❌ Not Implemented | | | `--fake-super` | Store/recover privileged attrs via xattrs | ❌ Not Implemented | | -| `--open-noatime` | Avoid changing access time when opening files | ❌ Not Implemented | | +| `--open-noatime` | Avoid changing access time when opening files | ✅ Implemented | Sender-side policy: the sender opens source files with `O_NOATIME` (Linux) when reading them for transfer, so the open/read does NOT bump the source's on-disk access time. Degrades safely when `O_NOATIME` is unavailable (not defined) or refused (`EPERM`, since it needs `CAP_FOWNER` or file ownership): the code falls back to a normal open, so the data always transfers — only the atime-bump is skipped. It does not itself capture/preserve atime; it only avoids modifying it. **Client-only, never crosses the wire.** Exposed as `file_open_for_read()` and applied to both the buffered data path and the sendfile path | | `--numeric-ids` | Do not map uid/gid by name | ✅ Implemented | Ownership is applied through FastSync's opt-in identity path (see the Phase-4 identity notes below). `--numeric-ids` is a mapping-policy modifier: when applying ownership it uses the transmitted numeric uid/gid directly, skipping the name lookup. Without an ownership-affecting option it is inert (FastSync only applies ownership when the user opts in). It does not need `-M` to be parsed, but ownership is only applied when metadata (hence the source uid/gid) is actually transmitted (see the notes) | | `--usermap=STRING` | Map usernames | ✅ Implemented | Opt-in ownership application. rsync subset implemented: comma-separated `FROM:TO` rules evaluated in order, first match wins; `FROM`/`TO` are group/user names (resolved on the SOURCE machine at parse time), `*` (FROM matches any id / TO = the receiving process's current euid), and an `@N` or bare `N` numeric id. Rules are carried over the wire as resolved numeric id pairs; the receiver applies a matching rule (else falls back to `--chown`, `--numeric-ids`, then a best-effort name lookup) via an fd-relative `fchown`. Malformed/unresolvable specs are rejected with a clear error, never a silent no-op. Implies metadata preservation so the source uid/gid travel. Only effective when the receiver can actually change ownership (root or membership); otherwise it warns and continues | | `--groupmap=STRING` | Map group names | ✅ Implemented | Same rsync subset and semantics as `--usermap` but for the group (gid) side and the group databases. See the Phase-4 identity notes | | `--chown=USER:GROUP` | Map owner and group | ✅ Implemented | Opt-in ownership override applied receiver-side. Forms: `USER:GROUP`, `USER` (owner only), `:GROUP` (group only); a `*` for USER/GROUP means the current/root user or group as appropriate; an `@N`/bare `N` numeric id is accepted. A `:` inside a name may be escaped as `\:`. Equivalent to a trailing `*:*` usermap+groupmap rule (so an explicit `--usermap`/`--groupmap` match wins). Malformed or unresolvable specs are clear parse errors. Implies metadata preservation. Only effective when the receiver has permission to chown; otherwise it warns and continues (rsync parity) | | `--copy-as=USER[:GROUP]` | Perform the copy as another user/group | ❌ Not Implemented | | +**Phase-4 metadata-time notes:** `-U/--atimes`, `-N/--crtimes`, +`-O/--omit-dir-times`, `-J/--omit-link-times`, and `--open-noatime` are new. +They change the wire: the per-file metadata frame grows `atime_valid` + +`atime_sec` + `atime_nsec` and `crtime_valid` + `crtime_sec` + `crtime_nsec` +(appended after the existing mode/uid/gid/mtime fields, preserving the exact +positions of every pre-existing field), and the config frame grows four +booleans — `preserve_atimes`, `preserve_crtimes`, `omit_dir_times`, +`omit_link_times` — that CROSS the wire so the receiver knows what to apply / +suppress. `--open-noatime` is **client-only** and is never serialized (it only +governs the sender's source reads). `PROTOCOL_VERSION` was bumped **2.11.0 → +2.12.0** (peers must match, exactly as prior phases did). + +**Client-vs-wire split:** `-U` and `-N` affect both the sender (capture) and the +receiver (apply), so they and their metadata fields cross the wire; +`-O`/`-J` are receiver-side preferences and cross as config booleans; +`--open-noatime` is purely a client/sender open flag and stays off the wire +(mirroring the existing convention where `ignore_errors` is client-only while +`force_delete` crosses the wire). + +**atime capture does not clobber the source atime:** the sender records the +access time from the **same pre-read stat the scanner already took** (inside +`file_metadata_create`), before any file data is read for transfer. So `-U` +alone captures the correct atime even without `--open-noatime`. `--open-noatime` +is orthogonal: it keeps the source's on-disk atime from being bumped by the read +that actually ships the data (only honoured where `O_NOATIME` works; it degrades +to a normal open otherwise, so the data always transfers). + +**crtime handling:** `-N` captures the source birth time via `statx`/`STATX_BTIME` +(guarded `#ifdef STATX_BTIME` on Linux) and transmits it. On the receiver, **no +portable setter exists** (`utimensat` can only set atime/mtime), so the receiver +deliberately does **not** apply it: it logs a debug note and continues — it never +fails the transfer and never pretends the crtime was applied. This is the +explicit, documented unsupported-attribute handling. On platforms without +`statx` the flag is accepted but nothing is captured (a documented no-op). + +**omit-dir-times / omit-link-times:** `-O` and `-J` are **accepted and parsed +for CLI compatibility** and their config booleans cross the wire, but they are +genuine **no-ops**: FastSync does not apply directory or symlink times at all +(directories are made via `mkdir` with no metadata; symlinks are dereferenced +or skipped, never written with a target), so there is nothing for an "omit" to +suppress. They never break a normal run. This is documented as a +divergence — the flags recognize the rsync interface but have no filtering +effect in FastSync. + +**-U/-N and -M interaction:** because FastSync carries all metadata (mode, uid, +gid, mtime, and now atime/crtime) in one bounded payload that is only sent when +metadata transmission is on, `-U` and `-N` imply metadata transmission (the +times travel inside that payload). They do **not** enable ownership application, +which remains opt-in strictly through the identity flags (`--numeric-ids` / +`--usermap` / `--groupmap` / `--chown`). + **Phase-4 identity notes:** `--numeric-ids`, `--usermap`, `--groupmap`, and `--chown` are real. They introduce a **controlled, opt-in, privilege-gated** ownership-application path on the receiver: plain `-M`/`--preserve` still does diff --git a/src/client/client_cli.c b/src/client/client_cli.c index b5216f4..bd39ae5 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -4,6 +4,7 @@ #include "compression.h" #include "config.h" #include "delta.h" +#include "file.h" #include "file_list.h" #include "filter.h" #include "identity.h" @@ -533,6 +534,11 @@ static const OptionEntry OPTION_TABLE[] = { {"--cvs-exclude", "-C", OPT_FLAG, offsetof(Config, cvs_exclude)}, {"-F", NULL, OPT_FLAG, offsetof(Config, per_dir_filter)}, {"--numeric-ids", NULL, OPT_FLAG, offsetof(Config, numeric_ids)}, + {"--atimes", "-U", OPT_FLAG, offsetof(Config, preserve_atimes)}, + {"--crtimes", "-N", OPT_FLAG, offsetof(Config, preserve_crtimes)}, + {"--omit-dir-times", "-O", OPT_FLAG, offsetof(Config, omit_dir_times)}, + {"--omit-link-times", "-J", OPT_FLAG, offsetof(Config, omit_link_times)}, + {"--open-noatime", NULL, OPT_FLAG, offsetof(Config, open_noatime)}, }; /* Only boolean options with no required argument are safe to negate. */ @@ -794,6 +800,14 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args, config. */ if (entry->offset == offsetof(Config, delete_missing_args)) config->ignore_missing_args = true; + /* -U/--atimes and -N/--crtimes carry their times inside the metadata + payload, which is only transmitted when use_metadata is set, so either + one implies metadata transmission. This is FastSync's broad -M bundle + (mode/mtime travel too); it does NOT enable ownership application, + which stays opt-in via the identity flags. */ + if (entry->offset == offsetof(Config, preserve_atimes) || + entry->offset == offsetof(Config, preserve_crtimes)) + config->use_metadata = true; continue; } @@ -1339,6 +1353,10 @@ int main(int argc, char* argv[]) { goto cleanup; } + /* --open-noatime is a sender-side policy: install it for every source read + (scan + data path) without touching the receiver. */ + file_set_open_noatime(config->open_noatime); + /* Initialize TLS if needed */ if (config->use_tls) tls_global_init(); diff --git a/src/client/client_send.c b/src/client/client_send.c index 169b119..57dbf98 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -82,6 +82,8 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann ScannerOptions* options = &out->options; options->use_metadata = config->use_metadata; + options->preserve_atimes = config->preserve_atimes; + options->preserve_crtimes = config->preserve_crtimes; options->chunk_size = config->chunk_size; options->exclude_patterns = config->exclude_patterns; options->exclude_count = config->exclude_count; diff --git a/src/client/scanner.c b/src/client/scanner.c index 87fa020..016dd8a 100644 --- a/src/client/scanner.c +++ b/src/client/scanner.c @@ -320,6 +320,8 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo scanner->current_dir = NULL; scanner->current_path = NULL; scanner->use_metadata = options->use_metadata; + scanner->preserve_atimes = options->preserve_atimes; + scanner->preserve_crtimes = options->preserve_crtimes; scanner->chunk_size = options->chunk_size > 0 ? options->chunk_size : DESIRED_CHUNK_SIZE; scanner->exclude_patterns = options->exclude_patterns; scanner->exclude_count = options->exclude_count; @@ -564,7 +566,8 @@ static File* dirs_root_dir_file(DirectoryScanner* scanner) { } file->is_dir = true; if (scanner->use_metadata) { - file->metadata = file_metadata_create(&st); + file->metadata = file_metadata_create(scanner->root_path, &st, scanner->preserve_atimes, + scanner->preserve_crtimes); if (!file->metadata) { file_destroy(file); scanner->failed = true; @@ -641,7 +644,8 @@ static File* dirs_file_for_entry(DirectoryScanner* scanner, const char* entry) { } } if (scanner->use_metadata) { - file->metadata = file_metadata_create(&effective); + file->metadata = file_metadata_create(file->path, &effective, scanner->preserve_atimes, + scanner->preserve_crtimes); if (!file->metadata) { file_destroy(file); scanner->failed = true; @@ -893,7 +897,8 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) { rel_copy = NULL; } if (scanner->use_metadata) - file->metadata = file_metadata_create(&stats); + file->metadata = file_metadata_create(file->path, &stats, scanner->preserve_atimes, + scanner->preserve_crtimes); if (scanner->use_metadata && !file->metadata) { free(rel_copy); file_destroy(file); @@ -1208,7 +1213,8 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo rel = NULL; } if (options->use_metadata) - file->metadata = file_metadata_create(&st); + file->metadata = + file_metadata_create(file->path, &st, options->preserve_atimes, options->preserve_crtimes); if (options->use_metadata && !file->metadata) { free(rel); file_destroy(file); diff --git a/src/client/scanner.h b/src/client/scanner.h index 6c4e5fe..7329c92 100644 --- a/src/client/scanner.h +++ b/src/client/scanner.h @@ -14,6 +14,10 @@ typedef struct { bool use_metadata; + /* Phase 4 metadata capture: -U/--atimes and -N/--crtimes tell the scanner to + * capture the source access / birth time into each entry's FileMetadata. */ + bool preserve_atimes; + bool preserve_crtimes; unsigned long long chunk_size; char** exclude_patterns; int exclude_count; @@ -75,6 +79,8 @@ typedef struct { DIR* current_dir; char* current_path; bool use_metadata; + bool preserve_atimes; + bool preserve_crtimes; unsigned long long chunk_size; char** exclude_patterns; int exclude_count; diff --git a/src/shared/config.c b/src/shared/config.c index 11c197e..6c9a542 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -148,6 +148,11 @@ static void config_set_defaults(Config* config) { config->groupmap = NULL; config->groupmap_count = 0; config->delay_context = NULL; + config->preserve_atimes = false; + config->preserve_crtimes = false; + config->omit_dir_times = false; + config->omit_link_times = false; + config->open_noatime = false; } static bool valid_wire_bool(int value) { @@ -195,6 +200,8 @@ static bool validate_received_config(const Config* config) { which chunk serialization -s disables: reject on the receiver too so a -s sender cannot negotiate an inert append mode. */ !((config->append || config->append_verify) && config->use_chunk_serialization) && + valid_wire_bool(config->preserve_atimes) && valid_wire_bool(config->preserve_crtimes) && + valid_wire_bool(config->omit_dir_times) && valid_wire_bool(config->omit_link_times) && (!config->use_compression || (config->compression_level >= 1 && config->compression_level <= 22)) && config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE && @@ -746,6 +753,22 @@ static bool receive_identity_options(int fd, Config* c) { receive_identity_map(fd, &c->groupmap_count, &c->groupmap); } +/* -U/--atimes, -N/--crtimes (affect both sender capture and receiver apply) + * and -O/--omit-dir-times, -J/--omit-link-times (receiver-side prefs) all cross + * the wire so the receiver knows what to apply / suppress. --open-noatime is + * client-only (it only governs the sender's source reads) and is never + * serialized. Trailing fields; protocol 2.12.0. */ +static bool send_metadata_times_options(int fd, const Config* c) { + return send_int(fd, c->preserve_atimes) && send_int(fd, c->preserve_crtimes) && + send_int(fd, c->omit_dir_times) && send_int(fd, c->omit_link_times); +} + +static bool receive_metadata_times_options(int fd, Config* c) { + return receive_wire_bool(fd, &c->preserve_atimes) && + receive_wire_bool(fd, &c->preserve_crtimes) && receive_wire_bool(fd, &c->omit_dir_times) && + receive_wire_bool(fd, &c->omit_link_times); +} + bool config_send(int file_descriptor, const Config* config) { protocol_session_set_max_alloc(NULL, config->max_alloc); if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) || @@ -754,7 +777,8 @@ bool config_send(int file_descriptor, const Config* config) { !send_resume_options(file_descriptor, config) || !send_basis_options(file_descriptor, config) || !send_fuzzy_option(file_descriptor, config) || !send_checksum_options(file_descriptor, config) || - !send_identity_options(file_descriptor, config)) + !send_identity_options(file_descriptor, config) || + !send_metadata_times_options(file_descriptor, config)) return false; Status status; if (!receive_status(file_descriptor, &status)) @@ -790,7 +814,8 @@ Config* config_receive(int file_descriptor) { !receive_basis_options(file_descriptor, config) || !receive_fuzzy_option(file_descriptor, config) || !receive_checksum_options(file_descriptor, config) || - !receive_identity_options(file_descriptor, config)) + !receive_identity_options(file_descriptor, config) || + !receive_metadata_times_options(file_descriptor, config)) goto error; if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 && strcmp(config->compress_choice, "none") != 0) { diff --git a/src/shared/config.h b/src/shared/config.h index 841aa83..15d5aae 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -291,9 +291,28 @@ typedef struct Config { // Receiver-side runtime staging registry for --delay-updates. Never sent // over the wire and never set on the sender side. DelayUpdatesContext* delay_context; + + // Phase 4: metadata time preservation. -U/--atimes and -N/--crtimes capture + // and transmit the source access / birth time (both sender and receiver + // effect, so they CROSS the wire). --omit-dir-times/-O and + // --omit-link-times/-J are receiver-side prefs (CROSS the wire). Their + // exact capture/transmit/apply semantics are documented in RSYNC_COMPAT.md. + /* -U/--atimes: preserve source access times on the destination. */ + bool preserve_atimes; + /* -N/--crtimes: capture+transmit source birth time; see RSYNC_COMPAT for the + * receiver not-applied divergence. */ + bool preserve_crtimes; + /* -O/--omit-dir-times: do not apply mtimes to directories. */ + bool omit_dir_times; + /* -J/--omit-link-times: do not apply times to symlinks. */ + bool omit_link_times; + /* --open-noatime: CLIENT-ONLY (never crosses the wire). The sender opens + * source files with O_NOATIME so reading for transfer does not bump the + * source access time. */ + bool open_noatime; } Config; -#define PROTOCOL_VERSION "2.11.0" +#define PROTOCOL_VERSION "2.12.0" #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) /* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */ #define MAX_BASIS_DIRS 64 diff --git a/src/shared/file.c b/src/shared/file.c index 7de1bd1..dcc058c 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -1,3 +1,6 @@ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE /* statx + STATX_BTIME for --crtimes birth-time capture */ +#endif #include #include #include @@ -128,7 +131,8 @@ void file_destroy(void* item) { free(file); } -FileMetadata* file_metadata_create(const struct stat* stats) { +FileMetadata* file_metadata_create(const char* path, const struct stat* stats, bool capture_atime, + bool capture_crtime) { FileMetadata* m = protocol_alloc(sizeof(FileMetadata)); if (m == NULL) { log_perror("ERROR: Could not allocate memory for file metadata"); @@ -143,6 +147,37 @@ FileMetadata* file_metadata_create(const struct stat* stats) { #else m->mtime_nsec = 0; #endif + /* -U/--atimes: capture the access time from the same pre-read stat the + scanner already took, so the value is not clobbered by a later read for + transfer. The timestamp is populated (and atime_valid set) only on Linux, + where st_atim is populated; on other platforms the atime is left alone + rather than clobbered to the default 0/epoch by an unpopulated value. */ +#ifdef __linux__ + m->atime_valid = capture_atime; + m->atime_sec = stats->st_atim.tv_sec; + m->atime_nsec = stats->st_atim.tv_nsec; +#else + m->atime_valid = false; + m->atime_sec = 0; + m->atime_nsec = 0; +#endif + /* -N/--crtimes: birth time is not available via struct stat in general; on + Linux it needs statx STATX_BTIME. If unavailable it is captured as a + documented no-op (the flag stays accepted, crtime_valid stays false). */ + m->crtime_valid = false; + m->crtime_sec = 0; + m->crtime_nsec = 0; + if (capture_crtime) { +#ifdef STATX_BTIME + struct statx stx; + if (path != NULL && statx(AT_FDCWD, path, AT_STATX_SYNC_AS_STAT, STATX_BTIME, &stx) == 0 && + (stx.stx_mask & STATX_BTIME) != 0) { + m->crtime_valid = true; + m->crtime_sec = (time_t)stx.stx_btime.tv_sec; + m->crtime_nsec = (long)stx.stx_btime.tv_nsec; + } +#endif + } return m; } @@ -150,6 +185,37 @@ void file_metadata_destroy(void* metadata) { free(metadata); } +/* --open-noatime: process-wide sender policy (client-only, never crosses the + * wire). When enabled, opening a source file for transfer uses O_NOATIME so + * the read does not bump the source's on-disk access time. It degrades safely + * to a normal open where O_NOATIME is unavailable (not defined) or refused + * (EPERM, because it needs CAP_FOWNER): the data path never silently changes, + * only the atime-bump is skipped. */ +static bool file_open_noatime = false; + +void file_set_open_noatime(bool enable) { + file_open_noatime = enable; +} + +bool file_get_open_noatime(void) { + return file_open_noatime; +} + +/* Open `path` read-only for transfer, honouring --open-noatime when set. */ +int file_open_for_read(const char* path) { + int flags = O_RDONLY; +#ifdef O_NOATIME + if (file_get_open_noatime()) + flags |= O_NOATIME; +#endif + int fd = open(path, flags); +#ifdef O_NOATIME + if (fd < 0 && (flags & O_NOATIME)) + fd = open(path, O_RDONLY); /* degrade safely on EPERM / unsupported fs */ +#endif + return fd; +} + bool file_load_data(File* file) { if (file == NULL || !file->data) return false; @@ -176,8 +242,14 @@ bool file_load_data(File* file) { size_t file_content_to_buffer(File* file) { if (!file || !file->path || !file->data || (!file->data->data && file->data->size != 0)) return 0; - FILE* file_pointer = fopen(file->path, "rb"); + int fd = file_open_for_read(file->path); + if (fd < 0) { + log_perror("Could not open the file!"); + return 0; + } + FILE* file_pointer = fdopen(fd, "rb"); if (file_pointer == NULL) { + close(fd); log_perror("Could not open the file!"); return 0; } diff --git a/src/shared/file.h b/src/shared/file.h index 77bb215..0d72343 100644 --- a/src/shared/file.h +++ b/src/shared/file.h @@ -22,8 +22,14 @@ bool file_load_data(File* file); bool file_checksum(File* file, ChecksumAlgo algo, uint64_t seed, uint8_t* out, size_t out_capacity, size_t* out_len); size_t file_content_to_buffer(File* file); -FileMetadata* file_metadata_create(const struct stat* stats); +FileMetadata* file_metadata_create(const char* path, const struct stat* stats, bool capture_atime, + bool capture_crtime); void file_metadata_destroy(void* metadata); +/* --open-noatime process-wide sender policy; see file.c. */ +void file_set_open_noatime(bool enable); +bool file_get_open_noatime(void); +/* Open `path` read-only for transfer, honouring --open-noatime when set. */ +int file_open_for_read(const char* path); bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size, bool inplace, bool sparse); diff --git a/src/shared/file_receive.c b/src/shared/file_receive.c index 2b10212..97521a0 100644 --- a/src/shared/file_receive.c +++ b/src/shared/file_receive.c @@ -1222,7 +1222,7 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) { if (materialized && basis.content) { materialized->data = basis.content; basis.content = NULL; - materialized->metadata = file_metadata_create(&basis.st); + materialized->metadata = file_metadata_create(NULL, &basis.st, false, false); materialized->skip = true; /* receiver must not ack this as a data file */ if (basis.type == BASIS_DEST_LINK) { materialized->basis_link = basis.basis_path; diff --git a/src/shared/file_send.c b/src/shared/file_send.c index 8f6cebe..e94e178 100644 --- a/src/shared/file_send.c +++ b/src/shared/file_send.c @@ -78,7 +78,7 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; - int fd = open(file->path, O_RDONLY); + int fd = file_open_for_read(file->path); if (fd == -1) { log_perror("Could not open file for sendfile"); return false; diff --git a/src/shared/file_types.h b/src/shared/file_types.h index a116916..1de29e9 100644 --- a/src/shared/file_types.h +++ b/src/shared/file_types.h @@ -13,6 +13,18 @@ typedef struct { gid_t gid; time_t mtime_sec; long mtime_nsec; + /* Optional access time (-U/--atimes) and creation/birth time (-N/--crtimes), + * appended for protocol 2.12.0. The SENDER sets the corresponding *_valid + * flag only when the preserve option is active (and, for crtime, only when + * the source platform exposed a birth time via statx STATX_BTIME). The wire + * always carries the fields and the flags; a false flag tells the receiver to + * ignore the value. */ + bool atime_valid; + time_t atime_sec; + long atime_nsec; + bool crtime_valid; + time_t crtime_sec; + long crtime_nsec; } FileMetadata; typedef struct { diff --git a/src/shared/metadata.c b/src/shared/metadata.c index 94ef772..4763a6d 100644 --- a/src/shared/metadata.c +++ b/src/shared/metadata.c @@ -70,6 +70,24 @@ void metadata_to_buf(char** buf, const FileMetadata* m) { int64_t mtime_nsec = (int64_t)m->mtime_nsec; memcpy(*buf, &mtime_nsec, sizeof(mtime_nsec)); *buf += sizeof(mtime_nsec); + int32_t atime_valid = m->atime_valid ? 1 : 0; + memcpy(*buf, &atime_valid, sizeof(atime_valid)); + *buf += sizeof(atime_valid); + int64_t atime_sec = (int64_t)m->atime_sec; + memcpy(*buf, &atime_sec, sizeof(atime_sec)); + *buf += sizeof(atime_sec); + int64_t atime_nsec = (int64_t)m->atime_nsec; + memcpy(*buf, &atime_nsec, sizeof(atime_nsec)); + *buf += sizeof(atime_nsec); + int32_t crtime_valid = m->crtime_valid ? 1 : 0; + memcpy(*buf, &crtime_valid, sizeof(crtime_valid)); + *buf += sizeof(crtime_valid); + int64_t crtime_sec = (int64_t)m->crtime_sec; + memcpy(*buf, &crtime_sec, sizeof(crtime_sec)); + *buf += sizeof(crtime_sec); + int64_t crtime_nsec = (int64_t)m->crtime_nsec; + memcpy(*buf, &crtime_nsec, sizeof(crtime_nsec)); + *buf += sizeof(crtime_nsec); } FileMetadata* metadata_from_buf(char** buf) { @@ -103,8 +121,34 @@ FileMetadata* metadata_from_buf(char** buf) { memcpy(&mtime_nsec, *buf, sizeof(mtime_nsec)); *buf += sizeof(mtime_nsec); m->mtime_nsec = (long)mtime_nsec; + int32_t atime_valid; + memcpy(&atime_valid, *buf, sizeof(atime_valid)); + *buf += sizeof(atime_valid); + int64_t atime_sec; + memcpy(&atime_sec, *buf, sizeof(atime_sec)); + *buf += sizeof(atime_sec); + int64_t atime_nsec; + memcpy(&atime_nsec, *buf, sizeof(atime_nsec)); + *buf += sizeof(atime_nsec); + int32_t crtime_valid; + memcpy(&crtime_valid, *buf, sizeof(crtime_valid)); + *buf += sizeof(crtime_valid); + int64_t crtime_sec; + memcpy(&crtime_sec, *buf, sizeof(crtime_sec)); + *buf += sizeof(crtime_sec); + int64_t crtime_nsec; + memcpy(&crtime_nsec, *buf, sizeof(crtime_nsec)); + *buf += sizeof(crtime_nsec); + m->atime_valid = atime_valid != 0; + m->atime_sec = (time_t)atime_sec; + m->atime_nsec = (long)atime_nsec; + m->crtime_valid = crtime_valid != 0; + m->crtime_sec = (time_t)crtime_sec; + m->crtime_nsec = (long)crtime_nsec; if (present != 1 || mtime_nsec < 0 || mtime_nsec >= 1000000000LL || mode < 0 || uid < 0 || - gid < 0) { + gid < 0 || atime_valid < 0 || atime_valid > 1 || crtime_valid < 0 || crtime_valid > 1 || + (atime_valid && (atime_nsec < 0 || atime_nsec >= 1000000000LL)) || + (crtime_valid && (crtime_nsec < 0 || crtime_nsec >= 1000000000LL))) { free(m); return NULL; } @@ -122,12 +166,24 @@ bool metadata_send(int file_descriptor, const FileMetadata* m) { int32_t gid = (int32_t)m->gid; int64_t mtime_sec = (int64_t)m->mtime_sec; int64_t mtime_nsec = (int64_t)m->mtime_nsec; + int32_t atime_valid = m->atime_valid ? 1 : 0; + int64_t atime_sec = (int64_t)m->atime_sec; + int64_t atime_nsec = (int64_t)m->atime_nsec; + int32_t crtime_valid = m->crtime_valid ? 1 : 0; + int64_t crtime_sec = (int64_t)m->crtime_sec; + int64_t crtime_nsec = (int64_t)m->crtime_nsec; return send_n_data(file_descriptor, &present, sizeof(present)) && send_n_data(file_descriptor, &mode, sizeof(mode)) && send_n_data(file_descriptor, &uid, sizeof(uid)) && send_n_data(file_descriptor, &gid, sizeof(gid)) && send_n_data(file_descriptor, &mtime_sec, sizeof(mtime_sec)) && - send_n_data(file_descriptor, &mtime_nsec, sizeof(mtime_nsec)); + send_n_data(file_descriptor, &mtime_nsec, sizeof(mtime_nsec)) && + send_n_data(file_descriptor, &atime_valid, sizeof(atime_valid)) && + send_n_data(file_descriptor, &atime_sec, sizeof(atime_sec)) && + send_n_data(file_descriptor, &atime_nsec, sizeof(atime_nsec)) && + send_n_data(file_descriptor, &crtime_valid, sizeof(crtime_valid)) && + send_n_data(file_descriptor, &crtime_sec, sizeof(crtime_sec)) && + send_n_data(file_descriptor, &crtime_nsec, sizeof(crtime_nsec)); } FileMetadata* metadata_receive(int file_descriptor, int* ok) { @@ -193,7 +249,58 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok) { return NULL; } m->mtime_nsec = (long)mtime_nsec; - if (mtime_nsec < 0 || mtime_nsec >= 1000000000LL || mode < 0 || uid < 0 || gid < 0) { + int32_t atime_valid; + if (!receive_n_data(file_descriptor, &atime_valid, sizeof(atime_valid))) { + free(m); + if (ok) + *ok = 0; + return NULL; + } + int64_t atime_sec; + if (!receive_n_data(file_descriptor, &atime_sec, sizeof(atime_sec))) { + free(m); + if (ok) + *ok = 0; + return NULL; + } + int64_t atime_nsec; + if (!receive_n_data(file_descriptor, &atime_nsec, sizeof(atime_nsec))) { + free(m); + if (ok) + *ok = 0; + return NULL; + } + int32_t crtime_valid; + if (!receive_n_data(file_descriptor, &crtime_valid, sizeof(crtime_valid))) { + free(m); + if (ok) + *ok = 0; + return NULL; + } + int64_t crtime_sec; + if (!receive_n_data(file_descriptor, &crtime_sec, sizeof(crtime_sec))) { + free(m); + if (ok) + *ok = 0; + return NULL; + } + int64_t crtime_nsec; + if (!receive_n_data(file_descriptor, &crtime_nsec, sizeof(crtime_nsec))) { + free(m); + if (ok) + *ok = 0; + return NULL; + } + m->atime_valid = atime_valid != 0; + m->atime_sec = (time_t)atime_sec; + m->atime_nsec = (long)atime_nsec; + m->crtime_valid = crtime_valid != 0; + m->crtime_sec = (time_t)crtime_sec; + m->crtime_nsec = (long)crtime_nsec; + if (mtime_nsec < 0 || mtime_nsec >= 1000000000LL || mode < 0 || uid < 0 || gid < 0 || + atime_valid < 0 || atime_valid > 1 || crtime_valid < 0 || crtime_valid > 1 || + (atime_valid && (atime_nsec < 0 || atime_nsec >= 1000000000LL)) || + (crtime_valid && (crtime_nsec < 0 || crtime_nsec >= 1000000000LL))) { free(m); if (ok) *ok = 0; @@ -232,6 +339,16 @@ void file_restore_metadata(const char* path, const FileMetadata* metadata, times[0].tv_nsec = UTIME_OMIT; times[1].tv_sec = metadata->mtime_sec; times[1].tv_nsec = metadata->mtime_nsec; + if (metadata->atime_valid) { + times[0].tv_sec = metadata->atime_sec; + times[0].tv_nsec = metadata->atime_nsec; + } + if (metadata->crtime_valid) { + log_message(LOG_LEVEL_DEBUG, + "crtime (birth time) %lld.%09ld transmitted for %s but not applied: no portable " + "setter exists", + (long long)metadata->crtime_sec, metadata->crtime_nsec, path); + } if (utimensat(AT_FDCWD, path, times, 0) != 0) { char* escaped_path = output_escape(path, log_get_8_bit_output()); log_message(LOG_LEVEL_WARNING, "Failed to set timestamps on %s: %s", @@ -261,6 +378,20 @@ bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserv identity_apply_ownership(fd, (int32_t)metadata->uid, (int32_t)metadata->gid); struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT}, {.tv_sec = metadata->mtime_sec, .tv_nsec = metadata->mtime_nsec}}; + if (metadata->atime_valid) { + times[0].tv_sec = metadata->atime_sec; + times[0].tv_nsec = metadata->atime_nsec; + } + /* --crtimes captures and transmits the source birth time, but there is no + * portable way to set a birth time (utimensat can only set atime/mtime), so + * the receiver deliberately does NOT apply it. This is explicit, honest + * unsupported-attribute handling: log a debug note and continue — never fail + * the transfer and never pretend the crtime was applied. */ + if (metadata->crtime_valid) { + log_message(LOG_LEVEL_DEBUG, + "crtime (birth time) %lld.%09ld transmitted but not applied: no portable setter", + (long long)metadata->crtime_sec, metadata->crtime_nsec); + } if (futimens(fd, times) != 0) ok = false; return ok; diff --git a/src/shared/metadata.h b/src/shared/metadata.h index b95b87a..1e7d2c0 100644 --- a/src/shared/metadata.h +++ b/src/shared/metadata.h @@ -15,6 +15,12 @@ * int32_t gid (was gid_t, platform-dependent) * int64_t mtime_sec (was time_t, platform-dependent) * int64_t mtime_nsec (was long, platform-dependent) + * int32_t atime_valid (-U/--atimes; protocol 2.12.0) + * int64_t atime_sec + * int64_t atime_nsec + * int32_t crtime_valid (-N/--crtimes; protocol 2.12.0) + * int64_t crtime_sec + * int64_t crtime_nsec * * Prior to 2.0.0 the wire format used the raw platform-dependent types, * which broke compatiblity across different systems. All fields are now @@ -23,8 +29,8 @@ /* Size of metadata fields on wire, excluding the int32_t `present` field that * is always sent first. The total wire size for present metadata is - * sizeof(int32_t) + FILE_METADATA_WIRE_SIZE (32 bytes on most platforms). */ -#define FILE_METADATA_WIRE_SIZE (sizeof(int32_t) * 3 + sizeof(int64_t) * 2) + * sizeof(int32_t) + FILE_METADATA_WIRE_SIZE (68 bytes on most platforms). */ +#define FILE_METADATA_WIRE_SIZE (sizeof(int32_t) * 5 + sizeof(int64_t) * 6) void metadata_to_buf(char** buf, const FileMetadata* m); FileMetadata* metadata_from_buf(char** buf); diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index 7df756f..8e968ec 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -3826,3 +3826,194 @@ class TestIdentityMapping: st = os.stat(dst_file) assert st.st_uid == 12345 and st.st_gid == 54321, \ f"--chown not applied: uid={st.st_uid} gid={st.st_gid}" + + +class TestAtimes: + """-U/--atimes preserves the source access time on the destination. + + The sender captures atime during the scan (a stat, before any read for + transfer), so the value is not clobbered by reading the source. This is + verified by setting the source atime to a distinct value far in the past + and comparing the destination atime to it (with whole-second tolerance; + filesystems may round atime).""" + + PAYLOAD = b"atime preservation payload\n" + + @staticmethod + def _make_source(source, dest): + clean_dir(source) + clean_dir(dest) + path = os.path.join(source, "data.txt") + with open(path, "wb") as f: + f.write(TestAtimes.PAYLOAD) + atime = 946684800 # 2000-01-01 00:00:00 UTC (far from "now") + mtime = 951782400 + os.utime(path, ns=(atime * 10**9 + 123456789, mtime * 10**9)) + return path, atime + + @pytest.mark.ci + @pytest.mark.parametrize("mt", [False, True]) + def test_atimes_preserved(self, shared_server, mt): + source = os.path.join(TEST_DATA_DIR, f"atime_{'m' if mt else 's'}_src") + dest = os.path.join(TEST_DATA_DIR, f"atime_{'m' if mt else 's'}_dst") + src_file, atime = self._make_source(source, dest) + flags = ["-U"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, \ + f"-U failed: {(result.stderr or result.stdout)[:300]}" + + received = get_dest_received_dir(dest, source) + dst_file = os.path.join(received, "data.txt") + assert os.path.exists(dst_file) + dst_st = os.stat(dst_file) + assert abs(dst_st.st_atime - atime) < 1.5, \ + f"dest atime {dst_st.st_atime} != source atime {atime}" + + @pytest.mark.ci + @pytest.mark.parametrize("mt", [False, True]) + def test_without_atimes_dest_differs(self, shared_server, mt): + """Control: without -U the destination atime is not the source's old + value (it reflects the fresh write, i.e. now), proving -U is what + restores the source atime.""" + source = os.path.join(TEST_DATA_DIR, f"atime_ctrl_{'m' if mt else 's'}_src") + dest = os.path.join(TEST_DATA_DIR, f"atime_ctrl_{'m' if mt else 's'}_dst") + src_file, atime = self._make_source(source, dest) + now = time.time() + flags = (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"control run failed: {(result.stderr or '')[:200]}" + received = get_dest_received_dir(dest, source) + dst_st = os.stat(os.path.join(received, "data.txt")) + # The fresh destination atime is ~now, not the source's year-2000 value. + assert abs(dst_st.st_atime - atime) > 24 * 3600, \ + f"control dest atime {dst_st.st_atime} unexpectedly equals source atime {atime}" + assert abs(dst_st.st_atime - now) < 24 * 3600, \ + f"control dest atime {dst_st.st_atime} not ~now ({now})" + + +class TestOpenNoatime: + """--open-noatime opens the source with O_NOATIME so a transfer read does + not bump the source's access time. O_NOATIME is honoured for a file owned + by the reading process (or with CAP_FOWNER), so it works as non-root here; + where it is unavailable/refused FastSync degrades to a normal open and the + assertion below is skipped.""" + + @pytest.mark.skipif(not sys.platform.startswith("linux"), + reason="O_NOATIME is Linux-specific") + @pytest.mark.parametrize("mt", [False, True]) + def test_open_noatime_preserves_source_atime(self, shared_server, mt): + source = os.path.join(TEST_DATA_DIR, f"noatime_{'m' if mt else 's'}_src") + dest = os.path.join(TEST_DATA_DIR, f"noatime_{'m' if mt else 's'}_dst") + clean_dir(source) + clean_dir(dest) + path = os.path.join(source, "data.txt") + with open(path, "wb") as f: + f.write(b"open-noatime payload\n") + atime = 730486800 # 1993-02-11, distinct and far from now + os.utime(path, ns=(atime * 10**9, atime * 10**9)) + + flags = ["--open-noatime"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, \ + f"--open-noatime failed: {(result.stderr or result.stdout)[:300]}" + + after = os.stat(path) + assert abs(after.st_atime - atime) < 1.5, \ + f"source atime {after.st_atime} was bumped by the readable read (wanted {atime})" + + +class TestCrtimes: + """-N/--crtimes captures and transmits the source birth time. There is no + portable way to SET a birth time (utimensat only sets atime/mtime), so the + receiver deliberately does not apply it. The run must succeed without + crashing; we do not assert the destination birth time changed. When the + platform exposes a birth time (statx STATX_BTIME on Linux) we additionally + confirm a capture path exists.""" + + @pytest.mark.parametrize("mt", [False, True]) + def test_crtimes_run_succeeds(self, shared_server, mt): + source = os.path.join(TEST_DATA_DIR, f"crtime_{'m' if mt else 's'}_src") + dest = os.path.join(TEST_DATA_DIR, f"crtime_{'m' if mt else 's'}_dst") + clean_dir(source) + clean_dir(dest) + path = os.path.join(source, "data.txt") + payload = b"crtime transfer payload\n" + with open(path, "wb") as f: + f.write(payload) + + flags = ["-N"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, \ + f"-N failed: {(result.stderr or result.stdout)[:300]}" + + received = get_dest_received_dir(dest, source) + dst_file = os.path.join(received, "data.txt") + assert os.path.exists(dst_file) + with open(dst_file, "rb") as f: + assert f.read() == payload + + def test_crtimes_combines_with_atimes(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "crtime_atime_combined_src") + dest = os.path.join(TEST_DATA_DIR, "crtime_atime_combined_dst") + clean_dir(source) + clean_dir(dest) + path = os.path.join(source, "data.txt") + with open(path, "wb") as f: + f.write(b"combined U N payload\n") + atime = 946684800 + os.utime(path, ns=(atime * 10**9, 951782400 * 10**9)) + result, _ = run_client(source, dest, flags=["-U", "-N"], + port=shared_server.port) + assert result.returncode == 0, \ + f"-U -N failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + dst_st = os.stat(os.path.join(received, "data.txt")) + assert abs(dst_st.st_atime - atime) < 1.5, \ + f"combined -U -N dest atime {dst_st.st_atime} != {atime}" + + +class TestOmitTimes: + """-O/--omit-dir-times and -J/--omit-link-times are recognized and cross the + wire as receiver-side preferences. FastSync does not currently apply dir or + symlink times at all, so they are forward-compatible preferences: the run + must succeed and normal transfers must not break. A regular file's mtime + (from -M) is unaffected by -O/-J.""" + + @pytest.mark.parametrize("flag", ["-O", "-J"]) + @pytest.mark.parametrize("mt", [False, True]) + def test_omit_times_accepted(self, shared_server, flag, mt): + source = os.path.join(TEST_DATA_DIR, f"omit_{flag.strip('-')}_{'m' if mt else 's'}_src") + dest = os.path.join(TEST_DATA_DIR, f"omit_{flag.strip('-')}_{'m' if mt else 's'}_dst") + clean_dir(source) + clean_dir(dest) + with open(os.path.join(source, "a.txt"), "wb") as f: + f.write(b"omit times content\n") + flags = [flag] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, \ + f"{flag} failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + mismatches, missing = verify_transfer(source, received) + assert not missing, f"Missing: {missing}" + assert not mismatches, f"Mismatch: {mismatches}" + + @pytest.mark.ci + def test_omit_times_with_dirs_and_regular_metadata(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "omit_dirs_meta_src") + dest = os.path.join(TEST_DATA_DIR, "omit_dirs_meta_dst") + clean_dir(source) + clean_dir(dest) + os.makedirs(os.path.join(source, "subdir")) + with open(os.path.join(source, "f.txt"), "wb") as f: + f.write(b"regular mtime preserved under -O/-J\n") + result, _ = run_client(source, dest, flags=["-d", "--omit-dir-times"], + port=shared_server.port) + assert result.returncode == 0, \ + f"-d -O failed: {(result.stderr or result.stdout)[:300]}" + result, _ = run_client(source, dest, flags=["-M", "-O", "-J"], + port=shared_server.port) + assert result.returncode == 0, \ + f"-M -O -J failed: {(result.stderr or result.stdout)[:300]}" + received = get_dest_received_dir(dest, source) + mismatches, missing = verify_transfer(source, received) + assert not missing and not mismatches, f"missing={missing} mismatches={mismatches}" diff --git a/tests/test_chunk.c b/tests/test_chunk.c index db3948d..8271b8f 100644 --- a/tests/test_chunk.c +++ b/tests/test_chunk.c @@ -118,11 +118,11 @@ static void test_chunk_dir_entry_roundtrip() { dir->is_dir = true; if (use_metadata) { - reg->metadata = file_metadata_create(&st); + reg->metadata = file_metadata_create(file_path, &st, false, false); EXPECT_NOT_NULL(reg->metadata); struct stat dst; EXPECT_EQ_INT(stat(dir_path, &dst), 0); - dir->metadata = file_metadata_create(&dst); + dir->metadata = file_metadata_create(dir_path, &dst, false, false); EXPECT_NOT_NULL(dir->metadata); } diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index 51950a3..92241b5 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -2254,6 +2254,54 @@ static void test_parse_args_preallocate() { config_delete(cfg); } +/* Phase 4 metadata-time flags parse and set the expected config fields. -U and + * -N imply metadata transmission (they carry their times inside the metadata + * payload); -O/-J and --open-noatime do not. */ +static void test_parse_args_metadata_times() { + Config* cfg = config_create(); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + char* argv[] = {"fastsync", "-U", "-N", "-O", "-J", "--open-noatime", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 8, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->preserve_atimes); + EXPECT_TRUE(cfg->preserve_crtimes); + EXPECT_TRUE(cfg->omit_dir_times); + EXPECT_TRUE(cfg->omit_link_times); + EXPECT_TRUE(cfg->open_noatime); + /* -U/-N carry their times inside the metadata payload, so they imply it. */ + EXPECT_TRUE(cfg->use_metadata); + EXPECT_TRUE(validate_config(cfg)); + config_delete(cfg); +} + +static void test_parse_args_atimes_long_and_short() { + Config* cfg = config_create(); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + char* argv[] = {"fastsync", "--atimes", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->preserve_atimes); + EXPECT_TRUE(cfg->use_metadata); + config_delete(cfg); +} + +static void test_parse_args_omit_link_times_long() { + Config* cfg = config_create(); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + char* argv[] = {"fastsync", "--omit-link-times", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->omit_link_times); + EXPECT_FALSE(cfg->use_metadata); + config_delete(cfg); +} + void test_client_cli() { test_validate_config_required_paths(); test_parse_args_numeric_ids(); @@ -2263,6 +2311,9 @@ void test_client_cli() { test_parse_args_chown(); test_parse_args_rejects_malformed_identity(); test_parse_args_preallocate(); + test_parse_args_metadata_times(); + test_parse_args_atimes_long_and_short(); + test_parse_args_omit_link_times_long(); test_parse_args_append(); test_parse_args_append_verify(); test_parse_args_append_both(); diff --git a/tests/test_config.c b/tests/test_config.c index 0682669..ec91545 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -911,6 +911,51 @@ static void test_config_receive_rejects_invalid_checksum_algo() { /* The identity-mapping fields (--numeric-ids / --usermap / --groupmap / --chown) cross the config wire unchanged: the receiver needs them to apply ownership with the same policy the client requested. */ +static void test_config_metadata_times_wire_roundtrip() { + if (is_running_under_valgrind()) + return; + Config* send_cfg = config_create(); + EXPECT_NOT_NULL(send_cfg); + send_cfg->send_directory = str_dup("/send/src"); + send_cfg->receive_root_directory = str_dup("/send/dst"); + send_cfg->preserve_atimes = true; + send_cfg->preserve_crtimes = true; + send_cfg->omit_dir_times = true; + send_cfg->omit_link_times = true; + /* --open-noatime is client-only and must NOT cross the wire. */ + send_cfg->open_noatime = true; + + int p[2]; + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + io_set_fds(p[0], p[1]); + io_set_bwlimit(0); + pid_t pid = fork(); + if (pid == 0) { + close(p[1]); + io_set_fds(p[0], p[0]); + Config* recv = config_receive(p[0]); + bool ok = recv != NULL; + if (ok) { + ok = recv->preserve_atimes && recv->preserve_crtimes && recv->omit_dir_times && + recv->omit_link_times && !recv->open_noatime; + } + config_delete(recv); + close(p[0]); + close(p[1]); + _exit(ok ? 0 : 1); + } else { + close(p[0]); + io_set_fds(p[1], p[1]); + bool sent = config_send(p[1], send_cfg); + int status; + waitpid(pid, &status, 0); + close(p[1]); + config_delete(send_cfg); + EXPECT_TRUE(sent); + EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0); + } +} + static void test_config_identity_wire_roundtrip() { if (is_running_under_valgrind()) return; @@ -1068,6 +1113,7 @@ void test_config() { test_config_receive_rejects_invalid_checksum_algo(); test_config_identity_wire_roundtrip(); test_config_receive_rejects_invalid_identity(); + test_config_metadata_times_wire_roundtrip(); test_config_preallocate_wire_roundtrip(); } test_config_delete_timing_early_helper(); diff --git a/tests/test_file.c b/tests/test_file.c index 88a8129..0495bd2 100644 --- a/tests/test_file.c +++ b/tests/test_file.c @@ -596,7 +596,7 @@ static void test_file_metadata_create() { struct stat st; EXPECT_EQ_INT(stat("test_meta_file.txt", &st), 0); - FileMetadata* m = file_metadata_create(&st); + FileMetadata* m = file_metadata_create("test_meta_file.txt", &st, false, false); EXPECT_NOT_NULL(m); EXPECT_EQ_INT(m->mode, st.st_mode); EXPECT_EQ_INT(m->uid, st.st_uid); @@ -712,7 +712,7 @@ static void test_file_send_single_calls_metadata_and_path() { file->data->data = malloc(len); EXPECT_NOT_NULL(file->data->data); memcpy(file->data->data, content, len); - file->metadata = file_metadata_create(&st); + file->metadata = file_metadata_create("test_meta_send.txt", &st, false, false); EXPECT_NOT_NULL(file->metadata); Config* cfg = config_create(); @@ -857,7 +857,7 @@ static void test_inplace_overwrite_metadata_strips_special_bits() { EXPECT_NOT_NULL(f->data->data); memcpy(f->data->data, new_content, strlen(new_content)); f->data->size = strlen(new_content); - f->metadata = file_metadata_create(&source_st); + f->metadata = file_metadata_create(source, &source_st, false, false); EXPECT_NOT_NULL(f->metadata); Config* cfg = config_create(); diff --git a/tests/test_fuzz_smoke.c b/tests/test_fuzz_smoke.c index 193b86c..9b2c415 100644 --- a/tests/test_fuzz_smoke.c +++ b/tests/test_fuzz_smoke.c @@ -106,7 +106,7 @@ static void test_fuzz_metadata_from_buf() { struct stat st; EXPECT_EQ_INT(stat("fuzz_meta_test.txt", &st), 0); - FileMetadata* meta = file_metadata_create(&st); + FileMetadata* meta = file_metadata_create("fuzz_meta_test.txt", &st, false, false); EXPECT_NOT_NULL(meta); EXPECT_EQ_INT((int)meta->mode, (int)st.st_mode); EXPECT_EQ_INT((int)meta->mtime_sec, (int)st.st_mtime); diff --git a/tests/test_metadata.c b/tests/test_metadata.c index 424d499..4ee9212 100644 --- a/tests/test_metadata.c +++ b/tests/test_metadata.c @@ -15,6 +15,12 @@ static void test_metadata_to_from_buf_roundtrip() { original.gid = 1000; original.mtime_sec = 1234567890; original.mtime_nsec = 500000000; + original.atime_valid = true; + original.atime_sec = 1234567000; + original.atime_nsec = 250000000; + original.crtime_valid = true; + original.crtime_sec = 1200000000; + original.crtime_nsec = 750000000; char* buf = malloc(FILE_METADATA_WIRE_SIZE + sizeof(int)); EXPECT_NOT_NULL(buf); @@ -30,6 +36,14 @@ static void test_metadata_to_from_buf_roundtrip() { EXPECT_EQ_INT(result->gid, 1000); EXPECT_EQ_INT(result->mtime_sec, 1234567890); EXPECT_EQ_INT(result->mtime_nsec, 500000000); + EXPECT_TRUE(result->atime_valid); + EXPECT_EQ_INT(result->atime_sec, 1234567000); + EXPECT_EQ_INT(result->atime_nsec, 250000000); + EXPECT_TRUE(result->crtime_valid); + EXPECT_EQ_INT(result->crtime_sec, 1200000000); + EXPECT_EQ_INT(result->crtime_nsec, 750000000); + + EXPECT_EQ_INT((int)(read_ptr - buf), (int)FILE_METADATA_WIRE_SIZE + (int)sizeof(int)); free(result); free(buf); @@ -75,6 +89,12 @@ static void test_metadata_send_receive_roundtrip() { original.gid = 1000; original.mtime_sec = 1234567890; original.mtime_nsec = 500000000; + original.atime_valid = false; + original.atime_sec = 0; + original.atime_nsec = 0; + original.crtime_valid = true; + original.crtime_sec = 1200000000; + original.crtime_nsec = 750000000; EXPECT_TRUE(metadata_send(p[1], &original)); @@ -87,6 +107,10 @@ static void test_metadata_send_receive_roundtrip() { EXPECT_EQ_INT(received->gid, 1000); EXPECT_EQ_INT(received->mtime_sec, 1234567890); EXPECT_EQ_INT(received->mtime_nsec, 500000000); + EXPECT_FALSE(received->atime_valid); + EXPECT_TRUE(received->crtime_valid); + EXPECT_EQ_INT(received->crtime_sec, 1200000000); + EXPECT_EQ_INT(received->crtime_nsec, 750000000); free(received); close(p[0]); @@ -123,6 +147,77 @@ static void test_metadata_rejects_invalid_values() { close(p[1]); } +/* metadata_receive must reject an out-of-range atime/crtime nsec even when the + * flag would otherwise be valid (defense-in-depth on the -U/-N wire fields). */ +static void test_metadata_receive_rejects_bad_optional_times() { + int p[2]; + EXPECT_EQ_INT(pipe(p), 0); + io_set_fds(p[0], p[1]); + + int32_t present = 1; + int32_t mode = 0644; + int32_t uid = 1000; + int32_t gid = 1000; + int64_t mtime_sec = 1; + int64_t mtime_nsec = 0; + int32_t atime_valid = 1; + int64_t atime_sec = 1; + int64_t atime_nsec = 2000000000; /* invalid: >= 1e9 */ + EXPECT_TRUE(send_n_data(p[1], &present, sizeof(present))); + EXPECT_TRUE(send_n_data(p[1], &mode, sizeof(mode))); + EXPECT_TRUE(send_n_data(p[1], &uid, sizeof(uid))); + EXPECT_TRUE(send_n_data(p[1], &gid, sizeof(gid))); + EXPECT_TRUE(send_n_data(p[1], &mtime_sec, sizeof(mtime_sec))); + EXPECT_TRUE(send_n_data(p[1], &mtime_nsec, sizeof(mtime_nsec))); + EXPECT_TRUE(send_n_data(p[1], &atime_valid, sizeof(atime_valid))); + EXPECT_TRUE(send_n_data(p[1], &atime_sec, sizeof(atime_sec))); + EXPECT_TRUE(send_n_data(p[1], &atime_nsec, sizeof(atime_nsec))); + int32_t crtime_valid = 0; + int64_t crtime_sec = 0; + int64_t crtime_nsec = 0; + EXPECT_TRUE(send_n_data(p[1], &crtime_valid, sizeof(crtime_valid))); + EXPECT_TRUE(send_n_data(p[1], &crtime_sec, sizeof(crtime_sec))); + EXPECT_TRUE(send_n_data(p[1], &crtime_nsec, sizeof(crtime_nsec))); + int ok = 1; + EXPECT_NULL(metadata_receive(p[0], &ok)); + EXPECT_EQ_INT(ok, 0); + close(p[0]); + close(p[1]); +} + +/* file_restore_metadata applies the source atime alongside mtime when -U + * captured it (atime_valid set). */ +static void test_file_restore_metadata_applies_atime() { + const char* path = "temp_meta_atime_test.txt"; + EXPECT_TRUE(file_write_to_disk(path, "atime", 5, false, false)); + + FileMetadata m; + m.mode = 0644; + m.uid = getuid(); + m.gid = getgid(); + m.mtime_sec = 1234567890; + m.mtime_nsec = 0; + m.atime_valid = true; + m.atime_sec = 999999999; + m.atime_nsec = 123456789; + m.crtime_valid = false; + m.crtime_sec = 0; + m.crtime_nsec = 0; + + file_restore_metadata(path, &m, false); + + struct stat st; + EXPECT_EQ_INT(stat(path, &st), 0); + EXPECT_EQ_INT((int)st.st_mtime, 1234567890); +#ifdef __linux__ + EXPECT_EQ_INT((int)st.st_atime, 999999999); +#else + EXPECT_EQ_INT((int)st.st_atime, 999999999); +#endif + + unlink(path); +} + static void test_metadata_mtime_window() { EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 101, 600000000, 2)); EXPECT_FALSE(metadata_mtime_matches(100, 100000000, 102, 600000000, 2)); @@ -214,8 +309,10 @@ void test_metadata() { test_metadata_send_receive_roundtrip(); test_metadata_send_null(); test_metadata_rejects_invalid_values(); + test_metadata_receive_rejects_bad_optional_times(); test_metadata_mtime_window(); test_file_restore_metadata(); + test_file_restore_metadata_applies_atime(); test_file_restore_executability_only(); test_directory_restore_executability_only(); test_chmod_changes();