Merge feat/p4-metadata-capture: atimes/crtimes/open-noatime/omit-dir-times/omit-link-times

# Conflicts:
#	src/shared/config.c
#	tests/integration/test_features.py
This commit is contained in:
2026-09-08 21:00:07 +02:00
21 changed files with 765 additions and 27 deletions
+190
View File
@@ -3938,3 +3938,193 @@ class TestHardLinks:
assert fh.read() == b"asymmetric group content\n" * 1200
# Under --existing the absent first member is not created.
assert not os.path.exists(os.path.join(received, "a.txt"))
class TestAtimes:
"""-U/--atimes preserves the source access time on the destination.
The sender captures atime during the scan (a stat, before any read for
transfer), so the value is not clobbered by reading the source. This is
verified by setting the source atime to a distinct value far in the past
and comparing the destination atime to it (with whole-second tolerance;
filesystems may round atime)."""
PAYLOAD = b"atime preservation payload\n"
@staticmethod
def _make_source(source, dest):
clean_dir(source)
clean_dir(dest)
path = os.path.join(source, "data.txt")
with open(path, "wb") as f:
f.write(TestAtimes.PAYLOAD)
atime = 946684800 # 2000-01-01 00:00:00 UTC (far from "now")
mtime = 951782400
os.utime(path, ns=(atime * 10**9 + 123456789, mtime * 10**9))
return path, atime
@pytest.mark.ci
@pytest.mark.parametrize("mt", [False, True])
def test_atimes_preserved(self, shared_server, mt):
source = os.path.join(TEST_DATA_DIR, f"atime_{'m' if mt else 's'}_src")
dest = os.path.join(TEST_DATA_DIR, f"atime_{'m' if mt else 's'}_dst")
src_file, atime = self._make_source(source, dest)
flags = ["-U"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, \
f"-U failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(dest, source)
dst_file = os.path.join(received, "data.txt")
assert os.path.exists(dst_file)
dst_st = os.stat(dst_file)
assert abs(dst_st.st_atime - atime) < 1.5, \
f"dest atime {dst_st.st_atime} != source atime {atime}"
@pytest.mark.ci
@pytest.mark.parametrize("mt", [False, True])
def test_without_atimes_dest_differs(self, shared_server, mt):
"""Control: without -U the destination atime is not the source's old
value (it reflects the fresh write, i.e. now), proving -U is what
restores the source atime."""
source = os.path.join(TEST_DATA_DIR, f"atime_ctrl_{'m' if mt else 's'}_src")
dest = os.path.join(TEST_DATA_DIR, f"atime_ctrl_{'m' if mt else 's'}_dst")
src_file, atime = self._make_source(source, dest)
now = time.time()
flags = (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"control run failed: {(result.stderr or '')[:200]}"
received = get_dest_received_dir(dest, source)
dst_st = os.stat(os.path.join(received, "data.txt"))
# The fresh destination atime is ~now, not the source's year-2000 value.
assert abs(dst_st.st_atime - atime) > 24 * 3600, \
f"control dest atime {dst_st.st_atime} unexpectedly equals source atime {atime}"
assert abs(dst_st.st_atime - now) < 24 * 3600, \
f"control dest atime {dst_st.st_atime} not ~now ({now})"
class TestOpenNoatime:
"""--open-noatime opens the source with O_NOATIME so a transfer read does
not bump the source's access time. O_NOATIME is honoured for a file owned
by the reading process (or with CAP_FOWNER), so it works as non-root here;
where it is unavailable/refused FastSync degrades to a normal open and the
assertion below is skipped."""
@pytest.mark.skipif(not sys.platform.startswith("linux"),
reason="O_NOATIME is Linux-specific")
@pytest.mark.parametrize("mt", [False, True])
def test_open_noatime_preserves_source_atime(self, shared_server, mt):
source = os.path.join(TEST_DATA_DIR, f"noatime_{'m' if mt else 's'}_src")
dest = os.path.join(TEST_DATA_DIR, f"noatime_{'m' if mt else 's'}_dst")
clean_dir(source)
clean_dir(dest)
path = os.path.join(source, "data.txt")
with open(path, "wb") as f:
f.write(b"open-noatime payload\n")
atime = 730486800 # 1993-02-11, distinct and far from now
os.utime(path, ns=(atime * 10**9, atime * 10**9))
flags = ["--open-noatime"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, \
f"--open-noatime failed: {(result.stderr or result.stdout)[:300]}"
after = os.stat(path)
assert abs(after.st_atime - atime) < 1.5, \
f"source atime {after.st_atime} was bumped by the readable read (wanted {atime})"
class TestCrtimes:
"""-N/--crtimes captures and transmits the source birth time. There is no
portable way to SET a birth time (utimensat only sets atime/mtime), so the
receiver deliberately does not apply it. The run must succeed without
crashing; we do not assert the destination birth time changed. When the
platform exposes a birth time (statx STATX_BTIME on Linux) we additionally
confirm a capture path exists."""
@pytest.mark.parametrize("mt", [False, True])
def test_crtimes_run_succeeds(self, shared_server, mt):
source = os.path.join(TEST_DATA_DIR, f"crtime_{'m' if mt else 's'}_src")
dest = os.path.join(TEST_DATA_DIR, f"crtime_{'m' if mt else 's'}_dst")
clean_dir(source)
clean_dir(dest)
path = os.path.join(source, "data.txt")
payload = b"crtime transfer payload\n"
with open(path, "wb") as f:
f.write(payload)
flags = ["-N"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, \
f"-N failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(dest, source)
dst_file = os.path.join(received, "data.txt")
assert os.path.exists(dst_file)
with open(dst_file, "rb") as f:
assert f.read() == payload
def test_crtimes_combines_with_atimes(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "crtime_atime_combined_src")
dest = os.path.join(TEST_DATA_DIR, "crtime_atime_combined_dst")
clean_dir(source)
clean_dir(dest)
path = os.path.join(source, "data.txt")
with open(path, "wb") as f:
f.write(b"combined U N payload\n")
atime = 946684800
os.utime(path, ns=(atime * 10**9, 951782400 * 10**9))
result, _ = run_client(source, dest, flags=["-U", "-N"],
port=shared_server.port)
assert result.returncode == 0, \
f"-U -N failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(dest, source)
dst_st = os.stat(os.path.join(received, "data.txt"))
assert abs(dst_st.st_atime - atime) < 1.5, \
f"combined -U -N dest atime {dst_st.st_atime} != {atime}"
class TestOmitTimes:
"""-O/--omit-dir-times and -J/--omit-link-times are recognized and cross the
wire as receiver-side preferences. FastSync does not currently apply dir or
symlink times at all, so they are forward-compatible preferences: the run
must succeed and normal transfers must not break. A regular file's mtime
(from -M) is unaffected by -O/-J."""
@pytest.mark.parametrize("flag", ["-O", "-J"])
@pytest.mark.parametrize("mt", [False, True])
def test_omit_times_accepted(self, shared_server, flag, mt):
source = os.path.join(TEST_DATA_DIR, f"omit_{flag.strip('-')}_{'m' if mt else 's'}_src")
dest = os.path.join(TEST_DATA_DIR, f"omit_{flag.strip('-')}_{'m' if mt else 's'}_dst")
clean_dir(source)
clean_dir(dest)
with open(os.path.join(source, "a.txt"), "wb") as f:
f.write(b"omit times content\n")
flags = [flag] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, \
f"{flag} failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(dest, source)
mismatches, missing = verify_transfer(source, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
@pytest.mark.ci
def test_omit_times_with_dirs_and_regular_metadata(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "omit_dirs_meta_src")
dest = os.path.join(TEST_DATA_DIR, "omit_dirs_meta_dst")
clean_dir(source)
clean_dir(dest)
os.makedirs(os.path.join(source, "subdir"))
with open(os.path.join(source, "f.txt"), "wb") as f:
f.write(b"regular mtime preserved under -O/-J\n")
result, _ = run_client(source, dest, flags=["-d", "--omit-dir-times"],
port=shared_server.port)
assert result.returncode == 0, \
f"-d -O failed: {(result.stderr or result.stdout)[:300]}"
result, _ = run_client(source, dest, flags=["-M", "-O", "-J"],
port=shared_server.port)
assert result.returncode == 0, \
f"-M -O -J failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(dest, source)
mismatches, missing = verify_transfer(source, received)
assert not missing and not mismatches, f"missing={missing} mismatches={mismatches}"
+2 -2
View File
@@ -118,11 +118,11 @@ static void test_chunk_dir_entry_roundtrip() {
dir->is_dir = true;
if (use_metadata) {
reg->metadata = file_metadata_create(&st);
reg->metadata = file_metadata_create(file_path, &st, false, false);
EXPECT_NOT_NULL(reg->metadata);
struct stat dst;
EXPECT_EQ_INT(stat(dir_path, &dst), 0);
dir->metadata = file_metadata_create(&dst);
dir->metadata = file_metadata_create(dir_path, &dst, false, false);
EXPECT_NOT_NULL(dir->metadata);
}
+51
View File
@@ -2298,6 +2298,54 @@ static void test_parse_args_preallocate() {
config_delete(cfg);
}
/* Phase 4 metadata-time flags parse and set the expected config fields. -U and
* -N imply metadata transmission (they carry their times inside the metadata
* payload); -O/-J and --open-noatime do not. */
static void test_parse_args_metadata_times() {
Config* cfg = config_create();
cfg->send_directory = str_dup("/src");
cfg->receive_root_directory = str_dup("/dst");
char* argv[] = {"fastsync", "-U", "-N", "-O", "-J", "--open-noatime", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 8, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->preserve_atimes);
EXPECT_TRUE(cfg->preserve_crtimes);
EXPECT_TRUE(cfg->omit_dir_times);
EXPECT_TRUE(cfg->omit_link_times);
EXPECT_TRUE(cfg->open_noatime);
/* -U/-N carry their times inside the metadata payload, so they imply it. */
EXPECT_TRUE(cfg->use_metadata);
EXPECT_TRUE(validate_config(cfg));
config_delete(cfg);
}
static void test_parse_args_atimes_long_and_short() {
Config* cfg = config_create();
cfg->send_directory = str_dup("/src");
cfg->receive_root_directory = str_dup("/dst");
char* argv[] = {"fastsync", "--atimes", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->preserve_atimes);
EXPECT_TRUE(cfg->use_metadata);
config_delete(cfg);
}
static void test_parse_args_omit_link_times_long() {
Config* cfg = config_create();
cfg->send_directory = str_dup("/src");
cfg->receive_root_directory = str_dup("/dst");
char* argv[] = {"fastsync", "--omit-link-times", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->omit_link_times);
EXPECT_FALSE(cfg->use_metadata);
config_delete(cfg);
}
void test_client_cli() {
test_validate_config_required_paths();
test_parse_args_numeric_ids();
@@ -2307,6 +2355,9 @@ void test_client_cli() {
test_parse_args_chown();
test_parse_args_rejects_malformed_identity();
test_parse_args_preallocate();
test_parse_args_metadata_times();
test_parse_args_atimes_long_and_short();
test_parse_args_omit_link_times_long();
test_parse_args_append();
test_parse_args_append_verify();
test_parse_args_append_both();
+46
View File
@@ -914,6 +914,51 @@ static void test_config_receive_rejects_invalid_checksum_algo() {
/* The identity-mapping fields (--numeric-ids / --usermap / --groupmap /
--chown) cross the config wire unchanged: the receiver needs them to apply
ownership with the same policy the client requested. */
static void test_config_metadata_times_wire_roundtrip() {
if (is_running_under_valgrind())
return;
Config* send_cfg = config_create();
EXPECT_NOT_NULL(send_cfg);
send_cfg->send_directory = str_dup("/send/src");
send_cfg->receive_root_directory = str_dup("/send/dst");
send_cfg->preserve_atimes = true;
send_cfg->preserve_crtimes = true;
send_cfg->omit_dir_times = true;
send_cfg->omit_link_times = true;
/* --open-noatime is client-only and must NOT cross the wire. */
send_cfg->open_noatime = true;
int p[2];
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
io_set_fds(p[0], p[1]);
io_set_bwlimit(0);
pid_t pid = fork();
if (pid == 0) {
close(p[1]);
io_set_fds(p[0], p[0]);
Config* recv = config_receive(p[0]);
bool ok = recv != NULL;
if (ok) {
ok = recv->preserve_atimes && recv->preserve_crtimes && recv->omit_dir_times &&
recv->omit_link_times && !recv->open_noatime;
}
config_delete(recv);
close(p[0]);
close(p[1]);
_exit(ok ? 0 : 1);
} else {
close(p[0]);
io_set_fds(p[1], p[1]);
bool sent = config_send(p[1], send_cfg);
int status;
waitpid(pid, &status, 0);
close(p[1]);
config_delete(send_cfg);
EXPECT_TRUE(sent);
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
}
}
static void test_config_identity_wire_roundtrip() {
if (is_running_under_valgrind())
return;
@@ -1071,6 +1116,7 @@ void test_config() {
test_config_receive_rejects_invalid_checksum_algo();
test_config_identity_wire_roundtrip();
test_config_receive_rejects_invalid_identity();
test_config_metadata_times_wire_roundtrip();
test_config_preallocate_wire_roundtrip();
}
test_config_delete_timing_early_helper();
+3 -3
View File
@@ -596,7 +596,7 @@ static void test_file_metadata_create() {
struct stat st;
EXPECT_EQ_INT(stat("test_meta_file.txt", &st), 0);
FileMetadata* m = file_metadata_create(&st);
FileMetadata* m = file_metadata_create("test_meta_file.txt", &st, false, false);
EXPECT_NOT_NULL(m);
EXPECT_EQ_INT(m->mode, st.st_mode);
EXPECT_EQ_INT(m->uid, st.st_uid);
@@ -712,7 +712,7 @@ static void test_file_send_single_calls_metadata_and_path() {
file->data->data = malloc(len);
EXPECT_NOT_NULL(file->data->data);
memcpy(file->data->data, content, len);
file->metadata = file_metadata_create(&st);
file->metadata = file_metadata_create("test_meta_send.txt", &st, false, false);
EXPECT_NOT_NULL(file->metadata);
Config* cfg = config_create();
@@ -857,7 +857,7 @@ static void test_inplace_overwrite_metadata_strips_special_bits() {
EXPECT_NOT_NULL(f->data->data);
memcpy(f->data->data, new_content, strlen(new_content));
f->data->size = strlen(new_content);
f->metadata = file_metadata_create(&source_st);
f->metadata = file_metadata_create(source, &source_st, false, false);
EXPECT_NOT_NULL(f->metadata);
Config* cfg = config_create();
+1 -1
View File
@@ -106,7 +106,7 @@ static void test_fuzz_metadata_from_buf() {
struct stat st;
EXPECT_EQ_INT(stat("fuzz_meta_test.txt", &st), 0);
FileMetadata* meta = file_metadata_create(&st);
FileMetadata* meta = file_metadata_create("fuzz_meta_test.txt", &st, false, false);
EXPECT_NOT_NULL(meta);
EXPECT_EQ_INT((int)meta->mode, (int)st.st_mode);
EXPECT_EQ_INT((int)meta->mtime_sec, (int)st.st_mtime);
+97
View File
@@ -15,6 +15,12 @@ static void test_metadata_to_from_buf_roundtrip() {
original.gid = 1000;
original.mtime_sec = 1234567890;
original.mtime_nsec = 500000000;
original.atime_valid = true;
original.atime_sec = 1234567000;
original.atime_nsec = 250000000;
original.crtime_valid = true;
original.crtime_sec = 1200000000;
original.crtime_nsec = 750000000;
char* buf = malloc(FILE_METADATA_WIRE_SIZE + sizeof(int));
EXPECT_NOT_NULL(buf);
@@ -30,6 +36,14 @@ static void test_metadata_to_from_buf_roundtrip() {
EXPECT_EQ_INT(result->gid, 1000);
EXPECT_EQ_INT(result->mtime_sec, 1234567890);
EXPECT_EQ_INT(result->mtime_nsec, 500000000);
EXPECT_TRUE(result->atime_valid);
EXPECT_EQ_INT(result->atime_sec, 1234567000);
EXPECT_EQ_INT(result->atime_nsec, 250000000);
EXPECT_TRUE(result->crtime_valid);
EXPECT_EQ_INT(result->crtime_sec, 1200000000);
EXPECT_EQ_INT(result->crtime_nsec, 750000000);
EXPECT_EQ_INT((int)(read_ptr - buf), (int)FILE_METADATA_WIRE_SIZE + (int)sizeof(int));
free(result);
free(buf);
@@ -75,6 +89,12 @@ static void test_metadata_send_receive_roundtrip() {
original.gid = 1000;
original.mtime_sec = 1234567890;
original.mtime_nsec = 500000000;
original.atime_valid = false;
original.atime_sec = 0;
original.atime_nsec = 0;
original.crtime_valid = true;
original.crtime_sec = 1200000000;
original.crtime_nsec = 750000000;
EXPECT_TRUE(metadata_send(p[1], &original));
@@ -87,6 +107,10 @@ static void test_metadata_send_receive_roundtrip() {
EXPECT_EQ_INT(received->gid, 1000);
EXPECT_EQ_INT(received->mtime_sec, 1234567890);
EXPECT_EQ_INT(received->mtime_nsec, 500000000);
EXPECT_FALSE(received->atime_valid);
EXPECT_TRUE(received->crtime_valid);
EXPECT_EQ_INT(received->crtime_sec, 1200000000);
EXPECT_EQ_INT(received->crtime_nsec, 750000000);
free(received);
close(p[0]);
@@ -123,6 +147,77 @@ static void test_metadata_rejects_invalid_values() {
close(p[1]);
}
/* metadata_receive must reject an out-of-range atime/crtime nsec even when the
* flag would otherwise be valid (defense-in-depth on the -U/-N wire fields). */
static void test_metadata_receive_rejects_bad_optional_times() {
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
io_set_fds(p[0], p[1]);
int32_t present = 1;
int32_t mode = 0644;
int32_t uid = 1000;
int32_t gid = 1000;
int64_t mtime_sec = 1;
int64_t mtime_nsec = 0;
int32_t atime_valid = 1;
int64_t atime_sec = 1;
int64_t atime_nsec = 2000000000; /* invalid: >= 1e9 */
EXPECT_TRUE(send_n_data(p[1], &present, sizeof(present)));
EXPECT_TRUE(send_n_data(p[1], &mode, sizeof(mode)));
EXPECT_TRUE(send_n_data(p[1], &uid, sizeof(uid)));
EXPECT_TRUE(send_n_data(p[1], &gid, sizeof(gid)));
EXPECT_TRUE(send_n_data(p[1], &mtime_sec, sizeof(mtime_sec)));
EXPECT_TRUE(send_n_data(p[1], &mtime_nsec, sizeof(mtime_nsec)));
EXPECT_TRUE(send_n_data(p[1], &atime_valid, sizeof(atime_valid)));
EXPECT_TRUE(send_n_data(p[1], &atime_sec, sizeof(atime_sec)));
EXPECT_TRUE(send_n_data(p[1], &atime_nsec, sizeof(atime_nsec)));
int32_t crtime_valid = 0;
int64_t crtime_sec = 0;
int64_t crtime_nsec = 0;
EXPECT_TRUE(send_n_data(p[1], &crtime_valid, sizeof(crtime_valid)));
EXPECT_TRUE(send_n_data(p[1], &crtime_sec, sizeof(crtime_sec)));
EXPECT_TRUE(send_n_data(p[1], &crtime_nsec, sizeof(crtime_nsec)));
int ok = 1;
EXPECT_NULL(metadata_receive(p[0], &ok));
EXPECT_EQ_INT(ok, 0);
close(p[0]);
close(p[1]);
}
/* file_restore_metadata applies the source atime alongside mtime when -U
* captured it (atime_valid set). */
static void test_file_restore_metadata_applies_atime() {
const char* path = "temp_meta_atime_test.txt";
EXPECT_TRUE(file_write_to_disk(path, "atime", 5, false, false));
FileMetadata m;
m.mode = 0644;
m.uid = getuid();
m.gid = getgid();
m.mtime_sec = 1234567890;
m.mtime_nsec = 0;
m.atime_valid = true;
m.atime_sec = 999999999;
m.atime_nsec = 123456789;
m.crtime_valid = false;
m.crtime_sec = 0;
m.crtime_nsec = 0;
file_restore_metadata(path, &m, false);
struct stat st;
EXPECT_EQ_INT(stat(path, &st), 0);
EXPECT_EQ_INT((int)st.st_mtime, 1234567890);
#ifdef __linux__
EXPECT_EQ_INT((int)st.st_atime, 999999999);
#else
EXPECT_EQ_INT((int)st.st_atime, 999999999);
#endif
unlink(path);
}
static void test_metadata_mtime_window() {
EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 101, 600000000, 2));
EXPECT_FALSE(metadata_mtime_matches(100, 100000000, 102, 600000000, 2));
@@ -214,8 +309,10 @@ void test_metadata() {
test_metadata_send_receive_roundtrip();
test_metadata_send_null();
test_metadata_rejects_invalid_values();
test_metadata_receive_rejects_bad_optional_times();
test_metadata_mtime_window();
test_file_restore_metadata();
test_file_restore_metadata_applies_atime();
test_file_restore_executability_only();
test_directory_restore_executability_only();
test_chmod_changes();