preallocate: --preallocate allocates dest space up front
CI / lint (pull_request) Failing after 3s
CI / build-and-test (pull_request) Skipped
CI / sanitizers (address) (pull_request) Skipped
CI / sanitizers (undefined) (pull_request) Skipped
CI / fuzz-build (pull_request) Skipped
CI / coverage (pull_request) Skipped
CI / valgrind (pull_request) Skipped
CI / lint (pull_request) Failing after 3s
CI / build-and-test (pull_request) Skipped
CI / sanitizers (address) (pull_request) Skipped
CI / sanitizers (undefined) (pull_request) Skipped
CI / fuzz-build (pull_request) Skipped
CI / coverage (pull_request) Skipped
CI / valgrind (pull_request) Skipped
Receiver allocates the destination file's full size before streaming data (posix_fallocate preferred, ftruncate fallback on EOPNOTSUPP/ENOSYS) so an out-of-space transfer fails fast instead of partway. Additive config bool crossing the wire; PROTOCOL_VERSION 2.10.0 -> 2.11.0. Threaded through all store paths (atomic, inplace, partial/delay-updates staging, link-dest copy fallback). Review hardening: explicit lseek(0) before the data write so correctness does not depend on posix_fallocate leaving the fd offset unchanged.
This commit is contained in:
@@ -226,6 +226,51 @@ class TestChmod:
|
||||
assert (os.stat(os.path.join(received, "small.txt")).st_mode & 0o777) == 0o644
|
||||
|
||||
|
||||
class TestPreallocate:
|
||||
"""--preallocate allocates the destination file space up front; the final
|
||||
destination content must be byte-identical to a normal run."""
|
||||
|
||||
def test_preallocate_transfer_succeeds(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "prealloc_source")
|
||||
dest = os.path.join(TEST_DATA_DIR, "prealloc_dest")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
payload = os.urandom(2 * 1024 * 1024 + 137)
|
||||
with open(os.path.join(source, "data.bin"), "wb") as f:
|
||||
f.write(payload)
|
||||
with open(os.path.join(source, "small.txt"), "wb") as f:
|
||||
f.write(b"hello\n")
|
||||
|
||||
result, _ = run_client(source, dest, flags=["--preallocate"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--preallocate failed: {(result.stderr or result.stdout)[:400]}"
|
||||
|
||||
received_dir = os.path.join(dest, os.path.abspath(source).lstrip(os.sep))
|
||||
data_path = os.path.join(received_dir, "data.bin")
|
||||
assert os.path.isfile(data_path), f"destination file not created: {data_path}"
|
||||
with open(data_path, "rb") as f:
|
||||
assert f.read() == payload, "destination content mismatch"
|
||||
small_path = os.path.join(received_dir, "small.txt")
|
||||
with open(small_path, "rb") as f:
|
||||
assert f.read() == b"hello\n", "small file content mismatch"
|
||||
|
||||
def test_preallocate_combines_with_partial(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "prealloc_partial_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "prealloc_partial_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
with open(os.path.join(source, "f.txt"), "wb") as f:
|
||||
f.write(b"partial + preallocate\n")
|
||||
result, _ = run_client(source, dest, flags=["--preallocate", "--partial"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--preallocate --partial failed: {(result.stderr or result.stdout)[:400]}"
|
||||
received_dir = os.path.join(dest, os.path.abspath(source).lstrip(os.sep))
|
||||
with open(os.path.join(received_dir, "f.txt"), "rb") as f:
|
||||
assert f.read() == b"partial + preallocate\n"
|
||||
|
||||
|
||||
class TestCompressionChoice:
|
||||
def test_zstd_choice_compresses(self, shared_server):
|
||||
clean_dir(DEST_DIR)
|
||||
|
||||
@@ -137,3 +137,7 @@ class TestSSHFeatures:
|
||||
r = _run_ssh_test("SSH Exclude (--exclude small.txt)",
|
||||
["--exclude", "small.txt"], expected_missing=["small.txt"])
|
||||
assert r["status"] == "Success", r["error"]
|
||||
|
||||
def test_preallocate(self):
|
||||
r = _run_ssh_test("SSH Preallocate (--preallocate)", ["--preallocate"])
|
||||
assert r["status"] == "Success", r["error"]
|
||||
|
||||
@@ -2083,8 +2083,24 @@ static void test_validate_config_append_verify_rejects_whole_file() {
|
||||
EXPECT_FALSE(validate_config(cfg));
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* --preallocate parses as a boolean flag and validates cleanly. */
|
||||
static void test_parse_args_preallocate() {
|
||||
Config* cfg = config_create();
|
||||
cfg->send_directory = str_dup("/src");
|
||||
cfg->receive_root_directory = str_dup("/dst");
|
||||
char* argv[] = {"fastsync", "--preallocate", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->preallocate);
|
||||
EXPECT_TRUE(validate_config(cfg));
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
void test_client_cli() {
|
||||
test_validate_config_required_paths();
|
||||
test_parse_args_preallocate();
|
||||
test_parse_args_append();
|
||||
test_parse_args_append_verify();
|
||||
test_parse_args_append_both();
|
||||
|
||||
@@ -908,6 +908,45 @@ static void test_config_receive_rejects_invalid_checksum_algo() {
|
||||
EXPECT_FALSE(roundtrip_config_ok(c));
|
||||
config_delete(c);
|
||||
}
|
||||
|
||||
/* --preallocate crosses the wire unchanged (receiver-side flag): the receiver
|
||||
must learn to allocate the destination file's space before data flows. */
|
||||
static void test_config_preallocate_wire_roundtrip() {
|
||||
struct {
|
||||
bool preallocate;
|
||||
} cases[] = {{false}, {true}};
|
||||
if (is_running_under_valgrind())
|
||||
return;
|
||||
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||
int p[2];
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
pid_t pid = fork();
|
||||
if (pid == 0) {
|
||||
close(p[1]);
|
||||
io_set_fds(p[0], p[0]);
|
||||
Config* recv = config_receive(p[0]);
|
||||
bool ok = recv != NULL && recv->preallocate == cases[i].preallocate;
|
||||
config_delete(recv);
|
||||
close(p[0]);
|
||||
_exit(ok ? 0 : 1);
|
||||
} else {
|
||||
close(p[0]);
|
||||
io_set_fds(p[1], p[1]);
|
||||
Config* send_cfg = config_create();
|
||||
EXPECT_NOT_NULL(send_cfg);
|
||||
send_cfg->send_directory = str_dup("/src");
|
||||
send_cfg->receive_root_directory = str_dup("/dst");
|
||||
send_cfg->preallocate = cases[i].preallocate;
|
||||
bool sent = config_send(p[1], send_cfg);
|
||||
int status;
|
||||
waitpid(pid, &status, 0);
|
||||
close(p[1]);
|
||||
config_delete(send_cfg);
|
||||
EXPECT_TRUE(sent);
|
||||
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
void test_config() {
|
||||
test_config_lifecycle();
|
||||
test_config_ssh_dest();
|
||||
@@ -932,6 +971,7 @@ void test_config() {
|
||||
test_config_basis_normalization();
|
||||
test_config_checksum_options_wire_roundtrip();
|
||||
test_config_receive_rejects_invalid_checksum_algo();
|
||||
test_config_preallocate_wire_roundtrip();
|
||||
}
|
||||
test_config_delete_timing_early_helper();
|
||||
test_config_is_remote_dest();
|
||||
|
||||
+40
-2
@@ -380,14 +380,50 @@ static void test_file_write_to_disk_basic() {
|
||||
static void test_file_write_to_disk_with_fsync() {
|
||||
const char* path = "test_file_write_to_disk_fsync.txt";
|
||||
const char* content = "fsync file content";
|
||||
EXPECT_TRUE(file_to_disk_secure_with_fsync(path, content, strlen(content), false, false, NULL,
|
||||
false, true, NULL));
|
||||
EXPECT_TRUE(file_to_disk_secure_with_fsync(path, content, strlen(content), false, false, false,
|
||||
NULL, false, true, NULL));
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
|
||||
unlink(path);
|
||||
}
|
||||
|
||||
static void test_file_write_to_disk_preallocate_atomic() {
|
||||
const char* path = "test_file_write_prealloc_atomic.txt";
|
||||
const char* content = "prealloc atomic content";
|
||||
EXPECT_TRUE(file_to_disk_secure(path, content, strlen(content), false, false, true, NULL, false,
|
||||
NULL));
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
|
||||
FILE* fp = fopen(path, "rb");
|
||||
EXPECT_NOT_NULL(fp);
|
||||
char buf[100];
|
||||
size_t nread = fread(buf, 1, sizeof(buf), fp);
|
||||
fclose(fp);
|
||||
EXPECT_EQ_INT((int)nread, (int)strlen(content));
|
||||
EXPECT_EQ_INT(memcmp(buf, content, strlen(content)), 0);
|
||||
unlink(path);
|
||||
}
|
||||
|
||||
static void test_file_write_to_disk_preallocate_inplace() {
|
||||
const char* path = "test_file_write_prealloc_inplace.txt";
|
||||
const char* content = "prealloc inplace content";
|
||||
EXPECT_TRUE(file_to_disk_secure(path, content, strlen(content), true, false, true, NULL, false,
|
||||
NULL));
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
|
||||
FILE* fp = fopen(path, "rb");
|
||||
EXPECT_NOT_NULL(fp);
|
||||
char buf[100];
|
||||
size_t nread = fread(buf, 1, sizeof(buf), fp);
|
||||
fclose(fp);
|
||||
EXPECT_EQ_INT((int)nread, (int)strlen(content));
|
||||
EXPECT_EQ_INT(memcmp(buf, content, strlen(content)), 0);
|
||||
unlink(path);
|
||||
}
|
||||
|
||||
static void test_file_write_to_disk_creates_dirs() {
|
||||
const char* content = "Nested dir test";
|
||||
EXPECT_TRUE(file_write_to_disk("test_nested_tmp/nested/file.txt", content, strlen(content), false,
|
||||
@@ -937,6 +973,8 @@ void test_file() {
|
||||
test_file_save_to_disk_reports_skips();
|
||||
test_file_write_to_disk_basic();
|
||||
test_file_write_to_disk_with_fsync();
|
||||
test_file_write_to_disk_preallocate_atomic();
|
||||
test_file_write_to_disk_preallocate_inplace();
|
||||
test_file_write_to_disk_creates_dirs();
|
||||
test_file_write_to_disk_does_not_follow_symlink();
|
||||
test_file_content_to_buffer();
|
||||
|
||||
Reference in New Issue
Block a user