Merge feat/p4-preallocate: --preallocate
# Conflicts: # tests/test_client_cli.c # tests/test_config.c
This commit is contained in:
+4
-1
@@ -323,7 +323,10 @@ unlike rsync, plain `-M` never applies ownership and `--usermap`/`--groupmap`/
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `-S`, `--sparse` | Sparse block handling | ⚠️ Partial | Flag is accepted, but full hole preservation is not implemented |
|
||||
| `--preallocate` | Allocate dest files before writing | ❌ Not Implemented | |
|
||||
| `--preallocate` | Allocate dest files before writing | ✅ Implemented | The receiver preallocates the destination file's full expected space before any data is written, so a transfer that would overflow disk fails fast at allocation time (a clean error, not a half-written file) and the file is laid out contiguously, avoiding fragmentation. Crosses the wire (the config frame carries a `preallocate` boolean; `PROTOCOL_VERSION` bumped **2.10.0 → 2.11.0**, peers must match) so the sender knows the receiver will preallocate and the receiver performs it. **Allocation approach:** `posix_fallocate()` is preferred because it reserves *real* disk blocks (true fail-fast on ENOSPC), falling back to plain `ftruncate()` only when the filesystem reports the allocation is unsupported (`EOPNOTSUPP`/`ENOSYS`); `ftruncate` still extends the logical size so the intent degrades gracefully. **Fallback/error semantics:** `EOPNOTSUPP`/`ENOSYS` → clean fallback to `ftruncate` (best-effort, preallocates the logical size and never fails a transfer on filesystems that lack `posix_fallocate`); a genuine allocation failure (`ENOSPC`/`EDQUOT`/`EFBIG`/…) aborts the file/receive with a distinct `preallocate failed ... transfer aborted` error — it does **not** fall back to a normal non-preallocated write, preserving the fail-fast purpose. **Size-known requirement:** preallocation only runs when the final size is already known up front (the normal regular-file case); unknown-length data is skipped (never failed). **Orthogonality:** applies uniformly across the atomic temp+rename store path, `--inplace`, `--partial`/`--partial-dir`, `--delay-updates` (the staged temp file is preallocated before data flows) and the `--link-dest` copy fallback; it neither implies nor conflicts with `-s`, `--append`, or delta. rsync-divergence: rsync signals that `--preallocate` is ignored with `--sparse`; FastSync simply preallocates first and still honours `--sparse`'s `ftruncate` sizing/trim, so the two combine rather than one being silently ignored. See the Phase-4 preallocate notes below |
|
||||
|
||||
**Preallocate notes (Phase 4, preallocate wave):** `--preallocate` is implemented as a real receiver-side allocation of the destination file's space before data is written. It is a plain boolean config flag that crosses the wire (serialized in the config frame's selection-options block, mirroring `--inplace`/`--append`/`--force`), so the run requires matching ends: `PROTOCOL_VERSION` was bumped **2.10.0 → 2.11.0** (peers must match or the version check fails). The allocation is performed on the exact destination fd, immediately after it is opened, before any bytes are streamed; `posix_fallocate` (and the `ftruncate` fallback) leave the fd's file offset untouched, so the subsequent data write at offset 0 is unaffected and complete. Because FastSync writes each file's byte payload in one in-memory batch, the "full expected size" is exactly the known `data_size`, which is what gets preallocated. Unknown-length/streamed payloads are skipped rather than failed. A failed allocation logs a distinct `preallocate failed` error and aborts the file (the atomic temp is unlinked, the inplace target is left untrimmed) so the run fails cleanly and never silently degrades to a non-preallocated write — preserving rsync's fail-fast intent on a full disk.
|
||||
|
||||
|
||||
## 11. Checksum & Comparison
|
||||
|
||||
|
||||
@@ -477,6 +477,7 @@ static const OptionEntry OPTION_TABLE[] = {
|
||||
{"--copy-unsafe-links", NULL, OPT_FLAG, offsetof(Config, copy_unsafe_links)},
|
||||
{"--sparse", "-S", OPT_FLAG, offsetof(Config, preserve_sparse)},
|
||||
{"--inplace", NULL, OPT_FLAG, offsetof(Config, inplace)},
|
||||
{"--preallocate", NULL, OPT_FLAG, offsetof(Config, preallocate)},
|
||||
{"--append", NULL, OPT_FLAG, offsetof(Config, append)},
|
||||
{"--append-verify", NULL, OPT_FLAG, offsetof(Config, append_verify)},
|
||||
{"--fsync", NULL, OPT_FLAG, offsetof(Config, use_fsync)},
|
||||
@@ -553,6 +554,7 @@ static const NegatableOption NEGATABLE_OPTIONS[] = {
|
||||
{"copy-unsafe-links", NULL, offsetof(Config, copy_unsafe_links)},
|
||||
{"sparse", "S", offsetof(Config, preserve_sparse)},
|
||||
{"inplace", NULL, offsetof(Config, inplace)},
|
||||
{"preallocate", NULL, offsetof(Config, preallocate)},
|
||||
{"checksum", NULL, offsetof(Config, checksum)},
|
||||
{"from0", NULL, offsetof(Config, from0)},
|
||||
{"cvs-exclude", NULL, offsetof(Config, cvs_exclude)},
|
||||
|
||||
@@ -182,6 +182,7 @@ void print_usage(void) {
|
||||
printf(" --copy-unsafe-links Only transform unsafe symlinks into referent files\n");
|
||||
printf(" -S, --sparse Handle sparse files efficiently\n");
|
||||
printf(" --inplace Update files in-place (no temp+rename)\n");
|
||||
printf(" --preallocate Allocate destination file space up front (fail-fast on full disk)\n");
|
||||
printf(" --append Resume a shorter destination by appending only its tail\n");
|
||||
printf(" (prefix is not verified; requires --incremental)\n");
|
||||
printf(" --append-verify Like --append, but verifies the retained prefix checksum\n");
|
||||
|
||||
+7
-3
@@ -92,6 +92,7 @@ static void config_set_defaults(Config* config) {
|
||||
config->use_fsync = false;
|
||||
config->append = false;
|
||||
config->append_verify = false;
|
||||
config->preallocate = false;
|
||||
config->delete_excluded = false;
|
||||
config->delete_after = false;
|
||||
config->max_delete = -1;
|
||||
@@ -178,8 +179,9 @@ static bool validate_received_config(const Config* config) {
|
||||
valid_wire_bool(config->existing) && valid_wire_bool(config->update) &&
|
||||
valid_wire_bool(config->inplace) && valid_wire_bool(config->append) &&
|
||||
valid_wire_bool(config->use_fsync) && valid_wire_bool(config->append_verify) &&
|
||||
valid_wire_bool(config->delete_excluded) && valid_wire_bool(config->force_delete) &&
|
||||
valid_wire_bool(config->delete_missing_args) && valid_wire_bool(config->delete_after) &&
|
||||
valid_wire_bool(config->delete_excluded) && valid_wire_bool(config->force_delete) &&
|
||||
valid_wire_bool(config->delete_missing_args) && valid_wire_bool(config->delete_after) &&
|
||||
valid_wire_bool(config->preallocate) &&
|
||||
valid_wire_bool(config->delete_delay) && valid_wire_bool(config->delete_during) &&
|
||||
valid_wire_bool(config->relative) && valid_wire_bool(config->prune_empty_dirs) &&
|
||||
valid_wire_bool(config->delay_updates) && valid_wire_bool(config->mkpath) &&
|
||||
@@ -448,6 +450,7 @@ static bool send_selection_options(int fd, const Config* c) {
|
||||
send_int(fd, c->use_fsync) && send_int(fd, c->append_verify) &&
|
||||
send_int(fd, c->delete_excluded) && send_int(fd, c->force_delete) &&
|
||||
send_int(fd, c->delete_missing_args) && send_int(fd, c->delete_after) &&
|
||||
send_int(fd, c->preallocate) &&
|
||||
send_n_data(fd, &c->max_delete, sizeof(c->max_delete)) && send_int(fd, c->relative) &&
|
||||
send_int(fd, c->prune_empty_dirs) && send_int(fd, c->mkpath) &&
|
||||
send_int(fd, c->delete_during) && send_int(fd, c->delete_delay);
|
||||
@@ -579,7 +582,8 @@ static bool receive_selection_options(int fd, Config* c) {
|
||||
&c->delete_excluded,
|
||||
&c->force_delete,
|
||||
&c->delete_missing_args,
|
||||
&c->delete_after};
|
||||
&c->delete_after,
|
||||
&c->preallocate};
|
||||
for (size_t i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
|
||||
if (!receive_wire_bool(fd, flags[i]))
|
||||
return false;
|
||||
|
||||
@@ -135,6 +135,11 @@ typedef struct Config {
|
||||
bool use_fsync;
|
||||
bool append;
|
||||
bool append_verify;
|
||||
/* --preallocate: allocates the destination file's full expected space up
|
||||
* front (before any data is written) so a transfer that would overflow disk
|
||||
* fails fast at allocation time and the file is laid out contiguously,
|
||||
* avoiding fragmentation. Receiver-side, crosses the wire. */
|
||||
bool preallocate;
|
||||
|
||||
// Issue #128: Extended delete options
|
||||
/* --delete-excluded: also delete destination entries that were excluded on
|
||||
|
||||
+94
-46
@@ -32,6 +32,29 @@ static bool write_all(int fd, const void* data, unsigned long long size) {
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Preallocate `size` bytes on `fd` before any data is written (--preallocate).
|
||||
* posix_fallocate reserves real disk blocks, so an out-of-space condition
|
||||
* (ENOSPC/EDQUOT) surfaces up front instead of partway through a transfer;
|
||||
* unavoidable fragmentation of a streamed file is also reduced. Some
|
||||
* filesystems (e.g. tmpfs, ZFS) do not support it and return EOPNOTSUPP/ENOSYS,
|
||||
* where we fall back to ftruncate, which still extends the logical size so the
|
||||
* fail-fast/contiguity intent degrades gracefully but never fails. Genuine
|
||||
* allocation failures are propagated as the error code (caller fails the write).
|
||||
* posix_fallocate leaves the fd's file offset unchanged, so the subsequent
|
||||
* write_all at offset 0 is unaffected. Returns 0 on success (including the
|
||||
* fallback) or a nonzero error code. */
|
||||
static int preallocate_fd(int fd, unsigned long long size) {
|
||||
if (size == 0)
|
||||
return 0;
|
||||
int rc = posix_fallocate(fd, 0, (off_t)size);
|
||||
if (rc == EOPNOTSUPP || rc == ENOSYS) {
|
||||
if (ftruncate(fd, (off_t)size) == 0)
|
||||
return 0;
|
||||
return errno;
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* Process-wide counter for scratch temp names. A --temp-dir scratch directory
|
||||
is flat: different destinations that share a basename must never race onto
|
||||
the same temp name. Deriving the trailing number from a global atomic
|
||||
@@ -510,9 +533,9 @@ int file_open_private_dir(const char* dir_path) {
|
||||
|
||||
static bool file_to_disk_secure_impl(const char* path, const void* data,
|
||||
unsigned long long data_size, bool inplace, bool sparse,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
bool update, bool no_replace, bool use_fsync,
|
||||
const char* temp_dir) {
|
||||
bool preallocate, const FileMetadata* metadata,
|
||||
bool preserve_executability, bool update, bool no_replace,
|
||||
bool use_fsync, const char* temp_dir) {
|
||||
char* leaf = NULL;
|
||||
int dirfd = file_open_secure_parent(path, &leaf, true);
|
||||
if (dirfd < 0)
|
||||
@@ -533,27 +556,39 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
||||
if (newer) {
|
||||
ok = true;
|
||||
} else {
|
||||
/* In-place overwrites: pre-size sparse targets and always trim the
|
||||
file to the new payload length afterwards so shorter payloads can
|
||||
never leave stale trailing bytes from a previous version. */
|
||||
if (sparse && data_size > 0)
|
||||
ok = ftruncate(fd, (off_t)data_size) == 0;
|
||||
if (ok || !sparse || data_size == 0)
|
||||
ok = write_all(fd, data, data_size);
|
||||
if (ok)
|
||||
ok = ftruncate(fd, (off_t)data_size) == 0;
|
||||
/* Normalize the mode: apply the metadata-derived safe mode when the
|
||||
sender supplied metadata (setuid/setgid/sticky are never honored);
|
||||
otherwise fall back to a safe default so dangerous bits on an
|
||||
existing destination cannot survive an overwrite. */
|
||||
if (ok) {
|
||||
if (metadata)
|
||||
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
||||
else if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) != 0)
|
||||
ok = false;
|
||||
/* Preallocate the expected payload size before writing so an
|
||||
out-of-space condition fails cleanly up front (--preallocate). */
|
||||
int prealloc_rc = 0;
|
||||
if (preallocate && data_size > 0) {
|
||||
prealloc_rc = preallocate_fd(fd, data_size);
|
||||
if (prealloc_rc != 0)
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"preallocate failed for '%s' (%s); transfer aborted",
|
||||
path, strerror(prealloc_rc));
|
||||
}
|
||||
if (prealloc_rc == 0) {
|
||||
/* posix_fallocate does not guarantee the fd's file offset is left
|
||||
unchanged, so seek back to 0 before the data write. */
|
||||
lseek(fd, 0, SEEK_SET);
|
||||
if (sparse && data_size > 0)
|
||||
ok = ftruncate(fd, (off_t)data_size) == 0;
|
||||
if (ok || !sparse || data_size == 0)
|
||||
ok = write_all(fd, data, data_size);
|
||||
if (ok)
|
||||
ok = ftruncate(fd, (off_t)data_size) == 0;
|
||||
/* Normalize the mode: apply the metadata-derived safe mode when the
|
||||
sender supplied metadata (setuid/setgid/sticky are never honored);
|
||||
otherwise fall back to a safe default so dangerous bits on an
|
||||
existing destination cannot survive an overwrite. */
|
||||
if (ok) {
|
||||
if (metadata)
|
||||
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
||||
else if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) != 0)
|
||||
ok = false;
|
||||
}
|
||||
if (ok && use_fsync)
|
||||
ok = fsync(fd) == 0;
|
||||
}
|
||||
if (ok && use_fsync)
|
||||
ok = fsync(fd) == 0;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -623,14 +658,24 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
||||
O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600);
|
||||
if (fd < 0)
|
||||
continue; /* EEXIST (or a transient open error): try a fresh name. */
|
||||
if (sparse && data_size > 0)
|
||||
ok = ftruncate(fd, (off_t)data_size) == 0;
|
||||
if (ok || (!sparse || data_size == 0))
|
||||
ok = write_all(fd, data, data_size);
|
||||
if (ok && metadata)
|
||||
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
||||
if (ok && use_fsync)
|
||||
ok = fsync(fd) == 0;
|
||||
int prealloc_rc = 0;
|
||||
if (preallocate && data_size > 0) {
|
||||
prealloc_rc = preallocate_fd(fd, data_size);
|
||||
if (prealloc_rc != 0)
|
||||
log_message(LOG_LEVEL_ERROR, "preallocate failed for '%s' (%s); transfer aborted",
|
||||
path, strerror(prealloc_rc));
|
||||
}
|
||||
if (prealloc_rc == 0) {
|
||||
lseek(fd, 0, SEEK_SET);
|
||||
if (sparse && data_size > 0)
|
||||
ok = ftruncate(fd, (off_t)data_size) == 0;
|
||||
if (ok || (!sparse || data_size == 0))
|
||||
ok = write_all(fd, data, data_size);
|
||||
if (ok && metadata)
|
||||
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
||||
if (ok && use_fsync)
|
||||
ok = fsync(fd) == 0;
|
||||
}
|
||||
if (close(fd) != 0)
|
||||
ok = false;
|
||||
fd = -1;
|
||||
@@ -678,32 +723,34 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
||||
}
|
||||
|
||||
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
|
||||
bool inplace, bool sparse, const FileMetadata* metadata,
|
||||
bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata,
|
||||
bool preserve_executability, const char* temp_dir) {
|
||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, metadata,
|
||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||
preserve_executability, false, false, false, temp_dir);
|
||||
}
|
||||
|
||||
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
|
||||
bool inplace, bool sparse, const FileMetadata* metadata,
|
||||
bool preserve_executability, const char* temp_dir) {
|
||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, metadata,
|
||||
bool inplace, bool sparse, bool preallocate,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
const char* temp_dir) {
|
||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||
preserve_executability, true, false, false, temp_dir);
|
||||
}
|
||||
|
||||
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
||||
unsigned long long data_size, bool inplace, bool sparse,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
bool use_fsync, const char* temp_dir) {
|
||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, metadata,
|
||||
bool preallocate, const FileMetadata* metadata,
|
||||
bool preserve_executability, bool use_fsync,
|
||||
const char* temp_dir) {
|
||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||
preserve_executability, false, false, use_fsync, temp_dir);
|
||||
}
|
||||
|
||||
bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
||||
unsigned long long data_size, bool sparse,
|
||||
unsigned long long data_size, bool sparse, bool preallocate,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
const char* temp_dir) {
|
||||
return file_to_disk_secure_impl(path, data, data_size, false, sparse, metadata,
|
||||
return file_to_disk_secure_impl(path, data, data_size, false, sparse, preallocate, metadata,
|
||||
preserve_executability, false, true, false, temp_dir);
|
||||
}
|
||||
|
||||
@@ -717,8 +764,9 @@ bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
||||
* (applying metadata through the shared inode would mutate the basis file).
|
||||
* Returns false only when both the link and the copy fallback fail. */
|
||||
bool file_to_disk_secure_link(const char* path, const char* basis_path, const void* data,
|
||||
unsigned long long data_size, const FileMetadata* metadata,
|
||||
bool preserve_executability, bool use_fsync, const char* temp_dir) {
|
||||
unsigned long long data_size, bool preallocate,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
bool use_fsync, const char* temp_dir) {
|
||||
if (!path || !basis_path)
|
||||
return false;
|
||||
char* leaf = NULL;
|
||||
@@ -796,8 +844,8 @@ bool file_to_disk_secure_link(const char* path, const char* basis_path, const vo
|
||||
free(leaf);
|
||||
/* The basis file could not be linked in (missing, cross-device, refused
|
||||
by the filesystem). Write a byte-identical local copy instead. */
|
||||
return file_to_disk_secure_with_fsync(path, data, data_size, false, false, metadata,
|
||||
preserve_executability, use_fsync, temp_dir);
|
||||
return file_to_disk_secure_with_fsync(path, data, data_size, false, false, preallocate,
|
||||
metadata, preserve_executability, use_fsync, temp_dir);
|
||||
}
|
||||
|
||||
if (scratch_dirfd >= 0)
|
||||
@@ -811,5 +859,5 @@ bool file_write_to_disk(const char* path, const void* data, unsigned long long d
|
||||
bool inplace, bool sparse) {
|
||||
if (!path || (!data && data_size != 0) || has_path_traversal(path))
|
||||
return false;
|
||||
return file_to_disk_secure(path, data, data_size, inplace, sparse, NULL, false, NULL);
|
||||
return file_to_disk_secure(path, data, data_size, inplace, sparse, false, NULL, false, NULL);
|
||||
}
|
||||
|
||||
+14
-9
@@ -57,28 +57,33 @@ int file_open_private_dir(const char* dir_path);
|
||||
silently copied into place. Pass NULL for the historical same-directory
|
||||
behavior. --inplace writes never use temp_dir. */
|
||||
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
|
||||
bool inplace, bool sparse, const FileMetadata* metadata,
|
||||
bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata,
|
||||
bool preserve_executability, const char* temp_dir);
|
||||
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
||||
unsigned long long data_size, bool inplace, bool sparse,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
bool use_fsync, const char* temp_dir);
|
||||
bool preallocate, const FileMetadata* metadata,
|
||||
bool preserve_executability, bool use_fsync,
|
||||
const char* temp_dir);
|
||||
/* With update enabled, an existing newer destination is left untouched. The
|
||||
check is descriptor-based for inplace writes; atomic replacement still has
|
||||
an unavoidable final rename race without filesystem locking. */
|
||||
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
|
||||
bool inplace, bool sparse, const FileMetadata* metadata,
|
||||
bool preserve_executability, const char* temp_dir);
|
||||
bool inplace, bool sparse, bool preallocate,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
const char* temp_dir);
|
||||
bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
||||
unsigned long long data_size, bool sparse,
|
||||
unsigned long long data_size, bool sparse, bool preallocate,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
const char* temp_dir);
|
||||
/* Atomic --link-dest install: replace `path` with a hard link to `basis_path`
|
||||
(via a temp name + rename); fall back to a byte-identical local copy from
|
||||
`data` when the link is impossible (EXDEV/EPERM/unsupported filesystem).
|
||||
`metadata` is applied only on the copy fallback. */
|
||||
`metadata` is applied only on the copy fallback. `preallocate` applies to
|
||||
that copy fallback only (a hard-linked file shares the basis inode and is
|
||||
never re-allocated). */
|
||||
bool file_to_disk_secure_link(const char* path, const char* basis_path, const void* data,
|
||||
unsigned long long data_size, const FileMetadata* metadata,
|
||||
bool preserve_executability, bool use_fsync, const char* temp_dir);
|
||||
unsigned long long data_size, bool preallocate,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
bool use_fsync, const char* temp_dir);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -81,11 +81,12 @@ static FileSaveResult file_stage_delayed_update(const char* root_directory,
|
||||
bool ok;
|
||||
if (file->basis_link) {
|
||||
ok = file_to_disk_secure_link(staged_path, file->basis_link, file->data->data, file->data->size,
|
||||
metadata, preserve_executability, config->use_fsync, NULL);
|
||||
config->preallocate, metadata, preserve_executability,
|
||||
config->use_fsync, NULL);
|
||||
} else {
|
||||
ok = file_to_disk_secure_with_fsync(staged_path, file->data->data, file->data->size, false,
|
||||
sparse, metadata, preserve_executability, config->use_fsync,
|
||||
NULL);
|
||||
sparse, config->preallocate, metadata,
|
||||
preserve_executability, config->use_fsync, NULL);
|
||||
}
|
||||
if (!ok) {
|
||||
free(staged_path);
|
||||
@@ -302,17 +303,20 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
|
||||
bool ok;
|
||||
if (config && file->basis_link) {
|
||||
ok = file_to_disk_secure_link(disk_path, file->basis_link, file->data->data, file->data->size,
|
||||
metadata, preserve_executability, config->use_fsync,
|
||||
confined_temp);
|
||||
config->preallocate, metadata, preserve_executability,
|
||||
config->use_fsync, confined_temp);
|
||||
} else {
|
||||
ok = config && config->ignore_existing
|
||||
? file_to_disk_secure_no_replace(disk_path, file->data->data, file->data->size, sparse,
|
||||
metadata, preserve_executability, confined_temp)
|
||||
config && config->preallocate, metadata,
|
||||
preserve_executability, confined_temp)
|
||||
: config && config->update
|
||||
? file_to_disk_secure_update(disk_path, file->data->data, file->data->size, inplace,
|
||||
sparse, metadata, preserve_executability, confined_temp)
|
||||
sparse, config && config->preallocate, metadata,
|
||||
preserve_executability, confined_temp)
|
||||
: file_to_disk_secure_with_fsync(disk_path, file->data->data, file->data->size,
|
||||
inplace, sparse, metadata, preserve_executability,
|
||||
inplace, sparse, config && config->preallocate,
|
||||
metadata, preserve_executability,
|
||||
config && config->use_fsync, confined_temp);
|
||||
}
|
||||
free(confined_temp);
|
||||
|
||||
@@ -233,6 +233,51 @@ class TestChmod:
|
||||
assert (os.stat(os.path.join(received, "small.txt")).st_mode & 0o777) == 0o644
|
||||
|
||||
|
||||
class TestPreallocate:
|
||||
"""--preallocate allocates the destination file space up front; the final
|
||||
destination content must be byte-identical to a normal run."""
|
||||
|
||||
def test_preallocate_transfer_succeeds(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "prealloc_source")
|
||||
dest = os.path.join(TEST_DATA_DIR, "prealloc_dest")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
payload = os.urandom(2 * 1024 * 1024 + 137)
|
||||
with open(os.path.join(source, "data.bin"), "wb") as f:
|
||||
f.write(payload)
|
||||
with open(os.path.join(source, "small.txt"), "wb") as f:
|
||||
f.write(b"hello\n")
|
||||
|
||||
result, _ = run_client(source, dest, flags=["--preallocate"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--preallocate failed: {(result.stderr or result.stdout)[:400]}"
|
||||
|
||||
received_dir = os.path.join(dest, os.path.abspath(source).lstrip(os.sep))
|
||||
data_path = os.path.join(received_dir, "data.bin")
|
||||
assert os.path.isfile(data_path), f"destination file not created: {data_path}"
|
||||
with open(data_path, "rb") as f:
|
||||
assert f.read() == payload, "destination content mismatch"
|
||||
small_path = os.path.join(received_dir, "small.txt")
|
||||
with open(small_path, "rb") as f:
|
||||
assert f.read() == b"hello\n", "small file content mismatch"
|
||||
|
||||
def test_preallocate_combines_with_partial(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "prealloc_partial_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "prealloc_partial_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
with open(os.path.join(source, "f.txt"), "wb") as f:
|
||||
f.write(b"partial + preallocate\n")
|
||||
result, _ = run_client(source, dest, flags=["--preallocate", "--partial"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--preallocate --partial failed: {(result.stderr or result.stdout)[:400]}"
|
||||
received_dir = os.path.join(dest, os.path.abspath(source).lstrip(os.sep))
|
||||
with open(os.path.join(received_dir, "f.txt"), "rb") as f:
|
||||
assert f.read() == b"partial + preallocate\n"
|
||||
|
||||
|
||||
class TestCompressionChoice:
|
||||
@pytest.mark.ci
|
||||
def test_zstd_choice_compresses(self, shared_server):
|
||||
|
||||
@@ -138,3 +138,7 @@ class TestSSHFeatures:
|
||||
r = _run_ssh_test("SSH Exclude (--exclude small.txt)",
|
||||
["--exclude", "small.txt"], expected_missing=["small.txt"])
|
||||
assert r["status"] == "Success", r["error"]
|
||||
|
||||
def test_preallocate(self):
|
||||
r = _run_ssh_test("SSH Preallocate (--preallocate)", ["--preallocate"])
|
||||
assert r["status"] == "Success", r["error"]
|
||||
|
||||
@@ -2237,6 +2237,18 @@ static void test_parse_args_rejects_malformed_identity() {
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 2, argv, positional_args, &positional_count), -1);
|
||||
|
||||
/* --preallocate parses as a boolean flag and validates cleanly. */
|
||||
static void test_parse_args_preallocate() {
|
||||
Config* cfg = config_create();
|
||||
cfg->send_directory = str_dup("/src");
|
||||
cfg->receive_root_directory = str_dup("/dst");
|
||||
char* argv[] = {"fastsync", "--preallocate", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->preallocate);
|
||||
EXPECT_TRUE(validate_config(cfg));
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
@@ -2248,6 +2260,7 @@ void test_client_cli() {
|
||||
test_parse_args_usermap_name_resolution();
|
||||
test_parse_args_chown();
|
||||
test_parse_args_rejects_malformed_identity();
|
||||
test_parse_args_preallocate();
|
||||
test_parse_args_append();
|
||||
test_parse_args_append_verify();
|
||||
test_parse_args_append_both();
|
||||
|
||||
@@ -1004,6 +1004,44 @@ static void test_config_receive_rejects_invalid_identity() {
|
||||
config_delete(c);
|
||||
}
|
||||
|
||||
/* --preallocate crosses the wire unchanged (receiver-side flag): the receiver
|
||||
must learn to allocate the destination file's space before data flows. */
|
||||
static void test_config_preallocate_wire_roundtrip() {
|
||||
struct {
|
||||
bool preallocate;
|
||||
} cases[] = {{false}, {true}};
|
||||
if (is_running_under_valgrind())
|
||||
return;
|
||||
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||
int p[2];
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
pid_t pid = fork();
|
||||
if (pid == 0) {
|
||||
close(p[1]);
|
||||
io_set_fds(p[0], p[0]);
|
||||
Config* recv = config_receive(p[0]);
|
||||
bool ok = recv != NULL && recv->preallocate == cases[i].preallocate;
|
||||
config_delete(recv);
|
||||
close(p[0]);
|
||||
_exit(ok ? 0 : 1);
|
||||
} else {
|
||||
close(p[0]);
|
||||
io_set_fds(p[1], p[1]);
|
||||
Config* send_cfg = config_create();
|
||||
EXPECT_NOT_NULL(send_cfg);
|
||||
send_cfg->send_directory = str_dup("/src");
|
||||
send_cfg->receive_root_directory = str_dup("/dst");
|
||||
send_cfg->preallocate = cases[i].preallocate;
|
||||
bool sent = config_send(p[1], send_cfg);
|
||||
int status;
|
||||
waitpid(pid, &status, 0);
|
||||
close(p[1]);
|
||||
config_delete(send_cfg);
|
||||
EXPECT_TRUE(sent);
|
||||
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
void test_config() {
|
||||
test_config_lifecycle();
|
||||
test_config_ssh_dest();
|
||||
@@ -1030,6 +1068,7 @@ void test_config() {
|
||||
test_config_receive_rejects_invalid_checksum_algo();
|
||||
test_config_identity_wire_roundtrip();
|
||||
test_config_receive_rejects_invalid_identity();
|
||||
test_config_preallocate_wire_roundtrip();
|
||||
}
|
||||
test_config_delete_timing_early_helper();
|
||||
test_config_is_remote_dest();
|
||||
|
||||
+40
-2
@@ -380,14 +380,50 @@ static void test_file_write_to_disk_basic() {
|
||||
static void test_file_write_to_disk_with_fsync() {
|
||||
const char* path = "test_file_write_to_disk_fsync.txt";
|
||||
const char* content = "fsync file content";
|
||||
EXPECT_TRUE(file_to_disk_secure_with_fsync(path, content, strlen(content), false, false, NULL,
|
||||
false, true, NULL));
|
||||
EXPECT_TRUE(file_to_disk_secure_with_fsync(path, content, strlen(content), false, false, false,
|
||||
NULL, false, true, NULL));
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
|
||||
unlink(path);
|
||||
}
|
||||
|
||||
static void test_file_write_to_disk_preallocate_atomic() {
|
||||
const char* path = "test_file_write_prealloc_atomic.txt";
|
||||
const char* content = "prealloc atomic content";
|
||||
EXPECT_TRUE(file_to_disk_secure(path, content, strlen(content), false, false, true, NULL, false,
|
||||
NULL));
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
|
||||
FILE* fp = fopen(path, "rb");
|
||||
EXPECT_NOT_NULL(fp);
|
||||
char buf[100];
|
||||
size_t nread = fread(buf, 1, sizeof(buf), fp);
|
||||
fclose(fp);
|
||||
EXPECT_EQ_INT((int)nread, (int)strlen(content));
|
||||
EXPECT_EQ_INT(memcmp(buf, content, strlen(content)), 0);
|
||||
unlink(path);
|
||||
}
|
||||
|
||||
static void test_file_write_to_disk_preallocate_inplace() {
|
||||
const char* path = "test_file_write_prealloc_inplace.txt";
|
||||
const char* content = "prealloc inplace content";
|
||||
EXPECT_TRUE(file_to_disk_secure(path, content, strlen(content), true, false, true, NULL, false,
|
||||
NULL));
|
||||
struct stat st;
|
||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
|
||||
FILE* fp = fopen(path, "rb");
|
||||
EXPECT_NOT_NULL(fp);
|
||||
char buf[100];
|
||||
size_t nread = fread(buf, 1, sizeof(buf), fp);
|
||||
fclose(fp);
|
||||
EXPECT_EQ_INT((int)nread, (int)strlen(content));
|
||||
EXPECT_EQ_INT(memcmp(buf, content, strlen(content)), 0);
|
||||
unlink(path);
|
||||
}
|
||||
|
||||
static void test_file_write_to_disk_creates_dirs() {
|
||||
const char* content = "Nested dir test";
|
||||
EXPECT_TRUE(file_write_to_disk("test_nested_tmp/nested/file.txt", content, strlen(content), false,
|
||||
@@ -937,6 +973,8 @@ void test_file() {
|
||||
test_file_save_to_disk_reports_skips();
|
||||
test_file_write_to_disk_basic();
|
||||
test_file_write_to_disk_with_fsync();
|
||||
test_file_write_to_disk_preallocate_atomic();
|
||||
test_file_write_to_disk_preallocate_inplace();
|
||||
test_file_write_to_disk_creates_dirs();
|
||||
test_file_write_to_disk_does_not_follow_symlink();
|
||||
test_file_content_to_buffer();
|
||||
|
||||
Reference in New Issue
Block a user