TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: implement rsync --temp-dir for atomic receiver installs

Browse Source
This commit is contained in:
TapTap
2026-09-06 12:21:36 +02:00
parent 06e83f2402
commit 19e6fc2205
10 changed files with 279 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/client_cli.c
+1
View File
@@ -405,6 +405,7 @@ static const OptionEntry OPTION_TABLE[] = {
{"--ca", NULL, OPT_STRING, offsetof(Config, tls_ca)},
{"--backup-dir", NULL, OPT_STRING, offsetof(Config, backup_dir)},
{"--fastsync-server-path", NULL, OPT_STRING, offsetof(Config, fastsync_server_path)},
{"--temp-dir", NULL, OPT_STRING, offsetof(Config, temp_dir)},
{"--partial-dir", NULL, OPT_STRING, offsetof(Config, partial_dir)},
{"--suffix", NULL, OPT_STRING, offsetof(Config, suffix)},
{"--compress-choice", "--zc", OPT_STRING, offsetof(Config, compress_choice)},
src/client/usage.c
+1
View File
@@ -87,6 +87,7 @@ void print_usage(void) {
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
printf(" --partial Keep partial files on interrupted transfer\n");
printf(" --partial-dir <dir> Directory for partial files\n");
printf(" --temp-dir <dir> Scratch dir for temp files before atomic install\n");
printf(" --fastsync-server-path <path>\n");
printf(" Path to fastsync-server on remote (default: fastsync-server)\n");
printf(
src/shared/file.c
+81 -16
View File
@@ -2,6 +2,7 @@
#include <fcntl.h>
#include <libgen.h>
#include <limits.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -30,6 +31,17 @@ static bool write_all(int fd, const void* data, unsigned long long size) {
return true;
}
/* Process-wide counter for scratch temp names. A --temp-dir scratch directory
is flat: different destinations that share a basename must never race onto
the same temp name. Deriving the trailing number from a global atomic
sequence keeps every temp name unique across the whole scratch directory
even when several threads write concurrently, so the O_EXCL creation loop
below almost never needs a retry. */
static unsigned long long next_temp_sequence(void) {
static atomic_ullong sequence;
return atomic_fetch_add_explicit(&sequence, 1, memory_order_relaxed);
}
bool file_checksum(File* file, uint64_t* checksum) {
if (!file || !checksum || !file->data)
return false;
@@ -326,10 +338,34 @@ bool file_rename_secure(const char* old_path, const char* new_path) {
return ok;
}
/* Open the configured --temp-dir scratch directory, creating it (and any
missing path components) on demand. scratch_path is expected to already be
confined below the authorized root by the caller; file_open_secure_parent
re-checks that confinement and rejects `..` components, so a scratch
directory can never be created or opened outside the destination root.
Returns an O_DIRECTORY|O_NOFOLLOW fd, or -1 on error. */
static int file_open_scratch_dir(const char* scratch_path) {
if (!scratch_path)
return -1;
char* leaf = NULL;
int parent_fd = file_open_secure_parent(scratch_path, &leaf, true);
if (parent_fd < 0)
return -1;
int fd = openat(parent_fd, leaf, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (fd < 0 && errno == ENOENT) {
if (mkdirat(parent_fd, leaf, 0755) == 0 || errno == EEXIST)
fd = openat(parent_fd, leaf, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
}
close(parent_fd);
free(leaf);
return fd;
}
static bool file_to_disk_secure_impl(const char* path, const void* data,
unsigned long long data_size, bool inplace, bool sparse,
const FileMetadata* metadata, bool preserve_executability,
bool update, bool no_replace, bool use_fsync) {
bool update, bool no_replace, bool use_fsync,
const char* temp_dir) {
char* leaf = NULL;
int dirfd = file_open_secure_parent(path, &leaf, true);
if (dirfd < 0)
@@ -337,6 +373,8 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
int fd = -1;
bool ok = false;
if (inplace) {
/* --inplace writes directly into the destination; a scratch --temp-dir
does not apply and must never redirect these writes. */
fd = openat(dirfd, leaf, O_WRONLY | O_CREAT | O_CLOEXEC | O_NOFOLLOW, 0644);
if (fd >= 0) {
struct stat destination_stat;
@@ -372,21 +410,39 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
}
}
} else {
/* Scratch directory for the temporary working copy. When NULL the temp
file is created in the destination directory, exactly as historically. */
int scratch_dirfd = temp_dir ? file_open_scratch_dir(temp_dir) : -1;
char tmp[NAME_MAX];
if (temp_dir && scratch_dirfd < 0) {
close(dirfd);
free(leaf);
return false;
}
if (update && metadata) {
/* This check protects the normal atomic path as far as possible. A
concurrent replacement can still occur before the final rename. */
struct stat destination_stat;
if (fstatat(dirfd, leaf, &destination_stat, AT_SYMLINK_NOFOLLOW) == 0 &&
S_ISREG(destination_stat.st_mode) && stat_is_newer(&destination_stat, metadata)) {
if (scratch_dirfd >= 0)
close(scratch_dirfd);
close(dirfd);
free(leaf);
return true;
}
}
for (unsigned int i = 0; i < 100 && !ok; ++i) {
snprintf(tmp, sizeof(tmp), ".%s.tmp.%ld.%u", leaf, (long)getpid(), i);
fd = openat(dirfd, tmp, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600);
/* The temp name is created inside the scratch directory (when one is
configured) and, on success, atomically renamed into the destination
directory. In a shared scratch directory the sequence number keeps
the name unique even for destinations with a common basename. */
if (scratch_dirfd >= 0)
snprintf(tmp, sizeof(tmp), ".%s.tmp.%ld.%llu", leaf, (long)getpid(), next_temp_sequence());
else
snprintf(tmp, sizeof(tmp), ".%s.tmp.%ld.%u", leaf, (long)getpid(), i);
fd = openat(scratch_dirfd >= 0 ? scratch_dirfd : dirfd, tmp,
O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600);
if (fd < 0)
continue;
if (sparse && data_size > 0)
@@ -404,19 +460,27 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
if (no_replace) {
/* The probe and commit cannot be one operation. A concurrent
creator may win; EEXIST is then the requested skip. */
if (linkat(dirfd, tmp, dirfd, leaf, 0) == 0 || errno == EEXIST) {
if (unlinkat(dirfd, tmp, 0) != 0 && errno != ENOENT)
if (linkat(scratch_dirfd >= 0 ? scratch_dirfd : dirfd, tmp, dirfd, leaf, 0) == 0 ||
errno == EEXIST) {
if (unlinkat(scratch_dirfd >= 0 ? scratch_dirfd : dirfd, tmp, 0) != 0 &&
errno != ENOENT)
ok = false;
} else {
ok = false;
}
} else if (renameat(dirfd, tmp, dirfd, leaf) != 0) {
} else if (renameat(scratch_dirfd >= 0 ? scratch_dirfd : dirfd, tmp, dirfd, leaf) != 0) {
if (scratch_dirfd >= 0 && errno == EXDEV)
log_message(LOG_LEVEL_ERROR,
"temp dir is on a different filesystem than the destination; cannot "
"atomically install file (EXDEV); no fallback copy is attempted");
ok = false;
}
}
if (!ok)
unlinkat(dirfd, tmp, 0);
unlinkat(scratch_dirfd >= 0 ? scratch_dirfd : dirfd, tmp, 0);
}
if (scratch_dirfd >= 0)
close(scratch_dirfd);
}
if (fd >= 0)
close(fd);
@@ -427,36 +491,37 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse, const FileMetadata* metadata,
bool preserve_executability) {
bool preserve_executability, const char* temp_dir) {
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, metadata,
preserve_executability, false, false, false);
preserve_executability, false, false, false, temp_dir);
}
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse, const FileMetadata* metadata,
bool preserve_executability) {
bool preserve_executability, const char* temp_dir) {
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, metadata,
preserve_executability, true, false, false);
preserve_executability, true, false, false, temp_dir);
}
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
unsigned long long data_size, bool inplace, bool sparse,
const FileMetadata* metadata, bool preserve_executability,
bool use_fsync) {
bool use_fsync, const char* temp_dir) {
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, metadata,
preserve_executability, false, false, use_fsync);
preserve_executability, false, false, use_fsync, temp_dir);
}
bool file_to_disk_secure_no_replace(const char* path, const void* data,
unsigned long long data_size, bool sparse,
const FileMetadata* metadata, bool preserve_executability) {
const FileMetadata* metadata, bool preserve_executability,
const char* temp_dir) {
return file_to_disk_secure_impl(path, data, data_size, false, sparse, metadata,
preserve_executability, false, true, false);
preserve_executability, false, true, false, temp_dir);
}
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse) {
if (!path || (!data && data_size != 0) || has_path_traversal(path))
return false;
return file_to_disk_secure(path, data, data_size, inplace, sparse, NULL, false);
return file_to_disk_secure(path, data, data_size, inplace, sparse, NULL, false, NULL);
}
src/shared/file.h
+15 -4
View File
@@ -31,21 +31,32 @@ bool file_destination_is_newer_secure(const char* path, const FileMetadata* meta
int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs);
bool file_ensure_directory_secure(const char* path);
bool file_rename_secure(const char* old_path, const char* new_path);
/* The file_to_disk_secure* variants write a temporary copy in the destination
directory and atomically rename it over `path`. temp_dir is an absolute,
root-confined scratch directory (already validated by the caller): when it
is non-NULL the temporary copy is instead created there (with a name unique
across the whole scratch directory) and atomically renamed into the
destination directory once fully written and fsynced. A rename across
filesystems (EXDEV) fails the write with an error; the file is never
silently copied into place. Pass NULL for the historical same-directory
behavior. --inplace writes never use temp_dir. */
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse, const FileMetadata* metadata,
bool preserve_executability);
bool preserve_executability, const char* temp_dir);
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
unsigned long long data_size, bool inplace, bool sparse,
const FileMetadata* metadata, bool preserve_executability,
bool use_fsync);
bool use_fsync, const char* temp_dir);
/* With update enabled, an existing newer destination is left untouched. The
check is descriptor-based for inplace writes; atomic replacement still has
an unavoidable final rename race without filesystem locking. */
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse, const FileMetadata* metadata,
bool preserve_executability);
bool preserve_executability, const char* temp_dir);
bool file_to_disk_secure_no_replace(const char* path, const void* data,
unsigned long long data_size, bool sparse,
const FileMetadata* metadata, bool preserve_executability);
const FileMetadata* metadata, bool preserve_executability,
const char* temp_dir);
#endif
src/shared/file_receive.c
+38 -11
View File
@@ -37,6 +37,7 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
const char* backup_suffix = (config && config->suffix) ? config->suffix : "~";
const char* backup_dir = (config && config->backup_dir) ? config->backup_dir : NULL;
const char* partial_dir = (config && config->partial_dir) ? config->partial_dir : NULL;
const char* temp_dir = (config && config->temp_dir) ? config->temp_dir : NULL;
bool use_partial_root = partial_dir && config && config->partial;
char *confined_backup = NULL, *confined_partial = NULL, *disk_path = NULL;
char* destination_path = NULL;
@@ -52,9 +53,13 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
}
/* These options arrive from the client. They are names below the server
root, never independent filesystem roots. */
root, never independent filesystem roots. --temp-dir is confined exactly
like --backup-dir/--partial-dir: an absolute or `..`-escaping scratch
directory is rejected outright so nothing is ever created outside the
authorized destination root. */
if ((backup_dir && (backup_dir[0] == '/' || has_path_traversal(backup_dir))) ||
(partial_dir && (partial_dir[0] == '/' || has_path_traversal(partial_dir))))
(partial_dir && (partial_dir[0] == '/' || has_path_traversal(partial_dir))) ||
(temp_dir && (temp_dir[0] == '/' || has_path_traversal(temp_dir))))
return FILE_SAVE_ERROR;
if (backup_dir && !(confined_backup = path_cat(root_directory, backup_dir)))
return FILE_SAVE_ERROR;
@@ -151,15 +156,37 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
goto fail;
metadata = &adjusted_metadata;
}
bool ok = config && config->ignore_existing
? file_to_disk_secure_no_replace(disk_path, file->data->data, file->data->size,
sparse, metadata, preserve_executability)
: config && config->update
? file_to_disk_secure_update(disk_path, file->data->data, file->data->size, inplace,
sparse, metadata, preserve_executability)
: file_to_disk_secure_with_fsync(disk_path, file->data->data, file->data->size,
inplace, sparse, metadata, preserve_executability,
config && config->use_fsync);
/* A configured --temp-dir sends the temporary working copy to a scratch
directory resolved below the receive root; the engine then atomically
renames the completed file into the final destination directory. The
partial-dir flow already keeps its working copy in a separate directory
and --inplace writes directly, so neither diverts through the scratch
dir (matching rsync, where --inplace/--partial-dir supersede --temp-dir). */
char* confined_temp = NULL;
bool use_temp_dir = temp_dir != NULL && !inplace && !use_partial_root;
if (use_temp_dir) {
confined_temp = path_cat(root_directory, temp_dir);
if (!confined_temp)
goto fail;
/* A user-supplied trailing slash would leave the scratch path ending in
"/", which has no final component to create/open. Normalize it away. */
size_t temp_len = strlen(confined_temp);
while (temp_len > 1 && confined_temp[temp_len - 1] == '/')
confined_temp[--temp_len] = '\0';
}
bool ok =
config && config->ignore_existing
? file_to_disk_secure_no_replace(disk_path, file->data->data, file->data->size, sparse,
metadata, preserve_executability, confined_temp)
: config && config->update
? file_to_disk_secure_update(disk_path, file->data->data, file->data->size, inplace,
sparse, metadata, preserve_executability, confined_temp)
: file_to_disk_secure_with_fsync(disk_path, file->data->data, file->data->size, inplace,
sparse, metadata, preserve_executability,
config && config->use_fsync, confined_temp);
free(confined_temp);
confined_temp = NULL;
if (!ok)
goto fail;