TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat: add -x/--one-file-system to stay within the source filesystem

Browse Source 
Capture the transfer root's device (st_dev) at scanner creation and skip
descending into any subdirectory on a different device (a mount point).
Implemented sender/client-side only: sequential BFS and parallel (-m) root
scan apply the same scanner_same_filesystem decision; no wire/protocol change
and default behavior is unchanged. Unit tests cover the pure decision, same
device scanning in both modes, and CLI parsing; integration tests prove -x
leaves a single-filesystem tree byte-identical and, when root can mount a
tmpfs, skips a genuine cross-device subtree.
This commit is contained in:
TapTap
2026-09-06 12:18:43 +02:00
parent 06e83f2402
commit 2bcc20aa08
11 changed files with 274 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
RSYNC_COMPAT.md
+1 -1
View File
@@ -64,7 +64,7 @@ This document maps rsync's full feature set to FastSync's current implementation
| `--existing` | Skip creating new files on receiver | ✅ Implemented | Existing destination files continue through normal update handling |
| `--ignore-existing` | Skip updating existing files | ❌ Not Implemented | |
| `--remove-source-files` | Sender removes regular files after confirmed transfer | ✅ Implemented | |
| `-x`, `--one-file-system` | Do not cross filesystem boundaries | ❌ Not Implemented | |
| `-x`, `--one-file-system` | Do not cross filesystem boundaries | ✅ Implemented | Sender scanner captures the root device and skips descending into mount-point crossings (`st_dev` differs) |
| `-F` | Add the default `.rsync-filter` rules | ❌ Not Implemented | |
## 4. Directory Options
src/client/client_cli.c
+1
View File
@@ -416,6 +416,7 @@ static const OptionEntry OPTION_TABLE[] = {
{"--max-size", NULL, OPT_ULL, offsetof(Config, max_size)},
{"--min-size", NULL, OPT_ULL, offsetof(Config, min_size)},
{"--one-file-system", "-x", OPT_FLAG, offsetof(Config, one_file_system)},
};
/* Only boolean options with no required argument are safe to negate. */
src/client/client_send.c
+8 -6
View File
@@ -40,12 +40,14 @@ static const char* display_bytes(unsigned long long bytes, bool human_readable,
}
static ScannerOptions scanner_options_from_config(const Config* config, int num_threads) {
ScannerOptions options = {
config->use_metadata, config->chunk_size, config->exclude_patterns,
config->exclude_count, config->include_patterns, config->include_count,
config->max_size, config->min_size, config->max_depth,
num_threads, config->follow_symlinks, config->copy_links,
config->safe_links, config->copy_unsafe_links, config->checksum};
ScannerOptions options = {config->use_metadata, config->chunk_size,
config->exclude_patterns, config->exclude_count,
config->include_patterns, config->include_count,
config->max_size, config->min_size,
config->max_depth, num_threads,
config->follow_symlinks, config->copy_links,
config->safe_links, config->copy_unsafe_links,
config->checksum, config->one_file_system};
return options;
}
src/client/scanner.c
+53 -10
View File
@@ -59,6 +59,13 @@ typedef struct {
bool is_directory;
} ScannerEntry;
/* --one-file-system (-x) decision. Only directories can carry a different
* device than their parent (mount points), so this is checked when a child
* directory is about to be descended into. */
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device) {
return !one_file_system || entry_device == root_device;
}
/* Inspect symlinks, resolve the entry type, and apply file filters once for both scanners. */
static int scanner_inspect_entry(const ScannerOptions* options, const char* source_root,
const char* containing_dir, const char* name,
@@ -156,7 +163,18 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
scanner->safe_links = options->safe_links;
scanner->copy_unsafe_links = options->copy_unsafe_links;
scanner->checksum = options->checksum;
scanner->one_file_system = options->one_file_system;
scanner->failed = false;
if (scanner->one_file_system) {
struct stat root_stats;
if (stat(root_directory, &root_stats) != 0) {
log_perror("Could not stat source directory");
queue_destroy(scanner->directories);
free(scanner);
return NULL;
}
scanner->root_dev = root_stats.st_dev;
}
DirEntry* root = dir_entry_create(root_directory, 0);
if (!root) {
queue_destroy(scanner->directories);
@@ -179,10 +197,14 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
unsigned long long min_size, int max_depth,
bool follow_symlinks, bool copy_links, bool safe_links,
bool copy_unsafe_links, bool checksum) {
ScannerOptions options = {
use_metadata, chunk_size, exclude_patterns, exclude_count, include_patterns,
include_count, max_size, min_size, max_depth, 0,
follow_symlinks, copy_links, safe_links, copy_unsafe_links, checksum};
ScannerOptions options = {use_metadata, chunk_size,
exclude_patterns, exclude_count,
include_patterns, include_count,
max_size, min_size,
max_depth, 0,
follow_symlinks, copy_links,
safe_links, copy_unsafe_links,
checksum, false};
return directory_scanner_create_with_options(root_directory, &options);
}
@@ -272,7 +294,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
scanner->max_depth, 0,
scanner->follow_symlinks, scanner->copy_links,
scanner->safe_links, scanner->copy_unsafe_links,
scanner->checksum};
scanner->checksum, scanner->one_file_system};
ScannerEntry inspected;
int inspection = scanner_inspect_entry(&options, scanner->current_path, scanner->current_path,
entry->d_name, &inspected);
@@ -286,6 +308,10 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
struct stat stats = inspected.stats;
if (inspected.is_directory) {
if (!scanner_same_filesystem(scanner->one_file_system, scanner->root_dev, stats.st_dev)) {
free(cur_path);
continue;
}
int next_depth = scanner->current_depth + 1;
if (scanner->max_depth <= 0 || next_depth < scanner->max_depth) {
DirEntry* de = dir_entry_create(cur_path, next_depth);
@@ -525,7 +551,7 @@ static Chunk* batch_files(ArrayList* files, unsigned long long chunk_size, Queue
/* Scan one root-directory entry into either the subdirs or files list. */
static void scan_root_entry(const ScannerOptions* options, const char* root_directory,
const struct dirent* entry, ArrayList* root_files, ArrayList* subdirs,
ParallelScanner* ps) {
dev_t root_dev, ParallelScanner* ps) {
ScannerEntry inspected;
int inspection =
scanner_inspect_entry(options, root_directory, root_directory, entry->d_name, &inspected);
@@ -538,6 +564,10 @@ static void scan_root_entry(const ScannerOptions* options, const char* root_dire
char* cur_path = inspected.path;
struct stat st = inspected.stats;
if (inspected.is_directory) {
if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) {
free(cur_path);
return;
}
if (!array_list_add(subdirs, cur_path)) {
free(cur_path);
ps->failed = true;
@@ -567,8 +597,8 @@ static void scan_root_entry(const ScannerOptions* options, const char* root_dire
/* Scan the root directory itself, collecting root files and subdirectories.
* Returns false if the root directory could not be opened. */
static bool scan_root_directory(ParallelScanner* ps, const char* root_directory,
const ScannerOptions* options, ArrayList* root_files,
ArrayList* subdirs) {
const ScannerOptions* options, dev_t root_dev,
ArrayList* root_files, ArrayList* subdirs) {
DIR* dir = opendir(root_directory);
if (!dir) {
log_perror("Could not open root directory for parallel scan");
@@ -578,7 +608,7 @@ static bool scan_root_directory(ParallelScanner* ps, const char* root_directory,
while ((entry = readdir(dir)) != NULL) {
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
continue;
scan_root_entry(options, root_directory, entry, root_files, subdirs, ps);
scan_root_entry(options, root_directory, entry, root_files, subdirs, root_dev, ps);
}
closedir(dir);
return true;
@@ -677,7 +707,20 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
return NULL;
}
if (!scan_root_directory(ps, root_directory, options, root_files, subdirs)) {
dev_t root_dev = 0;
if (options->one_file_system) {
struct stat root_stats;
if (stat(root_directory, &root_stats) != 0) {
log_perror("Could not stat source directory");
array_list_delete(root_files);
array_list_delete(subdirs);
parallel_scanner_destroy(ps);
return NULL;
}
root_dev = root_stats.st_dev;
}
if (!scan_root_directory(ps, root_directory, options, root_dev, root_files, subdirs)) {
array_list_delete(root_files);
array_list_delete(subdirs);
parallel_scanner_destroy(ps);
src/client/scanner.h
+10 -1
View File
@@ -6,8 +6,9 @@
#include "queue.h"
#include <dirent.h>
#include <stdbool.h>
#include <threads.h>
#include <stdatomic.h>
#include <sys/types.h>
#include <threads.h>
typedef struct {
bool use_metadata;
@@ -25,6 +26,7 @@ typedef struct {
bool safe_links;
bool copy_unsafe_links;
bool checksum;
bool one_file_system;
} ScannerOptions;
typedef struct {
@@ -46,6 +48,8 @@ typedef struct {
bool safe_links;
bool copy_unsafe_links;
bool checksum;
bool one_file_system;
dev_t root_dev;
bool failed;
} DirectoryScanner;
@@ -79,6 +83,11 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner);
bool directory_scanner_failed(const DirectoryScanner* scanner);
void directory_scanner_destroy(DirectoryScanner* scanner);
/* --one-file-system (-x) decision: a directory entry may be descended into
* only when the option is disabled or the entry lives on the same device as
* the transfer root. Exposed so tests can exercise the rule directly. */
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device);
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
const ScannerOptions* options,
ProtocolSession* allocation_session);
src/client/usage.c
+1
View File
@@ -83,6 +83,7 @@ void print_usage(void) {
printf(" --stats Print transfer statistics at end\n");
printf(" -h, --human-readable Print byte sizes in human-readable form\n");
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
printf(" -x, --one-file-system Do not cross filesystem boundaries\n");
printf(" --log-file <path> Write log messages to file\n");
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
printf(" --partial Keep partial files on interrupted transfer\n");
src/shared/config.c
+1
View File
@@ -93,6 +93,7 @@ static void config_set_defaults(Config* config) {
config->files_from = NULL;
config->cvs_exclude = false;
config->prune_empty_dirs = false;
config->one_file_system = false;
config->relative = false;
config->rsh_command = NULL;
config->rsync_path = NULL;
src/shared/config.h
+1
View File
@@ -104,6 +104,7 @@ typedef struct Config {
char* files_from;
bool cvs_exclude;
bool prune_empty_dirs;
bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */
bool relative;
// Issue #130: Remote shell/connection options
tests/integration/test_features.py
+73
View File
@@ -2,6 +2,7 @@
import filecmp
import os
import shutil
import subprocess
import sys
import time
import pytest
@@ -1021,3 +1022,75 @@ class TestLargeFile:
assert result.returncode == 0, f"Large-file sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert filecmp.cmp(source_file, os.path.join(received, "big.bin"), shallow=False)
class TestOneFileSystem:
def _make_tree(self, source):
clean_dir(source)
os.makedirs(os.path.join(source, "nested", "deeper"))
with open(os.path.join(source, "root.txt"), "wb") as f:
f.write(b"root")
with open(os.path.join(source, "nested", "inner.txt"), "wb") as f:
f.write(b"inner")
with open(os.path.join(source, "nested", "deeper", "deep.txt"), "wb") as f:
f.write(b"deep")
def _assert_full_tree_transferred(self, source, dest, port, flags):
clean_dir(dest)
result, _ = run_client(source, dest, flags=flags, port=port)
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
mismatches, missing = verify_transfer(source, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
def test_x_transfer_matches_plain_over_single_filesystem(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "ofs_src")
self._make_tree(source)
self._assert_full_tree_transferred(source, os.path.join(TEST_DATA_DIR, "ofs_dst"),
shared_server.port, ["-x"])
def test_x_multithreaded_transfer_matches_plain_over_single_filesystem(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "ofs_m_src")
self._make_tree(source)
self._assert_full_tree_transferred(source, os.path.join(TEST_DATA_DIR, "ofs_m_dst"),
shared_server.port, ["-m", "--one-file-system"])
def test_x_skips_other_device_mountpoint(self, shared_server):
if os.geteuid() != 0 or shutil.which("mount") is None or shutil.which("umount") is None:
pytest.skip("cross-device test requires root and mount(8)")
source = os.path.join(TEST_DATA_DIR, "ofs_mnt_src")
dest = os.path.join(TEST_DATA_DIR, "ofs_mnt_dst")
dest_plain = os.path.join(TEST_DATA_DIR, "ofs_mnt_plain_dst")
mountpoint = os.path.join(source, "external")
clean_dir(source)
os.makedirs(mountpoint)
os.makedirs(os.path.join(source, "nested"))
with open(os.path.join(source, "root.txt"), "wb") as f:
f.write(b"root")
with open(os.path.join(source, "nested", "inner.txt"), "wb") as f:
f.write(b"inner")
mounted = False
try:
mount = subprocess.run(["mount", "-t", "tmpfs", "tmpfs", mountpoint],
capture_output=True, text=True)
if mount.returncode != 0:
pytest.skip(f"cannot mount tmpfs: {mount.stderr.strip()}")
mounted = True
with open(os.path.join(mountpoint, "away.txt"), "wb") as f:
f.write(b"cross device")
result, _ = run_client(source, dest, flags=["-x"], port=shared_server.port)
assert result.returncode == 0, f"-x sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert os.path.isfile(os.path.join(received, "root.txt"))
assert os.path.isfile(os.path.join(received, "nested", "inner.txt"))
assert not os.path.exists(os.path.join(received, "external", "away.txt")), \
"-x must not cross into the mounted filesystem"
result, _ = run_client(source, dest_plain, port=shared_server.port)
assert result.returncode == 0, f"plain sync failed: {result.stderr[:200]}"
received_plain = get_dest_received_dir(dest_plain, source)
assert os.path.isfile(os.path.join(received_plain, "external", "away.txt")), \
"without -x the mounted subtree must be transferred"
finally:
if mounted:
subprocess.run(["umount", mountpoint], capture_output=True, text=True)
tests/test_client_cli.c
+30
View File
@@ -985,6 +985,35 @@ static void test_parse_args_whole_file() {
config_delete(cfg);
}
/* -x and --one-file-system enable client-side single-filesystem scanning. */
static void test_parse_args_one_file_system() {
Config* cfg = config_create();
EXPECT_FALSE(cfg->one_file_system);
char* argv[] = {"fastsync", "-x", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->one_file_system);
EXPECT_EQ_INT(positional_count, 2);
config_delete(cfg);
cfg = config_create();
char* long_argv[] = {"fastsync", "--one-file-system", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, long_argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->one_file_system);
config_delete(cfg);
cfg = config_create();
/* Flags never take a value: the "=value" form must be rejected. */
char* bad_argv[] = {"fastsync", "--one-file-system=yes", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, bad_argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
/* Test rsync-compatible compression-choice and compression-level aliases. */
static void test_parse_args_compression_aliases() {
Config* cfg = config_create();
@@ -1213,6 +1242,7 @@ void test_client_cli() {
test_parse_args_secluded_args();
test_parse_args_short_s_remains_chunk_serialization();
test_parse_args_whole_file();
test_parse_args_one_file_system();
test_parse_args_compression_aliases();
test_parse_args_compression_equals_and_none();
test_parse_args_compression_canonical_equals();
tests/test_scanner.c
+95 -2
View File
@@ -394,8 +394,8 @@ static void test_parallel_scanner_root_chunks_without_workers() {
create_test_file(file1, "a");
create_test_file(file2, "b");
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0,
0, 0, false, false, false, false, false};
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0,
0, 0, false, false, false, false, false, false};
ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options, NULL);
EXPECT_NOT_NULL(scanner);
@@ -414,6 +414,96 @@ static void test_parallel_scanner_root_chunks_without_workers() {
rmdir(dir);
}
/* --one-file-system (-x) decision is a pure device comparison. */
static void test_scanner_one_file_system_decision() {
/* Option disabled: every device is allowed (unchanged default behavior). */
EXPECT_TRUE(scanner_same_filesystem(false, 0, 123));
EXPECT_TRUE(scanner_same_filesystem(false, 7, 999));
/* Option enabled: only entries on the root device may be descended into. */
EXPECT_TRUE(scanner_same_filesystem(true, 7, 7));
EXPECT_FALSE(scanner_same_filesystem(true, 7, 8));
}
/* With -x over an ordinary tree (all one device) nothing may be skipped. */
static void test_scanner_one_file_system_same_device() {
const char* root = "test_scan_ofs";
const char* sub = "test_scan_ofs/sub";
const char* deeper = "test_scan_ofs/sub/deeper";
const char* root_file = "test_scan_ofs/root.txt";
const char* sub_file = "test_scan_ofs/sub/inner.txt";
const char* deep_file = "test_scan_ofs/sub/deeper/deep.txt";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
EXPECT_EQ_INT(mkdir(deeper, 0755), 0);
create_test_file(root_file, "root");
create_test_file(sub_file, "inner");
create_test_file(deep_file, "deep");
ScannerOptions options = {0};
options.one_file_system = true;
DirectoryScanner* scanner = directory_scanner_create_with_options(root, &options);
EXPECT_NOT_NULL(scanner);
int total_files = 0;
Chunk* chunk;
while ((chunk = directory_scanner_next(scanner)) != NULL) {
total_files += chunk->element_count;
chunk_destroy(chunk);
}
EXPECT_EQ_INT(total_files, 3);
EXPECT_FALSE(directory_scanner_failed(scanner));
directory_scanner_destroy(scanner);
unlink(root_file);
unlink(sub_file);
unlink(deep_file);
rmdir(deeper);
rmdir(sub);
rmdir(root);
}
/* Multithreaded (-m) scan with -x over a single-device tree must match the
* single-threaded result. */
static void test_parallel_scanner_one_file_system_same_device() {
const char* root = "test_parallel_scan_ofs";
const char* sub = "test_parallel_scan_ofs/sub";
const char* sub2 = "test_parallel_scan_ofs/sub2";
const char* root_file = "test_parallel_scan_ofs/root.txt";
const char* sub_file = "test_parallel_scan_ofs/sub/inner.txt";
const char* sub2_file = "test_parallel_scan_ofs/sub2/inner2.txt";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
EXPECT_EQ_INT(mkdir(sub2, 0755), 0);
create_test_file(root_file, "root");
create_test_file(sub_file, "inner");
create_test_file(sub2_file, "inner2");
ScannerOptions options = {0};
options.one_file_system = true;
options.num_threads = 2;
ParallelScanner* scanner = parallel_scanner_create_with_options(root, &options, NULL);
EXPECT_NOT_NULL(scanner);
int total_files = 0;
Chunk* chunk;
while ((chunk = parallel_scanner_next(scanner)) != NULL) {
total_files += chunk->element_count;
chunk_destroy(chunk);
}
EXPECT_EQ_INT(total_files, 3);
EXPECT_FALSE(parallel_scanner_failed(scanner));
parallel_scanner_destroy(scanner);
unlink(root_file);
unlink(sub_file);
unlink(sub2_file);
rmdir(sub);
rmdir(sub2);
rmdir(root);
}
void test_scanner() {
test_scanner_single_file();
test_scanner_multiple_files();
@@ -429,4 +519,7 @@ void test_scanner() {
test_scanner_mixed_patterns();
test_scanner_no_patterns();
test_parallel_scanner_root_chunks_without_workers();
test_scanner_one_file_system_decision();
test_scanner_one_file_system_same_device();
test_parallel_scanner_one_file_system_same_device();
}