Merge feat/p2-one-file-system: implement -x/--one-file-system

Sender-side scanner stays within the source filesystem (-x), single and
multithreaded; default unchanged; client-only, no wire change. Reviewed
(c-review APPROVE WITH NITS, all fixed); PR #261.
This commit is contained in:
2026-09-06 12:51:12 +02:00
11 changed files with 431 additions and 20 deletions
+1 -1
View File
@@ -64,7 +64,7 @@ This document maps rsync's full feature set to FastSync's current implementation
| `--existing` | Skip creating new files on receiver | ✅ Implemented | Existing destination files continue through normal update handling | | `--existing` | Skip creating new files on receiver | ✅ Implemented | Existing destination files continue through normal update handling |
| `--ignore-existing` | Skip updating existing files | ❌ Not Implemented | | | `--ignore-existing` | Skip updating existing files | ❌ Not Implemented | |
| `--remove-source-files` | Sender removes regular files after confirmed transfer | ✅ Implemented | | | `--remove-source-files` | Sender removes regular files after confirmed transfer | ✅ Implemented | |
| `-x`, `--one-file-system` | Do not cross filesystem boundaries | ❌ Not Implemented | | | `-x`, `--one-file-system` | Do not cross filesystem boundaries | ✅ Implemented | Sender scanner captures the root device and skips descending into mount-point crossings (`st_dev` differs); cross-filesystem mount-point subdirectories are dropped entirely, matching rsync |
| `-F` | Add the default `.rsync-filter` rules | ❌ Not Implemented | | | `-F` | Add the default `.rsync-filter` rules | ❌ Not Implemented | |
## 4. Directory Options ## 4. Directory Options
+1
View File
@@ -416,6 +416,7 @@ static const OptionEntry OPTION_TABLE[] = {
{"--max-size", NULL, OPT_ULL, offsetof(Config, max_size)}, {"--max-size", NULL, OPT_ULL, offsetof(Config, max_size)},
{"--min-size", NULL, OPT_ULL, offsetof(Config, min_size)}, {"--min-size", NULL, OPT_ULL, offsetof(Config, min_size)},
{"--one-file-system", "-x", OPT_FLAG, offsetof(Config, one_file_system)},
}; };
/* Only boolean options with no required argument are safe to negate. */ /* Only boolean options with no required argument are safe to negate. */
+8 -6
View File
@@ -40,12 +40,14 @@ static const char* display_bytes(unsigned long long bytes, bool human_readable,
} }
static ScannerOptions scanner_options_from_config(const Config* config, int num_threads) { static ScannerOptions scanner_options_from_config(const Config* config, int num_threads) {
ScannerOptions options = { ScannerOptions options = {config->use_metadata, config->chunk_size,
config->use_metadata, config->chunk_size, config->exclude_patterns, config->exclude_patterns, config->exclude_count,
config->exclude_count, config->include_patterns, config->include_count, config->include_patterns, config->include_count,
config->max_size, config->min_size, config->max_depth, config->max_size, config->min_size,
num_threads, config->follow_symlinks, config->copy_links, config->max_depth, num_threads,
config->safe_links, config->copy_unsafe_links, config->checksum}; config->follow_symlinks, config->copy_links,
config->safe_links, config->copy_unsafe_links,
config->checksum, config->one_file_system};
return options; return options;
} }
+53 -10
View File
@@ -59,6 +59,13 @@ typedef struct {
bool is_directory; bool is_directory;
} ScannerEntry; } ScannerEntry;
/* --one-file-system (-x) decision. Only directories can carry a different
* device than their parent (mount points), so this is checked when a child
* directory is about to be descended into. */
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device) {
return !one_file_system || entry_device == root_device;
}
/* Inspect symlinks, resolve the entry type, and apply file filters once for both scanners. */ /* Inspect symlinks, resolve the entry type, and apply file filters once for both scanners. */
static int scanner_inspect_entry(const ScannerOptions* options, const char* source_root, static int scanner_inspect_entry(const ScannerOptions* options, const char* source_root,
const char* containing_dir, const char* name, const char* containing_dir, const char* name,
@@ -156,7 +163,18 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
scanner->safe_links = options->safe_links; scanner->safe_links = options->safe_links;
scanner->copy_unsafe_links = options->copy_unsafe_links; scanner->copy_unsafe_links = options->copy_unsafe_links;
scanner->checksum = options->checksum; scanner->checksum = options->checksum;
scanner->one_file_system = options->one_file_system;
scanner->failed = false; scanner->failed = false;
if (scanner->one_file_system) {
struct stat root_stats;
if (stat(root_directory, &root_stats) != 0) {
log_perror("Could not stat source directory");
queue_destroy(scanner->directories);
free(scanner);
return NULL;
}
scanner->root_dev = root_stats.st_dev;
}
DirEntry* root = dir_entry_create(root_directory, 0); DirEntry* root = dir_entry_create(root_directory, 0);
if (!root) { if (!root) {
queue_destroy(scanner->directories); queue_destroy(scanner->directories);
@@ -179,10 +197,14 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
unsigned long long min_size, int max_depth, unsigned long long min_size, int max_depth,
bool follow_symlinks, bool copy_links, bool safe_links, bool follow_symlinks, bool copy_links, bool safe_links,
bool copy_unsafe_links, bool checksum) { bool copy_unsafe_links, bool checksum) {
ScannerOptions options = { ScannerOptions options = {use_metadata, chunk_size,
use_metadata, chunk_size, exclude_patterns, exclude_count, include_patterns, exclude_patterns, exclude_count,
include_count, max_size, min_size, max_depth, 0, include_patterns, include_count,
follow_symlinks, copy_links, safe_links, copy_unsafe_links, checksum}; max_size, min_size,
max_depth, 0,
follow_symlinks, copy_links,
safe_links, copy_unsafe_links,
checksum, false};
return directory_scanner_create_with_options(root_directory, &options); return directory_scanner_create_with_options(root_directory, &options);
} }
@@ -272,7 +294,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
scanner->max_depth, 0, scanner->max_depth, 0,
scanner->follow_symlinks, scanner->copy_links, scanner->follow_symlinks, scanner->copy_links,
scanner->safe_links, scanner->copy_unsafe_links, scanner->safe_links, scanner->copy_unsafe_links,
scanner->checksum}; scanner->checksum, scanner->one_file_system};
ScannerEntry inspected; ScannerEntry inspected;
int inspection = scanner_inspect_entry(&options, scanner->current_path, scanner->current_path, int inspection = scanner_inspect_entry(&options, scanner->current_path, scanner->current_path,
entry->d_name, &inspected); entry->d_name, &inspected);
@@ -286,6 +308,10 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
struct stat stats = inspected.stats; struct stat stats = inspected.stats;
if (inspected.is_directory) { if (inspected.is_directory) {
if (!scanner_same_filesystem(scanner->one_file_system, scanner->root_dev, stats.st_dev)) {
free(cur_path);
continue;
}
int next_depth = scanner->current_depth + 1; int next_depth = scanner->current_depth + 1;
if (scanner->max_depth <= 0 || next_depth < scanner->max_depth) { if (scanner->max_depth <= 0 || next_depth < scanner->max_depth) {
DirEntry* de = dir_entry_create(cur_path, next_depth); DirEntry* de = dir_entry_create(cur_path, next_depth);
@@ -525,7 +551,7 @@ static Chunk* batch_files(ArrayList* files, unsigned long long chunk_size, Queue
/* Scan one root-directory entry into either the subdirs or files list. */ /* Scan one root-directory entry into either the subdirs or files list. */
static void scan_root_entry(const ScannerOptions* options, const char* root_directory, static void scan_root_entry(const ScannerOptions* options, const char* root_directory,
const struct dirent* entry, ArrayList* root_files, ArrayList* subdirs, const struct dirent* entry, ArrayList* root_files, ArrayList* subdirs,
ParallelScanner* ps) { dev_t root_dev, ParallelScanner* ps) {
ScannerEntry inspected; ScannerEntry inspected;
int inspection = int inspection =
scanner_inspect_entry(options, root_directory, root_directory, entry->d_name, &inspected); scanner_inspect_entry(options, root_directory, root_directory, entry->d_name, &inspected);
@@ -538,6 +564,10 @@ static void scan_root_entry(const ScannerOptions* options, const char* root_dire
char* cur_path = inspected.path; char* cur_path = inspected.path;
struct stat st = inspected.stats; struct stat st = inspected.stats;
if (inspected.is_directory) { if (inspected.is_directory) {
if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) {
free(cur_path);
return;
}
if (!array_list_add(subdirs, cur_path)) { if (!array_list_add(subdirs, cur_path)) {
free(cur_path); free(cur_path);
ps->failed = true; ps->failed = true;
@@ -567,8 +597,8 @@ static void scan_root_entry(const ScannerOptions* options, const char* root_dire
/* Scan the root directory itself, collecting root files and subdirectories. /* Scan the root directory itself, collecting root files and subdirectories.
* Returns false if the root directory could not be opened. */ * Returns false if the root directory could not be opened. */
static bool scan_root_directory(ParallelScanner* ps, const char* root_directory, static bool scan_root_directory(ParallelScanner* ps, const char* root_directory,
const ScannerOptions* options, ArrayList* root_files, const ScannerOptions* options, dev_t root_dev,
ArrayList* subdirs) { ArrayList* root_files, ArrayList* subdirs) {
DIR* dir = opendir(root_directory); DIR* dir = opendir(root_directory);
if (!dir) { if (!dir) {
log_perror("Could not open root directory for parallel scan"); log_perror("Could not open root directory for parallel scan");
@@ -578,7 +608,7 @@ static bool scan_root_directory(ParallelScanner* ps, const char* root_directory,
while ((entry = readdir(dir)) != NULL) { while ((entry = readdir(dir)) != NULL) {
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
continue; continue;
scan_root_entry(options, root_directory, entry, root_files, subdirs, ps); scan_root_entry(options, root_directory, entry, root_files, subdirs, root_dev, ps);
} }
closedir(dir); closedir(dir);
return true; return true;
@@ -677,7 +707,20 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
return NULL; return NULL;
} }
if (!scan_root_directory(ps, root_directory, options, root_files, subdirs)) { dev_t root_dev = 0;
if (options->one_file_system) {
struct stat root_stats;
if (stat(root_directory, &root_stats) != 0) {
log_perror("Could not stat source directory");
array_list_delete(root_files);
array_list_delete(subdirs);
parallel_scanner_destroy(ps);
return NULL;
}
root_dev = root_stats.st_dev;
}
if (!scan_root_directory(ps, root_directory, options, root_dev, root_files, subdirs)) {
array_list_delete(root_files); array_list_delete(root_files);
array_list_delete(subdirs); array_list_delete(subdirs);
parallel_scanner_destroy(ps); parallel_scanner_destroy(ps);
+10 -1
View File
@@ -6,8 +6,9 @@
#include "queue.h" #include "queue.h"
#include <dirent.h> #include <dirent.h>
#include <stdbool.h> #include <stdbool.h>
#include <threads.h>
#include <stdatomic.h> #include <stdatomic.h>
#include <sys/types.h>
#include <threads.h>
typedef struct { typedef struct {
bool use_metadata; bool use_metadata;
@@ -25,6 +26,7 @@ typedef struct {
bool safe_links; bool safe_links;
bool copy_unsafe_links; bool copy_unsafe_links;
bool checksum; bool checksum;
bool one_file_system;
} ScannerOptions; } ScannerOptions;
typedef struct { typedef struct {
@@ -46,6 +48,8 @@ typedef struct {
bool safe_links; bool safe_links;
bool copy_unsafe_links; bool copy_unsafe_links;
bool checksum; bool checksum;
bool one_file_system;
dev_t root_dev;
bool failed; bool failed;
} DirectoryScanner; } DirectoryScanner;
@@ -79,6 +83,11 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner);
bool directory_scanner_failed(const DirectoryScanner* scanner); bool directory_scanner_failed(const DirectoryScanner* scanner);
void directory_scanner_destroy(DirectoryScanner* scanner); void directory_scanner_destroy(DirectoryScanner* scanner);
/* --one-file-system (-x) decision: a directory entry may be descended into
* only when the option is disabled or the entry lives on the same device as
* the transfer root. Exposed so tests can exercise the rule directly. */
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device);
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory, ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
const ScannerOptions* options, const ScannerOptions* options,
ProtocolSession* allocation_session); ProtocolSession* allocation_session);
+1
View File
@@ -83,6 +83,7 @@ void print_usage(void) {
printf(" --stats Print transfer statistics at end\n"); printf(" --stats Print transfer statistics at end\n");
printf(" -h, --human-readable Print byte sizes in human-readable form\n"); printf(" -h, --human-readable Print byte sizes in human-readable form\n");
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n"); printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
printf(" -x, --one-file-system Do not cross filesystem boundaries\n");
printf(" --log-file <path> Write log messages to file\n"); printf(" --log-file <path> Write log messages to file\n");
printf(" --stderr=MODE Route logging to stderr: errors or all\n"); printf(" --stderr=MODE Route logging to stderr: errors or all\n");
printf(" --partial Keep partial files on interrupted transfer\n"); printf(" --partial Keep partial files on interrupted transfer\n");
+1
View File
@@ -93,6 +93,7 @@ static void config_set_defaults(Config* config) {
config->files_from = NULL; config->files_from = NULL;
config->cvs_exclude = false; config->cvs_exclude = false;
config->prune_empty_dirs = false; config->prune_empty_dirs = false;
config->one_file_system = false;
config->relative = false; config->relative = false;
config->rsh_command = NULL; config->rsh_command = NULL;
config->rsync_path = NULL; config->rsync_path = NULL;
+1
View File
@@ -104,6 +104,7 @@ typedef struct Config {
char* files_from; char* files_from;
bool cvs_exclude; bool cvs_exclude;
bool prune_empty_dirs; bool prune_empty_dirs;
bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */
bool relative; bool relative;
// Issue #130: Remote shell/connection options // Issue #130: Remote shell/connection options
+80
View File
@@ -2,6 +2,7 @@
import filecmp import filecmp
import os import os
import shutil import shutil
import subprocess
import sys import sys
import time import time
import pytest import pytest
@@ -1021,3 +1022,82 @@ class TestLargeFile:
assert result.returncode == 0, f"Large-file sync failed: {result.stderr[:200]}" assert result.returncode == 0, f"Large-file sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source) received = get_dest_received_dir(dest, source)
assert filecmp.cmp(source_file, os.path.join(received, "big.bin"), shallow=False) assert filecmp.cmp(source_file, os.path.join(received, "big.bin"), shallow=False)
class TestOneFileSystem:
def _make_tree(self, source):
clean_dir(source)
os.makedirs(os.path.join(source, "nested", "deeper"))
with open(os.path.join(source, "root.txt"), "wb") as f:
f.write(b"root")
with open(os.path.join(source, "nested", "inner.txt"), "wb") as f:
f.write(b"inner")
with open(os.path.join(source, "nested", "deeper", "deep.txt"), "wb") as f:
f.write(b"deep")
def _assert_full_tree_transferred(self, source, dest, port, flags):
clean_dir(dest)
result, _ = run_client(source, dest, flags=flags, port=port)
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
mismatches, missing = verify_transfer(source, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
def test_x_transfer_matches_plain_over_single_filesystem(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "ofs_src")
self._make_tree(source)
self._assert_full_tree_transferred(source, os.path.join(TEST_DATA_DIR, "ofs_dst"),
shared_server.port, ["-x"])
def test_x_multithreaded_transfer_matches_plain_over_single_filesystem(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "ofs_m_src")
self._make_tree(source)
self._assert_full_tree_transferred(source, os.path.join(TEST_DATA_DIR, "ofs_m_dst"),
shared_server.port, ["-m", "--one-file-system"])
def test_x_skips_other_device_mountpoint(self, shared_server):
if os.geteuid() != 0 or shutil.which("mount") is None or shutil.which("umount") is None:
pytest.skip("cross-device test requires root and mount(8)")
source = os.path.join(TEST_DATA_DIR, "ofs_mnt_src")
dest = os.path.join(TEST_DATA_DIR, "ofs_mnt_dst")
dest_plain = os.path.join(TEST_DATA_DIR, "ofs_mnt_plain_dst")
mountpoint = os.path.join(source, "external")
clean_dir(source)
clean_dir(dest)
clean_dir(dest_plain)
os.makedirs(mountpoint)
os.makedirs(os.path.join(source, "nested"))
with open(os.path.join(source, "root.txt"), "wb") as f:
f.write(b"root")
with open(os.path.join(source, "nested", "inner.txt"), "wb") as f:
f.write(b"inner")
mounted = False
unmount_error = ""
try:
mount = subprocess.run(["mount", "-t", "tmpfs", "tmpfs", mountpoint],
capture_output=True, text=True)
if mount.returncode != 0:
pytest.skip(f"cannot mount tmpfs: {mount.stderr.strip()}")
mounted = True
with open(os.path.join(mountpoint, "away.txt"), "wb") as f:
f.write(b"cross device")
result, _ = run_client(source, dest, flags=["-x"], port=shared_server.port)
assert result.returncode == 0, f"-x sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert os.path.isfile(os.path.join(received, "root.txt"))
assert os.path.isfile(os.path.join(received, "nested", "inner.txt"))
assert not os.path.exists(os.path.join(received, "external", "away.txt")), \
"-x must not cross into the mounted filesystem"
result, _ = run_client(source, dest_plain, port=shared_server.port)
assert result.returncode == 0, f"plain sync failed: {result.stderr[:200]}"
received_plain = get_dest_received_dir(dest_plain, source)
assert os.path.isfile(os.path.join(received_plain, "external", "away.txt")), \
"without -x the mounted subtree must be transferred"
finally:
if mounted:
umount = subprocess.run(["umount", mountpoint], capture_output=True, text=True)
if umount.returncode != 0:
unmount_error = umount.stderr.strip()
if unmount_error:
pytest.fail(f"test mountpoint {mountpoint} still mounted after umount: {unmount_error}")
+30
View File
@@ -985,6 +985,35 @@ static void test_parse_args_whole_file() {
config_delete(cfg); config_delete(cfg);
} }
/* -x and --one-file-system enable client-side single-filesystem scanning. */
static void test_parse_args_one_file_system() {
Config* cfg = config_create();
EXPECT_FALSE(cfg->one_file_system);
char* argv[] = {"fastsync", "-x", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->one_file_system);
EXPECT_EQ_INT(positional_count, 2);
config_delete(cfg);
cfg = config_create();
char* long_argv[] = {"fastsync", "--one-file-system", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, long_argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->one_file_system);
config_delete(cfg);
cfg = config_create();
/* Flags never take a value: the "=value" form must be rejected. */
char* bad_argv[] = {"fastsync", "--one-file-system=yes", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, bad_argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
/* Test rsync-compatible compression-choice and compression-level aliases. */ /* Test rsync-compatible compression-choice and compression-level aliases. */
static void test_parse_args_compression_aliases() { static void test_parse_args_compression_aliases() {
Config* cfg = config_create(); Config* cfg = config_create();
@@ -1213,6 +1242,7 @@ void test_client_cli() {
test_parse_args_secluded_args(); test_parse_args_secluded_args();
test_parse_args_short_s_remains_chunk_serialization(); test_parse_args_short_s_remains_chunk_serialization();
test_parse_args_whole_file(); test_parse_args_whole_file();
test_parse_args_one_file_system();
test_parse_args_compression_aliases(); test_parse_args_compression_aliases();
test_parse_args_compression_equals_and_none(); test_parse_args_compression_equals_and_none();
test_parse_args_compression_canonical_equals(); test_parse_args_compression_canonical_equals();
+245 -2
View File
@@ -394,8 +394,8 @@ static void test_parallel_scanner_root_chunks_without_workers() {
create_test_file(file1, "a"); create_test_file(file1, "a");
create_test_file(file2, "b"); create_test_file(file2, "b");
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0, ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0,
0, 0, false, false, false, false, false}; 0, 0, false, false, false, false, false, false};
ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options, NULL); ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options, NULL);
EXPECT_NOT_NULL(scanner); EXPECT_NOT_NULL(scanner);
@@ -414,6 +414,245 @@ static void test_parallel_scanner_root_chunks_without_workers() {
rmdir(dir); rmdir(dir);
} }
/* --one-file-system (-x) decision is a pure device comparison. */
static void test_scanner_one_file_system_decision() {
/* Option disabled: every device is allowed (unchanged default behavior). */
EXPECT_TRUE(scanner_same_filesystem(false, 0, 123));
EXPECT_TRUE(scanner_same_filesystem(false, 7, 999));
/* Option enabled: only entries on the root device may be descended into. */
EXPECT_TRUE(scanner_same_filesystem(true, 7, 7));
EXPECT_FALSE(scanner_same_filesystem(true, 7, 8));
}
/* With -x over an ordinary tree (all one device) nothing may be skipped. */
static void test_scanner_one_file_system_same_device() {
const char* root = "test_scan_ofs";
const char* sub = "test_scan_ofs/sub";
const char* deeper = "test_scan_ofs/sub/deeper";
const char* root_file = "test_scan_ofs/root.txt";
const char* sub_file = "test_scan_ofs/sub/inner.txt";
const char* deep_file = "test_scan_ofs/sub/deeper/deep.txt";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
EXPECT_EQ_INT(mkdir(deeper, 0755), 0);
create_test_file(root_file, "root");
create_test_file(sub_file, "inner");
create_test_file(deep_file, "deep");
ScannerOptions options = {0};
options.one_file_system = true;
DirectoryScanner* scanner = directory_scanner_create_with_options(root, &options);
EXPECT_NOT_NULL(scanner);
int total_files = 0;
Chunk* chunk;
while ((chunk = directory_scanner_next(scanner)) != NULL) {
total_files += chunk->element_count;
chunk_destroy(chunk);
}
EXPECT_EQ_INT(total_files, 3);
EXPECT_FALSE(directory_scanner_failed(scanner));
directory_scanner_destroy(scanner);
unlink(root_file);
unlink(sub_file);
unlink(deep_file);
rmdir(deeper);
rmdir(sub);
rmdir(root);
}
/* Multithreaded (-m) scan with -x over a single-device tree must match the
* single-threaded result. */
static void test_parallel_scanner_one_file_system_same_device() {
const char* root = "test_parallel_scan_ofs";
const char* sub = "test_parallel_scan_ofs/sub";
const char* sub2 = "test_parallel_scan_ofs/sub2";
const char* root_file = "test_parallel_scan_ofs/root.txt";
const char* sub_file = "test_parallel_scan_ofs/sub/inner.txt";
const char* sub2_file = "test_parallel_scan_ofs/sub2/inner2.txt";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
EXPECT_EQ_INT(mkdir(sub2, 0755), 0);
create_test_file(root_file, "root");
create_test_file(sub_file, "inner");
create_test_file(sub2_file, "inner2");
ScannerOptions options = {0};
options.one_file_system = true;
options.num_threads = 2;
ParallelScanner* scanner = parallel_scanner_create_with_options(root, &options, NULL);
EXPECT_NOT_NULL(scanner);
int total_files = 0;
Chunk* chunk;
while ((chunk = parallel_scanner_next(scanner)) != NULL) {
total_files += chunk->element_count;
chunk_destroy(chunk);
}
EXPECT_EQ_INT(total_files, 3);
EXPECT_FALSE(parallel_scanner_failed(scanner));
parallel_scanner_destroy(scanner);
unlink(root_file);
unlink(sub_file);
unlink(sub2_file);
rmdir(sub);
rmdir(sub2);
rmdir(root);
}
/* Scan a tree with copy_links semantics, collecting every emitted path.
* Returns 0 on success, -1 on scanner failure. */
static int collect_directory_scan(const char* root, bool one_file_system, const char* needle,
bool* found, int* total) {
ScannerOptions options = {0};
options.copy_links = true;
options.one_file_system = one_file_system;
DirectoryScanner* scanner = directory_scanner_create_with_options(root, &options);
if (!scanner)
return -1;
*found = false;
*total = 0;
Chunk* chunk;
while ((chunk = directory_scanner_next(scanner)) != NULL) {
for (int i = 0; i < chunk->element_count; i++) {
(*total)++;
if (strstr(chunk->items[i]->path, needle) != NULL)
*found = true;
}
chunk_destroy(chunk);
}
bool failed = directory_scanner_failed(scanner);
directory_scanner_destroy(scanner);
return failed ? -1 : 0;
}
static int collect_parallel_scan(const char* root, bool one_file_system, const char* needle,
bool* found, int* total) {
ScannerOptions options = {0};
options.copy_links = true;
options.one_file_system = one_file_system;
options.num_threads = 2;
ParallelScanner* scanner = parallel_scanner_create_with_options(root, &options, NULL);
if (!scanner)
return -1;
*found = false;
*total = 0;
Chunk* chunk;
while ((chunk = parallel_scanner_next(scanner)) != NULL) {
for (int i = 0; i < chunk->element_count; i++) {
(*total)++;
if (strstr(chunk->items[i]->path, needle) != NULL)
*found = true;
}
chunk_destroy(chunk);
}
bool failed = parallel_scanner_failed(scanner);
parallel_scanner_destroy(scanner);
return failed ? -1 : 0;
}
/* Rootless cross-filesystem test: a symlink nested under the scan root points
* at a directory on another device (typically /dev/shm, a tmpfs distinct from
* the build filesystem). With --copy-links semantics the scanner resolves the
* link and must descend into it only when -x is off. The nested placement
* exercises the skip decision in the sequential walker and in the parallel
* worker (depth > 1). Skips when no cross-device target is available. */
static void test_scanner_one_file_system_cross_device() {
struct stat local_stat;
if (stat(".", &local_stat) != 0)
return;
char shm_dir[64] = "/dev/shm/fastsync_ofs_shm_XXXXXX";
if (mkdtemp(shm_dir) == NULL)
return;
struct stat shm_stat;
if (stat(shm_dir, &shm_stat) != 0 || shm_stat.st_dev == local_stat.st_dev) {
rmdir(shm_dir);
return;
}
char root_dir[64] = "./fastsync_ofs_root_XXXXXX";
if (mkdtemp(root_dir) == NULL) {
rmdir(shm_dir);
return;
}
char nested[96];
snprintf(nested, sizeof(nested), "%s/nested", root_dir);
char link_path[128];
snprintf(link_path, sizeof(link_path), "%s/link", nested);
char root_file[96];
snprintf(root_file, sizeof(root_file), "%s/keep.txt", root_dir);
char shm_file[96];
snprintf(shm_file, sizeof(shm_file), "%s/inside.txt", shm_dir);
bool ready = mkdir(nested, 0755) == 0 && symlink(shm_dir, link_path) == 0;
if (ready)
create_test_file(root_file, "keep");
if (ready)
create_test_file(shm_file, "cross");
int rc, total;
bool found;
int seq_off_rc, seq_off_total, seq_on_rc, seq_on_total;
bool seq_off_found, seq_on_found;
int par_off_rc, par_off_total, par_on_rc, par_on_total;
bool par_off_found, par_on_found;
if (!ready) {
seq_off_rc = seq_on_rc = par_off_rc = par_on_rc = -1;
seq_off_total = seq_on_total = par_off_total = par_on_total = 0;
seq_off_found = seq_on_found = par_off_found = par_on_found = false;
} else {
rc = collect_directory_scan(root_dir, false, "inside.txt", &found, &total);
seq_off_rc = rc;
seq_off_total = total;
seq_off_found = found;
rc = collect_directory_scan(root_dir, true, "inside.txt", &found, &total);
seq_on_rc = rc;
seq_on_total = total;
seq_on_found = found;
rc = collect_parallel_scan(root_dir, false, "inside.txt", &found, &total);
par_off_rc = rc;
par_off_total = total;
par_off_found = found;
rc = collect_parallel_scan(root_dir, true, "inside.txt", &found, &total);
par_on_rc = rc;
par_on_total = total;
par_on_found = found;
}
/* Hermetic cleanup regardless of scan outcome, before any assertions. */
unlink(shm_file);
rmdir(shm_dir);
unlink(link_path);
unlink(root_file);
rmdir(nested);
rmdir(root_dir);
if (!ready)
return;
/* Sequential: without -x the symlinked foreign subtree is included. */
EXPECT_EQ_INT(seq_off_rc, 0);
EXPECT_TRUE(seq_off_found);
EXPECT_EQ_INT(seq_off_total, 2);
/* Sequential: with -x the cross-device subtree is dropped, keep.txt remains. */
EXPECT_EQ_INT(seq_on_rc, 0);
EXPECT_FALSE(seq_on_found);
EXPECT_EQ_INT(seq_on_total, 1);
/* Parallel: same behavior, worker path (depth > 1). */
EXPECT_EQ_INT(par_off_rc, 0);
EXPECT_TRUE(par_off_found);
EXPECT_EQ_INT(par_off_total, 2);
EXPECT_EQ_INT(par_on_rc, 0);
EXPECT_FALSE(par_on_found);
EXPECT_EQ_INT(par_on_total, 1);
}
void test_scanner() { void test_scanner() {
test_scanner_single_file(); test_scanner_single_file();
test_scanner_multiple_files(); test_scanner_multiple_files();
@@ -429,4 +668,8 @@ void test_scanner() {
test_scanner_mixed_patterns(); test_scanner_mixed_patterns();
test_scanner_no_patterns(); test_scanner_no_patterns();
test_parallel_scanner_root_chunks_without_workers(); test_parallel_scanner_root_chunks_without_workers();
test_scanner_one_file_system_decision();
test_scanner_one_file_system_same_device();
test_parallel_scanner_one_file_system_same_device();
test_scanner_one_file_system_cross_device();
} }