Merge feat/p2-one-file-system: implement -x/--one-file-system
Sender-side scanner stays within the source filesystem (-x), single and multithreaded; default unchanged; client-only, no wire change. Reviewed (c-review APPROVE WITH NITS, all fixed); PR #261.
This commit is contained in:
+1
-1
@@ -64,7 +64,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|
||||
| `--existing` | Skip creating new files on receiver | ✅ Implemented | Existing destination files continue through normal update handling |
|
||||
| `--ignore-existing` | Skip updating existing files | ❌ Not Implemented | |
|
||||
| `--remove-source-files` | Sender removes regular files after confirmed transfer | ✅ Implemented | |
|
||||
| `-x`, `--one-file-system` | Do not cross filesystem boundaries | ❌ Not Implemented | |
|
||||
| `-x`, `--one-file-system` | Do not cross filesystem boundaries | ✅ Implemented | Sender scanner captures the root device and skips descending into mount-point crossings (`st_dev` differs); cross-filesystem mount-point subdirectories are dropped entirely, matching rsync |
|
||||
| `-F` | Add the default `.rsync-filter` rules | ❌ Not Implemented | |
|
||||
|
||||
## 4. Directory Options
|
||||
|
||||
@@ -416,6 +416,7 @@ static const OptionEntry OPTION_TABLE[] = {
|
||||
|
||||
{"--max-size", NULL, OPT_ULL, offsetof(Config, max_size)},
|
||||
{"--min-size", NULL, OPT_ULL, offsetof(Config, min_size)},
|
||||
{"--one-file-system", "-x", OPT_FLAG, offsetof(Config, one_file_system)},
|
||||
};
|
||||
|
||||
/* Only boolean options with no required argument are safe to negate. */
|
||||
|
||||
@@ -40,12 +40,14 @@ static const char* display_bytes(unsigned long long bytes, bool human_readable,
|
||||
}
|
||||
|
||||
static ScannerOptions scanner_options_from_config(const Config* config, int num_threads) {
|
||||
ScannerOptions options = {
|
||||
config->use_metadata, config->chunk_size, config->exclude_patterns,
|
||||
config->exclude_count, config->include_patterns, config->include_count,
|
||||
config->max_size, config->min_size, config->max_depth,
|
||||
num_threads, config->follow_symlinks, config->copy_links,
|
||||
config->safe_links, config->copy_unsafe_links, config->checksum};
|
||||
ScannerOptions options = {config->use_metadata, config->chunk_size,
|
||||
config->exclude_patterns, config->exclude_count,
|
||||
config->include_patterns, config->include_count,
|
||||
config->max_size, config->min_size,
|
||||
config->max_depth, num_threads,
|
||||
config->follow_symlinks, config->copy_links,
|
||||
config->safe_links, config->copy_unsafe_links,
|
||||
config->checksum, config->one_file_system};
|
||||
return options;
|
||||
}
|
||||
|
||||
|
||||
+53
-10
@@ -59,6 +59,13 @@ typedef struct {
|
||||
bool is_directory;
|
||||
} ScannerEntry;
|
||||
|
||||
/* --one-file-system (-x) decision. Only directories can carry a different
|
||||
* device than their parent (mount points), so this is checked when a child
|
||||
* directory is about to be descended into. */
|
||||
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device) {
|
||||
return !one_file_system || entry_device == root_device;
|
||||
}
|
||||
|
||||
/* Inspect symlinks, resolve the entry type, and apply file filters once for both scanners. */
|
||||
static int scanner_inspect_entry(const ScannerOptions* options, const char* source_root,
|
||||
const char* containing_dir, const char* name,
|
||||
@@ -156,7 +163,18 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
||||
scanner->safe_links = options->safe_links;
|
||||
scanner->copy_unsafe_links = options->copy_unsafe_links;
|
||||
scanner->checksum = options->checksum;
|
||||
scanner->one_file_system = options->one_file_system;
|
||||
scanner->failed = false;
|
||||
if (scanner->one_file_system) {
|
||||
struct stat root_stats;
|
||||
if (stat(root_directory, &root_stats) != 0) {
|
||||
log_perror("Could not stat source directory");
|
||||
queue_destroy(scanner->directories);
|
||||
free(scanner);
|
||||
return NULL;
|
||||
}
|
||||
scanner->root_dev = root_stats.st_dev;
|
||||
}
|
||||
DirEntry* root = dir_entry_create(root_directory, 0);
|
||||
if (!root) {
|
||||
queue_destroy(scanner->directories);
|
||||
@@ -179,10 +197,14 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
|
||||
unsigned long long min_size, int max_depth,
|
||||
bool follow_symlinks, bool copy_links, bool safe_links,
|
||||
bool copy_unsafe_links, bool checksum) {
|
||||
ScannerOptions options = {
|
||||
use_metadata, chunk_size, exclude_patterns, exclude_count, include_patterns,
|
||||
include_count, max_size, min_size, max_depth, 0,
|
||||
follow_symlinks, copy_links, safe_links, copy_unsafe_links, checksum};
|
||||
ScannerOptions options = {use_metadata, chunk_size,
|
||||
exclude_patterns, exclude_count,
|
||||
include_patterns, include_count,
|
||||
max_size, min_size,
|
||||
max_depth, 0,
|
||||
follow_symlinks, copy_links,
|
||||
safe_links, copy_unsafe_links,
|
||||
checksum, false};
|
||||
return directory_scanner_create_with_options(root_directory, &options);
|
||||
}
|
||||
|
||||
@@ -272,7 +294,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
scanner->max_depth, 0,
|
||||
scanner->follow_symlinks, scanner->copy_links,
|
||||
scanner->safe_links, scanner->copy_unsafe_links,
|
||||
scanner->checksum};
|
||||
scanner->checksum, scanner->one_file_system};
|
||||
ScannerEntry inspected;
|
||||
int inspection = scanner_inspect_entry(&options, scanner->current_path, scanner->current_path,
|
||||
entry->d_name, &inspected);
|
||||
@@ -286,6 +308,10 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||
struct stat stats = inspected.stats;
|
||||
|
||||
if (inspected.is_directory) {
|
||||
if (!scanner_same_filesystem(scanner->one_file_system, scanner->root_dev, stats.st_dev)) {
|
||||
free(cur_path);
|
||||
continue;
|
||||
}
|
||||
int next_depth = scanner->current_depth + 1;
|
||||
if (scanner->max_depth <= 0 || next_depth < scanner->max_depth) {
|
||||
DirEntry* de = dir_entry_create(cur_path, next_depth);
|
||||
@@ -525,7 +551,7 @@ static Chunk* batch_files(ArrayList* files, unsigned long long chunk_size, Queue
|
||||
/* Scan one root-directory entry into either the subdirs or files list. */
|
||||
static void scan_root_entry(const ScannerOptions* options, const char* root_directory,
|
||||
const struct dirent* entry, ArrayList* root_files, ArrayList* subdirs,
|
||||
ParallelScanner* ps) {
|
||||
dev_t root_dev, ParallelScanner* ps) {
|
||||
ScannerEntry inspected;
|
||||
int inspection =
|
||||
scanner_inspect_entry(options, root_directory, root_directory, entry->d_name, &inspected);
|
||||
@@ -538,6 +564,10 @@ static void scan_root_entry(const ScannerOptions* options, const char* root_dire
|
||||
char* cur_path = inspected.path;
|
||||
struct stat st = inspected.stats;
|
||||
if (inspected.is_directory) {
|
||||
if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) {
|
||||
free(cur_path);
|
||||
return;
|
||||
}
|
||||
if (!array_list_add(subdirs, cur_path)) {
|
||||
free(cur_path);
|
||||
ps->failed = true;
|
||||
@@ -567,8 +597,8 @@ static void scan_root_entry(const ScannerOptions* options, const char* root_dire
|
||||
/* Scan the root directory itself, collecting root files and subdirectories.
|
||||
* Returns false if the root directory could not be opened. */
|
||||
static bool scan_root_directory(ParallelScanner* ps, const char* root_directory,
|
||||
const ScannerOptions* options, ArrayList* root_files,
|
||||
ArrayList* subdirs) {
|
||||
const ScannerOptions* options, dev_t root_dev,
|
||||
ArrayList* root_files, ArrayList* subdirs) {
|
||||
DIR* dir = opendir(root_directory);
|
||||
if (!dir) {
|
||||
log_perror("Could not open root directory for parallel scan");
|
||||
@@ -578,7 +608,7 @@ static bool scan_root_directory(ParallelScanner* ps, const char* root_directory,
|
||||
while ((entry = readdir(dir)) != NULL) {
|
||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||
continue;
|
||||
scan_root_entry(options, root_directory, entry, root_files, subdirs, ps);
|
||||
scan_root_entry(options, root_directory, entry, root_files, subdirs, root_dev, ps);
|
||||
}
|
||||
closedir(dir);
|
||||
return true;
|
||||
@@ -677,7 +707,20 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (!scan_root_directory(ps, root_directory, options, root_files, subdirs)) {
|
||||
dev_t root_dev = 0;
|
||||
if (options->one_file_system) {
|
||||
struct stat root_stats;
|
||||
if (stat(root_directory, &root_stats) != 0) {
|
||||
log_perror("Could not stat source directory");
|
||||
array_list_delete(root_files);
|
||||
array_list_delete(subdirs);
|
||||
parallel_scanner_destroy(ps);
|
||||
return NULL;
|
||||
}
|
||||
root_dev = root_stats.st_dev;
|
||||
}
|
||||
|
||||
if (!scan_root_directory(ps, root_directory, options, root_dev, root_files, subdirs)) {
|
||||
array_list_delete(root_files);
|
||||
array_list_delete(subdirs);
|
||||
parallel_scanner_destroy(ps);
|
||||
|
||||
+10
-1
@@ -6,8 +6,9 @@
|
||||
#include "queue.h"
|
||||
#include <dirent.h>
|
||||
#include <stdbool.h>
|
||||
#include <threads.h>
|
||||
#include <stdatomic.h>
|
||||
#include <sys/types.h>
|
||||
#include <threads.h>
|
||||
|
||||
typedef struct {
|
||||
bool use_metadata;
|
||||
@@ -25,6 +26,7 @@ typedef struct {
|
||||
bool safe_links;
|
||||
bool copy_unsafe_links;
|
||||
bool checksum;
|
||||
bool one_file_system;
|
||||
} ScannerOptions;
|
||||
|
||||
typedef struct {
|
||||
@@ -46,6 +48,8 @@ typedef struct {
|
||||
bool safe_links;
|
||||
bool copy_unsafe_links;
|
||||
bool checksum;
|
||||
bool one_file_system;
|
||||
dev_t root_dev;
|
||||
bool failed;
|
||||
} DirectoryScanner;
|
||||
|
||||
@@ -79,6 +83,11 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner);
|
||||
bool directory_scanner_failed(const DirectoryScanner* scanner);
|
||||
void directory_scanner_destroy(DirectoryScanner* scanner);
|
||||
|
||||
/* --one-file-system (-x) decision: a directory entry may be descended into
|
||||
* only when the option is disabled or the entry lives on the same device as
|
||||
* the transfer root. Exposed so tests can exercise the rule directly. */
|
||||
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device);
|
||||
|
||||
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
||||
const ScannerOptions* options,
|
||||
ProtocolSession* allocation_session);
|
||||
|
||||
@@ -83,6 +83,7 @@ void print_usage(void) {
|
||||
printf(" --stats Print transfer statistics at end\n");
|
||||
printf(" -h, --human-readable Print byte sizes in human-readable form\n");
|
||||
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
|
||||
printf(" -x, --one-file-system Do not cross filesystem boundaries\n");
|
||||
printf(" --log-file <path> Write log messages to file\n");
|
||||
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
|
||||
printf(" --partial Keep partial files on interrupted transfer\n");
|
||||
|
||||
@@ -93,6 +93,7 @@ static void config_set_defaults(Config* config) {
|
||||
config->files_from = NULL;
|
||||
config->cvs_exclude = false;
|
||||
config->prune_empty_dirs = false;
|
||||
config->one_file_system = false;
|
||||
config->relative = false;
|
||||
config->rsh_command = NULL;
|
||||
config->rsync_path = NULL;
|
||||
|
||||
@@ -104,6 +104,7 @@ typedef struct Config {
|
||||
char* files_from;
|
||||
bool cvs_exclude;
|
||||
bool prune_empty_dirs;
|
||||
bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */
|
||||
bool relative;
|
||||
|
||||
// Issue #130: Remote shell/connection options
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import filecmp
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import pytest
|
||||
@@ -1021,3 +1022,82 @@ class TestLargeFile:
|
||||
assert result.returncode == 0, f"Large-file sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert filecmp.cmp(source_file, os.path.join(received, "big.bin"), shallow=False)
|
||||
|
||||
|
||||
class TestOneFileSystem:
|
||||
def _make_tree(self, source):
|
||||
clean_dir(source)
|
||||
os.makedirs(os.path.join(source, "nested", "deeper"))
|
||||
with open(os.path.join(source, "root.txt"), "wb") as f:
|
||||
f.write(b"root")
|
||||
with open(os.path.join(source, "nested", "inner.txt"), "wb") as f:
|
||||
f.write(b"inner")
|
||||
with open(os.path.join(source, "nested", "deeper", "deep.txt"), "wb") as f:
|
||||
f.write(b"deep")
|
||||
|
||||
def _assert_full_tree_transferred(self, source, dest, port, flags):
|
||||
clean_dir(dest)
|
||||
result, _ = run_client(source, dest, flags=flags, port=port)
|
||||
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
mismatches, missing = verify_transfer(source, received)
|
||||
assert not missing, f"Missing: {missing}"
|
||||
assert not mismatches, f"Mismatch: {mismatches}"
|
||||
|
||||
def test_x_transfer_matches_plain_over_single_filesystem(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "ofs_src")
|
||||
self._make_tree(source)
|
||||
self._assert_full_tree_transferred(source, os.path.join(TEST_DATA_DIR, "ofs_dst"),
|
||||
shared_server.port, ["-x"])
|
||||
|
||||
def test_x_multithreaded_transfer_matches_plain_over_single_filesystem(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "ofs_m_src")
|
||||
self._make_tree(source)
|
||||
self._assert_full_tree_transferred(source, os.path.join(TEST_DATA_DIR, "ofs_m_dst"),
|
||||
shared_server.port, ["-m", "--one-file-system"])
|
||||
|
||||
def test_x_skips_other_device_mountpoint(self, shared_server):
|
||||
if os.geteuid() != 0 or shutil.which("mount") is None or shutil.which("umount") is None:
|
||||
pytest.skip("cross-device test requires root and mount(8)")
|
||||
source = os.path.join(TEST_DATA_DIR, "ofs_mnt_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "ofs_mnt_dst")
|
||||
dest_plain = os.path.join(TEST_DATA_DIR, "ofs_mnt_plain_dst")
|
||||
mountpoint = os.path.join(source, "external")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
clean_dir(dest_plain)
|
||||
os.makedirs(mountpoint)
|
||||
os.makedirs(os.path.join(source, "nested"))
|
||||
with open(os.path.join(source, "root.txt"), "wb") as f:
|
||||
f.write(b"root")
|
||||
with open(os.path.join(source, "nested", "inner.txt"), "wb") as f:
|
||||
f.write(b"inner")
|
||||
mounted = False
|
||||
unmount_error = ""
|
||||
try:
|
||||
mount = subprocess.run(["mount", "-t", "tmpfs", "tmpfs", mountpoint],
|
||||
capture_output=True, text=True)
|
||||
if mount.returncode != 0:
|
||||
pytest.skip(f"cannot mount tmpfs: {mount.stderr.strip()}")
|
||||
mounted = True
|
||||
with open(os.path.join(mountpoint, "away.txt"), "wb") as f:
|
||||
f.write(b"cross device")
|
||||
result, _ = run_client(source, dest, flags=["-x"], port=shared_server.port)
|
||||
assert result.returncode == 0, f"-x sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.path.isfile(os.path.join(received, "root.txt"))
|
||||
assert os.path.isfile(os.path.join(received, "nested", "inner.txt"))
|
||||
assert not os.path.exists(os.path.join(received, "external", "away.txt")), \
|
||||
"-x must not cross into the mounted filesystem"
|
||||
result, _ = run_client(source, dest_plain, port=shared_server.port)
|
||||
assert result.returncode == 0, f"plain sync failed: {result.stderr[:200]}"
|
||||
received_plain = get_dest_received_dir(dest_plain, source)
|
||||
assert os.path.isfile(os.path.join(received_plain, "external", "away.txt")), \
|
||||
"without -x the mounted subtree must be transferred"
|
||||
finally:
|
||||
if mounted:
|
||||
umount = subprocess.run(["umount", mountpoint], capture_output=True, text=True)
|
||||
if umount.returncode != 0:
|
||||
unmount_error = umount.stderr.strip()
|
||||
if unmount_error:
|
||||
pytest.fail(f"test mountpoint {mountpoint} still mounted after umount: {unmount_error}")
|
||||
|
||||
@@ -985,6 +985,35 @@ static void test_parse_args_whole_file() {
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* -x and --one-file-system enable client-side single-filesystem scanning. */
|
||||
static void test_parse_args_one_file_system() {
|
||||
Config* cfg = config_create();
|
||||
EXPECT_FALSE(cfg->one_file_system);
|
||||
|
||||
char* argv[] = {"fastsync", "-x", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->one_file_system);
|
||||
EXPECT_EQ_INT(positional_count, 2);
|
||||
|
||||
config_delete(cfg);
|
||||
cfg = config_create();
|
||||
char* long_argv[] = {"fastsync", "--one-file-system", "/src", "/dst"};
|
||||
positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, long_argv, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->one_file_system);
|
||||
|
||||
config_delete(cfg);
|
||||
cfg = config_create();
|
||||
/* Flags never take a value: the "=value" form must be rejected. */
|
||||
char* bad_argv[] = {"fastsync", "--one-file-system=yes", "/src", "/dst"};
|
||||
positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, bad_argv, positional_args, &positional_count), -1);
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* Test rsync-compatible compression-choice and compression-level aliases. */
|
||||
static void test_parse_args_compression_aliases() {
|
||||
Config* cfg = config_create();
|
||||
@@ -1213,6 +1242,7 @@ void test_client_cli() {
|
||||
test_parse_args_secluded_args();
|
||||
test_parse_args_short_s_remains_chunk_serialization();
|
||||
test_parse_args_whole_file();
|
||||
test_parse_args_one_file_system();
|
||||
test_parse_args_compression_aliases();
|
||||
test_parse_args_compression_equals_and_none();
|
||||
test_parse_args_compression_canonical_equals();
|
||||
|
||||
+244
-1
@@ -395,7 +395,7 @@ static void test_parallel_scanner_root_chunks_without_workers() {
|
||||
create_test_file(file2, "b");
|
||||
|
||||
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0,
|
||||
0, 0, false, false, false, false, false};
|
||||
0, 0, false, false, false, false, false, false};
|
||||
ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options, NULL);
|
||||
EXPECT_NOT_NULL(scanner);
|
||||
|
||||
@@ -414,6 +414,245 @@ static void test_parallel_scanner_root_chunks_without_workers() {
|
||||
rmdir(dir);
|
||||
}
|
||||
|
||||
/* --one-file-system (-x) decision is a pure device comparison. */
|
||||
static void test_scanner_one_file_system_decision() {
|
||||
/* Option disabled: every device is allowed (unchanged default behavior). */
|
||||
EXPECT_TRUE(scanner_same_filesystem(false, 0, 123));
|
||||
EXPECT_TRUE(scanner_same_filesystem(false, 7, 999));
|
||||
/* Option enabled: only entries on the root device may be descended into. */
|
||||
EXPECT_TRUE(scanner_same_filesystem(true, 7, 7));
|
||||
EXPECT_FALSE(scanner_same_filesystem(true, 7, 8));
|
||||
}
|
||||
|
||||
/* With -x over an ordinary tree (all one device) nothing may be skipped. */
|
||||
static void test_scanner_one_file_system_same_device() {
|
||||
const char* root = "test_scan_ofs";
|
||||
const char* sub = "test_scan_ofs/sub";
|
||||
const char* deeper = "test_scan_ofs/sub/deeper";
|
||||
const char* root_file = "test_scan_ofs/root.txt";
|
||||
const char* sub_file = "test_scan_ofs/sub/inner.txt";
|
||||
const char* deep_file = "test_scan_ofs/sub/deeper/deep.txt";
|
||||
|
||||
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
|
||||
EXPECT_EQ_INT(mkdir(deeper, 0755), 0);
|
||||
create_test_file(root_file, "root");
|
||||
create_test_file(sub_file, "inner");
|
||||
create_test_file(deep_file, "deep");
|
||||
|
||||
ScannerOptions options = {0};
|
||||
options.one_file_system = true;
|
||||
DirectoryScanner* scanner = directory_scanner_create_with_options(root, &options);
|
||||
EXPECT_NOT_NULL(scanner);
|
||||
|
||||
int total_files = 0;
|
||||
Chunk* chunk;
|
||||
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||
total_files += chunk->element_count;
|
||||
chunk_destroy(chunk);
|
||||
}
|
||||
EXPECT_EQ_INT(total_files, 3);
|
||||
EXPECT_FALSE(directory_scanner_failed(scanner));
|
||||
|
||||
directory_scanner_destroy(scanner);
|
||||
unlink(root_file);
|
||||
unlink(sub_file);
|
||||
unlink(deep_file);
|
||||
rmdir(deeper);
|
||||
rmdir(sub);
|
||||
rmdir(root);
|
||||
}
|
||||
|
||||
/* Multithreaded (-m) scan with -x over a single-device tree must match the
|
||||
* single-threaded result. */
|
||||
static void test_parallel_scanner_one_file_system_same_device() {
|
||||
const char* root = "test_parallel_scan_ofs";
|
||||
const char* sub = "test_parallel_scan_ofs/sub";
|
||||
const char* sub2 = "test_parallel_scan_ofs/sub2";
|
||||
const char* root_file = "test_parallel_scan_ofs/root.txt";
|
||||
const char* sub_file = "test_parallel_scan_ofs/sub/inner.txt";
|
||||
const char* sub2_file = "test_parallel_scan_ofs/sub2/inner2.txt";
|
||||
|
||||
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
|
||||
EXPECT_EQ_INT(mkdir(sub2, 0755), 0);
|
||||
create_test_file(root_file, "root");
|
||||
create_test_file(sub_file, "inner");
|
||||
create_test_file(sub2_file, "inner2");
|
||||
|
||||
ScannerOptions options = {0};
|
||||
options.one_file_system = true;
|
||||
options.num_threads = 2;
|
||||
ParallelScanner* scanner = parallel_scanner_create_with_options(root, &options, NULL);
|
||||
EXPECT_NOT_NULL(scanner);
|
||||
|
||||
int total_files = 0;
|
||||
Chunk* chunk;
|
||||
while ((chunk = parallel_scanner_next(scanner)) != NULL) {
|
||||
total_files += chunk->element_count;
|
||||
chunk_destroy(chunk);
|
||||
}
|
||||
EXPECT_EQ_INT(total_files, 3);
|
||||
EXPECT_FALSE(parallel_scanner_failed(scanner));
|
||||
|
||||
parallel_scanner_destroy(scanner);
|
||||
unlink(root_file);
|
||||
unlink(sub_file);
|
||||
unlink(sub2_file);
|
||||
rmdir(sub);
|
||||
rmdir(sub2);
|
||||
rmdir(root);
|
||||
}
|
||||
|
||||
/* Scan a tree with copy_links semantics, collecting every emitted path.
|
||||
* Returns 0 on success, -1 on scanner failure. */
|
||||
static int collect_directory_scan(const char* root, bool one_file_system, const char* needle,
|
||||
bool* found, int* total) {
|
||||
ScannerOptions options = {0};
|
||||
options.copy_links = true;
|
||||
options.one_file_system = one_file_system;
|
||||
DirectoryScanner* scanner = directory_scanner_create_with_options(root, &options);
|
||||
if (!scanner)
|
||||
return -1;
|
||||
*found = false;
|
||||
*total = 0;
|
||||
Chunk* chunk;
|
||||
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
(*total)++;
|
||||
if (strstr(chunk->items[i]->path, needle) != NULL)
|
||||
*found = true;
|
||||
}
|
||||
chunk_destroy(chunk);
|
||||
}
|
||||
bool failed = directory_scanner_failed(scanner);
|
||||
directory_scanner_destroy(scanner);
|
||||
return failed ? -1 : 0;
|
||||
}
|
||||
|
||||
static int collect_parallel_scan(const char* root, bool one_file_system, const char* needle,
|
||||
bool* found, int* total) {
|
||||
ScannerOptions options = {0};
|
||||
options.copy_links = true;
|
||||
options.one_file_system = one_file_system;
|
||||
options.num_threads = 2;
|
||||
ParallelScanner* scanner = parallel_scanner_create_with_options(root, &options, NULL);
|
||||
if (!scanner)
|
||||
return -1;
|
||||
*found = false;
|
||||
*total = 0;
|
||||
Chunk* chunk;
|
||||
while ((chunk = parallel_scanner_next(scanner)) != NULL) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
(*total)++;
|
||||
if (strstr(chunk->items[i]->path, needle) != NULL)
|
||||
*found = true;
|
||||
}
|
||||
chunk_destroy(chunk);
|
||||
}
|
||||
bool failed = parallel_scanner_failed(scanner);
|
||||
parallel_scanner_destroy(scanner);
|
||||
return failed ? -1 : 0;
|
||||
}
|
||||
|
||||
/* Rootless cross-filesystem test: a symlink nested under the scan root points
|
||||
* at a directory on another device (typically /dev/shm, a tmpfs distinct from
|
||||
* the build filesystem). With --copy-links semantics the scanner resolves the
|
||||
* link and must descend into it only when -x is off. The nested placement
|
||||
* exercises the skip decision in the sequential walker and in the parallel
|
||||
* worker (depth > 1). Skips when no cross-device target is available. */
|
||||
static void test_scanner_one_file_system_cross_device() {
|
||||
struct stat local_stat;
|
||||
if (stat(".", &local_stat) != 0)
|
||||
return;
|
||||
|
||||
char shm_dir[64] = "/dev/shm/fastsync_ofs_shm_XXXXXX";
|
||||
if (mkdtemp(shm_dir) == NULL)
|
||||
return;
|
||||
struct stat shm_stat;
|
||||
if (stat(shm_dir, &shm_stat) != 0 || shm_stat.st_dev == local_stat.st_dev) {
|
||||
rmdir(shm_dir);
|
||||
return;
|
||||
}
|
||||
|
||||
char root_dir[64] = "./fastsync_ofs_root_XXXXXX";
|
||||
if (mkdtemp(root_dir) == NULL) {
|
||||
rmdir(shm_dir);
|
||||
return;
|
||||
}
|
||||
|
||||
char nested[96];
|
||||
snprintf(nested, sizeof(nested), "%s/nested", root_dir);
|
||||
char link_path[128];
|
||||
snprintf(link_path, sizeof(link_path), "%s/link", nested);
|
||||
char root_file[96];
|
||||
snprintf(root_file, sizeof(root_file), "%s/keep.txt", root_dir);
|
||||
char shm_file[96];
|
||||
snprintf(shm_file, sizeof(shm_file), "%s/inside.txt", shm_dir);
|
||||
|
||||
bool ready = mkdir(nested, 0755) == 0 && symlink(shm_dir, link_path) == 0;
|
||||
if (ready)
|
||||
create_test_file(root_file, "keep");
|
||||
if (ready)
|
||||
create_test_file(shm_file, "cross");
|
||||
|
||||
int rc, total;
|
||||
bool found;
|
||||
int seq_off_rc, seq_off_total, seq_on_rc, seq_on_total;
|
||||
bool seq_off_found, seq_on_found;
|
||||
int par_off_rc, par_off_total, par_on_rc, par_on_total;
|
||||
bool par_off_found, par_on_found;
|
||||
if (!ready) {
|
||||
seq_off_rc = seq_on_rc = par_off_rc = par_on_rc = -1;
|
||||
seq_off_total = seq_on_total = par_off_total = par_on_total = 0;
|
||||
seq_off_found = seq_on_found = par_off_found = par_on_found = false;
|
||||
} else {
|
||||
rc = collect_directory_scan(root_dir, false, "inside.txt", &found, &total);
|
||||
seq_off_rc = rc;
|
||||
seq_off_total = total;
|
||||
seq_off_found = found;
|
||||
rc = collect_directory_scan(root_dir, true, "inside.txt", &found, &total);
|
||||
seq_on_rc = rc;
|
||||
seq_on_total = total;
|
||||
seq_on_found = found;
|
||||
rc = collect_parallel_scan(root_dir, false, "inside.txt", &found, &total);
|
||||
par_off_rc = rc;
|
||||
par_off_total = total;
|
||||
par_off_found = found;
|
||||
rc = collect_parallel_scan(root_dir, true, "inside.txt", &found, &total);
|
||||
par_on_rc = rc;
|
||||
par_on_total = total;
|
||||
par_on_found = found;
|
||||
}
|
||||
|
||||
/* Hermetic cleanup regardless of scan outcome, before any assertions. */
|
||||
unlink(shm_file);
|
||||
rmdir(shm_dir);
|
||||
unlink(link_path);
|
||||
unlink(root_file);
|
||||
rmdir(nested);
|
||||
rmdir(root_dir);
|
||||
|
||||
if (!ready)
|
||||
return;
|
||||
|
||||
/* Sequential: without -x the symlinked foreign subtree is included. */
|
||||
EXPECT_EQ_INT(seq_off_rc, 0);
|
||||
EXPECT_TRUE(seq_off_found);
|
||||
EXPECT_EQ_INT(seq_off_total, 2);
|
||||
/* Sequential: with -x the cross-device subtree is dropped, keep.txt remains. */
|
||||
EXPECT_EQ_INT(seq_on_rc, 0);
|
||||
EXPECT_FALSE(seq_on_found);
|
||||
EXPECT_EQ_INT(seq_on_total, 1);
|
||||
/* Parallel: same behavior, worker path (depth > 1). */
|
||||
EXPECT_EQ_INT(par_off_rc, 0);
|
||||
EXPECT_TRUE(par_off_found);
|
||||
EXPECT_EQ_INT(par_off_total, 2);
|
||||
EXPECT_EQ_INT(par_on_rc, 0);
|
||||
EXPECT_FALSE(par_on_found);
|
||||
EXPECT_EQ_INT(par_on_total, 1);
|
||||
}
|
||||
|
||||
void test_scanner() {
|
||||
test_scanner_single_file();
|
||||
test_scanner_multiple_files();
|
||||
@@ -429,4 +668,8 @@ void test_scanner() {
|
||||
test_scanner_mixed_patterns();
|
||||
test_scanner_no_patterns();
|
||||
test_parallel_scanner_root_chunks_without_workers();
|
||||
test_scanner_one_file_system_decision();
|
||||
test_scanner_one_file_system_same_device();
|
||||
test_parallel_scanner_one_file_system_same_device();
|
||||
test_scanner_one_file_system_cross_device();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user