feat: add --files-from/-0, --filter, -C and -F filter layer
CI / lint (pull_request) Successful in 23s
CI / sanitizers (address) (pull_request) Successful in 40s
CI / sanitizers (undefined) (pull_request) Successful in 39s
CI / fuzz-build (pull_request) Successful in 17s
CI / coverage (pull_request) Successful in 34s
CI / build-and-test (pull_request) Successful in 1m28s
CI / valgrind (pull_request) Successful in 36s
CI / lint (pull_request) Successful in 23s
CI / sanitizers (address) (pull_request) Successful in 40s
CI / sanitizers (undefined) (pull_request) Successful in 39s
CI / fuzz-build (pull_request) Successful in 17s
CI / coverage (pull_request) Successful in 34s
CI / build-and-test (pull_request) Successful in 1m28s
CI / valgrind (pull_request) Successful in 36s
Implement the RSYNC_COMPAT Phase-2 filter/parser feature group: - --files-from=FILE (repeatable) plus -0/--from0 NUL delimiters: parse the source file list relative to the source root into a shared read-only allow-set; the scanner transfers listed files and the whole subtree of listed directories and prunes everything else in single- and multithreaded mode. Absolute/'..' entries and missing files are hard CLI errors. - --filter=RULE: rsync-style +/- rules (anchored '/', dir-only trailing '/', word include/exclude forms) evaluated first-match-wins with a default of include, as an independent layer from legacy --exclude/--include. Unsupported directives (merge/hide/... ) are rejected explicitly. -f stays sendfile. - -C/--cvs-exclude: well-known rsync CVS default exclude set. - -F: per-directory .rsync-filter files read during traversal and applied to the owning directory's subtree (single + parallel), never transferred. - Delete manifest still derives from what was actually sent. Client-only config fields; no wire/protocol change. Adds unit coverage (CLI parse, allow-set and filter scanning single+parallel) and integration tests (TestFilesFrom, TestFilters). RSYNC_COMPAT matrix rows updated: 5 rows move to Implemented (Summary 62/3/5/1/76 = 147).
This commit is contained in:
@@ -9,7 +9,7 @@ import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import (
|
||||
PROJECT_ROOT, BUILD_DIR, TEST_DATA_DIR,
|
||||
PROJECT_ROOT, BUILD_DIR, TEST_DATA_DIR, ServerManager,
|
||||
run_client,
|
||||
generate_test_files, verify_transfer, clean_dir, make_result,
|
||||
get_dest_received_dir, CLIENT_CMD,
|
||||
@@ -1405,3 +1405,228 @@ class TestLogFileFormat:
|
||||
expected = {f"{os.path.join(source, rel)} {len(data)}" for rel, data in files.items()}
|
||||
for line in expected:
|
||||
assert line in content, f"log file (-m) missing {line!r}"
|
||||
|
||||
|
||||
class TestFilesFrom:
|
||||
"""--files-from transfers exactly the listed files; a listed directory
|
||||
transfers its whole subtree. The manifest (and thus --delete) derives from
|
||||
what was actually sent."""
|
||||
|
||||
def _make_source(self, name):
|
||||
source = os.path.join(TEST_DATA_DIR, name)
|
||||
clean_dir(source)
|
||||
entries = {
|
||||
"top.txt": b"top\n",
|
||||
"sub/a.txt": b"a\n",
|
||||
"sub/b.txt": b"b\n",
|
||||
"other/c.txt": b"c\n",
|
||||
}
|
||||
for rel, content in entries.items():
|
||||
full = os.path.join(source, rel)
|
||||
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||
with open(full, "wb") as fh:
|
||||
fh.write(content)
|
||||
return source
|
||||
|
||||
def _write_list(self, rel_text):
|
||||
path = os.path.join(TEST_DATA_DIR, "files_from.list")
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(rel_text)
|
||||
return path
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_files_from_exact_subset(self, shared_server, mt):
|
||||
source = self._make_source("ff_subset_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "ff_subset_dst")
|
||||
clean_dir(dest)
|
||||
lst = self._write_list(b"top.txt\nsub/a.txt\n")
|
||||
flags = ["--files-from", lst] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||
assert result.returncode == 0, f"files-from sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.path.isfile(os.path.join(received, "top.txt"))
|
||||
assert os.path.isfile(os.path.join(received, "sub", "a.txt"))
|
||||
assert not os.path.exists(os.path.join(received, "sub", "b.txt")), \
|
||||
"unlisted sub/b.txt must not be transferred"
|
||||
assert not os.path.exists(os.path.join(received, "other")), \
|
||||
"unlisted other/ subtree must not be transferred"
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_files_from_listed_directory_transfers_subtree(self, shared_server, mt):
|
||||
source = self._make_source("ff_subdir_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "ff_subdir_dst")
|
||||
clean_dir(dest)
|
||||
lst = self._write_list(b"sub\n")
|
||||
flags = ["--files-from", lst] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||
assert result.returncode == 0, f"files-from dir sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.path.isfile(os.path.join(received, "sub", "a.txt"))
|
||||
assert os.path.isfile(os.path.join(received, "sub", "b.txt"))
|
||||
assert not os.path.exists(os.path.join(received, "top.txt"))
|
||||
assert not os.path.exists(os.path.join(received, "other"))
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_files_from_nul_separated(self, shared_server, mt):
|
||||
source = self._make_source("ff_nul_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "ff_nul_dst")
|
||||
clean_dir(dest)
|
||||
lst = self._write_list(b"top.txt\0other/c.txt\0")
|
||||
flags = ["--files-from", lst, "--from0"] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||
assert result.returncode == 0, f"files-from -0 sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.path.isfile(os.path.join(received, "top.txt"))
|
||||
assert os.path.isfile(os.path.join(received, "other", "c.txt"))
|
||||
assert not os.path.exists(os.path.join(received, "sub"))
|
||||
|
||||
def test_files_from_missing_list_file_rejected(self):
|
||||
result, _ = run_client(
|
||||
os.path.join(TEST_DATA_DIR, "nowhere_src"),
|
||||
os.path.join(TEST_DATA_DIR, "nowhere_dst"),
|
||||
flags=["--files-from", os.path.join(TEST_DATA_DIR, "no_such_list.txt")],
|
||||
)
|
||||
assert result.returncode != 0, "missing --files-from file must be rejected"
|
||||
assert "--files-from" in result.stderr
|
||||
|
||||
def test_files_from_delete_deletes_unlisted(self):
|
||||
source = self._make_source("ff_delete_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "ff_delete_dst")
|
||||
clean_dir(dest)
|
||||
server = ServerManager()
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
try:
|
||||
# Full transfer first.
|
||||
result, _ = run_client(source, dest, port=server.port)
|
||||
assert result.returncode == 0, f"full sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.path.isfile(os.path.join(received, "top.txt"))
|
||||
|
||||
# A subset sync with --delete deletes everything not in the sent
|
||||
# manifest (which is derived from what was actually sent).
|
||||
lst = self._write_list(b"sub/a.txt\n")
|
||||
result, _ = run_client(source, dest, flags=["--files-from", lst, "--delete"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, f"files-from delete sync failed: {result.stderr[:200]}"
|
||||
assert os.path.isfile(os.path.join(received, "sub", "a.txt"))
|
||||
assert not os.path.exists(os.path.join(received, "top.txt")), \
|
||||
"unlisted file not deleted"
|
||||
assert not os.path.exists(os.path.join(received, "other")), \
|
||||
"unlisted subtree not deleted"
|
||||
finally:
|
||||
server.stop()
|
||||
|
||||
|
||||
class TestFilters:
|
||||
"""--filter/-C/-F rule layer: excludes prune, ordering is first-match-wins,
|
||||
the default with no matching rule is include, and legacy --exclude remains
|
||||
an independent layer."""
|
||||
|
||||
def _make_tree(self, name, with_filter_file=False):
|
||||
source = os.path.join(TEST_DATA_DIR, name)
|
||||
clean_dir(source)
|
||||
entries = {
|
||||
"data/keep.txt": b"keep\n",
|
||||
"data/drop.tmp": b"drop\n",
|
||||
"nested/deep.tmp": b"deep\n",
|
||||
"nested/ok.log": b"log\n",
|
||||
"top.bin": b"bin\n",
|
||||
}
|
||||
for rel, content in entries.items():
|
||||
full = os.path.join(source, rel)
|
||||
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||
with open(full, "wb") as fh:
|
||||
fh.write(content)
|
||||
if with_filter_file:
|
||||
with open(os.path.join(source, ".rsync-filter"), "wb") as fh:
|
||||
fh.write(b"- *.tmp\n")
|
||||
return source
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_filter_excludes_glob(self, shared_server, mt):
|
||||
source = self._make_tree("filter_tmp_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "filter_tmp_dst")
|
||||
clean_dir(dest)
|
||||
flags = ["--filter", "- *.tmp"] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||
assert result.returncode == 0, f"filter sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.path.isfile(os.path.join(received, "data", "keep.txt"))
|
||||
assert os.path.isfile(os.path.join(received, "nested", "ok.log"))
|
||||
assert os.path.isfile(os.path.join(received, "top.bin"))
|
||||
assert not os.path.exists(os.path.join(received, "data", "drop.tmp"))
|
||||
assert not os.path.exists(os.path.join(received, "nested", "deep.tmp"))
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_filter_anchored_include_overrides_exclude_all(self, shared_server, mt):
|
||||
source = os.path.join(TEST_DATA_DIR, "filter_keep_src")
|
||||
clean_dir(source)
|
||||
with open(os.path.join(source, "a.keepme"), "wb") as fh:
|
||||
fh.write(b"keep me")
|
||||
with open(os.path.join(source, "b.other"), "wb") as fh:
|
||||
fh.write(b"drop me")
|
||||
dest = os.path.join(TEST_DATA_DIR, "filter_keep_dst")
|
||||
clean_dir(dest)
|
||||
# First match wins: the anchored include beats the catch-all exclude.
|
||||
flags = ["--filter=+ /a.keepme", "--filter=- *"] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||
assert result.returncode == 0, f"filter include sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.path.isfile(os.path.join(received, "a.keepme"))
|
||||
assert not os.path.exists(os.path.join(received, "b.other"))
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_cvs_exclude_ignores_scm_and_build_artifacts(self, shared_server, mt):
|
||||
source = os.path.join(TEST_DATA_DIR, "filter_cvs_src")
|
||||
clean_dir(source)
|
||||
entries = {
|
||||
"src/main.c": b"int main() {}\n",
|
||||
"src/main.o": b"obj\n",
|
||||
"src/notes.txt~": b"backup\n",
|
||||
".git/HEAD": b"ref\n",
|
||||
".git/config": b"cfg\n",
|
||||
"README.md": b"readme\n",
|
||||
}
|
||||
for rel, content in entries.items():
|
||||
full = os.path.join(source, rel)
|
||||
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||
with open(full, "wb") as fh:
|
||||
fh.write(content)
|
||||
dest = os.path.join(TEST_DATA_DIR, "filter_cvs_dst")
|
||||
clean_dir(dest)
|
||||
flags = ["-C"] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||
assert result.returncode == 0, f"-C sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.path.isfile(os.path.join(received, "src", "main.c"))
|
||||
assert os.path.isfile(os.path.join(received, "README.md"))
|
||||
assert not os.path.exists(os.path.join(received, ".git")), ".git/ must be pruned"
|
||||
assert not os.path.exists(os.path.join(received, "src", "main.o")), "*.o must be pruned"
|
||||
assert not os.path.exists(os.path.join(received, "src", "notes.txt~")), "*~ must be pruned"
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_per_dir_filter_file(self, shared_server, mt):
|
||||
source = self._make_tree("filter_file_src", with_filter_file=True)
|
||||
dest = os.path.join(TEST_DATA_DIR, "filter_file_dst")
|
||||
clean_dir(dest)
|
||||
flags = ["-F"] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||
assert result.returncode == 0, f"-F sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.path.isfile(os.path.join(received, "data", "keep.txt"))
|
||||
assert os.path.isfile(os.path.join(received, "nested", "ok.log"))
|
||||
assert not os.path.exists(os.path.join(received, "data", "drop.tmp"))
|
||||
assert not os.path.exists(os.path.join(received, "nested", "deep.tmp"))
|
||||
assert not os.path.exists(os.path.join(received, ".rsync-filter")), \
|
||||
".rsync-filter must not be transferred"
|
||||
|
||||
def test_filter_leaves_default_behavior_unchanged(self, shared_server):
|
||||
source = self._make_tree("filter_default_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "filter_default_dst")
|
||||
clean_dir(dest)
|
||||
result, _ = run_client(source, dest, port=shared_server.port)
|
||||
assert result.returncode == 0, f"plain sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
mismatches, missing = verify_transfer(source, received)
|
||||
assert not missing, f"Missing: {missing}"
|
||||
assert not mismatches, f"Mismatch: {mismatches}"
|
||||
|
||||
+135
-3
@@ -2,6 +2,7 @@
|
||||
#include "client_validation.h"
|
||||
#include "chmod.h"
|
||||
#include "config.h"
|
||||
#include "file_list.h"
|
||||
#include "log.h"
|
||||
#include "test_utils.h"
|
||||
#include "utils.h"
|
||||
@@ -607,9 +608,6 @@ static void test_parse_args_rejects_unimplemented_options() {
|
||||
"--delete-excluded",
|
||||
"--delete-after",
|
||||
"--max-delete",
|
||||
"--filter",
|
||||
"--files-from",
|
||||
"--cvs-exclude",
|
||||
"--prune-empty-dirs",
|
||||
"-R",
|
||||
"--relative",
|
||||
@@ -1265,6 +1263,137 @@ static void test_parse_args_log_file_format() {
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* --filter is repeatable and accepts both "--filter RULE" and "--filter=RULE". */
|
||||
static void test_parse_args_filter_rules() {
|
||||
Config* cfg = config_create();
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
char* argv[] = {"fastsync", "--filter", "- *.tmp", "--filter=+ /keep.txt", "/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 6, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_NOT_NULL(cfg->filters);
|
||||
EXPECT_EQ_INT(cfg->filters->size, 2);
|
||||
EXPECT_EQ_STR((char*)cfg->filters->items[0], "- *.tmp");
|
||||
EXPECT_EQ_STR((char*)cfg->filters->items[1], "+ /keep.txt");
|
||||
config_delete(cfg);
|
||||
|
||||
/* An unsupported rsync rule type is rejected with a clear error. */
|
||||
cfg = config_create();
|
||||
positional_count = 0;
|
||||
char* bad_argv[] = {"fastsync", "--filter=merge /tmp/excludes", "/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, bad_argv, positional_args, &positional_count), -1);
|
||||
config_delete(cfg);
|
||||
|
||||
/* A trailing --filter with no rule is a missing-argument error. */
|
||||
cfg = config_create();
|
||||
positional_count = 0;
|
||||
char* missing_argv[] = {"fastsync", "/src", "/dst", "--filter"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, missing_argv, positional_args, &positional_count), -1);
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* -0/--from0, -C/--cvs-exclude and -F wire into their config flags. */
|
||||
static void test_parse_args_from0_cvs_filter_file_flags() {
|
||||
static const struct {
|
||||
const char* arg;
|
||||
bool from0;
|
||||
bool cvs;
|
||||
bool per_dir;
|
||||
} cases[] = {
|
||||
{"--from0", true, false, false},
|
||||
{"-0", true, false, false},
|
||||
{"--cvs-exclude", false, true, false},
|
||||
{"-C", false, true, false},
|
||||
{"-F", false, false, true},
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", (char*)cases[i].arg, "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_INT(cfg->from0, cases[i].from0);
|
||||
EXPECT_EQ_INT(cfg->cvs_exclude, cases[i].cvs);
|
||||
EXPECT_EQ_INT(cfg->per_dir_filter, cases[i].per_dir);
|
||||
config_delete(cfg);
|
||||
}
|
||||
}
|
||||
|
||||
static void write_file_bytes(const char* path, const char* bytes, size_t len) {
|
||||
FILE* fp = fopen(path, "wb");
|
||||
EXPECT_NOT_NULL(fp);
|
||||
EXPECT_EQ_INT((int)fwrite(bytes, 1, len, fp), (int)len);
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
/* --files-from is validated and parsed after the full argument scan, so -0 may
|
||||
* appear before or after it. */
|
||||
static void test_parse_args_files_from() {
|
||||
const char* list_path = "cli_files_from_list.txt";
|
||||
write_file_bytes(list_path, "a.txt\nsub/b.bin\n\n./c.txt\n", 25);
|
||||
Config* cfg = config_create();
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
char* argv[] = {"fastsync", "--files-from", (char*)list_path, "/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_STR(cfg->files_from, list_path);
|
||||
EXPECT_NOT_NULL(cfg->files_from_set);
|
||||
FileListSet* set = (FileListSet*)cfg->files_from_set;
|
||||
EXPECT_TRUE(file_list_affects(set, "a.txt"));
|
||||
EXPECT_TRUE(file_list_affects(set, "sub/b.bin"));
|
||||
EXPECT_TRUE(file_list_affects(set, "sub/b.bin/x"));
|
||||
EXPECT_TRUE(file_list_affects(set, "sub"));
|
||||
EXPECT_TRUE(file_list_affects(set, "c.txt"));
|
||||
EXPECT_FALSE(file_list_affects(set, "other.txt"));
|
||||
config_delete(cfg);
|
||||
remove(list_path);
|
||||
|
||||
/* -0 switches the separator to NUL regardless of argument order. */
|
||||
write_file_bytes(list_path, "x.txt\0y/z.bin\0", 14);
|
||||
cfg = config_create();
|
||||
positional_count = 0;
|
||||
char* nul_argv[] = {"fastsync",
|
||||
"--files-from="
|
||||
"cli_files_from_list.txt",
|
||||
"-0", "/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, nul_argv, positional_args, &positional_count), 0);
|
||||
set = (FileListSet*)cfg->files_from_set;
|
||||
EXPECT_NOT_NULL(set);
|
||||
EXPECT_TRUE(file_list_affects(set, "x.txt"));
|
||||
EXPECT_TRUE(file_list_affects(set, "y/z.bin"));
|
||||
EXPECT_TRUE(file_list_affects(set, "y"));
|
||||
EXPECT_FALSE(file_list_affects(set, "z.txt"));
|
||||
config_delete(cfg);
|
||||
remove(list_path);
|
||||
|
||||
/* A missing list file is a hard parse-time error. */
|
||||
cfg = config_create();
|
||||
positional_count = 0;
|
||||
char* missing_argv[] = {"fastsync", "--files-from", "does_not_exist_ff.txt", "/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, missing_argv, positional_args, &positional_count), -1);
|
||||
config_delete(cfg);
|
||||
|
||||
/* Absolute and traversal entries are rejected. */
|
||||
write_file_bytes(list_path, "/abs/path\n", 10);
|
||||
cfg = config_create();
|
||||
positional_count = 0;
|
||||
char* abs_argv[] = {"fastsync",
|
||||
"--files-from="
|
||||
"cli_files_from_list.txt",
|
||||
"/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, abs_argv, positional_args, &positional_count), -1);
|
||||
config_delete(cfg);
|
||||
write_file_bytes(list_path, "../escape\n", 10);
|
||||
cfg = config_create();
|
||||
positional_count = 0;
|
||||
char* trav_argv[] = {"fastsync",
|
||||
"--files-from="
|
||||
"cli_files_from_list.txt",
|
||||
"/src", "/dst"};
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, trav_argv, positional_args, &positional_count), -1);
|
||||
config_delete(cfg);
|
||||
remove(list_path);
|
||||
}
|
||||
|
||||
void test_client_cli() {
|
||||
test_validate_config_required_paths();
|
||||
test_validate_config_incompatible_options();
|
||||
@@ -1344,4 +1473,7 @@ void test_client_cli() {
|
||||
test_parse_args_checksum_choice_aliases();
|
||||
test_parse_args_checksum_choice_requires_value();
|
||||
test_parse_args_temp_dir();
|
||||
test_parse_args_filter_rules();
|
||||
test_parse_args_from0_cvs_filter_file_flags();
|
||||
test_parse_args_files_from();
|
||||
}
|
||||
|
||||
+354
-2
@@ -1,7 +1,10 @@
|
||||
#include "test_utils.h"
|
||||
#include "scanner.h"
|
||||
#include "file.h"
|
||||
#include "file_list.h"
|
||||
#include "filter.h"
|
||||
#include "utils.h"
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
@@ -394,8 +397,8 @@ static void test_parallel_scanner_root_chunks_without_workers() {
|
||||
create_test_file(file1, "a");
|
||||
create_test_file(file2, "b");
|
||||
|
||||
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0,
|
||||
0, 0, false, false, false, false, false, false};
|
||||
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0, 0, 0,
|
||||
false, false, false, false, false, false, NULL, NULL, false};
|
||||
ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options, NULL);
|
||||
EXPECT_NOT_NULL(scanner);
|
||||
|
||||
@@ -653,6 +656,345 @@ static void test_scanner_one_file_system_cross_device() {
|
||||
EXPECT_EQ_INT(par_on_total, 1);
|
||||
}
|
||||
|
||||
/* Collect emitted file paths (relative to `root`) from a sequential scan.
|
||||
* Returns 0 on success with *out and *count set (caller frees *out). */
|
||||
static int collect_files(const char* root, const ScannerOptions* options, char*** out,
|
||||
int* out_count) {
|
||||
DirectoryScanner* scanner = directory_scanner_create_with_options(root, options);
|
||||
if (!scanner)
|
||||
return -1;
|
||||
size_t root_len = strlen(root);
|
||||
while (root_len > 0 && root[root_len - 1] == '/')
|
||||
root_len--;
|
||||
int cap = 16;
|
||||
int count = 0;
|
||||
char** paths = malloc((size_t)cap * sizeof(char*));
|
||||
if (!paths) {
|
||||
directory_scanner_destroy(scanner);
|
||||
return -1;
|
||||
}
|
||||
Chunk* chunk;
|
||||
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
const char* rel = chunk->items[i]->path + root_len;
|
||||
if (*rel == '/')
|
||||
rel++;
|
||||
if (count == cap) {
|
||||
cap *= 2;
|
||||
char** grown = realloc(paths, (size_t)cap * sizeof(char*));
|
||||
if (!grown) {
|
||||
for (int k = 0; k < count; k++)
|
||||
free(paths[k]);
|
||||
free(paths);
|
||||
chunk_destroy(chunk);
|
||||
directory_scanner_destroy(scanner);
|
||||
return -1;
|
||||
}
|
||||
paths = grown;
|
||||
}
|
||||
paths[count++] = str_dup(rel);
|
||||
}
|
||||
chunk_destroy(chunk);
|
||||
}
|
||||
bool failed = directory_scanner_failed(scanner);
|
||||
directory_scanner_destroy(scanner);
|
||||
if (failed) {
|
||||
for (int k = 0; k < count; k++)
|
||||
free(paths[k]);
|
||||
free(paths);
|
||||
return -1;
|
||||
}
|
||||
*out = paths;
|
||||
*out_count = count;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int collect_files_parallel(const char* root, const ScannerOptions* options, char*** out,
|
||||
int* out_count) {
|
||||
ParallelScanner* scanner = parallel_scanner_create_with_options(root, options, NULL);
|
||||
if (!scanner)
|
||||
return -1;
|
||||
size_t root_len = strlen(root);
|
||||
while (root_len > 0 && root[root_len - 1] == '/')
|
||||
root_len--;
|
||||
int cap = 16;
|
||||
int count = 0;
|
||||
char** paths = malloc((size_t)cap * sizeof(char*));
|
||||
if (!paths) {
|
||||
parallel_scanner_destroy(scanner);
|
||||
return -1;
|
||||
}
|
||||
Chunk* chunk;
|
||||
while ((chunk = parallel_scanner_next(scanner)) != NULL) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
const char* rel = chunk->items[i]->path + root_len;
|
||||
if (*rel == '/')
|
||||
rel++;
|
||||
if (count == cap) {
|
||||
cap *= 2;
|
||||
char** grown = realloc(paths, (size_t)cap * sizeof(char*));
|
||||
if (!grown) {
|
||||
for (int k = 0; k < count; k++)
|
||||
free(paths[k]);
|
||||
free(paths);
|
||||
chunk_destroy(chunk);
|
||||
parallel_scanner_destroy(scanner);
|
||||
return -1;
|
||||
}
|
||||
paths = grown;
|
||||
}
|
||||
paths[count++] = str_dup(rel);
|
||||
}
|
||||
chunk_destroy(chunk);
|
||||
}
|
||||
bool failed = parallel_scanner_failed(scanner);
|
||||
parallel_scanner_destroy(scanner);
|
||||
if (failed) {
|
||||
for (int k = 0; k < count; k++)
|
||||
free(paths[k]);
|
||||
free(paths);
|
||||
return -1;
|
||||
}
|
||||
*out = paths;
|
||||
*out_count = count;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static bool has_path(char** paths, int count, const char* rel) {
|
||||
for (int i = 0; i < count; i++)
|
||||
if (strcmp(paths[i], rel) == 0)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
static void free_paths(char** paths, int count) {
|
||||
for (int i = 0; i < count; i++)
|
||||
free(paths[i]);
|
||||
free(paths);
|
||||
}
|
||||
|
||||
static const char* FILE_LIST_PATH = "test_scan_files_from.txt";
|
||||
|
||||
/* --files-from: only the listed files (and the subtree of a listed directory)
|
||||
* are emitted; unrelated files and directories are pruned. */
|
||||
static void test_files_from_subset(bool parallel) {
|
||||
const char* root = "test_scan_ff";
|
||||
const char* sub = "test_scan_ff/sub";
|
||||
const char* other = "test_scan_ff/other";
|
||||
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
|
||||
EXPECT_EQ_INT(mkdir(other, 0755), 0);
|
||||
create_test_file("test_scan_ff/root.txt", "root");
|
||||
create_test_file("test_scan_ff/sub/keep.txt", "keep");
|
||||
create_test_file("test_scan_ff/sub/skip.bin", "skip");
|
||||
create_test_file("test_scan_ff/other/unrelated.txt", "unrelated");
|
||||
|
||||
/* List a root file and a file under sub: sub is descended but its other file
|
||||
* is not listed, and the whole `other` directory is pruned. */
|
||||
create_test_file(FILE_LIST_PATH, "root.txt\nsub/keep.txt\n");
|
||||
char err[160];
|
||||
FileListSet* set = file_list_load(FILE_LIST_PATH, false, err, sizeof(err));
|
||||
EXPECT_NOT_NULL(set);
|
||||
|
||||
ScannerOptions options = {0};
|
||||
options.file_list = set;
|
||||
if (parallel)
|
||||
options.num_threads = 2;
|
||||
char** paths = NULL;
|
||||
int count = 0;
|
||||
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||
: collect_files(root, &options, &paths, &count);
|
||||
EXPECT_EQ_INT(rc, 0);
|
||||
EXPECT_EQ_INT(count, 2);
|
||||
EXPECT_TRUE(has_path(paths, count, "root.txt"));
|
||||
EXPECT_TRUE(has_path(paths, count, "sub/keep.txt"));
|
||||
EXPECT_FALSE(has_path(paths, count, "sub/skip.bin"));
|
||||
EXPECT_FALSE(has_path(paths, count, "other/unrelated.txt"));
|
||||
free_paths(paths, count);
|
||||
file_list_destroy(set);
|
||||
remove(FILE_LIST_PATH);
|
||||
|
||||
/* Listing a directory transfers its whole subtree. */
|
||||
create_test_file(FILE_LIST_PATH, "sub\n");
|
||||
set = file_list_load(FILE_LIST_PATH, false, err, sizeof(err));
|
||||
EXPECT_NOT_NULL(set);
|
||||
options.file_list = set;
|
||||
rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||
: collect_files(root, &options, &paths, &count);
|
||||
EXPECT_EQ_INT(rc, 0);
|
||||
EXPECT_EQ_INT(count, 2);
|
||||
EXPECT_TRUE(has_path(paths, count, "sub/keep.txt"));
|
||||
EXPECT_TRUE(has_path(paths, count, "sub/skip.bin"));
|
||||
EXPECT_FALSE(has_path(paths, count, "root.txt"));
|
||||
EXPECT_FALSE(has_path(paths, count, "other/unrelated.txt"));
|
||||
free_paths(paths, count);
|
||||
file_list_destroy(set);
|
||||
remove(FILE_LIST_PATH);
|
||||
|
||||
unlink("test_scan_ff/root.txt");
|
||||
unlink("test_scan_ff/sub/keep.txt");
|
||||
unlink("test_scan_ff/sub/skip.bin");
|
||||
unlink("test_scan_ff/other/unrelated.txt");
|
||||
rmdir(other);
|
||||
rmdir(sub);
|
||||
rmdir(root);
|
||||
}
|
||||
|
||||
/* Filter layer: '-' excludes, first-match-wins ordering with '+', anchored
|
||||
* rules, and dir-only rules all prune during the scan. */
|
||||
static void test_filter_rules(bool parallel) {
|
||||
const char* root = "test_scan_filter";
|
||||
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||
create_test_file("test_scan_filter/a.txt", "a");
|
||||
create_test_file("test_scan_filter/b.tmp", "b");
|
||||
create_test_file("test_scan_filter/c.txt", "c");
|
||||
|
||||
/* - *.tmp excludes only the tmp file; other files remain (default include). */
|
||||
const char* exclude_only[] = {"- *.tmp"};
|
||||
char err[160];
|
||||
FilterRuleList* base = filter_base_build(exclude_only, 1, false, err, sizeof(err));
|
||||
EXPECT_NOT_NULL(base);
|
||||
ScannerOptions options = {0};
|
||||
options.base_filters = base;
|
||||
if (parallel)
|
||||
options.num_threads = 2;
|
||||
char** paths = NULL;
|
||||
int count = 0;
|
||||
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||
: collect_files(root, &options, &paths, &count);
|
||||
EXPECT_EQ_INT(rc, 0);
|
||||
EXPECT_EQ_INT(count, 2);
|
||||
EXPECT_TRUE(has_path(paths, count, "a.txt"));
|
||||
EXPECT_TRUE(has_path(paths, count, "c.txt"));
|
||||
EXPECT_FALSE(has_path(paths, count, "b.tmp"));
|
||||
free_paths(paths, count);
|
||||
filter_rule_list_free(base);
|
||||
|
||||
/* Anchored include then exclude-all: only root-level keep* survives. */
|
||||
const char* anchored[] = {"+ /a.txt", "- *"};
|
||||
base = filter_base_build(anchored, 2, false, err, sizeof(err));
|
||||
EXPECT_NOT_NULL(base);
|
||||
options.base_filters = base;
|
||||
rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||
: collect_files(root, &options, &paths, &count);
|
||||
EXPECT_EQ_INT(rc, 0);
|
||||
EXPECT_EQ_INT(count, 1);
|
||||
EXPECT_TRUE(has_path(paths, count, "a.txt"));
|
||||
free_paths(paths, count);
|
||||
filter_rule_list_free(base);
|
||||
|
||||
unlink("test_scan_filter/a.txt");
|
||||
unlink("test_scan_filter/b.tmp");
|
||||
unlink("test_scan_filter/c.txt");
|
||||
rmdir(root);
|
||||
}
|
||||
|
||||
/* Anchored dir-only rules prune a whole subtree. */
|
||||
static void test_filter_dir_only_and_anchored(bool parallel) {
|
||||
const char* root = "test_scan_filter_dir";
|
||||
const char* sub = "test_scan_filter_dir/sub";
|
||||
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
|
||||
create_test_file("test_scan_filter_dir/sub/inner.txt", "x");
|
||||
create_test_file("test_scan_filter_dir/keep.txt", "keep");
|
||||
|
||||
const char* rules[] = {"- /sub/"};
|
||||
char err[160];
|
||||
FilterRuleList* base = filter_base_build(rules, 1, false, err, sizeof(err));
|
||||
EXPECT_NOT_NULL(base);
|
||||
ScannerOptions options = {0};
|
||||
options.base_filters = base;
|
||||
if (parallel)
|
||||
options.num_threads = 2;
|
||||
char** paths = NULL;
|
||||
int count = 0;
|
||||
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||
: collect_files(root, &options, &paths, &count);
|
||||
EXPECT_EQ_INT(rc, 0);
|
||||
EXPECT_EQ_INT(count, 1);
|
||||
EXPECT_TRUE(has_path(paths, count, "keep.txt"));
|
||||
EXPECT_FALSE(has_path(paths, count, "sub/inner.txt"));
|
||||
free_paths(paths, count);
|
||||
filter_rule_list_free(base);
|
||||
|
||||
unlink("test_scan_filter_dir/sub/inner.txt");
|
||||
unlink("test_scan_filter_dir/keep.txt");
|
||||
rmdir(sub);
|
||||
rmdir(root);
|
||||
}
|
||||
|
||||
/* -C default CVS excludes prune .git/ directories and *.o files. */
|
||||
static void test_cvs_defaults(bool parallel) {
|
||||
const char* root = "test_scan_cvs";
|
||||
const char* git = "test_scan_cvs/.git";
|
||||
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||
EXPECT_EQ_INT(mkdir(git, 0755), 0);
|
||||
create_test_file("test_scan_cvs/.git/config", "cfg");
|
||||
create_test_file("test_scan_cvs/object.o", "o");
|
||||
create_test_file("test_scan_cvs/keep.txt", "keep");
|
||||
|
||||
char err[160];
|
||||
FilterRuleList* base = filter_base_build(NULL, 0, true, err, sizeof(err));
|
||||
EXPECT_NOT_NULL(base);
|
||||
ScannerOptions options = {0};
|
||||
options.base_filters = base;
|
||||
if (parallel)
|
||||
options.num_threads = 2;
|
||||
char** paths = NULL;
|
||||
int count = 0;
|
||||
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||
: collect_files(root, &options, &paths, &count);
|
||||
EXPECT_EQ_INT(rc, 0);
|
||||
EXPECT_EQ_INT(count, 1);
|
||||
EXPECT_TRUE(has_path(paths, count, "keep.txt"));
|
||||
EXPECT_FALSE(has_path(paths, count, ".git/config"));
|
||||
EXPECT_FALSE(has_path(paths, count, "object.o"));
|
||||
free_paths(paths, count);
|
||||
filter_rule_list_free(base);
|
||||
|
||||
unlink("test_scan_cvs/.git/config");
|
||||
unlink("test_scan_cvs/object.o");
|
||||
unlink("test_scan_cvs/keep.txt");
|
||||
rmdir(git);
|
||||
rmdir(root);
|
||||
}
|
||||
|
||||
/* -F: a .rsync-filter placed in a directory governs its subtree and the file
|
||||
* itself is never transferred. */
|
||||
static void test_per_dir_filter(bool parallel) {
|
||||
const char* root = "test_scan_perdir";
|
||||
const char* sub = "test_scan_perdir/sub";
|
||||
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
|
||||
create_test_file("test_scan_perdir/drop.tmp", "tmp");
|
||||
create_test_file("test_scan_perdir/keep.txt", "keep");
|
||||
create_test_file("test_scan_perdir/sub/nested.tmp", "tmp");
|
||||
create_test_file("test_scan_perdir/.rsync-filter", "- *.tmp\n");
|
||||
|
||||
ScannerOptions options = {0};
|
||||
options.per_dir_filters = true;
|
||||
if (parallel)
|
||||
options.num_threads = 2;
|
||||
char** paths = NULL;
|
||||
int count = 0;
|
||||
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||
: collect_files(root, &options, &paths, &count);
|
||||
EXPECT_EQ_INT(rc, 0);
|
||||
EXPECT_EQ_INT(count, 1);
|
||||
EXPECT_TRUE(has_path(paths, count, "keep.txt"));
|
||||
EXPECT_FALSE(has_path(paths, count, "drop.tmp"));
|
||||
EXPECT_FALSE(has_path(paths, count, "sub/nested.tmp"));
|
||||
EXPECT_FALSE(has_path(paths, count, ".rsync-filter"));
|
||||
free_paths(paths, count);
|
||||
|
||||
unlink("test_scan_perdir/drop.tmp");
|
||||
unlink("test_scan_perdir/keep.txt");
|
||||
unlink("test_scan_perdir/sub/nested.tmp");
|
||||
unlink("test_scan_perdir/.rsync-filter");
|
||||
rmdir(sub);
|
||||
rmdir(root);
|
||||
}
|
||||
|
||||
void test_scanner() {
|
||||
test_scanner_single_file();
|
||||
test_scanner_multiple_files();
|
||||
@@ -672,4 +1014,14 @@ void test_scanner() {
|
||||
test_scanner_one_file_system_same_device();
|
||||
test_parallel_scanner_one_file_system_same_device();
|
||||
test_scanner_one_file_system_cross_device();
|
||||
test_files_from_subset(false);
|
||||
test_files_from_subset(true);
|
||||
test_filter_rules(false);
|
||||
test_filter_rules(true);
|
||||
test_filter_dir_only_and_anchored(false);
|
||||
test_filter_dir_only_and_anchored(true);
|
||||
test_cvs_defaults(false);
|
||||
test_cvs_defaults(true);
|
||||
test_per_dir_filter(false);
|
||||
test_per_dir_filter(true);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user