feat: add --files-from/-0, --filter, -C and -F filter layer
CI / lint (pull_request) Successful in 23s
CI / sanitizers (address) (pull_request) Successful in 40s
CI / sanitizers (undefined) (pull_request) Successful in 39s
CI / fuzz-build (pull_request) Successful in 17s
CI / coverage (pull_request) Successful in 34s
CI / build-and-test (pull_request) Successful in 1m28s
CI / valgrind (pull_request) Successful in 36s

Implement the RSYNC_COMPAT Phase-2 filter/parser feature group:
- --files-from=FILE (repeatable) plus -0/--from0 NUL delimiters: parse the
  source file list relative to the source root into a shared read-only
  allow-set; the scanner transfers listed files and the whole subtree of
  listed directories and prunes everything else in single- and
  multithreaded mode. Absolute/'..' entries and missing files are hard
  CLI errors.
- --filter=RULE: rsync-style +/- rules (anchored '/', dir-only trailing '/',
  word include/exclude forms) evaluated first-match-wins with a default of
  include, as an independent layer from legacy --exclude/--include.
  Unsupported directives (merge/hide/... ) are rejected explicitly. -f stays
  sendfile.
- -C/--cvs-exclude: well-known rsync CVS default exclude set.
- -F: per-directory .rsync-filter files read during traversal and applied to
  the owning directory's subtree (single + parallel), never transferred.
- Delete manifest still derives from what was actually sent.

Client-only config fields; no wire/protocol change. Adds unit coverage
(CLI parse, allow-set and filter scanning single+parallel) and integration
tests (TestFilesFrom, TestFilters). RSYNC_COMPAT matrix rows updated:
5 rows move to Implemented (Summary 62/3/5/1/76 = 147).
This commit is contained in:
2026-09-06 13:43:34 +02:00
parent 8577f95550
commit f4c15a7b78
15 changed files with 1917 additions and 64 deletions
+226 -1
View File
@@ -9,7 +9,7 @@ import pytest
sys.path.insert(0, os.path.dirname(__file__))
from common import (
PROJECT_ROOT, BUILD_DIR, TEST_DATA_DIR,
PROJECT_ROOT, BUILD_DIR, TEST_DATA_DIR, ServerManager,
run_client,
generate_test_files, verify_transfer, clean_dir, make_result,
get_dest_received_dir, CLIENT_CMD,
@@ -1405,3 +1405,228 @@ class TestLogFileFormat:
expected = {f"{os.path.join(source, rel)} {len(data)}" for rel, data in files.items()}
for line in expected:
assert line in content, f"log file (-m) missing {line!r}"
class TestFilesFrom:
"""--files-from transfers exactly the listed files; a listed directory
transfers its whole subtree. The manifest (and thus --delete) derives from
what was actually sent."""
def _make_source(self, name):
source = os.path.join(TEST_DATA_DIR, name)
clean_dir(source)
entries = {
"top.txt": b"top\n",
"sub/a.txt": b"a\n",
"sub/b.txt": b"b\n",
"other/c.txt": b"c\n",
}
for rel, content in entries.items():
full = os.path.join(source, rel)
os.makedirs(os.path.dirname(full), exist_ok=True)
with open(full, "wb") as fh:
fh.write(content)
return source
def _write_list(self, rel_text):
path = os.path.join(TEST_DATA_DIR, "files_from.list")
with open(path, "wb") as fh:
fh.write(rel_text)
return path
@pytest.mark.parametrize("mt", [False, True])
def test_files_from_exact_subset(self, shared_server, mt):
source = self._make_source("ff_subset_src")
dest = os.path.join(TEST_DATA_DIR, "ff_subset_dst")
clean_dir(dest)
lst = self._write_list(b"top.txt\nsub/a.txt\n")
flags = ["--files-from", lst] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"files-from sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert os.path.isfile(os.path.join(received, "top.txt"))
assert os.path.isfile(os.path.join(received, "sub", "a.txt"))
assert not os.path.exists(os.path.join(received, "sub", "b.txt")), \
"unlisted sub/b.txt must not be transferred"
assert not os.path.exists(os.path.join(received, "other")), \
"unlisted other/ subtree must not be transferred"
@pytest.mark.parametrize("mt", [False, True])
def test_files_from_listed_directory_transfers_subtree(self, shared_server, mt):
source = self._make_source("ff_subdir_src")
dest = os.path.join(TEST_DATA_DIR, "ff_subdir_dst")
clean_dir(dest)
lst = self._write_list(b"sub\n")
flags = ["--files-from", lst] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"files-from dir sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert os.path.isfile(os.path.join(received, "sub", "a.txt"))
assert os.path.isfile(os.path.join(received, "sub", "b.txt"))
assert not os.path.exists(os.path.join(received, "top.txt"))
assert not os.path.exists(os.path.join(received, "other"))
@pytest.mark.parametrize("mt", [False, True])
def test_files_from_nul_separated(self, shared_server, mt):
source = self._make_source("ff_nul_src")
dest = os.path.join(TEST_DATA_DIR, "ff_nul_dst")
clean_dir(dest)
lst = self._write_list(b"top.txt\0other/c.txt\0")
flags = ["--files-from", lst, "--from0"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"files-from -0 sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert os.path.isfile(os.path.join(received, "top.txt"))
assert os.path.isfile(os.path.join(received, "other", "c.txt"))
assert not os.path.exists(os.path.join(received, "sub"))
def test_files_from_missing_list_file_rejected(self):
result, _ = run_client(
os.path.join(TEST_DATA_DIR, "nowhere_src"),
os.path.join(TEST_DATA_DIR, "nowhere_dst"),
flags=["--files-from", os.path.join(TEST_DATA_DIR, "no_such_list.txt")],
)
assert result.returncode != 0, "missing --files-from file must be rejected"
assert "--files-from" in result.stderr
def test_files_from_delete_deletes_unlisted(self):
source = self._make_source("ff_delete_src")
dest = os.path.join(TEST_DATA_DIR, "ff_delete_dst")
clean_dir(dest)
server = ServerManager()
server.start(extra_args=["--allow-delete"])
try:
# Full transfer first.
result, _ = run_client(source, dest, port=server.port)
assert result.returncode == 0, f"full sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert os.path.isfile(os.path.join(received, "top.txt"))
# A subset sync with --delete deletes everything not in the sent
# manifest (which is derived from what was actually sent).
lst = self._write_list(b"sub/a.txt\n")
result, _ = run_client(source, dest, flags=["--files-from", lst, "--delete"],
port=server.port)
assert result.returncode == 0, f"files-from delete sync failed: {result.stderr[:200]}"
assert os.path.isfile(os.path.join(received, "sub", "a.txt"))
assert not os.path.exists(os.path.join(received, "top.txt")), \
"unlisted file not deleted"
assert not os.path.exists(os.path.join(received, "other")), \
"unlisted subtree not deleted"
finally:
server.stop()
class TestFilters:
"""--filter/-C/-F rule layer: excludes prune, ordering is first-match-wins,
the default with no matching rule is include, and legacy --exclude remains
an independent layer."""
def _make_tree(self, name, with_filter_file=False):
source = os.path.join(TEST_DATA_DIR, name)
clean_dir(source)
entries = {
"data/keep.txt": b"keep\n",
"data/drop.tmp": b"drop\n",
"nested/deep.tmp": b"deep\n",
"nested/ok.log": b"log\n",
"top.bin": b"bin\n",
}
for rel, content in entries.items():
full = os.path.join(source, rel)
os.makedirs(os.path.dirname(full), exist_ok=True)
with open(full, "wb") as fh:
fh.write(content)
if with_filter_file:
with open(os.path.join(source, ".rsync-filter"), "wb") as fh:
fh.write(b"- *.tmp\n")
return source
@pytest.mark.parametrize("mt", [False, True])
def test_filter_excludes_glob(self, shared_server, mt):
source = self._make_tree("filter_tmp_src")
dest = os.path.join(TEST_DATA_DIR, "filter_tmp_dst")
clean_dir(dest)
flags = ["--filter", "- *.tmp"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"filter sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert os.path.isfile(os.path.join(received, "data", "keep.txt"))
assert os.path.isfile(os.path.join(received, "nested", "ok.log"))
assert os.path.isfile(os.path.join(received, "top.bin"))
assert not os.path.exists(os.path.join(received, "data", "drop.tmp"))
assert not os.path.exists(os.path.join(received, "nested", "deep.tmp"))
@pytest.mark.parametrize("mt", [False, True])
def test_filter_anchored_include_overrides_exclude_all(self, shared_server, mt):
source = os.path.join(TEST_DATA_DIR, "filter_keep_src")
clean_dir(source)
with open(os.path.join(source, "a.keepme"), "wb") as fh:
fh.write(b"keep me")
with open(os.path.join(source, "b.other"), "wb") as fh:
fh.write(b"drop me")
dest = os.path.join(TEST_DATA_DIR, "filter_keep_dst")
clean_dir(dest)
# First match wins: the anchored include beats the catch-all exclude.
flags = ["--filter=+ /a.keepme", "--filter=- *"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"filter include sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert os.path.isfile(os.path.join(received, "a.keepme"))
assert not os.path.exists(os.path.join(received, "b.other"))
@pytest.mark.parametrize("mt", [False, True])
def test_cvs_exclude_ignores_scm_and_build_artifacts(self, shared_server, mt):
source = os.path.join(TEST_DATA_DIR, "filter_cvs_src")
clean_dir(source)
entries = {
"src/main.c": b"int main() {}\n",
"src/main.o": b"obj\n",
"src/notes.txt~": b"backup\n",
".git/HEAD": b"ref\n",
".git/config": b"cfg\n",
"README.md": b"readme\n",
}
for rel, content in entries.items():
full = os.path.join(source, rel)
os.makedirs(os.path.dirname(full), exist_ok=True)
with open(full, "wb") as fh:
fh.write(content)
dest = os.path.join(TEST_DATA_DIR, "filter_cvs_dst")
clean_dir(dest)
flags = ["-C"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"-C sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert os.path.isfile(os.path.join(received, "src", "main.c"))
assert os.path.isfile(os.path.join(received, "README.md"))
assert not os.path.exists(os.path.join(received, ".git")), ".git/ must be pruned"
assert not os.path.exists(os.path.join(received, "src", "main.o")), "*.o must be pruned"
assert not os.path.exists(os.path.join(received, "src", "notes.txt~")), "*~ must be pruned"
@pytest.mark.parametrize("mt", [False, True])
def test_per_dir_filter_file(self, shared_server, mt):
source = self._make_tree("filter_file_src", with_filter_file=True)
dest = os.path.join(TEST_DATA_DIR, "filter_file_dst")
clean_dir(dest)
flags = ["-F"] + (["-m"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"-F sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
assert os.path.isfile(os.path.join(received, "data", "keep.txt"))
assert os.path.isfile(os.path.join(received, "nested", "ok.log"))
assert not os.path.exists(os.path.join(received, "data", "drop.tmp"))
assert not os.path.exists(os.path.join(received, "nested", "deep.tmp"))
assert not os.path.exists(os.path.join(received, ".rsync-filter")), \
".rsync-filter must not be transferred"
def test_filter_leaves_default_behavior_unchanged(self, shared_server):
source = self._make_tree("filter_default_src")
dest = os.path.join(TEST_DATA_DIR, "filter_default_dst")
clean_dir(dest)
result, _ = run_client(source, dest, port=shared_server.port)
assert result.returncode == 0, f"plain sync failed: {result.stderr[:200]}"
received = get_dest_received_dir(dest, source)
mismatches, missing = verify_transfer(source, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
+135 -3
View File
@@ -2,6 +2,7 @@
#include "client_validation.h"
#include "chmod.h"
#include "config.h"
#include "file_list.h"
#include "log.h"
#include "test_utils.h"
#include "utils.h"
@@ -607,9 +608,6 @@ static void test_parse_args_rejects_unimplemented_options() {
"--delete-excluded",
"--delete-after",
"--max-delete",
"--filter",
"--files-from",
"--cvs-exclude",
"--prune-empty-dirs",
"-R",
"--relative",
@@ -1265,6 +1263,137 @@ static void test_parse_args_log_file_format() {
config_delete(cfg);
}
/* --filter is repeatable and accepts both "--filter RULE" and "--filter=RULE". */
static void test_parse_args_filter_rules() {
Config* cfg = config_create();
int positional_args[2];
int positional_count = 0;
char* argv[] = {"fastsync", "--filter", "- *.tmp", "--filter=+ /keep.txt", "/src", "/dst"};
EXPECT_EQ_INT(parse_args(cfg, 6, argv, positional_args, &positional_count), 0);
EXPECT_NOT_NULL(cfg->filters);
EXPECT_EQ_INT(cfg->filters->size, 2);
EXPECT_EQ_STR((char*)cfg->filters->items[0], "- *.tmp");
EXPECT_EQ_STR((char*)cfg->filters->items[1], "+ /keep.txt");
config_delete(cfg);
/* An unsupported rsync rule type is rejected with a clear error. */
cfg = config_create();
positional_count = 0;
char* bad_argv[] = {"fastsync", "--filter=merge /tmp/excludes", "/src", "/dst"};
EXPECT_EQ_INT(parse_args(cfg, 4, bad_argv, positional_args, &positional_count), -1);
config_delete(cfg);
/* A trailing --filter with no rule is a missing-argument error. */
cfg = config_create();
positional_count = 0;
char* missing_argv[] = {"fastsync", "/src", "/dst", "--filter"};
EXPECT_EQ_INT(parse_args(cfg, 4, missing_argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
/* -0/--from0, -C/--cvs-exclude and -F wire into their config flags. */
static void test_parse_args_from0_cvs_filter_file_flags() {
static const struct {
const char* arg;
bool from0;
bool cvs;
bool per_dir;
} cases[] = {
{"--from0", true, false, false},
{"-0", true, false, false},
{"--cvs-exclude", false, true, false},
{"-C", false, true, false},
{"-F", false, false, true},
};
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", (char*)cases[i].arg, "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->from0, cases[i].from0);
EXPECT_EQ_INT(cfg->cvs_exclude, cases[i].cvs);
EXPECT_EQ_INT(cfg->per_dir_filter, cases[i].per_dir);
config_delete(cfg);
}
}
static void write_file_bytes(const char* path, const char* bytes, size_t len) {
FILE* fp = fopen(path, "wb");
EXPECT_NOT_NULL(fp);
EXPECT_EQ_INT((int)fwrite(bytes, 1, len, fp), (int)len);
fclose(fp);
}
/* --files-from is validated and parsed after the full argument scan, so -0 may
* appear before or after it. */
static void test_parse_args_files_from() {
const char* list_path = "cli_files_from_list.txt";
write_file_bytes(list_path, "a.txt\nsub/b.bin\n\n./c.txt\n", 25);
Config* cfg = config_create();
int positional_args[2];
int positional_count = 0;
char* argv[] = {"fastsync", "--files-from", (char*)list_path, "/src", "/dst"};
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
EXPECT_EQ_STR(cfg->files_from, list_path);
EXPECT_NOT_NULL(cfg->files_from_set);
FileListSet* set = (FileListSet*)cfg->files_from_set;
EXPECT_TRUE(file_list_affects(set, "a.txt"));
EXPECT_TRUE(file_list_affects(set, "sub/b.bin"));
EXPECT_TRUE(file_list_affects(set, "sub/b.bin/x"));
EXPECT_TRUE(file_list_affects(set, "sub"));
EXPECT_TRUE(file_list_affects(set, "c.txt"));
EXPECT_FALSE(file_list_affects(set, "other.txt"));
config_delete(cfg);
remove(list_path);
/* -0 switches the separator to NUL regardless of argument order. */
write_file_bytes(list_path, "x.txt\0y/z.bin\0", 14);
cfg = config_create();
positional_count = 0;
char* nul_argv[] = {"fastsync",
"--files-from="
"cli_files_from_list.txt",
"-0", "/src", "/dst"};
EXPECT_EQ_INT(parse_args(cfg, 5, nul_argv, positional_args, &positional_count), 0);
set = (FileListSet*)cfg->files_from_set;
EXPECT_NOT_NULL(set);
EXPECT_TRUE(file_list_affects(set, "x.txt"));
EXPECT_TRUE(file_list_affects(set, "y/z.bin"));
EXPECT_TRUE(file_list_affects(set, "y"));
EXPECT_FALSE(file_list_affects(set, "z.txt"));
config_delete(cfg);
remove(list_path);
/* A missing list file is a hard parse-time error. */
cfg = config_create();
positional_count = 0;
char* missing_argv[] = {"fastsync", "--files-from", "does_not_exist_ff.txt", "/src", "/dst"};
EXPECT_EQ_INT(parse_args(cfg, 5, missing_argv, positional_args, &positional_count), -1);
config_delete(cfg);
/* Absolute and traversal entries are rejected. */
write_file_bytes(list_path, "/abs/path\n", 10);
cfg = config_create();
positional_count = 0;
char* abs_argv[] = {"fastsync",
"--files-from="
"cli_files_from_list.txt",
"/src", "/dst"};
EXPECT_EQ_INT(parse_args(cfg, 4, abs_argv, positional_args, &positional_count), -1);
config_delete(cfg);
write_file_bytes(list_path, "../escape\n", 10);
cfg = config_create();
positional_count = 0;
char* trav_argv[] = {"fastsync",
"--files-from="
"cli_files_from_list.txt",
"/src", "/dst"};
EXPECT_EQ_INT(parse_args(cfg, 4, trav_argv, positional_args, &positional_count), -1);
config_delete(cfg);
remove(list_path);
}
void test_client_cli() {
test_validate_config_required_paths();
test_validate_config_incompatible_options();
@@ -1344,4 +1473,7 @@ void test_client_cli() {
test_parse_args_checksum_choice_aliases();
test_parse_args_checksum_choice_requires_value();
test_parse_args_temp_dir();
test_parse_args_filter_rules();
test_parse_args_from0_cvs_filter_file_flags();
test_parse_args_files_from();
}
+354 -2
View File
@@ -1,7 +1,10 @@
#include "test_utils.h"
#include "scanner.h"
#include "file.h"
#include "file_list.h"
#include "filter.h"
#include "utils.h"
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
@@ -394,8 +397,8 @@ static void test_parallel_scanner_root_chunks_without_workers() {
create_test_file(file1, "a");
create_test_file(file2, "b");
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0,
0, 0, false, false, false, false, false, false};
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0, 0, 0,
false, false, false, false, false, false, NULL, NULL, false};
ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options, NULL);
EXPECT_NOT_NULL(scanner);
@@ -653,6 +656,345 @@ static void test_scanner_one_file_system_cross_device() {
EXPECT_EQ_INT(par_on_total, 1);
}
/* Collect emitted file paths (relative to `root`) from a sequential scan.
* Returns 0 on success with *out and *count set (caller frees *out). */
static int collect_files(const char* root, const ScannerOptions* options, char*** out,
int* out_count) {
DirectoryScanner* scanner = directory_scanner_create_with_options(root, options);
if (!scanner)
return -1;
size_t root_len = strlen(root);
while (root_len > 0 && root[root_len - 1] == '/')
root_len--;
int cap = 16;
int count = 0;
char** paths = malloc((size_t)cap * sizeof(char*));
if (!paths) {
directory_scanner_destroy(scanner);
return -1;
}
Chunk* chunk;
while ((chunk = directory_scanner_next(scanner)) != NULL) {
for (int i = 0; i < chunk->element_count; i++) {
const char* rel = chunk->items[i]->path + root_len;
if (*rel == '/')
rel++;
if (count == cap) {
cap *= 2;
char** grown = realloc(paths, (size_t)cap * sizeof(char*));
if (!grown) {
for (int k = 0; k < count; k++)
free(paths[k]);
free(paths);
chunk_destroy(chunk);
directory_scanner_destroy(scanner);
return -1;
}
paths = grown;
}
paths[count++] = str_dup(rel);
}
chunk_destroy(chunk);
}
bool failed = directory_scanner_failed(scanner);
directory_scanner_destroy(scanner);
if (failed) {
for (int k = 0; k < count; k++)
free(paths[k]);
free(paths);
return -1;
}
*out = paths;
*out_count = count;
return 0;
}
static int collect_files_parallel(const char* root, const ScannerOptions* options, char*** out,
int* out_count) {
ParallelScanner* scanner = parallel_scanner_create_with_options(root, options, NULL);
if (!scanner)
return -1;
size_t root_len = strlen(root);
while (root_len > 0 && root[root_len - 1] == '/')
root_len--;
int cap = 16;
int count = 0;
char** paths = malloc((size_t)cap * sizeof(char*));
if (!paths) {
parallel_scanner_destroy(scanner);
return -1;
}
Chunk* chunk;
while ((chunk = parallel_scanner_next(scanner)) != NULL) {
for (int i = 0; i < chunk->element_count; i++) {
const char* rel = chunk->items[i]->path + root_len;
if (*rel == '/')
rel++;
if (count == cap) {
cap *= 2;
char** grown = realloc(paths, (size_t)cap * sizeof(char*));
if (!grown) {
for (int k = 0; k < count; k++)
free(paths[k]);
free(paths);
chunk_destroy(chunk);
parallel_scanner_destroy(scanner);
return -1;
}
paths = grown;
}
paths[count++] = str_dup(rel);
}
chunk_destroy(chunk);
}
bool failed = parallel_scanner_failed(scanner);
parallel_scanner_destroy(scanner);
if (failed) {
for (int k = 0; k < count; k++)
free(paths[k]);
free(paths);
return -1;
}
*out = paths;
*out_count = count;
return 0;
}
static bool has_path(char** paths, int count, const char* rel) {
for (int i = 0; i < count; i++)
if (strcmp(paths[i], rel) == 0)
return true;
return false;
}
static void free_paths(char** paths, int count) {
for (int i = 0; i < count; i++)
free(paths[i]);
free(paths);
}
static const char* FILE_LIST_PATH = "test_scan_files_from.txt";
/* --files-from: only the listed files (and the subtree of a listed directory)
* are emitted; unrelated files and directories are pruned. */
static void test_files_from_subset(bool parallel) {
const char* root = "test_scan_ff";
const char* sub = "test_scan_ff/sub";
const char* other = "test_scan_ff/other";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
EXPECT_EQ_INT(mkdir(other, 0755), 0);
create_test_file("test_scan_ff/root.txt", "root");
create_test_file("test_scan_ff/sub/keep.txt", "keep");
create_test_file("test_scan_ff/sub/skip.bin", "skip");
create_test_file("test_scan_ff/other/unrelated.txt", "unrelated");
/* List a root file and a file under sub: sub is descended but its other file
* is not listed, and the whole `other` directory is pruned. */
create_test_file(FILE_LIST_PATH, "root.txt\nsub/keep.txt\n");
char err[160];
FileListSet* set = file_list_load(FILE_LIST_PATH, false, err, sizeof(err));
EXPECT_NOT_NULL(set);
ScannerOptions options = {0};
options.file_list = set;
if (parallel)
options.num_threads = 2;
char** paths = NULL;
int count = 0;
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 2);
EXPECT_TRUE(has_path(paths, count, "root.txt"));
EXPECT_TRUE(has_path(paths, count, "sub/keep.txt"));
EXPECT_FALSE(has_path(paths, count, "sub/skip.bin"));
EXPECT_FALSE(has_path(paths, count, "other/unrelated.txt"));
free_paths(paths, count);
file_list_destroy(set);
remove(FILE_LIST_PATH);
/* Listing a directory transfers its whole subtree. */
create_test_file(FILE_LIST_PATH, "sub\n");
set = file_list_load(FILE_LIST_PATH, false, err, sizeof(err));
EXPECT_NOT_NULL(set);
options.file_list = set;
rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 2);
EXPECT_TRUE(has_path(paths, count, "sub/keep.txt"));
EXPECT_TRUE(has_path(paths, count, "sub/skip.bin"));
EXPECT_FALSE(has_path(paths, count, "root.txt"));
EXPECT_FALSE(has_path(paths, count, "other/unrelated.txt"));
free_paths(paths, count);
file_list_destroy(set);
remove(FILE_LIST_PATH);
unlink("test_scan_ff/root.txt");
unlink("test_scan_ff/sub/keep.txt");
unlink("test_scan_ff/sub/skip.bin");
unlink("test_scan_ff/other/unrelated.txt");
rmdir(other);
rmdir(sub);
rmdir(root);
}
/* Filter layer: '-' excludes, first-match-wins ordering with '+', anchored
* rules, and dir-only rules all prune during the scan. */
static void test_filter_rules(bool parallel) {
const char* root = "test_scan_filter";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
create_test_file("test_scan_filter/a.txt", "a");
create_test_file("test_scan_filter/b.tmp", "b");
create_test_file("test_scan_filter/c.txt", "c");
/* - *.tmp excludes only the tmp file; other files remain (default include). */
const char* exclude_only[] = {"- *.tmp"};
char err[160];
FilterRuleList* base = filter_base_build(exclude_only, 1, false, err, sizeof(err));
EXPECT_NOT_NULL(base);
ScannerOptions options = {0};
options.base_filters = base;
if (parallel)
options.num_threads = 2;
char** paths = NULL;
int count = 0;
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 2);
EXPECT_TRUE(has_path(paths, count, "a.txt"));
EXPECT_TRUE(has_path(paths, count, "c.txt"));
EXPECT_FALSE(has_path(paths, count, "b.tmp"));
free_paths(paths, count);
filter_rule_list_free(base);
/* Anchored include then exclude-all: only root-level keep* survives. */
const char* anchored[] = {"+ /a.txt", "- *"};
base = filter_base_build(anchored, 2, false, err, sizeof(err));
EXPECT_NOT_NULL(base);
options.base_filters = base;
rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 1);
EXPECT_TRUE(has_path(paths, count, "a.txt"));
free_paths(paths, count);
filter_rule_list_free(base);
unlink("test_scan_filter/a.txt");
unlink("test_scan_filter/b.tmp");
unlink("test_scan_filter/c.txt");
rmdir(root);
}
/* Anchored dir-only rules prune a whole subtree. */
static void test_filter_dir_only_and_anchored(bool parallel) {
const char* root = "test_scan_filter_dir";
const char* sub = "test_scan_filter_dir/sub";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
create_test_file("test_scan_filter_dir/sub/inner.txt", "x");
create_test_file("test_scan_filter_dir/keep.txt", "keep");
const char* rules[] = {"- /sub/"};
char err[160];
FilterRuleList* base = filter_base_build(rules, 1, false, err, sizeof(err));
EXPECT_NOT_NULL(base);
ScannerOptions options = {0};
options.base_filters = base;
if (parallel)
options.num_threads = 2;
char** paths = NULL;
int count = 0;
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 1);
EXPECT_TRUE(has_path(paths, count, "keep.txt"));
EXPECT_FALSE(has_path(paths, count, "sub/inner.txt"));
free_paths(paths, count);
filter_rule_list_free(base);
unlink("test_scan_filter_dir/sub/inner.txt");
unlink("test_scan_filter_dir/keep.txt");
rmdir(sub);
rmdir(root);
}
/* -C default CVS excludes prune .git/ directories and *.o files. */
static void test_cvs_defaults(bool parallel) {
const char* root = "test_scan_cvs";
const char* git = "test_scan_cvs/.git";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(git, 0755), 0);
create_test_file("test_scan_cvs/.git/config", "cfg");
create_test_file("test_scan_cvs/object.o", "o");
create_test_file("test_scan_cvs/keep.txt", "keep");
char err[160];
FilterRuleList* base = filter_base_build(NULL, 0, true, err, sizeof(err));
EXPECT_NOT_NULL(base);
ScannerOptions options = {0};
options.base_filters = base;
if (parallel)
options.num_threads = 2;
char** paths = NULL;
int count = 0;
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 1);
EXPECT_TRUE(has_path(paths, count, "keep.txt"));
EXPECT_FALSE(has_path(paths, count, ".git/config"));
EXPECT_FALSE(has_path(paths, count, "object.o"));
free_paths(paths, count);
filter_rule_list_free(base);
unlink("test_scan_cvs/.git/config");
unlink("test_scan_cvs/object.o");
unlink("test_scan_cvs/keep.txt");
rmdir(git);
rmdir(root);
}
/* -F: a .rsync-filter placed in a directory governs its subtree and the file
* itself is never transferred. */
static void test_per_dir_filter(bool parallel) {
const char* root = "test_scan_perdir";
const char* sub = "test_scan_perdir/sub";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
create_test_file("test_scan_perdir/drop.tmp", "tmp");
create_test_file("test_scan_perdir/keep.txt", "keep");
create_test_file("test_scan_perdir/sub/nested.tmp", "tmp");
create_test_file("test_scan_perdir/.rsync-filter", "- *.tmp\n");
ScannerOptions options = {0};
options.per_dir_filters = true;
if (parallel)
options.num_threads = 2;
char** paths = NULL;
int count = 0;
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 1);
EXPECT_TRUE(has_path(paths, count, "keep.txt"));
EXPECT_FALSE(has_path(paths, count, "drop.tmp"));
EXPECT_FALSE(has_path(paths, count, "sub/nested.tmp"));
EXPECT_FALSE(has_path(paths, count, ".rsync-filter"));
free_paths(paths, count);
unlink("test_scan_perdir/drop.tmp");
unlink("test_scan_perdir/keep.txt");
unlink("test_scan_perdir/sub/nested.tmp");
unlink("test_scan_perdir/.rsync-filter");
rmdir(sub);
rmdir(root);
}
void test_scanner() {
test_scanner_single_file();
test_scanner_multiple_files();
@@ -672,4 +1014,14 @@ void test_scanner() {
test_scanner_one_file_system_same_device();
test_parallel_scanner_one_file_system_same_device();
test_scanner_one_file_system_cross_device();
test_files_from_subset(false);
test_files_from_subset(true);
test_filter_rules(false);
test_filter_rules(true);
test_filter_dir_only_and_anchored(false);
test_filter_dir_only_and_anchored(true);
test_cvs_defaults(false);
test_cvs_defaults(true);
test_per_dir_filter(false);
test_per_dir_filter(true);
}