feat: add --files-from/-0, --filter, -C and -F filter layer
CI / lint (pull_request) Successful in 23s
CI / sanitizers (address) (pull_request) Successful in 40s
CI / sanitizers (undefined) (pull_request) Successful in 39s
CI / fuzz-build (pull_request) Successful in 17s
CI / coverage (pull_request) Successful in 34s
CI / build-and-test (pull_request) Successful in 1m28s
CI / valgrind (pull_request) Successful in 36s

Implement the RSYNC_COMPAT Phase-2 filter/parser feature group:
- --files-from=FILE (repeatable) plus -0/--from0 NUL delimiters: parse the
  source file list relative to the source root into a shared read-only
  allow-set; the scanner transfers listed files and the whole subtree of
  listed directories and prunes everything else in single- and
  multithreaded mode. Absolute/'..' entries and missing files are hard
  CLI errors.
- --filter=RULE: rsync-style +/- rules (anchored '/', dir-only trailing '/',
  word include/exclude forms) evaluated first-match-wins with a default of
  include, as an independent layer from legacy --exclude/--include.
  Unsupported directives (merge/hide/... ) are rejected explicitly. -f stays
  sendfile.
- -C/--cvs-exclude: well-known rsync CVS default exclude set.
- -F: per-directory .rsync-filter files read during traversal and applied to
  the owning directory's subtree (single + parallel), never transferred.
- Delete manifest still derives from what was actually sent.

Client-only config fields; no wire/protocol change. Adds unit coverage
(CLI parse, allow-set and filter scanning single+parallel) and integration
tests (TestFilesFrom, TestFilters). RSYNC_COMPAT matrix rows updated:
5 rows move to Implemented (Summary 62/3/5/1/76 = 147).
This commit is contained in:
2026-09-06 13:43:34 +02:00
parent 8577f95550
commit f4c15a7b78
15 changed files with 1917 additions and 64 deletions
+354 -2
View File
@@ -1,7 +1,10 @@
#include "test_utils.h"
#include "scanner.h"
#include "file.h"
#include "file_list.h"
#include "filter.h"
#include "utils.h"
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
@@ -394,8 +397,8 @@ static void test_parallel_scanner_root_chunks_without_workers() {
create_test_file(file1, "a");
create_test_file(file2, "b");
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0,
0, 0, false, false, false, false, false, false};
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0, 0, 0,
false, false, false, false, false, false, NULL, NULL, false};
ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options, NULL);
EXPECT_NOT_NULL(scanner);
@@ -653,6 +656,345 @@ static void test_scanner_one_file_system_cross_device() {
EXPECT_EQ_INT(par_on_total, 1);
}
/* Collect emitted file paths (relative to `root`) from a sequential scan.
* Returns 0 on success with *out and *count set (caller frees *out). */
static int collect_files(const char* root, const ScannerOptions* options, char*** out,
int* out_count) {
DirectoryScanner* scanner = directory_scanner_create_with_options(root, options);
if (!scanner)
return -1;
size_t root_len = strlen(root);
while (root_len > 0 && root[root_len - 1] == '/')
root_len--;
int cap = 16;
int count = 0;
char** paths = malloc((size_t)cap * sizeof(char*));
if (!paths) {
directory_scanner_destroy(scanner);
return -1;
}
Chunk* chunk;
while ((chunk = directory_scanner_next(scanner)) != NULL) {
for (int i = 0; i < chunk->element_count; i++) {
const char* rel = chunk->items[i]->path + root_len;
if (*rel == '/')
rel++;
if (count == cap) {
cap *= 2;
char** grown = realloc(paths, (size_t)cap * sizeof(char*));
if (!grown) {
for (int k = 0; k < count; k++)
free(paths[k]);
free(paths);
chunk_destroy(chunk);
directory_scanner_destroy(scanner);
return -1;
}
paths = grown;
}
paths[count++] = str_dup(rel);
}
chunk_destroy(chunk);
}
bool failed = directory_scanner_failed(scanner);
directory_scanner_destroy(scanner);
if (failed) {
for (int k = 0; k < count; k++)
free(paths[k]);
free(paths);
return -1;
}
*out = paths;
*out_count = count;
return 0;
}
static int collect_files_parallel(const char* root, const ScannerOptions* options, char*** out,
int* out_count) {
ParallelScanner* scanner = parallel_scanner_create_with_options(root, options, NULL);
if (!scanner)
return -1;
size_t root_len = strlen(root);
while (root_len > 0 && root[root_len - 1] == '/')
root_len--;
int cap = 16;
int count = 0;
char** paths = malloc((size_t)cap * sizeof(char*));
if (!paths) {
parallel_scanner_destroy(scanner);
return -1;
}
Chunk* chunk;
while ((chunk = parallel_scanner_next(scanner)) != NULL) {
for (int i = 0; i < chunk->element_count; i++) {
const char* rel = chunk->items[i]->path + root_len;
if (*rel == '/')
rel++;
if (count == cap) {
cap *= 2;
char** grown = realloc(paths, (size_t)cap * sizeof(char*));
if (!grown) {
for (int k = 0; k < count; k++)
free(paths[k]);
free(paths);
chunk_destroy(chunk);
parallel_scanner_destroy(scanner);
return -1;
}
paths = grown;
}
paths[count++] = str_dup(rel);
}
chunk_destroy(chunk);
}
bool failed = parallel_scanner_failed(scanner);
parallel_scanner_destroy(scanner);
if (failed) {
for (int k = 0; k < count; k++)
free(paths[k]);
free(paths);
return -1;
}
*out = paths;
*out_count = count;
return 0;
}
static bool has_path(char** paths, int count, const char* rel) {
for (int i = 0; i < count; i++)
if (strcmp(paths[i], rel) == 0)
return true;
return false;
}
static void free_paths(char** paths, int count) {
for (int i = 0; i < count; i++)
free(paths[i]);
free(paths);
}
static const char* FILE_LIST_PATH = "test_scan_files_from.txt";
/* --files-from: only the listed files (and the subtree of a listed directory)
* are emitted; unrelated files and directories are pruned. */
static void test_files_from_subset(bool parallel) {
const char* root = "test_scan_ff";
const char* sub = "test_scan_ff/sub";
const char* other = "test_scan_ff/other";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
EXPECT_EQ_INT(mkdir(other, 0755), 0);
create_test_file("test_scan_ff/root.txt", "root");
create_test_file("test_scan_ff/sub/keep.txt", "keep");
create_test_file("test_scan_ff/sub/skip.bin", "skip");
create_test_file("test_scan_ff/other/unrelated.txt", "unrelated");
/* List a root file and a file under sub: sub is descended but its other file
* is not listed, and the whole `other` directory is pruned. */
create_test_file(FILE_LIST_PATH, "root.txt\nsub/keep.txt\n");
char err[160];
FileListSet* set = file_list_load(FILE_LIST_PATH, false, err, sizeof(err));
EXPECT_NOT_NULL(set);
ScannerOptions options = {0};
options.file_list = set;
if (parallel)
options.num_threads = 2;
char** paths = NULL;
int count = 0;
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 2);
EXPECT_TRUE(has_path(paths, count, "root.txt"));
EXPECT_TRUE(has_path(paths, count, "sub/keep.txt"));
EXPECT_FALSE(has_path(paths, count, "sub/skip.bin"));
EXPECT_FALSE(has_path(paths, count, "other/unrelated.txt"));
free_paths(paths, count);
file_list_destroy(set);
remove(FILE_LIST_PATH);
/* Listing a directory transfers its whole subtree. */
create_test_file(FILE_LIST_PATH, "sub\n");
set = file_list_load(FILE_LIST_PATH, false, err, sizeof(err));
EXPECT_NOT_NULL(set);
options.file_list = set;
rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 2);
EXPECT_TRUE(has_path(paths, count, "sub/keep.txt"));
EXPECT_TRUE(has_path(paths, count, "sub/skip.bin"));
EXPECT_FALSE(has_path(paths, count, "root.txt"));
EXPECT_FALSE(has_path(paths, count, "other/unrelated.txt"));
free_paths(paths, count);
file_list_destroy(set);
remove(FILE_LIST_PATH);
unlink("test_scan_ff/root.txt");
unlink("test_scan_ff/sub/keep.txt");
unlink("test_scan_ff/sub/skip.bin");
unlink("test_scan_ff/other/unrelated.txt");
rmdir(other);
rmdir(sub);
rmdir(root);
}
/* Filter layer: '-' excludes, first-match-wins ordering with '+', anchored
* rules, and dir-only rules all prune during the scan. */
static void test_filter_rules(bool parallel) {
const char* root = "test_scan_filter";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
create_test_file("test_scan_filter/a.txt", "a");
create_test_file("test_scan_filter/b.tmp", "b");
create_test_file("test_scan_filter/c.txt", "c");
/* - *.tmp excludes only the tmp file; other files remain (default include). */
const char* exclude_only[] = {"- *.tmp"};
char err[160];
FilterRuleList* base = filter_base_build(exclude_only, 1, false, err, sizeof(err));
EXPECT_NOT_NULL(base);
ScannerOptions options = {0};
options.base_filters = base;
if (parallel)
options.num_threads = 2;
char** paths = NULL;
int count = 0;
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 2);
EXPECT_TRUE(has_path(paths, count, "a.txt"));
EXPECT_TRUE(has_path(paths, count, "c.txt"));
EXPECT_FALSE(has_path(paths, count, "b.tmp"));
free_paths(paths, count);
filter_rule_list_free(base);
/* Anchored include then exclude-all: only root-level keep* survives. */
const char* anchored[] = {"+ /a.txt", "- *"};
base = filter_base_build(anchored, 2, false, err, sizeof(err));
EXPECT_NOT_NULL(base);
options.base_filters = base;
rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 1);
EXPECT_TRUE(has_path(paths, count, "a.txt"));
free_paths(paths, count);
filter_rule_list_free(base);
unlink("test_scan_filter/a.txt");
unlink("test_scan_filter/b.tmp");
unlink("test_scan_filter/c.txt");
rmdir(root);
}
/* Anchored dir-only rules prune a whole subtree. */
static void test_filter_dir_only_and_anchored(bool parallel) {
const char* root = "test_scan_filter_dir";
const char* sub = "test_scan_filter_dir/sub";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
create_test_file("test_scan_filter_dir/sub/inner.txt", "x");
create_test_file("test_scan_filter_dir/keep.txt", "keep");
const char* rules[] = {"- /sub/"};
char err[160];
FilterRuleList* base = filter_base_build(rules, 1, false, err, sizeof(err));
EXPECT_NOT_NULL(base);
ScannerOptions options = {0};
options.base_filters = base;
if (parallel)
options.num_threads = 2;
char** paths = NULL;
int count = 0;
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 1);
EXPECT_TRUE(has_path(paths, count, "keep.txt"));
EXPECT_FALSE(has_path(paths, count, "sub/inner.txt"));
free_paths(paths, count);
filter_rule_list_free(base);
unlink("test_scan_filter_dir/sub/inner.txt");
unlink("test_scan_filter_dir/keep.txt");
rmdir(sub);
rmdir(root);
}
/* -C default CVS excludes prune .git/ directories and *.o files. */
static void test_cvs_defaults(bool parallel) {
const char* root = "test_scan_cvs";
const char* git = "test_scan_cvs/.git";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(git, 0755), 0);
create_test_file("test_scan_cvs/.git/config", "cfg");
create_test_file("test_scan_cvs/object.o", "o");
create_test_file("test_scan_cvs/keep.txt", "keep");
char err[160];
FilterRuleList* base = filter_base_build(NULL, 0, true, err, sizeof(err));
EXPECT_NOT_NULL(base);
ScannerOptions options = {0};
options.base_filters = base;
if (parallel)
options.num_threads = 2;
char** paths = NULL;
int count = 0;
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 1);
EXPECT_TRUE(has_path(paths, count, "keep.txt"));
EXPECT_FALSE(has_path(paths, count, ".git/config"));
EXPECT_FALSE(has_path(paths, count, "object.o"));
free_paths(paths, count);
filter_rule_list_free(base);
unlink("test_scan_cvs/.git/config");
unlink("test_scan_cvs/object.o");
unlink("test_scan_cvs/keep.txt");
rmdir(git);
rmdir(root);
}
/* -F: a .rsync-filter placed in a directory governs its subtree and the file
* itself is never transferred. */
static void test_per_dir_filter(bool parallel) {
const char* root = "test_scan_perdir";
const char* sub = "test_scan_perdir/sub";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
create_test_file("test_scan_perdir/drop.tmp", "tmp");
create_test_file("test_scan_perdir/keep.txt", "keep");
create_test_file("test_scan_perdir/sub/nested.tmp", "tmp");
create_test_file("test_scan_perdir/.rsync-filter", "- *.tmp\n");
ScannerOptions options = {0};
options.per_dir_filters = true;
if (parallel)
options.num_threads = 2;
char** paths = NULL;
int count = 0;
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 1);
EXPECT_TRUE(has_path(paths, count, "keep.txt"));
EXPECT_FALSE(has_path(paths, count, "drop.tmp"));
EXPECT_FALSE(has_path(paths, count, "sub/nested.tmp"));
EXPECT_FALSE(has_path(paths, count, ".rsync-filter"));
free_paths(paths, count);
unlink("test_scan_perdir/drop.tmp");
unlink("test_scan_perdir/keep.txt");
unlink("test_scan_perdir/sub/nested.tmp");
unlink("test_scan_perdir/.rsync-filter");
rmdir(sub);
rmdir(root);
}
void test_scanner() {
test_scanner_single_file();
test_scanner_multiple_files();
@@ -672,4 +1014,14 @@ void test_scanner() {
test_scanner_one_file_system_same_device();
test_parallel_scanner_one_file_system_same_device();
test_scanner_one_file_system_cross_device();
test_files_from_subset(false);
test_files_from_subset(true);
test_filter_rules(false);
test_filter_rules(true);
test_filter_dir_only_and_anchored(false);
test_filter_dir_only_and_anchored(true);
test_cvs_defaults(false);
test_cvs_defaults(true);
test_per_dir_filter(false);
test_per_dir_filter(true);
}