diff --git a/RSYNC_COMPAT.md b/RSYNC_COMPAT.md index efdd5a9..1dd8959 100644 --- a/RSYNC_COMPAT.md +++ b/RSYNC_COMPAT.md @@ -6,11 +6,11 @@ This document maps rsync's full feature set to FastSync's current implementation | Status | Count | Description | |--------|-------|-------------| -| ✅ Implemented | 57 | Feature works end-to-end | +| ✅ Implemented | 62 | Feature works end-to-end | | 🔀 Alt Arg | 3 | Functionality exists but under different flag/semantics | | ⚠️ Partial | 5 | Flag parsed/stored but behavior incomplete | | 🔄 Compatibility No-op | 1 | Flag is accepted for CLI compatibility but has no effect | -| ❌ Not Implemented | 81 | Flag not recognized or no behavior | +| ❌ Not Implemented | 76 | Flag not recognized or no behavior | | **Total** | **147** | | --- @@ -30,7 +30,7 @@ This document maps rsync's full feature set to FastSync's current implementation | `--no-motd` | Suppress daemon MOTD | ❌ Not Implemented | | | `--exclude=PATTERN` | Exclude files matching pattern | ✅ Implemented | Glob matching in scanner | | `--include=PATTERN` | Include files matching pattern | ✅ Implemented | Glob matching in scanner | -| `-C`, `--cvs-exclude` | Auto-ignore CVS files | ❌ Not Implemented | Removed because it had no effect | +| `-C`, `--cvs-exclude` | Auto-ignore CVS files | ✅ Implemented | Applies the well-known rsync default exclude set as exclude rules during scanning (RCS SCCS CVS CVS.adm RCSLOG cvslog.* tags TAGS .make.state .nse_depinfo *~ #* .#* ,* _$* *$ *.old *.bak *.BAK *.orig *.rej .del-* *.a *.olb *.o *.obj *.so *.exe *.Z *.elc *.ln core .svn/ .git/ .hg/ .bzr/); `.git/`-style repo dirs are pruned without descending | ## 2. Modifying Output @@ -53,9 +53,9 @@ This document maps rsync's full feature set to FastSync's current implementation |------|-------------------|-----------------|-------| | `--exclude-from=FILE` | Read exclude patterns from file | ✅ Implemented | Reads patterns from file | | `--include-from=FILE` | Read include patterns from file | ✅ Implemented | Reads patterns from file | -| `--filter=RULE` | Add file-filtering rule | ❌ Not Implemented | Removed because it had no effect | -| `--files-from=FILE` | Read source file list from file | ❌ Not Implemented | Removed because it had no effect | -| `-0`, `--from0` | Delimit *-from files with NULs | ❌ Not Implemented | | +| `--filter=RULE` | Add file-filtering rule | ✅ Implemented | Long option only: rsync's short `-f` conflicts with FastSync sendfile (see FastSync-specific list), so `-f` is not reassigned. Supported subset: `+`/`-` include/exclude, implicit-exclude patterns, `include`/`exclude` word forms, a leading `/` anchor (to the transfer root, or to a `.rsync-filter` file's directory), and a trailing `/` for dir-only rules; first match wins with a default of include inside the filter layer. `merge`/`dir-merge`/`hide`/`show`/`protect`/`risk`/`clear` and rule modifiers are rejected with a clear error. Filters are an independent layer from `--exclude`/`--include` (an entry must pass both) | +| `--files-from=FILE` | Read source file list from file | ✅ Implemented | Entries are paths relative to the source root (leading `./` stripped, `..`/absolute entries rejected at parse time, blank lines ignored). A listed regular file is transferred; a listed directory transfers its whole subtree (FastSync recursion is always on, unlike rsync's non-recursive default). Non-listed paths and their subtrees are pruned by the scanner. The delete manifest still derives from what was actually sent, so `--delete` stays consistent with the subset | +| `-0`, `--from0` | Delimit *-from files with NULs | ✅ Implemented | `--files-from` entries become NUL-delimited; the flag may appear before or after `--files-from` on the command line | | `--max-size=SIZE` | Skip files larger than SIZE | ✅ Implemented | `max_size` in scanner | | `--min-size=SIZE` | Skip files smaller than SIZE | ✅ Implemented | `min_size` in scanner | | `-I`, `--ignore-times` | Don't skip files matching size+time | ❌ Not Implemented | | @@ -65,7 +65,7 @@ This document maps rsync's full feature set to FastSync's current implementation | `--ignore-existing` | Skip updating existing files | ❌ Not Implemented | | | `--remove-source-files` | Sender removes regular files after confirmed transfer | ✅ Implemented | | | `-x`, `--one-file-system` | Do not cross filesystem boundaries | ✅ Implemented | Sender scanner captures the root device and skips descending into mount-point crossings (`st_dev` differs); cross-filesystem mount-point subdirectories are dropped entirely, matching rsync | -| `-F` | Add the default `.rsync-filter` rules | ❌ Not Implemented | | +| `-F` | Add the default `.rsync-filter` rules | ✅ Implemented | Reads one filter rule per line from each directory's `.rsync-filter` file during traversal and applies it to that directory's subtree (rules inherit into deeper directories and are overridable by deeper files); `.rsync-filter` files are never transferred. The rsync `-FF` behavior (also `.cvsignore`) is out of scope; unsupported rule types inside the file abort with a clear error | ## 4. Directory Options diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 469034e..b380ef6 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -4,6 +4,8 @@ #include "compression.h" #include "config.h" #include "delta.h" +#include "file_list.h" +#include "filter.h" #include "log.h" #include "protocol.h" #include "transport_tcp.h" @@ -313,6 +315,31 @@ static int config_add_pattern(char*** patterns, int* count, const char* value, return 0; } +/* Validate and append one --filter=RULE string. Returns 0 on success, -1 on error. */ +static int config_add_filter(Config* config, const char* rule) { + char err[160]; + FilterRule* parsed = filter_rule_parse(rule, err, sizeof(err)); + if (!parsed) { + log_message(LOG_LEVEL_ERROR, "invalid --filter rule '%s': %s", rule, err); + return -1; + } + filter_rule_free(parsed); + if (!config->filters) { + config->filters = array_list_create(free); + if (!config->filters) { + log_message(LOG_LEVEL_ERROR, "memory allocation failed for --filter"); + return -1; + } + } + char* dup = str_dup(rule); + if (!dup || !array_list_add(config->filters, dup)) { + free(dup); + log_message(LOG_LEVEL_ERROR, "memory allocation failed for --filter"); + return -1; + } + return 0; +} + static int parse_skip_compress(Config* config, const char* value) { char* list = str_dup(value); if (!list) @@ -422,6 +449,9 @@ static const OptionEntry OPTION_TABLE[] = { {"--max-size", NULL, OPT_ULL, offsetof(Config, max_size)}, {"--min-size", NULL, OPT_ULL, offsetof(Config, min_size)}, {"--one-file-system", "-x", OPT_FLAG, offsetof(Config, one_file_system)}, + {"--from0", "-0", OPT_FLAG, offsetof(Config, from0)}, + {"--cvs-exclude", "-C", OPT_FLAG, offsetof(Config, cvs_exclude)}, + {"-F", NULL, OPT_FLAG, offsetof(Config, per_dir_filter)}, }; /* Only boolean options with no required argument are safe to negate. */ @@ -861,6 +891,26 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args, if (read_patterns_from_file(argv[++i], &config->include_patterns, &config->include_count) != 0) return -1; + } else if (strncmp(argv[i], "--filter=", 9) == 0) { + if (config_add_filter(config, argv[i] + 9) != 0) + return -1; + } else if (opt_is(argv[i], "--filter", NULL)) { + if (i + 1 >= argc) { + log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]); + return -1; + } + if (config_add_filter(config, argv[++i]) != 0) + return -1; + } else if (strncmp(argv[i], "--files-from=", 13) == 0) { + if (set_string_option(&config->files_from, argv[i] + 13, "--files-from") != 0) + return -1; + } else if (opt_is(argv[i], "--files-from", NULL)) { + if (i + 1 >= argc) { + log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]); + return -1; + } + if (set_string_option(&config->files_from, argv[++i], "--files-from") != 0) + return -1; } else if (opt_is(argv[i], "-v", "--verbose")) { verbose = true; set_log_level(LOG_LEVEL_DEBUG); @@ -935,6 +985,20 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args, if (config->compress_choice) config->use_compression = strcmp(config->compress_choice, "zstd") == 0; + /* --files-from is loaded after every argument is seen so that -0/--from0 may + * appear anywhere on the command line. A missing or unreadable file, and + * invalid (absolute / traversal) entries, are hard CLI errors. */ + if (config->files_from) { + char err[256]; + FileListSet* set = file_list_load(config->files_from, config->from0, err, sizeof(err)); + if (!set) { + log_message(LOG_LEVEL_ERROR, "--files-from: %s", err); + return -1; + } + file_list_destroy((FileListSet*)config->files_from_set); + config->files_from_set = set; + } + /* Incremental and delta transfers need metadata unless the user disabled it. */ if ((config->use_incremental || config->use_delta) && !config->use_metadata && !config->metadata_explicitly_disabled) { diff --git a/src/client/client_send.c b/src/client/client_send.c index cc9392a..d5c4717 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -7,6 +7,8 @@ #include "data.h" #include "delta.h" #include "file.h" +#include "file_list.h" +#include "filter.h" #include "metadata.h" #include "log.h" #include "multiprocessing.h" @@ -40,16 +42,72 @@ static const char* display_bytes(unsigned long long bytes, bool human_readable, return buffer; } -static ScannerOptions scanner_options_from_config(const Config* config, int num_threads) { - ScannerOptions options = {config->use_metadata, config->chunk_size, - config->exclude_patterns, config->exclude_count, - config->include_patterns, config->include_count, - config->max_size, config->min_size, - config->max_depth, num_threads, - config->follow_symlinks, config->copy_links, - config->safe_links, config->copy_unsafe_links, - config->checksum, config->one_file_system}; - return options; +/* Compiled scanner inputs that are shared read-only across scanner instances + * and, in -m mode, across worker threads. `base_filters` owns the compiled + * command-line + -C rules; the FileListSet allow-set lives in the Config. */ +typedef struct { + ScannerOptions options; + FilterRuleList* base_filters; /* owned; may be NULL */ +} PreparedScanner; + +/* Build the scanner options for one scan. Returns false and logs on failure. */ +static bool prepare_scanner(const Config* config, int num_threads, PreparedScanner* out) { + if (!out) + return false; + out->base_filters = NULL; + memset(&out->options, 0, sizeof(out->options)); + + int rule_count = config->filters ? config->filters->size : 0; + const char** texts = NULL; + if (rule_count > 0) { + texts = malloc((size_t)rule_count * sizeof(char*)); + if (!texts) { + log_message(LOG_LEVEL_ERROR, "memory allocation failed for filter rules"); + return false; + } + for (int i = 0; i < rule_count; i++) + texts[i] = (const char*)config->filters->items[i]; + } + if (rule_count > 0 || config->cvs_exclude) { + char err[160]; + out->base_filters = filter_base_build(texts, rule_count, config->cvs_exclude, err, sizeof(err)); + free(texts); + if (!out->base_filters) { + log_message(LOG_LEVEL_ERROR, "invalid filter rule: %s", err); + return false; + } + } else { + free(texts); + } + + ScannerOptions* options = &out->options; + options->use_metadata = config->use_metadata; + options->chunk_size = config->chunk_size; + options->exclude_patterns = config->exclude_patterns; + options->exclude_count = config->exclude_count; + options->include_patterns = config->include_patterns; + options->include_count = config->include_count; + options->max_size = config->max_size; + options->min_size = config->min_size; + options->max_depth = config->max_depth; + options->num_threads = num_threads; + options->follow_symlinks = config->follow_symlinks; + options->copy_links = config->copy_links; + options->safe_links = config->safe_links; + options->copy_unsafe_links = config->copy_unsafe_links; + options->checksum = config->checksum; + options->one_file_system = config->one_file_system; + options->file_list = (const FileListSet*)config->files_from_set; + options->base_filters = out->base_filters; + options->per_dir_filters = config->per_dir_filter; + return true; +} + +static void prepared_scanner_destroy(PreparedScanner* prepared) { + if (!prepared) + return; + filter_rule_list_free(prepared->base_filters); + prepared->base_filters = NULL; } /* Select the configured transport for both transfer execution paths. */ @@ -249,11 +307,15 @@ static void pipeline_cancel(PipelineContextSender* context) { /* Print dry-run manifest showing files that would be transferred. Returns 0 on success. */ static int send_dry_run_manifest(const Config* config) { - ScannerOptions options = scanner_options_from_config(config, 0); - DirectoryScanner* scanner = - directory_scanner_create_with_options(config->send_directory, &options); - if (!scanner) + PreparedScanner prepared; + if (!prepare_scanner(config, 0, &prepared)) return -1; + DirectoryScanner* scanner = + directory_scanner_create_with_options(config->send_directory, &prepared.options); + if (!scanner) { + prepared_scanner_destroy(&prepared); + return -1; + } Chunk* chunk; int file_count = 0; unsigned long long total_bytes = 0; @@ -267,6 +329,7 @@ static int send_dry_run_manifest(const Config* config) { if (!escaped_path) { chunk_destroy(chunk); directory_scanner_destroy(scanner); + prepared_scanner_destroy(&prepared); return -1; } if (config->human_readable) @@ -283,6 +346,7 @@ static int send_dry_run_manifest(const Config* config) { chunk_destroy(chunk); } directory_scanner_destroy(scanner); + prepared_scanner_destroy(&prepared); if (!config->quiet) { if (config->human_readable) printf("Total: %d files, %s\n", file_count, @@ -319,12 +383,16 @@ static int compare_list_entries(const void* left, const void* right) { * Directory lines are not printed because the scanner only yields regular * transfer candidates. Returns 0 on success, 1 on error. */ static int send_list_only(const Config* config) { - ScannerOptions options = scanner_options_from_config(config, 0); - options.use_metadata = true; /* capture mode + mtime for the listing */ - DirectoryScanner* scanner = - directory_scanner_create_with_options(config->send_directory, &options); - if (!scanner) + PreparedScanner prepared; + if (!prepare_scanner(config, 0, &prepared)) return 1; + prepared.options.use_metadata = true; /* capture mode + mtime for the listing */ + DirectoryScanner* scanner = + directory_scanner_create_with_options(config->send_directory, &prepared.options); + if (!scanner) { + prepared_scanner_destroy(&prepared); + return 1; + } ListEntry* entries = NULL; size_t count = 0; size_t capacity = 0; @@ -378,6 +446,7 @@ static int send_list_only(const Config* config) { } bool failed = oom || directory_scanner_failed(scanner); directory_scanner_destroy(scanner); + prepared_scanner_destroy(&prepared); if (failed) { list_entries_destroy(entries, count); if (oom) @@ -770,14 +839,20 @@ static int send_chunks_multithreaded(void* pipeline_context) { static int scan_directory_multithreaded(void* pipeline_context) { PipelineContextSender* context = (PipelineContextSender*)pipeline_context; protocol_session_bind(&context->allocation_session); - ScannerOptions options = scanner_options_from_config(context->config, 4); + PreparedScanner prepared; + if (!prepare_scanner(context->config, 4, &prepared)) { + pipeline_cancel(context); + protocol_session_unbind(); + return thrd_error; + } ParallelScanner* scanner = parallel_scanner_create_with_options( - context->config->send_directory, &options, &context->allocation_session); + context->config->send_directory, &prepared.options, &context->allocation_session); Chunk* current_chunk; if (scanner == NULL) { log_message(LOG_LEVEL_ERROR, "Failed to create parallel scanner"); pipeline_cancel(context); + prepared_scanner_destroy(&prepared); protocol_session_unbind(); return thrd_error; } @@ -790,6 +865,7 @@ static int scan_directory_multithreaded(void* pipeline_context) { pipeline_cancel(context); chunk_destroy(current_chunk); parallel_scanner_destroy(scanner); + prepared_scanner_destroy(&prepared); protocol_session_unbind(); return thrd_error; } @@ -801,12 +877,14 @@ static int scan_directory_multithreaded(void* pipeline_context) { chunk_destroy(current_chunk); pipeline_cancel(context); parallel_scanner_destroy(scanner); + prepared_scanner_destroy(&prepared); protocol_session_unbind(); return thrd_error; } } if (parallel_scanner_failed(scanner)) { parallel_scanner_destroy(scanner); + prepared_scanner_destroy(&prepared); mtx_lock(&context->mutex_scanner); context->scanner_done = true; cnd_broadcast(&context->condition_not_empty_scanner); @@ -822,6 +900,7 @@ static int scan_directory_multithreaded(void* pipeline_context) { mtx_unlock(&context->mutex_scanner); parallel_scanner_destroy(scanner); + prepared_scanner_destroy(&prepared); protocol_session_unbind(); return thrd_success; } @@ -939,10 +1018,13 @@ int send_files(Config* config) { DirectoryScanner* scanner = NULL; ArrayList* manifest = NULL; ArrayList* remove_sources = NULL; + PreparedScanner prepared; + memset(&prepared, 0, sizeof(prepared)); if (!config_send(client->file_descriptor, config)) goto send_fail; - ScannerOptions scanner_options = scanner_options_from_config(config, 0); - scanner = directory_scanner_create_with_options(config->send_directory, &scanner_options); + if (!prepare_scanner(config, 0, &prepared)) + goto send_fail; + scanner = directory_scanner_create_with_options(config->send_directory, &prepared.options); manifest = create_transfer_manifest(config); if (config->remove_source_files) remove_sources = array_list_create(source_file_destroy); @@ -1043,6 +1125,7 @@ send_fail: array_list_delete(remove_sources); if (scanner) directory_scanner_destroy(scanner); + prepared_scanner_destroy(&prepared); disconnect_transfer_client(client); protocol_session_unbind(); return ret; diff --git a/src/client/scanner.c b/src/client/scanner.c index dfa8b20..70fbbf9 100644 --- a/src/client/scanner.c +++ b/src/client/scanner.c @@ -17,8 +17,56 @@ typedef struct { char* path; int depth; + FilterNode* context; /* inherited per-directory filter context */ } DirEntry; +/* A chain node: `own` holds the .rsync-filter rules of one directory, `parent` + * the context that directory inherited (nearest ancestor with a filter file). + * Rules are evaluated base-first, then from the outermost node inward. */ +struct FilterNode { + FilterNode* parent; + FilterRuleList* own; +}; + +static void filter_node_destroy(void* item) { + if (item) { + FilterNode* node = (FilterNode*)item; + if (node->own) + filter_rule_list_free(node->own); + free(node); + } +} + +static FilterNode* filter_node_alloc(FilterNode* parent, FilterRuleList* own) { + FilterNode* node = malloc(sizeof(FilterNode)); + if (!node) + return NULL; + node->parent = parent; + node->own = own; + return node; +} + +/* Evaluate a rule chain (base rules, then per-directory nodes outermost + * first). Returns FILTER_ACTION_NONE when nothing matched. */ +static FilterAction chain_rules_apply(const FilterRuleList* base, const FilterNode* node, + const char* rel, const char* leaf, bool is_dir) { + if (node) { + FilterAction parent_action = chain_rules_apply(base, node->parent, rel, leaf, is_dir); + if (parent_action != FILTER_ACTION_NONE) + return parent_action; + return filter_rules_apply(node->own, rel, leaf, is_dir); + } + return base ? filter_rules_apply(base, rel, leaf, is_dir) : FILTER_ACTION_NONE; +} + +static bool entry_allowed(const FilterRuleList* base, const FilterNode* node, const char* rel, + const char* leaf, bool is_dir, bool per_dir_filters) { + /* -F: per-directory .rsync-filter files are never transferred. */ + if (per_dir_filters && !is_dir && strcmp(leaf, ".rsync-filter") == 0) + return false; + return chain_rules_apply(base, node, rel, leaf, is_dir) != FILTER_ACTION_EXCLUDE; +} + static void dir_entry_destroy(void* item) { if (item) { DirEntry* de = (DirEntry*)item; @@ -27,7 +75,7 @@ static void dir_entry_destroy(void* item) { } } -static DirEntry* dir_entry_create(const char* path, int depth) { +static DirEntry* dir_entry_create(const char* path, int depth, FilterNode* context) { DirEntry* de = malloc(sizeof(DirEntry)); if (!de) return NULL; @@ -37,6 +85,7 @@ static DirEntry* dir_entry_create(const char* path, int depth) { return NULL; } de->depth = depth; + de->context = context; return de; } @@ -66,6 +115,73 @@ bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entr return !one_file_system || entry_device == root_device; } +/* Relative path of an on-disk path below `root`. The transfer root may be + * given with a trailing slash; the returned rel path never has one and is "" + * for the root itself. */ +static char* rel_for_fs_path(const char* root, const char* fs_path) { + size_t root_len = strlen(root); + while (root_len > 1 && root[root_len - 1] == '/') + root_len--; + if (strncmp(root, fs_path, root_len) != 0) + return NULL; + if (fs_path[root_len] == '\0') + return str_dup(""); + if (fs_path[root_len] != '/') + return NULL; + return str_dup(fs_path + root_len + 1); +} + +/* Relative path of a child entry below the current directory. */ +static char* child_rel_path(const char* parent_rel, const char* name) { + if (!parent_rel || parent_rel[0] == '\0') + return str_dup(name); + return path_cat(parent_rel, name); +} + +/* Apply the --files-from allow-set and the filter layer to one entry. */ +static bool entry_passes_selection(const FileListSet* file_list, const FilterRuleList* base, + const FilterNode* node, const char* rel, const char* leaf, + bool is_dir, bool per_dir_filters) { + if (file_list && !file_list_affects(file_list, rel)) + return false; + if (base || per_dir_filters) + return entry_allowed(base, node, rel, leaf, is_dir, per_dir_filters); + return true; +} + +/* Merge the open directory's own .rsync-filter rules into the inherited + * context, returning the context used for this directory's entries. On a parse + * error the scanner is marked failed. Returns 0 on success, -1 on failure. */ +static int open_directory_filter_context(DirectoryScanner* scanner, const FilterNode* inherited) { + if (!scanner->per_dir_filters) { + scanner->current_node = (FilterNode*)inherited; + return 0; + } + char err[256]; + bool exists = false; + FilterRuleList* own = + filter_file_read(scanner->current_path, scanner->current_rel ? scanner->current_rel : "", + &exists, err, sizeof(err)); + if (!own) { + log_message(LOG_LEVEL_ERROR, "invalid .rsync-filter in %s: %s", scanner->current_path, err); + scanner->failed = true; + return -1; + } + if (exists && own->count > 0) { + FilterNode* node = filter_node_alloc((FilterNode*)inherited, own); + if (!node || !array_list_add(scanner->filter_nodes, node)) { + filter_node_destroy(node); + scanner->failed = true; + return -1; + } + scanner->current_node = node; + } else { + filter_rule_list_free(own); + scanner->current_node = (FilterNode*)inherited; + } + return 0; +} + /* Inspect symlinks, resolve the entry type, and apply file filters once for both scanners. */ static int scanner_inspect_entry(const ScannerOptions* options, const char* source_root, const char* containing_dir, const char* name, @@ -165,25 +281,54 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo scanner->checksum = options->checksum; scanner->one_file_system = options->one_file_system; scanner->failed = false; + scanner->root_path = str_dup(root_directory); + if (!scanner->root_path) { + queue_destroy(scanner->directories); + free(scanner); + return NULL; + } + scanner->current_rel = NULL; + scanner->at_seed_dir = true; + scanner->seed_node = NULL; + scanner->current_node = NULL; + scanner->file_list = options->file_list; + scanner->base_filters = options->base_filters; + scanner->per_dir_filters = options->per_dir_filters; + scanner->filter_nodes = NULL; + if (scanner->base_filters || scanner->per_dir_filters) { + scanner->filter_nodes = array_list_create(filter_node_destroy); + if (!scanner->filter_nodes) { + free(scanner->root_path); + queue_destroy(scanner->directories); + free(scanner); + return NULL; + } + } if (scanner->one_file_system) { struct stat root_stats; if (stat(root_directory, &root_stats) != 0) { log_perror("Could not stat source directory"); + free(scanner->root_path); queue_destroy(scanner->directories); + array_list_delete(scanner->filter_nodes); free(scanner); return NULL; } scanner->root_dev = root_stats.st_dev; } - DirEntry* root = dir_entry_create(root_directory, 0); + DirEntry* root = dir_entry_create(root_directory, 0, NULL); if (!root) { + free(scanner->root_path); queue_destroy(scanner->directories); + array_list_delete(scanner->filter_nodes); free(scanner); return NULL; } if (!queue_enqueue(scanner->directories, root)) { dir_entry_destroy(root); + free(scanner->root_path); queue_destroy(scanner->directories); + array_list_delete(scanner->filter_nodes); free(scanner); return NULL; } @@ -197,14 +342,25 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_ unsigned long long min_size, int max_depth, bool follow_symlinks, bool copy_links, bool safe_links, bool copy_unsafe_links, bool checksum) { - ScannerOptions options = {use_metadata, chunk_size, - exclude_patterns, exclude_count, - include_patterns, include_count, - max_size, min_size, - max_depth, 0, - follow_symlinks, copy_links, - safe_links, copy_unsafe_links, - checksum, false}; + ScannerOptions options = {use_metadata, + chunk_size, + exclude_patterns, + exclude_count, + include_patterns, + include_count, + max_size, + min_size, + max_depth, + 0, + follow_symlinks, + copy_links, + safe_links, + copy_unsafe_links, + checksum, + false, + NULL, + NULL, + false}; return directory_scanner_create_with_options(root_directory, &options); } @@ -216,6 +372,9 @@ void directory_scanner_destroy(DirectoryScanner* scanner) { scanner->current_dir = NULL; } free(scanner->current_path); + free(scanner->current_rel); + free(scanner->root_path); + array_list_delete(scanner->filter_nodes); queue_destroy(scanner->directories); free(scanner); } @@ -246,7 +405,21 @@ static int open_next_directory(DirectoryScanner* scanner) { DirEntry* de = (DirEntry*)queue_dequeue(scanner->directories); scanner->current_path = de->path; scanner->current_depth = de->depth; + /* The seed directory inherits the scanner's configured context (the root + * .rsync-filter context in parallel mode); other dirs inherit the context of + * the directory that enqueued them. */ + const FilterNode* inherited = scanner->at_seed_dir ? scanner->seed_node : de->context; + scanner->at_seed_dir = false; free(de); + + free(scanner->current_rel); + scanner->current_rel = rel_for_fs_path(scanner->root_path, scanner->current_path); + if (!scanner->current_rel) { + log_message(LOG_LEVEL_ERROR, "Could not compute relative path under %s", scanner->root_path); + scanner->failed = true; + return -1; + } + scanner->current_dir = opendir(scanner->current_path); if (scanner->current_dir == NULL) { log_perror("Could not open directory"); @@ -255,6 +428,11 @@ static int open_next_directory(DirectoryScanner* scanner) { scanner->failed = true; return -1; } + if (open_directory_filter_context(scanner, inherited) != 0) { + closedir(scanner->current_dir); + scanner->current_dir = NULL; + return -1; + } return 1; } @@ -294,7 +472,9 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) { scanner->max_depth, 0, scanner->follow_symlinks, scanner->copy_links, scanner->safe_links, scanner->copy_unsafe_links, - scanner->checksum, scanner->one_file_system}; + scanner->checksum, scanner->one_file_system, + scanner->file_list, scanner->base_filters, + scanner->per_dir_filters}; ScannerEntry inspected; int inspection = scanner_inspect_entry(&options, scanner->current_path, scanner->current_path, entry->d_name, &inspected); @@ -307,14 +487,32 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) { char* cur_path = inspected.path; struct stat stats = inspected.stats; - if (inspected.is_directory) { + /* --files-from allow-set and the filter layer apply to files and to + * directories (an excluded directory is not descended into). */ + bool is_dir = inspected.is_directory; + char* rel = child_rel_path(scanner->current_rel, entry->d_name); + if (!rel) { + free(cur_path); + scanner->failed = true; + break; + } + bool passes_selection = + entry_passes_selection(scanner->file_list, scanner->base_filters, scanner->current_node, + rel, entry->d_name, is_dir, scanner->per_dir_filters); + free(rel); + if (!passes_selection) { + free(cur_path); + continue; + } + + if (is_dir) { if (!scanner_same_filesystem(scanner->one_file_system, scanner->root_dev, stats.st_dev)) { free(cur_path); continue; } int next_depth = scanner->current_depth + 1; if (scanner->max_depth <= 0 || next_depth < scanner->max_depth) { - DirEntry* de = dir_entry_create(cur_path, next_depth); + DirEntry* de = dir_entry_create(cur_path, next_depth, scanner->current_node); if (!de || !queue_enqueue(scanner->directories, de)) { dir_entry_destroy(de); scanner->failed = true; @@ -376,6 +574,7 @@ typedef struct { ParallelScanner* ps; char** dirs; int dir_count; + char* root_dir; /* the transfer root, for relative-path computation */ ScannerOptions options; ProtocolSession* allocation_session; } ParallelWorkerArg; @@ -398,6 +597,13 @@ static int parallel_worker_thread(void* arg) { free(wa->dirs[j]); break; } + /* Root .rsync-filter rules (parsed by the parallel scanner) apply to the + * contents of every assigned subdirectory. Relative paths (used by the + * allow-set and per-directory rules) are computed against the transfer + * root, not the subdirectory the worker is seeded with. */ + free(ds->root_path); + ds->root_path = str_dup(wa->root_dir); + ds->seed_node = wa->ps->root_filter_node; Chunk* chunk; while ((chunk = directory_scanner_next(ds)) != NULL) { if (!queue_enqueue_multithreaded_cancel(wa->ps->result_queue, chunk, &wa->ps->result_mutex, @@ -419,6 +625,7 @@ static int parallel_worker_thread(void* arg) { free(wa->dirs[i]); } ParallelScanner* ps = wa->ps; + free(wa->root_dir); free(wa->dirs); free(wa); mtx_lock(&ps->result_mutex); @@ -549,9 +756,10 @@ static Chunk* batch_files(ArrayList* files, unsigned long long chunk_size, Queue } /* Scan one root-directory entry into either the subdirs or files list. */ -static void scan_root_entry(const ScannerOptions* options, const char* root_directory, - const struct dirent* entry, ArrayList* root_files, ArrayList* subdirs, - dev_t root_dev, ParallelScanner* ps) { +static void scan_root_entry(const ScannerOptions* options, const FilterNode* root_node, + const char* root_directory, const struct dirent* entry, + ArrayList* root_files, ArrayList* subdirs, dev_t root_dev, + ParallelScanner* ps) { ScannerEntry inspected; int inspection = scanner_inspect_entry(options, root_directory, root_directory, entry->d_name, &inspected); @@ -563,7 +771,21 @@ static void scan_root_entry(const ScannerOptions* options, const char* root_dire return; char* cur_path = inspected.path; struct stat st = inspected.stats; - if (inspected.is_directory) { + bool is_dir = inspected.is_directory; + char* rel = str_dup(entry->d_name); + if (!rel) { + free(cur_path); + ps->failed = true; + return; + } + bool passes = entry_passes_selection(options->file_list, options->base_filters, root_node, rel, + entry->d_name, is_dir, options->per_dir_filters); + free(rel); + if (!passes) { + free(cur_path); + return; + } + if (is_dir) { if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) { free(cur_path); return; @@ -597,8 +819,8 @@ static void scan_root_entry(const ScannerOptions* options, const char* root_dire /* Scan the root directory itself, collecting root files and subdirectories. * Returns false if the root directory could not be opened. */ static bool scan_root_directory(ParallelScanner* ps, const char* root_directory, - const ScannerOptions* options, dev_t root_dev, - ArrayList* root_files, ArrayList* subdirs) { + const ScannerOptions* options, const FilterNode* root_node, + dev_t root_dev, ArrayList* root_files, ArrayList* subdirs) { DIR* dir = opendir(root_directory); if (!dir) { log_perror("Could not open root directory for parallel scan"); @@ -608,7 +830,7 @@ static bool scan_root_directory(ParallelScanner* ps, const char* root_directory, while ((entry = readdir(dir)) != NULL) { if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) continue; - scan_root_entry(options, root_directory, entry, root_files, subdirs, root_dev, ps); + scan_root_entry(options, root_node, root_directory, entry, root_files, subdirs, root_dev, ps); } closedir(dir); return true; @@ -616,7 +838,8 @@ static bool scan_root_directory(ParallelScanner* ps, const char* root_directory, /* Spawn worker threads, one per group of subdirectories. */ static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs, - const ScannerOptions* options, unsigned long long cs) { + const ScannerOptions* options, const char* root_directory, + unsigned long long cs) { if (subdirs->size <= 0) return; int n = options->num_threads > 0 ? options->num_threads : 4; @@ -647,7 +870,10 @@ static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs, } wa->ps = ps; wa->dirs = calloc(count, sizeof(char*)); - if (!wa->dirs) { + wa->root_dir = str_dup(root_directory); + if (!wa->dirs || !wa->root_dir) { + free(wa->root_dir); + free(wa->dirs); free(wa); parallel_scanner_creation_failed(ps); break; @@ -661,6 +887,7 @@ static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs, if (!dup_ok) { for (int j = 0; j < count; j++) free(wa->dirs[j]); + free(wa->root_dir); free(wa->dirs); free(wa); parallel_scanner_creation_failed(ps); @@ -674,6 +901,7 @@ static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs, if (thrd_create(&ps->threads[t], parallel_worker_thread, wa) != thrd_success) { for (int j = 0; j < count; j++) free(wa->dirs[j]); + free(wa->root_dir); free(wa->dirs); free(wa); parallel_scanner_creation_failed(ps); @@ -720,7 +948,37 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory root_dev = root_stats.st_dev; } - if (!scan_root_directory(ps, root_directory, options, root_dev, root_files, subdirs)) { + /* Build the root directory's .rsync-filter context once; workers seed their + * scanners with it so per-dir rules behave identically to the sequential + * scanner. */ + FilterNode* root_node = NULL; + if (options->per_dir_filters) { + char err[256]; + bool exists = false; + FilterRuleList* own = filter_file_read(root_directory, "", &exists, err, sizeof(err)); + if (!own) { + log_message(LOG_LEVEL_ERROR, "invalid .rsync-filter in %s: %s", root_directory, err); + array_list_delete(root_files); + array_list_delete(subdirs); + parallel_scanner_destroy(ps); + return NULL; + } + if (exists && own->count > 0) { + root_node = filter_node_alloc(NULL, own); + if (!root_node) { + filter_rule_list_free(own); + array_list_delete(root_files); + array_list_delete(subdirs); + parallel_scanner_destroy(ps); + return NULL; + } + } else { + filter_rule_list_free(own); + } + } + ps->root_filter_node = root_node; + + if (!scan_root_directory(ps, root_directory, options, root_node, root_dev, root_files, subdirs)) { array_list_delete(root_files); array_list_delete(subdirs); parallel_scanner_destroy(ps); @@ -731,7 +989,7 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory ps->initial_chunk = batch_files(root_files, cs, ps->result_queue, &ps->failed); array_list_delete(root_files); - spawn_parallel_workers(ps, subdirs, options, cs); + spawn_parallel_workers(ps, subdirs, options, root_directory, cs); array_list_delete(subdirs); return ps; } @@ -774,6 +1032,8 @@ void parallel_scanner_destroy(ParallelScanner* ps) { for (int i = 0; i < ps->num_threads; i++) thrd_join(ps->threads[i], NULL); free(ps->threads); + if (ps->root_filter_node) + filter_node_destroy(ps->root_filter_node); if (ps->initial_chunk) chunk_destroy(ps->initial_chunk); queue_destroy(ps->result_queue); diff --git a/src/client/scanner.h b/src/client/scanner.h index 5922cba..3122cad 100644 --- a/src/client/scanner.h +++ b/src/client/scanner.h @@ -2,6 +2,8 @@ #define SCANNER_H #include "chunk.h" +#include "file_list.h" +#include "filter.h" #include "protocol.h" #include "queue.h" #include @@ -27,8 +29,18 @@ typedef struct { bool copy_unsafe_links; bool checksum; bool one_file_system; + /* Phase 2 (files-from / filter layer). All pointers are shared read-only + * across scanner instances and worker threads; ownership stays with the + * caller (client_send). */ + const FileListSet* file_list; /* --files-from allow-set, or NULL */ + const FilterRuleList* base_filters; /* command-line + -C rules, or NULL */ + bool per_dir_filters; /* -F: read .rsync-filter per directory */ } ScannerOptions; +/* Internal per-scanner filter state. FilterNode chains represent the ordered + * per-directory .rsync-filter rules that apply below a directory. */ +typedef struct FilterNode FilterNode; + typedef struct { Queue* directories; DIR* current_dir; @@ -51,6 +63,16 @@ typedef struct { bool one_file_system; dev_t root_dev; bool failed; + /* Phase 2 (files-from / filter layer). */ + char* root_path; /* transfer root (fs path) for rel computation */ + char* current_rel; /* rel path of the open directory ("" == root) */ + bool at_seed_dir; /* next open is the seed directory */ + FilterNode* seed_node; /* inherited context of the seed dir, or NULL */ + FilterNode* current_node; /* filter context of the open directory */ + ArrayList* filter_nodes; /* owned FilterNode arena (may be NULL) */ + const FileListSet* file_list; + const FilterRuleList* base_filters; + bool per_dir_filters; } DirectoryScanner; typedef struct { @@ -68,6 +90,7 @@ typedef struct { int completed; Chunk* initial_chunk; ProtocolSession* allocation_session; + FilterNode* root_filter_node; /* root .rsync-filter context (owned by ps) */ } ParallelScanner; DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata, diff --git a/src/client/usage.c b/src/client/usage.c index 25f6732..88782db 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -32,6 +32,13 @@ void print_usage(void) { printf(" --include Only include files matching pattern\n"); printf(" --exclude-from Read exclude patterns from file\n"); printf(" --include-from Read include patterns from file\n"); + printf(" --files-from Read the source file list from FILE (paths relative to the " + "source root)\n"); + printf(" -0, --from0 Entries in --files-from are NUL-delimited\n"); + printf(" --filter=RULE rsync-style filter rule (+/- include/exclude; repeatable; the\n"); + printf(" rsync -f short form conflicts with FastSync sendfile -f)\n"); + printf(" -C, --cvs-exclude Auto-ignore common CVS/SCM files (.git/, .svn/, *.o, *~, ...)\n"); + printf(" -F Apply per-directory .rsync-filter files during the scan\n"); printf(" --max-size Skip files larger than n bytes\n"); printf(" --min-size Skip files smaller than n bytes\n"); printf(" --max-alloc Maximum single allocation (default: 1G)\n"); diff --git a/src/shared/config.c b/src/shared/config.c index acf9147..3b2e33f 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -1,6 +1,7 @@ #include "config.h" #include "chmod.h" #include "delta.h" +#include "file_list.h" #include "log.h" #include "protocol.h" #include "utils.h" @@ -92,7 +93,10 @@ static void config_set_defaults(Config* config) { config->max_delete = 0; config->filters = NULL; config->files_from = NULL; + config->files_from_set = NULL; + config->from0 = false; config->cvs_exclude = false; + config->per_dir_filter = false; config->prune_empty_dirs = false; config->one_file_system = false; config->relative = false; @@ -227,6 +231,7 @@ void config_delete(Config* config) { free(config->out_format); free(config->log_file_format); free(config->files_from); + file_list_destroy((FileListSet*)config->files_from_set); free(config->rsh_command); free(config->rsync_path); free(config->temp_dir); diff --git a/src/shared/config.h b/src/shared/config.h index 366f0c5..56b9e13 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -100,10 +100,14 @@ typedef struct Config { bool delete_after; int max_delete; - // Issue #129: Advanced file selection - ArrayList* filters; - char* files_from; - bool cvs_exclude; + // Issue #129: Advanced file selection. These fields are CLIENT-ONLY: they are + // never serialized to the wire (the receiver must not learn them). + ArrayList* filters; /* --filter=RULE rule strings, in order */ + char* files_from; /* --files-from path (may be NULL) */ + void* files_from_set; /* parsed FileListSet* allow-set, or NULL */ + bool from0; /* -0/--from0: NUL-delimited *-from files */ + bool cvs_exclude; /* -C/--cvs-exclude: standard CVS ignore set */ + bool per_dir_filter; /* -F: apply per-directory .rsync-filter files */ bool prune_empty_dirs; bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */ bool relative; diff --git a/src/shared/file_list.c b/src/shared/file_list.c new file mode 100644 index 0000000..ee2f7e7 --- /dev/null +++ b/src/shared/file_list.c @@ -0,0 +1,188 @@ +#include "file_list.h" +#include "log.h" +#include "utils.h" +#include +#include +#include +#include + +typedef struct { + char** items; + int count; + int capacity; +} StringList; + +static void string_list_destroy(StringList* list) { + if (!list) + return; + for (int i = 0; i < list->count; i++) + free(list->items[i]); + free(list->items); +} + +static bool string_list_add(StringList* list, const char* text) { + if (list->count == list->capacity) { + int new_cap = list->capacity > 0 ? list->capacity * 2 : 16; + char** grown = realloc(list->items, (size_t)new_cap * sizeof(char*)); + if (!grown) + return false; + list->items = grown; + list->capacity = new_cap; + } + list->items[list->count] = str_dup(text); + if (!list->items[list->count]) + return false; + list->count++; + return true; +} + +/* Validate and normalize one entry. Returns: + * 1 -> added to `out` + * 0 -> blank entry, skip + * -1 -> invalid (message set in `err`) */ +static int normalize_entry(const char* raw, size_t len, StringList* out, char* err, + size_t err_size) { + /* Trim the trailing newline/carriage-return from line mode. */ + while (len > 0 && (raw[len - 1] == '\n' || raw[len - 1] == '\r')) + len--; + if (len == 0) + return 0; + if (raw[0] == '/') { + snprintf(err, err_size, "absolute path entries are not allowed: '%s'", raw); + return -1; + } + /* Reject NUL bytes inside a token defensively (NUL-delimited mode splits on + * them, so this only guards against embedded garbage). */ + char* dup = malloc(len + 1); + if (!dup) { + snprintf(err, err_size, "memory allocation failed"); + return -1; + } + memcpy(dup, raw, len); + dup[len] = '\0'; + + /* Rebuild the path token-by-token: skip '.' and empty segments, reject '..'. */ + size_t out_len = 0; + for (const char* part = dup;;) { + const char* slash = strchr(part, '/'); + size_t part_len = slash ? (size_t)(slash - part) : strlen(part); + if (part_len == 1 && part[0] == '.') { + /* skip "." segment */ + } else if (part_len == 2 && part[0] == '.' && part[1] == '.') { + snprintf(err, err_size, "path traversal entry is not allowed: '%s'", dup); + free(dup); + return -1; + } else if (part_len > 0) { + if (out_len > 0) + dup[out_len++] = '/'; + memmove(dup + out_len, part, part_len); + out_len += part_len; + } + if (!slash) + break; + part = slash + 1; + } + dup[out_len] = '\0'; + + int result; + if (out_len == 0) { + /* "." / "./" lists the source root: the whole tree is transferred. */ + result = string_list_add(out, "") ? 1 : -1; + if (result < 0) + snprintf(err, err_size, "memory allocation failed"); + } else { + result = string_list_add(out, dup) ? 1 : -1; + if (result < 0) + snprintf(err, err_size, "memory allocation failed"); + } + free(dup); + return result; +} + +static FileListSet* string_list_to_set(StringList* raw, char* err, size_t err_size) { + FileListSet* set = malloc(sizeof(FileListSet)); + if (!set) { + snprintf(err, err_size, "memory allocation failed"); + return NULL; + } + set->count = raw->count; + set->entries = raw->items; + raw->items = NULL; + raw->count = 0; + return set; +} + +FileListSet* file_list_load(const char* path, bool null_separated, char* err, size_t err_size) { + if (err && err_size > 0) + err[0] = '\0'; + if (!path || !*path) { + snprintf(err, err_size, "no file given"); + return NULL; + } + FILE* fp = fopen(path, "r"); + if (!fp) { + char* escaped = output_escape(path, false); + snprintf(err, err_size, "could not open '%s': %s", escaped ? escaped : path, strerror(errno)); + free(escaped); + return NULL; + } + + StringList raw = {0}; + char* line = NULL; + size_t line_cap = 0; + ssize_t n; + bool ok = true; + char delim = null_separated ? '\0' : '\n'; + while (ok && (n = getdelim(&line, &line_cap, delim, fp)) != -1) { + int r = normalize_entry(line, (size_t)n, &raw, err, err_size); + if (r < 0) { + ok = false; + break; + } + } + free(line); + fclose(fp); + if (!ok) { + string_list_destroy(&raw); + return NULL; + } + FileListSet* set = string_list_to_set(&raw, err, err_size); + if (!set) + string_list_destroy(&raw); + return set; +} + +void file_list_destroy(FileListSet* set) { + if (!set) + return; + for (int i = 0; i < set->count; i++) + free(set->entries[i]); + free(set->entries); + free(set); +} + +static bool path_has_prefix(const char* path, const char* prefix) { + size_t plen = strlen(prefix); + if (strncmp(path, prefix, plen) != 0) + return false; + return path[plen] == '/' || path[plen] == '\0'; +} + +bool file_list_affects(const FileListSet* set, const char* rel) { + if (!set) + return true; + if (!rel) + return false; + for (int i = 0; i < set->count; i++) { + const char* entry = set->entries[i]; + if (entry[0] == '\0') + return true; /* whole tree listed */ + if (strcmp(rel, entry) == 0) + return true; /* the entry itself is listed */ + if (path_has_prefix(rel, entry)) + return true; /* rel lives under a listed directory */ + if (path_has_prefix(entry, rel)) + return true; /* rel is an ancestor directory of a listed entry */ + } + return false; +} diff --git a/src/shared/file_list.h b/src/shared/file_list.h new file mode 100644 index 0000000..0ced332 --- /dev/null +++ b/src/shared/file_list.h @@ -0,0 +1,35 @@ +#ifndef FILE_LIST_H +#define FILE_LIST_H + +#include +#include + +/* --files-from allow-set. The file lists source paths RELATIVE to the source + * root. A listed regular file is transferred; a listed directory transfers its + * whole subtree (FastSync's recursion is always on). Blank lines are ignored. + * + * Entries are normalized: leading "./" and duplicate "/" are removed, an entry + * of "." means the whole tree, absolute entries and ".." traversal are + * rejected at parse time. The set is immutable and shared read-only across + * scanner worker threads. + */ + +typedef struct { + char** entries; /* normalized rel paths; "" means the whole tree */ + int count; +} FileListSet; + +/* Load and validate a --files-from file. When `null_separated` (-0/--from0) + * entries are delimited by NUL instead of newlines. Returns NULL with a message + * in `err` on open/validation failure. An empty file yields an empty set + * (nothing is transferred). */ +FileListSet* file_list_load(const char* path, bool null_separated, char* err, size_t err_size); +void file_list_destroy(FileListSet* set); + +/* True when `rel` (path relative to the source root, "" == root) is a listed + * entry, lives under a listed directory, or is an ancestor directory of a + * listed entry. Used to prune scanning: directories are descended only when + * this returns true, files are transferred only when it returns true. */ +bool file_list_affects(const FileListSet* set, const char* rel); + +#endif diff --git a/src/shared/filter.c b/src/shared/filter.c new file mode 100644 index 0000000..666d8da --- /dev/null +++ b/src/shared/filter.c @@ -0,0 +1,396 @@ +#include "filter.h" +#include "log.h" +#include "utils.h" +#include +#include +#include +#include + +/* ---- Single rule parsing ---- */ + +static bool rule_text_is_unsupported_word(const char* p, size_t len) { + static const char* const words[] = {"merge", "dir-merge", "hide", "show", + "protect", "risk", "clear"}; + for (size_t i = 0; i < sizeof(words) / sizeof(words[0]); i++) { + size_t wl = strlen(words[i]); + if (len == wl && strncmp(p, words[i], wl) == 0) + return true; + } + return false; +} + +FilterRule* filter_rule_parse(const char* line, char* err, size_t err_size) { + if (err && err_size > 0) + err[0] = '\0'; + if (!line) + return NULL; + char* text = str_dup(line); + if (!text) { + if (err) + snprintf(err, err_size, "memory allocation failed"); + return NULL; + } + size_t len = strlen(text); + while (len > 0 && (text[len - 1] == '\n' || text[len - 1] == '\r')) + text[--len] = '\0'; + + FilterAction action = FILTER_ACTION_EXCLUDE; + const char* p = text; + while (*p == ' ' || *p == '\t') + p++; + if (*p == '\0') { + snprintf(err, err_size, "empty filter rule"); + free(text); + return NULL; + } + + if (*p == '+' || *p == '-') { + action = *p == '+' ? FILTER_ACTION_INCLUDE : FILTER_ACTION_EXCLUDE; + p++; + /* Accept the rsync word forms include/exclude. */ + } else { + const char* sp = p; + while (*sp != '\0' && *sp != ' ' && *sp != '\t') + sp++; + size_t word_len = (size_t)(sp - p); + if (rule_text_is_unsupported_word(p, word_len)) { + snprintf(err, err_size, + "'%.*s' filter directives are not supported (only +/- include/exclude rules " + "with an optional '/' anchor and trailing '/' dir marker)", + (int)word_len, p); + free(text); + return NULL; + } + if (word_len == strlen("include") && strncmp(p, "include", word_len) == 0) { + action = FILTER_ACTION_INCLUDE; + p = sp; + } else if (word_len == strlen("exclude") && strncmp(p, "exclude", word_len) == 0) { + action = FILTER_ACTION_EXCLUDE; + p = sp; + } + } + + while (*p == ' ' || *p == '\t') + p++; + if (*p == '\0') { + snprintf(err, err_size, "filter rule has no pattern"); + free(text); + return NULL; + } + + bool anchored = false; + if (*p == '/') { + anchored = true; + p++; + while (*p == ' ' || *p == '\t') + p++; + } + if (*p == '\0') { + snprintf(err, err_size, "filter rule has no pattern after '/' anchor"); + free(text); + return NULL; + } + + /* Pattern runs to the end of the rule; a single trailing '/' marks dir-only. */ + size_t pat_len = strlen(p); + bool dir_only = false; + if (pat_len > 1 && p[pat_len - 1] == '/') { + dir_only = true; + pat_len--; + } else if (pat_len == 1 && p[0] == '/') { + /* "//" anchored with nothing after: meaningless. */ + snprintf(err, err_size, "filter rule has no pattern"); + free(text); + return NULL; + } + + FilterRule* rule = calloc(1, sizeof(FilterRule)); + if (!rule) { + snprintf(err, err_size, "memory allocation failed"); + free(text); + return NULL; + } + rule->pattern = malloc(pat_len + 1); + if (!rule->pattern) { + free(rule); + snprintf(err, err_size, "memory allocation failed"); + free(text); + return NULL; + } + memcpy(rule->pattern, p, pat_len); + rule->pattern[pat_len] = '\0'; + rule->action = action; + rule->anchored = anchored; + rule->dir_only = dir_only; + rule->owner = NULL; + free(text); + return rule; +} + +void filter_rule_free(FilterRule* rule) { + if (!rule) + return; + free(rule->pattern); + free(rule->owner); + free(rule); +} + +/* ---- Ordered rule lists ---- */ + +FilterRuleList* filter_rule_list_create(void) { + return calloc(1, sizeof(FilterRuleList)); +} + +bool filter_rule_list_add(FilterRuleList* list, FilterRule* rule) { + if (!list || !rule) + return false; + if (list->count == list->capacity) { + int new_cap = list->capacity > 0 ? list->capacity * 2 : 8; + FilterRule** grown = realloc(list->items, (size_t)new_cap * sizeof(FilterRule*)); + if (!grown) + return false; + list->items = grown; + list->capacity = new_cap; + } + list->items[list->count++] = rule; + return true; +} + +bool filter_rule_list_parse_append(FilterRuleList* list, const char* line, char* err, + size_t err_size) { + FilterRule* rule = filter_rule_parse(line, err, err_size); + if (!rule) + return false; + if (!filter_rule_list_add(list, rule)) { + filter_rule_free(rule); + snprintf(err, err_size, "memory allocation failed"); + return false; + } + return true; +} + +void filter_rule_list_free(FilterRuleList* list) { + if (!list) + return; + for (int i = 0; i < list->count; i++) + filter_rule_free(list->items[i]); + free(list->items); + free(list); +} + +static bool set_rule_owner(FilterRule* rule, const char* owner) { + char* dup = str_dup(owner ? owner : ""); + if (!dup) + return false; + free(rule->owner); + rule->owner = dup; + return true; +} + +/* ---- CVS default excludes (-C) ---- */ + +typedef struct { + const char* pattern; + bool dir_only; +} CvsDefaultRule; + +static const CvsDefaultRule CVS_DEFAULTS[] = { + {"RCS", false}, {"SCCS", false}, {"CVS", false}, {"CVS.adm", false}, + {"RCSLOG", false}, {"cvslog.*", false}, {"tags", false}, {"TAGS", false}, + {".make.state", false}, {".nse_depinfo", false}, {"*~", false}, {"#*", false}, + {".#*", false}, {",*", false}, {"_$*", false}, {"*$", false}, + {"*.old", false}, {"*.bak", false}, {"*.BAK", false}, {"*.orig", false}, + {"*.rej", false}, {".del-*", false}, {"*.a", false}, {"*.olb", false}, + {"*.o", false}, {"*.obj", false}, {"*.so", false}, {"*.exe", false}, + {"*.Z", false}, {"*.elc", false}, {"*.ln", false}, {"core", false}, + {".svn/", true}, {".git/", true}, {".hg/", true}, {".bzr/", true}, +}; + +static bool cvs_rule_list_append(FilterRuleList* list) { + for (size_t i = 0; i < sizeof(CVS_DEFAULTS) / sizeof(CVS_DEFAULTS[0]); i++) { + FilterRule* rule = calloc(1, sizeof(FilterRule)); + if (!rule) + return false; + rule->action = FILTER_ACTION_EXCLUDE; + rule->dir_only = CVS_DEFAULTS[i].dir_only; + size_t plen = strlen(CVS_DEFAULTS[i].pattern); + if (rule->dir_only && plen > 0 && CVS_DEFAULTS[i].pattern[plen - 1] == '/') + plen--; /* keep the cleaned pattern, matching filter_rule_parse */ + rule->pattern = malloc(plen + 1); + if (!rule->pattern) { + free(rule); + return false; + } + memcpy(rule->pattern, CVS_DEFAULTS[i].pattern, plen); + rule->pattern[plen] = '\0'; + if (!set_rule_owner(rule, "")) { + filter_rule_free(rule); + return false; + } + if (!filter_rule_list_add(list, rule)) { + filter_rule_free(rule); + return false; + } + } + return true; +} + +FilterRuleList* filter_base_build(const char* const* rule_texts, int rule_count, bool cvs_exclude, + char* err, size_t err_size) { + if (err && err_size > 0) + err[0] = '\0'; + FilterRuleList* list = filter_rule_list_create(); + if (!list) { + snprintf(err, err_size, "memory allocation failed"); + return NULL; + } + for (int i = 0; i < rule_count; i++) { + if (!rule_texts || !rule_texts[i]) + continue; + FilterRule* rule = filter_rule_parse(rule_texts[i], err, err_size); + if (!rule) { + filter_rule_list_free(list); + return NULL; + } + if (!set_rule_owner(rule, "")) { + filter_rule_free(rule); + filter_rule_list_free(list); + snprintf(err, err_size, "memory allocation failed"); + return NULL; + } + if (!filter_rule_list_add(list, rule)) { + filter_rule_free(rule); + filter_rule_list_free(list); + snprintf(err, err_size, "memory allocation failed"); + return NULL; + } + } + if (cvs_exclude && !cvs_rule_list_append(list)) { + filter_rule_list_free(list); + snprintf(err, err_size, "memory allocation failed"); + return NULL; + } + return list; +} + +/* ---- Per-directory .rsync-filter files ---- */ + +FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bool* exists, + char* err, size_t err_size) { + if (err && err_size > 0) + err[0] = '\0'; + if (exists) + *exists = false; + char* filter_path = path_cat(dir_path, ".rsync-filter"); + if (!filter_path) { + snprintf(err, err_size, "memory allocation failed"); + return NULL; + } + FILE* fp = fopen(filter_path, "r"); + free(filter_path); + if (!fp) { + if (errno == ENOENT || errno == ENOTDIR) + return filter_rule_list_create(); + log_message(LOG_LEVEL_WARNING, "Could not read .rsync-filter in %s: %s", dir_path, + strerror(errno)); + return filter_rule_list_create(); + } + if (exists) + *exists = true; + FilterRuleList* list = filter_rule_list_create(); + if (!list) { + fclose(fp); + snprintf(err, err_size, "memory allocation failed"); + return NULL; + } + char* line = NULL; + size_t line_cap = 0; + ssize_t n; + bool ok = true; + while ((n = getline(&line, &line_cap, fp)) != -1) { + const char* p = line; + while (*p == ' ' || *p == '\t') + p++; + if (*p == '\0' || *p == '\n' || *p == '\r' || *p == '#') + continue; + FilterRule* rule = filter_rule_parse(p, err, err_size); + if (!rule) { + ok = false; + break; + } + if (!set_rule_owner(rule, owner_rel)) { + filter_rule_free(rule); + snprintf(err, err_size, "memory allocation failed"); + ok = false; + break; + } + if (!filter_rule_list_add(list, rule)) { + filter_rule_free(rule); + snprintf(err, err_size, "memory allocation failed"); + ok = false; + break; + } + } + free(line); + fclose(fp); + if (!ok) { + filter_rule_list_free(list); + return NULL; + } + return list; +} + +/* ---- Rule matching ---- */ + +/* Match a pattern that contains '/' (non-anchored) against the end of the + * relative path, starting at any path-component boundary. */ +static bool glob_suffix_match(const char* pattern, const char* str) { + if (glob_match(pattern, str)) + return true; + for (const char* slash = strchr(str, '/'); slash; slash = strchr(slash + 1, '/')) { + if (glob_match(pattern, slash + 1)) + return true; + } + return false; +} + +static FilterAction rule_matches(const FilterRule* rule, const char* rel_path, const char* leaf, + bool is_dir) { + if (!rule || !rule->pattern) + return FILTER_ACTION_NONE; + if (rule->dir_only && !is_dir) + return FILTER_ACTION_NONE; + /* A rule applies only to entries below its owner directory. */ + const char* rel2 = rel_path; + if (rule->owner && rule->owner[0] != '\0') { + size_t owner_len = strlen(rule->owner); + if (strncmp(rule->owner, rel_path, owner_len) != 0) + return FILTER_ACTION_NONE; + if (rel_path[owner_len] != '/') + return FILTER_ACTION_NONE; + rel2 = rel_path + owner_len + 1; + } + if (rel2[0] == '\0') + return FILTER_ACTION_NONE; + bool matched; + if (rule->anchored) { + matched = glob_match(rule->pattern, rel2); + } else if (strchr(rule->pattern, '/') != NULL) { + matched = glob_suffix_match(rule->pattern, rel2); + } else { + matched = glob_match(rule->pattern, leaf); + } + return matched ? rule->action : FILTER_ACTION_NONE; +} + +FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf, + bool is_dir) { + if (!list) + return FILTER_ACTION_NONE; + for (int i = 0; i < list->count; i++) { + FilterAction action = rule_matches(list->items[i], rel_path, leaf, is_dir); + if (action != FILTER_ACTION_NONE) + return action; + } + return FILTER_ACTION_NONE; +} diff --git a/src/shared/filter.h b/src/shared/filter.h new file mode 100644 index 0000000..cf7b5c0 --- /dev/null +++ b/src/shared/filter.h @@ -0,0 +1,79 @@ +#ifndef FILTER_H +#define FILTER_H + +#include +#include + +/* rsync-style filter rule engine (client-side file selection). + * + * Supported rule syntax (documented subset): + * [+|-] [anchored '/' prefix] pattern [trailing '/' for dir-only] + * + * "+ PATTERN" include rule (first match wins) + * "- PATTERN" exclude rule + * "PATTERN" implicit exclude rule (rsync default) + * "include PATTERN" / "exclude PATTERN" word forms + * leading '/' after the +/- anchors the pattern to its owner directory + * (the transfer root for command-line/-C rules, the directory that + * contains a .rsync-filter file for per-directory rules) + * a trailing '/' makes the rule match directories only + * + * Unsupported rsync rule types (merge/dir-merge/hide/show/protect/risk/clear, + * rule modifiers other than '/') are rejected with a clear error. + */ + +typedef enum { + FILTER_ACTION_NONE = 0, /* no rule matched */ + FILTER_ACTION_EXCLUDE = -1, + FILTER_ACTION_INCLUDE = 1 +} FilterAction; + +typedef struct { + FilterAction action; + bool anchored; /* pattern anchored to the rule's owner directory */ + bool dir_only; /* pattern had a trailing '/': matches directories only */ + char* owner; /* owning directory rel path ("" == transfer root) */ + char* pattern; /* cleaned glob pattern (no leading '/', no trailing '/') */ +} FilterRule; + +typedef struct { + FilterRule** items; /* owned array of rule pointers */ + int count; + int capacity; +} FilterRuleList; + +/* Parse a single filter-rule line (no trailing newline required). Returns an + * owned rule, or NULL on unsupported/invalid syntax with a message in `err`. */ +FilterRule* filter_rule_parse(const char* line, char* err, size_t err_size); +void filter_rule_free(FilterRule* rule); + +FilterRuleList* filter_rule_list_create(void); +/* Append a fully-parsed rule (takes ownership). Returns false on OOM. */ +bool filter_rule_list_add(FilterRuleList* list, FilterRule* rule); +/* Parse `line` and append it. Returns false and fills `err` on bad syntax. */ +bool filter_rule_list_parse_append(FilterRuleList* list, const char* line, char* err, + size_t err_size); +void filter_rule_list_free(FilterRuleList* list); + +/* Build the command-line filter set: `rule_texts` (--filter=RULE in the order + * given, 0..rule_count) followed by the -C CVS default excludes when + * cvs_exclude is true. All rules are owned by "" (the transfer root). + * Returns NULL on unsupported rule text (message in `err`). */ +FilterRuleList* filter_base_build(const char* const* rule_texts, int rule_count, bool cvs_exclude, + char* err, size_t err_size); + +/* Read "/.rsync-filter" and return its rules, each owned by + * `owner_rel`. A missing file yields an empty list with *exists=false; an + * unreadable file is treated as missing. Returns NULL only on parse or + * allocation failure (message in `err`). */ +FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bool* exists, + char* err, size_t err_size); + +/* Evaluate an entry against one ordered rule list. Returns FILTER_ACTION_NONE + * when no rule matched, otherwise the first matching rule's action. + * `rel_path` is the entry's path relative to the transfer root ("" == root), + * `leaf` its final name, `is_dir` whether it is a directory. */ +FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf, + bool is_dir); + +#endif diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index 3b21a75..00a27c4 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -9,7 +9,7 @@ import pytest sys.path.insert(0, os.path.dirname(__file__)) from common import ( - PROJECT_ROOT, BUILD_DIR, TEST_DATA_DIR, + PROJECT_ROOT, BUILD_DIR, TEST_DATA_DIR, ServerManager, run_client, generate_test_files, verify_transfer, clean_dir, make_result, get_dest_received_dir, CLIENT_CMD, @@ -1405,3 +1405,228 @@ class TestLogFileFormat: expected = {f"{os.path.join(source, rel)} {len(data)}" for rel, data in files.items()} for line in expected: assert line in content, f"log file (-m) missing {line!r}" + + +class TestFilesFrom: + """--files-from transfers exactly the listed files; a listed directory + transfers its whole subtree. The manifest (and thus --delete) derives from + what was actually sent.""" + + def _make_source(self, name): + source = os.path.join(TEST_DATA_DIR, name) + clean_dir(source) + entries = { + "top.txt": b"top\n", + "sub/a.txt": b"a\n", + "sub/b.txt": b"b\n", + "other/c.txt": b"c\n", + } + for rel, content in entries.items(): + full = os.path.join(source, rel) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "wb") as fh: + fh.write(content) + return source + + def _write_list(self, rel_text): + path = os.path.join(TEST_DATA_DIR, "files_from.list") + with open(path, "wb") as fh: + fh.write(rel_text) + return path + + @pytest.mark.parametrize("mt", [False, True]) + def test_files_from_exact_subset(self, shared_server, mt): + source = self._make_source("ff_subset_src") + dest = os.path.join(TEST_DATA_DIR, "ff_subset_dst") + clean_dir(dest) + lst = self._write_list(b"top.txt\nsub/a.txt\n") + flags = ["--files-from", lst] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"files-from sync failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + assert os.path.isfile(os.path.join(received, "top.txt")) + assert os.path.isfile(os.path.join(received, "sub", "a.txt")) + assert not os.path.exists(os.path.join(received, "sub", "b.txt")), \ + "unlisted sub/b.txt must not be transferred" + assert not os.path.exists(os.path.join(received, "other")), \ + "unlisted other/ subtree must not be transferred" + + @pytest.mark.parametrize("mt", [False, True]) + def test_files_from_listed_directory_transfers_subtree(self, shared_server, mt): + source = self._make_source("ff_subdir_src") + dest = os.path.join(TEST_DATA_DIR, "ff_subdir_dst") + clean_dir(dest) + lst = self._write_list(b"sub\n") + flags = ["--files-from", lst] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"files-from dir sync failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + assert os.path.isfile(os.path.join(received, "sub", "a.txt")) + assert os.path.isfile(os.path.join(received, "sub", "b.txt")) + assert not os.path.exists(os.path.join(received, "top.txt")) + assert not os.path.exists(os.path.join(received, "other")) + + @pytest.mark.parametrize("mt", [False, True]) + def test_files_from_nul_separated(self, shared_server, mt): + source = self._make_source("ff_nul_src") + dest = os.path.join(TEST_DATA_DIR, "ff_nul_dst") + clean_dir(dest) + lst = self._write_list(b"top.txt\0other/c.txt\0") + flags = ["--files-from", lst, "--from0"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"files-from -0 sync failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + assert os.path.isfile(os.path.join(received, "top.txt")) + assert os.path.isfile(os.path.join(received, "other", "c.txt")) + assert not os.path.exists(os.path.join(received, "sub")) + + def test_files_from_missing_list_file_rejected(self): + result, _ = run_client( + os.path.join(TEST_DATA_DIR, "nowhere_src"), + os.path.join(TEST_DATA_DIR, "nowhere_dst"), + flags=["--files-from", os.path.join(TEST_DATA_DIR, "no_such_list.txt")], + ) + assert result.returncode != 0, "missing --files-from file must be rejected" + assert "--files-from" in result.stderr + + def test_files_from_delete_deletes_unlisted(self): + source = self._make_source("ff_delete_src") + dest = os.path.join(TEST_DATA_DIR, "ff_delete_dst") + clean_dir(dest) + server = ServerManager() + server.start(extra_args=["--allow-delete"]) + try: + # Full transfer first. + result, _ = run_client(source, dest, port=server.port) + assert result.returncode == 0, f"full sync failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + assert os.path.isfile(os.path.join(received, "top.txt")) + + # A subset sync with --delete deletes everything not in the sent + # manifest (which is derived from what was actually sent). + lst = self._write_list(b"sub/a.txt\n") + result, _ = run_client(source, dest, flags=["--files-from", lst, "--delete"], + port=server.port) + assert result.returncode == 0, f"files-from delete sync failed: {result.stderr[:200]}" + assert os.path.isfile(os.path.join(received, "sub", "a.txt")) + assert not os.path.exists(os.path.join(received, "top.txt")), \ + "unlisted file not deleted" + assert not os.path.exists(os.path.join(received, "other")), \ + "unlisted subtree not deleted" + finally: + server.stop() + + +class TestFilters: + """--filter/-C/-F rule layer: excludes prune, ordering is first-match-wins, + the default with no matching rule is include, and legacy --exclude remains + an independent layer.""" + + def _make_tree(self, name, with_filter_file=False): + source = os.path.join(TEST_DATA_DIR, name) + clean_dir(source) + entries = { + "data/keep.txt": b"keep\n", + "data/drop.tmp": b"drop\n", + "nested/deep.tmp": b"deep\n", + "nested/ok.log": b"log\n", + "top.bin": b"bin\n", + } + for rel, content in entries.items(): + full = os.path.join(source, rel) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "wb") as fh: + fh.write(content) + if with_filter_file: + with open(os.path.join(source, ".rsync-filter"), "wb") as fh: + fh.write(b"- *.tmp\n") + return source + + @pytest.mark.parametrize("mt", [False, True]) + def test_filter_excludes_glob(self, shared_server, mt): + source = self._make_tree("filter_tmp_src") + dest = os.path.join(TEST_DATA_DIR, "filter_tmp_dst") + clean_dir(dest) + flags = ["--filter", "- *.tmp"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"filter sync failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + assert os.path.isfile(os.path.join(received, "data", "keep.txt")) + assert os.path.isfile(os.path.join(received, "nested", "ok.log")) + assert os.path.isfile(os.path.join(received, "top.bin")) + assert not os.path.exists(os.path.join(received, "data", "drop.tmp")) + assert not os.path.exists(os.path.join(received, "nested", "deep.tmp")) + + @pytest.mark.parametrize("mt", [False, True]) + def test_filter_anchored_include_overrides_exclude_all(self, shared_server, mt): + source = os.path.join(TEST_DATA_DIR, "filter_keep_src") + clean_dir(source) + with open(os.path.join(source, "a.keepme"), "wb") as fh: + fh.write(b"keep me") + with open(os.path.join(source, "b.other"), "wb") as fh: + fh.write(b"drop me") + dest = os.path.join(TEST_DATA_DIR, "filter_keep_dst") + clean_dir(dest) + # First match wins: the anchored include beats the catch-all exclude. + flags = ["--filter=+ /a.keepme", "--filter=- *"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"filter include sync failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + assert os.path.isfile(os.path.join(received, "a.keepme")) + assert not os.path.exists(os.path.join(received, "b.other")) + + @pytest.mark.parametrize("mt", [False, True]) + def test_cvs_exclude_ignores_scm_and_build_artifacts(self, shared_server, mt): + source = os.path.join(TEST_DATA_DIR, "filter_cvs_src") + clean_dir(source) + entries = { + "src/main.c": b"int main() {}\n", + "src/main.o": b"obj\n", + "src/notes.txt~": b"backup\n", + ".git/HEAD": b"ref\n", + ".git/config": b"cfg\n", + "README.md": b"readme\n", + } + for rel, content in entries.items(): + full = os.path.join(source, rel) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "wb") as fh: + fh.write(content) + dest = os.path.join(TEST_DATA_DIR, "filter_cvs_dst") + clean_dir(dest) + flags = ["-C"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"-C sync failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + assert os.path.isfile(os.path.join(received, "src", "main.c")) + assert os.path.isfile(os.path.join(received, "README.md")) + assert not os.path.exists(os.path.join(received, ".git")), ".git/ must be pruned" + assert not os.path.exists(os.path.join(received, "src", "main.o")), "*.o must be pruned" + assert not os.path.exists(os.path.join(received, "src", "notes.txt~")), "*~ must be pruned" + + @pytest.mark.parametrize("mt", [False, True]) + def test_per_dir_filter_file(self, shared_server, mt): + source = self._make_tree("filter_file_src", with_filter_file=True) + dest = os.path.join(TEST_DATA_DIR, "filter_file_dst") + clean_dir(dest) + flags = ["-F"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"-F sync failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + assert os.path.isfile(os.path.join(received, "data", "keep.txt")) + assert os.path.isfile(os.path.join(received, "nested", "ok.log")) + assert not os.path.exists(os.path.join(received, "data", "drop.tmp")) + assert not os.path.exists(os.path.join(received, "nested", "deep.tmp")) + assert not os.path.exists(os.path.join(received, ".rsync-filter")), \ + ".rsync-filter must not be transferred" + + def test_filter_leaves_default_behavior_unchanged(self, shared_server): + source = self._make_tree("filter_default_src") + dest = os.path.join(TEST_DATA_DIR, "filter_default_dst") + clean_dir(dest) + result, _ = run_client(source, dest, port=shared_server.port) + assert result.returncode == 0, f"plain sync failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + mismatches, missing = verify_transfer(source, received) + assert not missing, f"Missing: {missing}" + assert not mismatches, f"Mismatch: {mismatches}" diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index 8c83df6..576337d 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -2,6 +2,7 @@ #include "client_validation.h" #include "chmod.h" #include "config.h" +#include "file_list.h" #include "log.h" #include "test_utils.h" #include "utils.h" @@ -607,9 +608,6 @@ static void test_parse_args_rejects_unimplemented_options() { "--delete-excluded", "--delete-after", "--max-delete", - "--filter", - "--files-from", - "--cvs-exclude", "--prune-empty-dirs", "-R", "--relative", @@ -1265,6 +1263,137 @@ static void test_parse_args_log_file_format() { config_delete(cfg); } +/* --filter is repeatable and accepts both "--filter RULE" and "--filter=RULE". */ +static void test_parse_args_filter_rules() { + Config* cfg = config_create(); + int positional_args[2]; + int positional_count = 0; + char* argv[] = {"fastsync", "--filter", "- *.tmp", "--filter=+ /keep.txt", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 6, argv, positional_args, &positional_count), 0); + EXPECT_NOT_NULL(cfg->filters); + EXPECT_EQ_INT(cfg->filters->size, 2); + EXPECT_EQ_STR((char*)cfg->filters->items[0], "- *.tmp"); + EXPECT_EQ_STR((char*)cfg->filters->items[1], "+ /keep.txt"); + config_delete(cfg); + + /* An unsupported rsync rule type is rejected with a clear error. */ + cfg = config_create(); + positional_count = 0; + char* bad_argv[] = {"fastsync", "--filter=merge /tmp/excludes", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 4, bad_argv, positional_args, &positional_count), -1); + config_delete(cfg); + + /* A trailing --filter with no rule is a missing-argument error. */ + cfg = config_create(); + positional_count = 0; + char* missing_argv[] = {"fastsync", "/src", "/dst", "--filter"}; + EXPECT_EQ_INT(parse_args(cfg, 4, missing_argv, positional_args, &positional_count), -1); + config_delete(cfg); +} + +/* -0/--from0, -C/--cvs-exclude and -F wire into their config flags. */ +static void test_parse_args_from0_cvs_filter_file_flags() { + static const struct { + const char* arg; + bool from0; + bool cvs; + bool per_dir; + } cases[] = { + {"--from0", true, false, false}, + {"-0", true, false, false}, + {"--cvs-exclude", false, true, false}, + {"-C", false, true, false}, + {"-F", false, false, true}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + Config* cfg = config_create(); + char* argv[] = {"fastsync", (char*)cases[i].arg, "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); + EXPECT_EQ_INT(cfg->from0, cases[i].from0); + EXPECT_EQ_INT(cfg->cvs_exclude, cases[i].cvs); + EXPECT_EQ_INT(cfg->per_dir_filter, cases[i].per_dir); + config_delete(cfg); + } +} + +static void write_file_bytes(const char* path, const char* bytes, size_t len) { + FILE* fp = fopen(path, "wb"); + EXPECT_NOT_NULL(fp); + EXPECT_EQ_INT((int)fwrite(bytes, 1, len, fp), (int)len); + fclose(fp); +} + +/* --files-from is validated and parsed after the full argument scan, so -0 may + * appear before or after it. */ +static void test_parse_args_files_from() { + const char* list_path = "cli_files_from_list.txt"; + write_file_bytes(list_path, "a.txt\nsub/b.bin\n\n./c.txt\n", 25); + Config* cfg = config_create(); + int positional_args[2]; + int positional_count = 0; + char* argv[] = {"fastsync", "--files-from", (char*)list_path, "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_EQ_STR(cfg->files_from, list_path); + EXPECT_NOT_NULL(cfg->files_from_set); + FileListSet* set = (FileListSet*)cfg->files_from_set; + EXPECT_TRUE(file_list_affects(set, "a.txt")); + EXPECT_TRUE(file_list_affects(set, "sub/b.bin")); + EXPECT_TRUE(file_list_affects(set, "sub/b.bin/x")); + EXPECT_TRUE(file_list_affects(set, "sub")); + EXPECT_TRUE(file_list_affects(set, "c.txt")); + EXPECT_FALSE(file_list_affects(set, "other.txt")); + config_delete(cfg); + remove(list_path); + + /* -0 switches the separator to NUL regardless of argument order. */ + write_file_bytes(list_path, "x.txt\0y/z.bin\0", 14); + cfg = config_create(); + positional_count = 0; + char* nul_argv[] = {"fastsync", + "--files-from=" + "cli_files_from_list.txt", + "-0", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 5, nul_argv, positional_args, &positional_count), 0); + set = (FileListSet*)cfg->files_from_set; + EXPECT_NOT_NULL(set); + EXPECT_TRUE(file_list_affects(set, "x.txt")); + EXPECT_TRUE(file_list_affects(set, "y/z.bin")); + EXPECT_TRUE(file_list_affects(set, "y")); + EXPECT_FALSE(file_list_affects(set, "z.txt")); + config_delete(cfg); + remove(list_path); + + /* A missing list file is a hard parse-time error. */ + cfg = config_create(); + positional_count = 0; + char* missing_argv[] = {"fastsync", "--files-from", "does_not_exist_ff.txt", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 5, missing_argv, positional_args, &positional_count), -1); + config_delete(cfg); + + /* Absolute and traversal entries are rejected. */ + write_file_bytes(list_path, "/abs/path\n", 10); + cfg = config_create(); + positional_count = 0; + char* abs_argv[] = {"fastsync", + "--files-from=" + "cli_files_from_list.txt", + "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 4, abs_argv, positional_args, &positional_count), -1); + config_delete(cfg); + write_file_bytes(list_path, "../escape\n", 10); + cfg = config_create(); + positional_count = 0; + char* trav_argv[] = {"fastsync", + "--files-from=" + "cli_files_from_list.txt", + "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 4, trav_argv, positional_args, &positional_count), -1); + config_delete(cfg); + remove(list_path); +} + void test_client_cli() { test_validate_config_required_paths(); test_validate_config_incompatible_options(); @@ -1344,4 +1473,7 @@ void test_client_cli() { test_parse_args_checksum_choice_aliases(); test_parse_args_checksum_choice_requires_value(); test_parse_args_temp_dir(); + test_parse_args_filter_rules(); + test_parse_args_from0_cvs_filter_file_flags(); + test_parse_args_files_from(); } diff --git a/tests/test_scanner.c b/tests/test_scanner.c index 0044f87..4627ffb 100644 --- a/tests/test_scanner.c +++ b/tests/test_scanner.c @@ -1,7 +1,10 @@ #include "test_utils.h" #include "scanner.h" #include "file.h" +#include "file_list.h" +#include "filter.h" #include "utils.h" +#include #include #include #include @@ -394,8 +397,8 @@ static void test_parallel_scanner_root_chunks_without_workers() { create_test_file(file1, "a"); create_test_file(file2, "b"); - ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0, - 0, 0, false, false, false, false, false, false}; + ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0, 0, 0, + false, false, false, false, false, false, NULL, NULL, false}; ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options, NULL); EXPECT_NOT_NULL(scanner); @@ -653,6 +656,345 @@ static void test_scanner_one_file_system_cross_device() { EXPECT_EQ_INT(par_on_total, 1); } +/* Collect emitted file paths (relative to `root`) from a sequential scan. + * Returns 0 on success with *out and *count set (caller frees *out). */ +static int collect_files(const char* root, const ScannerOptions* options, char*** out, + int* out_count) { + DirectoryScanner* scanner = directory_scanner_create_with_options(root, options); + if (!scanner) + return -1; + size_t root_len = strlen(root); + while (root_len > 0 && root[root_len - 1] == '/') + root_len--; + int cap = 16; + int count = 0; + char** paths = malloc((size_t)cap * sizeof(char*)); + if (!paths) { + directory_scanner_destroy(scanner); + return -1; + } + Chunk* chunk; + while ((chunk = directory_scanner_next(scanner)) != NULL) { + for (int i = 0; i < chunk->element_count; i++) { + const char* rel = chunk->items[i]->path + root_len; + if (*rel == '/') + rel++; + if (count == cap) { + cap *= 2; + char** grown = realloc(paths, (size_t)cap * sizeof(char*)); + if (!grown) { + for (int k = 0; k < count; k++) + free(paths[k]); + free(paths); + chunk_destroy(chunk); + directory_scanner_destroy(scanner); + return -1; + } + paths = grown; + } + paths[count++] = str_dup(rel); + } + chunk_destroy(chunk); + } + bool failed = directory_scanner_failed(scanner); + directory_scanner_destroy(scanner); + if (failed) { + for (int k = 0; k < count; k++) + free(paths[k]); + free(paths); + return -1; + } + *out = paths; + *out_count = count; + return 0; +} + +static int collect_files_parallel(const char* root, const ScannerOptions* options, char*** out, + int* out_count) { + ParallelScanner* scanner = parallel_scanner_create_with_options(root, options, NULL); + if (!scanner) + return -1; + size_t root_len = strlen(root); + while (root_len > 0 && root[root_len - 1] == '/') + root_len--; + int cap = 16; + int count = 0; + char** paths = malloc((size_t)cap * sizeof(char*)); + if (!paths) { + parallel_scanner_destroy(scanner); + return -1; + } + Chunk* chunk; + while ((chunk = parallel_scanner_next(scanner)) != NULL) { + for (int i = 0; i < chunk->element_count; i++) { + const char* rel = chunk->items[i]->path + root_len; + if (*rel == '/') + rel++; + if (count == cap) { + cap *= 2; + char** grown = realloc(paths, (size_t)cap * sizeof(char*)); + if (!grown) { + for (int k = 0; k < count; k++) + free(paths[k]); + free(paths); + chunk_destroy(chunk); + parallel_scanner_destroy(scanner); + return -1; + } + paths = grown; + } + paths[count++] = str_dup(rel); + } + chunk_destroy(chunk); + } + bool failed = parallel_scanner_failed(scanner); + parallel_scanner_destroy(scanner); + if (failed) { + for (int k = 0; k < count; k++) + free(paths[k]); + free(paths); + return -1; + } + *out = paths; + *out_count = count; + return 0; +} + +static bool has_path(char** paths, int count, const char* rel) { + for (int i = 0; i < count; i++) + if (strcmp(paths[i], rel) == 0) + return true; + return false; +} + +static void free_paths(char** paths, int count) { + for (int i = 0; i < count; i++) + free(paths[i]); + free(paths); +} + +static const char* FILE_LIST_PATH = "test_scan_files_from.txt"; + +/* --files-from: only the listed files (and the subtree of a listed directory) + * are emitted; unrelated files and directories are pruned. */ +static void test_files_from_subset(bool parallel) { + const char* root = "test_scan_ff"; + const char* sub = "test_scan_ff/sub"; + const char* other = "test_scan_ff/other"; + EXPECT_EQ_INT(mkdir(root, 0755), 0); + EXPECT_EQ_INT(mkdir(sub, 0755), 0); + EXPECT_EQ_INT(mkdir(other, 0755), 0); + create_test_file("test_scan_ff/root.txt", "root"); + create_test_file("test_scan_ff/sub/keep.txt", "keep"); + create_test_file("test_scan_ff/sub/skip.bin", "skip"); + create_test_file("test_scan_ff/other/unrelated.txt", "unrelated"); + + /* List a root file and a file under sub: sub is descended but its other file + * is not listed, and the whole `other` directory is pruned. */ + create_test_file(FILE_LIST_PATH, "root.txt\nsub/keep.txt\n"); + char err[160]; + FileListSet* set = file_list_load(FILE_LIST_PATH, false, err, sizeof(err)); + EXPECT_NOT_NULL(set); + + ScannerOptions options = {0}; + options.file_list = set; + if (parallel) + options.num_threads = 2; + char** paths = NULL; + int count = 0; + int rc = parallel ? collect_files_parallel(root, &options, &paths, &count) + : collect_files(root, &options, &paths, &count); + EXPECT_EQ_INT(rc, 0); + EXPECT_EQ_INT(count, 2); + EXPECT_TRUE(has_path(paths, count, "root.txt")); + EXPECT_TRUE(has_path(paths, count, "sub/keep.txt")); + EXPECT_FALSE(has_path(paths, count, "sub/skip.bin")); + EXPECT_FALSE(has_path(paths, count, "other/unrelated.txt")); + free_paths(paths, count); + file_list_destroy(set); + remove(FILE_LIST_PATH); + + /* Listing a directory transfers its whole subtree. */ + create_test_file(FILE_LIST_PATH, "sub\n"); + set = file_list_load(FILE_LIST_PATH, false, err, sizeof(err)); + EXPECT_NOT_NULL(set); + options.file_list = set; + rc = parallel ? collect_files_parallel(root, &options, &paths, &count) + : collect_files(root, &options, &paths, &count); + EXPECT_EQ_INT(rc, 0); + EXPECT_EQ_INT(count, 2); + EXPECT_TRUE(has_path(paths, count, "sub/keep.txt")); + EXPECT_TRUE(has_path(paths, count, "sub/skip.bin")); + EXPECT_FALSE(has_path(paths, count, "root.txt")); + EXPECT_FALSE(has_path(paths, count, "other/unrelated.txt")); + free_paths(paths, count); + file_list_destroy(set); + remove(FILE_LIST_PATH); + + unlink("test_scan_ff/root.txt"); + unlink("test_scan_ff/sub/keep.txt"); + unlink("test_scan_ff/sub/skip.bin"); + unlink("test_scan_ff/other/unrelated.txt"); + rmdir(other); + rmdir(sub); + rmdir(root); +} + +/* Filter layer: '-' excludes, first-match-wins ordering with '+', anchored + * rules, and dir-only rules all prune during the scan. */ +static void test_filter_rules(bool parallel) { + const char* root = "test_scan_filter"; + EXPECT_EQ_INT(mkdir(root, 0755), 0); + create_test_file("test_scan_filter/a.txt", "a"); + create_test_file("test_scan_filter/b.tmp", "b"); + create_test_file("test_scan_filter/c.txt", "c"); + + /* - *.tmp excludes only the tmp file; other files remain (default include). */ + const char* exclude_only[] = {"- *.tmp"}; + char err[160]; + FilterRuleList* base = filter_base_build(exclude_only, 1, false, err, sizeof(err)); + EXPECT_NOT_NULL(base); + ScannerOptions options = {0}; + options.base_filters = base; + if (parallel) + options.num_threads = 2; + char** paths = NULL; + int count = 0; + int rc = parallel ? collect_files_parallel(root, &options, &paths, &count) + : collect_files(root, &options, &paths, &count); + EXPECT_EQ_INT(rc, 0); + EXPECT_EQ_INT(count, 2); + EXPECT_TRUE(has_path(paths, count, "a.txt")); + EXPECT_TRUE(has_path(paths, count, "c.txt")); + EXPECT_FALSE(has_path(paths, count, "b.tmp")); + free_paths(paths, count); + filter_rule_list_free(base); + + /* Anchored include then exclude-all: only root-level keep* survives. */ + const char* anchored[] = {"+ /a.txt", "- *"}; + base = filter_base_build(anchored, 2, false, err, sizeof(err)); + EXPECT_NOT_NULL(base); + options.base_filters = base; + rc = parallel ? collect_files_parallel(root, &options, &paths, &count) + : collect_files(root, &options, &paths, &count); + EXPECT_EQ_INT(rc, 0); + EXPECT_EQ_INT(count, 1); + EXPECT_TRUE(has_path(paths, count, "a.txt")); + free_paths(paths, count); + filter_rule_list_free(base); + + unlink("test_scan_filter/a.txt"); + unlink("test_scan_filter/b.tmp"); + unlink("test_scan_filter/c.txt"); + rmdir(root); +} + +/* Anchored dir-only rules prune a whole subtree. */ +static void test_filter_dir_only_and_anchored(bool parallel) { + const char* root = "test_scan_filter_dir"; + const char* sub = "test_scan_filter_dir/sub"; + EXPECT_EQ_INT(mkdir(root, 0755), 0); + EXPECT_EQ_INT(mkdir(sub, 0755), 0); + create_test_file("test_scan_filter_dir/sub/inner.txt", "x"); + create_test_file("test_scan_filter_dir/keep.txt", "keep"); + + const char* rules[] = {"- /sub/"}; + char err[160]; + FilterRuleList* base = filter_base_build(rules, 1, false, err, sizeof(err)); + EXPECT_NOT_NULL(base); + ScannerOptions options = {0}; + options.base_filters = base; + if (parallel) + options.num_threads = 2; + char** paths = NULL; + int count = 0; + int rc = parallel ? collect_files_parallel(root, &options, &paths, &count) + : collect_files(root, &options, &paths, &count); + EXPECT_EQ_INT(rc, 0); + EXPECT_EQ_INT(count, 1); + EXPECT_TRUE(has_path(paths, count, "keep.txt")); + EXPECT_FALSE(has_path(paths, count, "sub/inner.txt")); + free_paths(paths, count); + filter_rule_list_free(base); + + unlink("test_scan_filter_dir/sub/inner.txt"); + unlink("test_scan_filter_dir/keep.txt"); + rmdir(sub); + rmdir(root); +} + +/* -C default CVS excludes prune .git/ directories and *.o files. */ +static void test_cvs_defaults(bool parallel) { + const char* root = "test_scan_cvs"; + const char* git = "test_scan_cvs/.git"; + EXPECT_EQ_INT(mkdir(root, 0755), 0); + EXPECT_EQ_INT(mkdir(git, 0755), 0); + create_test_file("test_scan_cvs/.git/config", "cfg"); + create_test_file("test_scan_cvs/object.o", "o"); + create_test_file("test_scan_cvs/keep.txt", "keep"); + + char err[160]; + FilterRuleList* base = filter_base_build(NULL, 0, true, err, sizeof(err)); + EXPECT_NOT_NULL(base); + ScannerOptions options = {0}; + options.base_filters = base; + if (parallel) + options.num_threads = 2; + char** paths = NULL; + int count = 0; + int rc = parallel ? collect_files_parallel(root, &options, &paths, &count) + : collect_files(root, &options, &paths, &count); + EXPECT_EQ_INT(rc, 0); + EXPECT_EQ_INT(count, 1); + EXPECT_TRUE(has_path(paths, count, "keep.txt")); + EXPECT_FALSE(has_path(paths, count, ".git/config")); + EXPECT_FALSE(has_path(paths, count, "object.o")); + free_paths(paths, count); + filter_rule_list_free(base); + + unlink("test_scan_cvs/.git/config"); + unlink("test_scan_cvs/object.o"); + unlink("test_scan_cvs/keep.txt"); + rmdir(git); + rmdir(root); +} + +/* -F: a .rsync-filter placed in a directory governs its subtree and the file + * itself is never transferred. */ +static void test_per_dir_filter(bool parallel) { + const char* root = "test_scan_perdir"; + const char* sub = "test_scan_perdir/sub"; + EXPECT_EQ_INT(mkdir(root, 0755), 0); + EXPECT_EQ_INT(mkdir(sub, 0755), 0); + create_test_file("test_scan_perdir/drop.tmp", "tmp"); + create_test_file("test_scan_perdir/keep.txt", "keep"); + create_test_file("test_scan_perdir/sub/nested.tmp", "tmp"); + create_test_file("test_scan_perdir/.rsync-filter", "- *.tmp\n"); + + ScannerOptions options = {0}; + options.per_dir_filters = true; + if (parallel) + options.num_threads = 2; + char** paths = NULL; + int count = 0; + int rc = parallel ? collect_files_parallel(root, &options, &paths, &count) + : collect_files(root, &options, &paths, &count); + EXPECT_EQ_INT(rc, 0); + EXPECT_EQ_INT(count, 1); + EXPECT_TRUE(has_path(paths, count, "keep.txt")); + EXPECT_FALSE(has_path(paths, count, "drop.tmp")); + EXPECT_FALSE(has_path(paths, count, "sub/nested.tmp")); + EXPECT_FALSE(has_path(paths, count, ".rsync-filter")); + free_paths(paths, count); + + unlink("test_scan_perdir/drop.tmp"); + unlink("test_scan_perdir/keep.txt"); + unlink("test_scan_perdir/sub/nested.tmp"); + unlink("test_scan_perdir/.rsync-filter"); + rmdir(sub); + rmdir(root); +} + void test_scanner() { test_scanner_single_file(); test_scanner_multiple_files(); @@ -672,4 +1014,14 @@ void test_scanner() { test_scanner_one_file_system_same_device(); test_parallel_scanner_one_file_system_same_device(); test_scanner_one_file_system_cross_device(); + test_files_from_subset(false); + test_files_from_subset(true); + test_filter_rules(false); + test_filter_rules(true); + test_filter_dir_only_and_anchored(false); + test_filter_dir_only_and_anchored(true); + test_cvs_defaults(false); + test_cvs_defaults(true); + test_per_dir_filter(false); + test_per_dir_filter(true); }