Merge feat/p2-files-from-filter: files-from/from0/filter/-F/-C
Client-side file-list selection and filter-rule layer (client-only; no wire change). rsync-consistent inner-first per-dir filter precedence; listed-but- missing files-from entries hard-error; root '/' scan regression fixed; Summary recounted (63 implemented / 75 not). c-review REQUEST CHANGES -> blockers fixed; PR #265.
This commit is contained in:
+7
-7
@@ -6,11 +6,11 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
|
|
||||||
| Status | Count | Description |
|
| Status | Count | Description |
|
||||||
|--------|-------|-------------|
|
|--------|-------|-------------|
|
||||||
| ✅ Implemented | 58 | Feature works end-to-end |
|
| ✅ Implemented | 63 | Feature works end-to-end |
|
||||||
| 🔀 Alt Arg | 3 | Functionality exists but under different flag/semantics |
|
| 🔀 Alt Arg | 3 | Functionality exists but under different flag/semantics |
|
||||||
| ⚠️ Partial | 5 | Flag parsed/stored but behavior incomplete |
|
| ⚠️ Partial | 5 | Flag parsed/stored but behavior incomplete |
|
||||||
| 🔄 Compatibility No-op | 1 | Flag is accepted for CLI compatibility but has no effect |
|
| 🔄 Compatibility No-op | 1 | Flag is accepted for CLI compatibility but has no effect |
|
||||||
| ❌ Not Implemented | 80 | Flag not recognized or no behavior |
|
| ❌ Not Implemented | 75 | Flag not recognized or no behavior |
|
||||||
| **Total** | **147** | |
|
| **Total** | **147** | |
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -30,7 +30,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
| `--no-motd` | Suppress daemon MOTD | ❌ Not Implemented | |
|
| `--no-motd` | Suppress daemon MOTD | ❌ Not Implemented | |
|
||||||
| `--exclude=PATTERN` | Exclude files matching pattern | ✅ Implemented | Glob matching in scanner |
|
| `--exclude=PATTERN` | Exclude files matching pattern | ✅ Implemented | Glob matching in scanner |
|
||||||
| `--include=PATTERN` | Include files matching pattern | ✅ Implemented | Glob matching in scanner |
|
| `--include=PATTERN` | Include files matching pattern | ✅ Implemented | Glob matching in scanner |
|
||||||
| `-C`, `--cvs-exclude` | Auto-ignore CVS files | ❌ Not Implemented | Removed because it had no effect |
|
| `-C`, `--cvs-exclude` | Auto-ignore CVS files | ✅ Implemented | Applies the well-known rsync default exclude set as exclude rules during scanning (RCS SCCS CVS CVS.adm RCSLOG cvslog.* tags TAGS .make.state .nse_depinfo *~ #* .#* ,* _$* *$ *.old *.bak *.BAK *.orig *.rej .del-* *.a *.olb *.o *.obj *.so *.exe *.Z *.elc *.ln core .svn/ .git/ .hg/ .bzr/); `.git/`-style repo dirs are pruned without descending |
|
||||||
|
|
||||||
## 2. Modifying Output
|
## 2. Modifying Output
|
||||||
|
|
||||||
@@ -53,9 +53,9 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `--exclude-from=FILE` | Read exclude patterns from file | ✅ Implemented | Reads patterns from file |
|
| `--exclude-from=FILE` | Read exclude patterns from file | ✅ Implemented | Reads patterns from file |
|
||||||
| `--include-from=FILE` | Read include patterns from file | ✅ Implemented | Reads patterns from file |
|
| `--include-from=FILE` | Read include patterns from file | ✅ Implemented | Reads patterns from file |
|
||||||
| `--filter=RULE` | Add file-filtering rule | ❌ Not Implemented | Removed because it had no effect |
|
| `--filter=RULE` | Add file-filtering rule | ✅ Implemented | Long option only: rsync's short `-f` conflicts with FastSync sendfile (see FastSync-specific list), so `-f` is not reassigned. Supported subset: `+`/`-` include/exclude, implicit-exclude patterns, `include`/`exclude` word forms, a leading `/` anchor (to the transfer root, or to a `.rsync-filter` file's directory), and a trailing `/` for dir-only rules; first match wins with a default of include inside the filter layer. Filters are an independent layer from `--exclude`/`--include` (an entry must pass both). Rejected with a clear error (no silent no-ops): `merge`/`dir-merge`/`hide`/`show`/`protect`/`risk`/`clear` words, rules that begin with `:`/`.`/`!` (merge/dir-merge/list-clear shorthands), and include/exclude modifiers other than `/` (`! C s r p x`) |
|
||||||
| `--files-from=FILE` | Read source file list from file | ❌ Not Implemented | Removed because it had no effect |
|
| `--files-from=FILE` | Read source file list from file | ✅ Implemented | Entries are paths relative to the source root (leading `./` stripped, `..`/absolute entries rejected at parse time, blank lines ignored; NUL-delimited with `-0`). A listed regular file is transferred; a listed directory transfers its whole subtree (FastSync recursion is always on, unlike rsync's non-recursive default). Non-listed paths and their subtrees are pruned by the scanner. A listed entry that does not exist under the source (and an empty list) is a hard error reported before any transfer; listing `.` (whole tree) and empty listed directories are fine. Scalability note: `file_list_affects` is O(list size) per scanned entry, so a very large `--files-from` list against a huge tree is quadratic; lists are typically small enough that this is acceptable, but it is the documented bound. The delete manifest still derives from what was actually sent, so `--delete` stays consistent with the subset |
|
||||||
| `-0`, `--from0` | Delimit *-from files with NULs | ❌ Not Implemented | |
|
| `-0`, `--from0` | Delimit *-from files with NULs | ✅ Implemented | `--files-from` entries become NUL-delimited; the flag may appear before or after `--files-from` on the command line. NUL mode preserves entry bytes exactly (trailing CR/LF are part of the name; only newline mode trims them) |
|
||||||
| `--max-size=SIZE` | Skip files larger than SIZE | ✅ Implemented | `max_size` in scanner |
|
| `--max-size=SIZE` | Skip files larger than SIZE | ✅ Implemented | `max_size` in scanner |
|
||||||
| `--min-size=SIZE` | Skip files smaller than SIZE | ✅ Implemented | `min_size` in scanner |
|
| `--min-size=SIZE` | Skip files smaller than SIZE | ✅ Implemented | `min_size` in scanner |
|
||||||
| `-I`, `--ignore-times` | Don't skip files matching size+time | ❌ Not Implemented | |
|
| `-I`, `--ignore-times` | Don't skip files matching size+time | ❌ Not Implemented | |
|
||||||
@@ -65,7 +65,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
| `--ignore-existing` | Skip updating existing files | ❌ Not Implemented | |
|
| `--ignore-existing` | Skip updating existing files | ❌ Not Implemented | |
|
||||||
| `--remove-source-files` | Sender removes regular files after confirmed transfer | ✅ Implemented | |
|
| `--remove-source-files` | Sender removes regular files after confirmed transfer | ✅ Implemented | |
|
||||||
| `-x`, `--one-file-system` | Do not cross filesystem boundaries | ✅ Implemented | Sender scanner captures the root device and skips descending into mount-point crossings (`st_dev` differs); cross-filesystem mount-point subdirectories are dropped entirely, matching rsync |
|
| `-x`, `--one-file-system` | Do not cross filesystem boundaries | ✅ Implemented | Sender scanner captures the root device and skips descending into mount-point crossings (`st_dev` differs); cross-filesystem mount-point subdirectories are dropped entirely, matching rsync |
|
||||||
| `-F` | Add the default `.rsync-filter` rules | ❌ Not Implemented | |
|
| `-F` | Add the default `.rsync-filter` rules | ✅ Implemented | Reads one filter rule per line from each directory's `.rsync-filter` file during traversal and applies it to that directory's subtree; the current directory's rules are evaluated before its ancestors', so deeper files override shallower ones and per-directory files override the command-line `--filter`/`-C` base by default (matching rsync's first-match-wins precedence); `.rsync-filter` files are never transferred. The rsync `-FF` behavior (also `.cvsignore`) is out of scope; unsupported rule types inside the file abort with a clear error |
|
||||||
|
|
||||||
## 4. Directory Options
|
## 4. Directory Options
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
#include "compression.h"
|
#include "compression.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
#include "delta.h"
|
#include "delta.h"
|
||||||
|
#include "file_list.h"
|
||||||
|
#include "filter.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "transport_tcp.h"
|
#include "transport_tcp.h"
|
||||||
@@ -313,6 +315,31 @@ static int config_add_pattern(char*** patterns, int* count, const char* value,
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Validate and append one --filter=RULE string. Returns 0 on success, -1 on error. */
|
||||||
|
static int config_add_filter(Config* config, const char* rule) {
|
||||||
|
char err[160];
|
||||||
|
FilterRule* parsed = filter_rule_parse(rule, err, sizeof(err));
|
||||||
|
if (!parsed) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "invalid --filter rule '%s': %s", rule, err);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
filter_rule_free(parsed);
|
||||||
|
if (!config->filters) {
|
||||||
|
config->filters = array_list_create(free);
|
||||||
|
if (!config->filters) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed for --filter");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
char* dup = str_dup(rule);
|
||||||
|
if (!dup || !array_list_add(config->filters, dup)) {
|
||||||
|
free(dup);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed for --filter");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
static int parse_skip_compress(Config* config, const char* value) {
|
static int parse_skip_compress(Config* config, const char* value) {
|
||||||
char* list = str_dup(value);
|
char* list = str_dup(value);
|
||||||
if (!list)
|
if (!list)
|
||||||
@@ -423,6 +450,9 @@ static const OptionEntry OPTION_TABLE[] = {
|
|||||||
{"--max-size", NULL, OPT_ULL, offsetof(Config, max_size)},
|
{"--max-size", NULL, OPT_ULL, offsetof(Config, max_size)},
|
||||||
{"--min-size", NULL, OPT_ULL, offsetof(Config, min_size)},
|
{"--min-size", NULL, OPT_ULL, offsetof(Config, min_size)},
|
||||||
{"--one-file-system", "-x", OPT_FLAG, offsetof(Config, one_file_system)},
|
{"--one-file-system", "-x", OPT_FLAG, offsetof(Config, one_file_system)},
|
||||||
|
{"--from0", "-0", OPT_FLAG, offsetof(Config, from0)},
|
||||||
|
{"--cvs-exclude", "-C", OPT_FLAG, offsetof(Config, cvs_exclude)},
|
||||||
|
{"-F", NULL, OPT_FLAG, offsetof(Config, per_dir_filter)},
|
||||||
};
|
};
|
||||||
|
|
||||||
/* Only boolean options with no required argument are safe to negate. */
|
/* Only boolean options with no required argument are safe to negate. */
|
||||||
@@ -444,6 +474,8 @@ static const NegatableOption NEGATABLE_OPTIONS[] = {
|
|||||||
{"sparse", "S", offsetof(Config, preserve_sparse)},
|
{"sparse", "S", offsetof(Config, preserve_sparse)},
|
||||||
{"inplace", NULL, offsetof(Config, inplace)},
|
{"inplace", NULL, offsetof(Config, inplace)},
|
||||||
{"checksum", NULL, offsetof(Config, checksum)},
|
{"checksum", NULL, offsetof(Config, checksum)},
|
||||||
|
{"from0", NULL, offsetof(Config, from0)},
|
||||||
|
{"cvs-exclude", NULL, offsetof(Config, cvs_exclude)},
|
||||||
|
|
||||||
/* These options are also implied by --archive or handled outside the table. */
|
/* These options are also implied by --archive or handled outside the table. */
|
||||||
{"compress", "c", offsetof(Config, use_compression)},
|
{"compress", "c", offsetof(Config, use_compression)},
|
||||||
@@ -862,6 +894,26 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
if (read_patterns_from_file(argv[++i], &config->include_patterns, &config->include_count) !=
|
if (read_patterns_from_file(argv[++i], &config->include_patterns, &config->include_count) !=
|
||||||
0)
|
0)
|
||||||
return -1;
|
return -1;
|
||||||
|
} else if (strncmp(argv[i], "--filter=", 9) == 0) {
|
||||||
|
if (config_add_filter(config, argv[i] + 9) != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (opt_is(argv[i], "--filter", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (config_add_filter(config, argv[++i]) != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (strncmp(argv[i], "--files-from=", 13) == 0) {
|
||||||
|
if (set_string_option(&config->files_from, argv[i] + 13, "--files-from") != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (opt_is(argv[i], "--files-from", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (set_string_option(&config->files_from, argv[++i], "--files-from") != 0)
|
||||||
|
return -1;
|
||||||
} else if (opt_is(argv[i], "-v", "--verbose")) {
|
} else if (opt_is(argv[i], "-v", "--verbose")) {
|
||||||
verbose = true;
|
verbose = true;
|
||||||
set_log_level(LOG_LEVEL_DEBUG);
|
set_log_level(LOG_LEVEL_DEBUG);
|
||||||
@@ -936,6 +988,20 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
if (config->compress_choice)
|
if (config->compress_choice)
|
||||||
config->use_compression = strcmp(config->compress_choice, "zstd") == 0;
|
config->use_compression = strcmp(config->compress_choice, "zstd") == 0;
|
||||||
|
|
||||||
|
/* --files-from is loaded after every argument is seen so that -0/--from0 may
|
||||||
|
* appear anywhere on the command line. A missing or unreadable file, and
|
||||||
|
* invalid (absolute / traversal) entries, are hard CLI errors. */
|
||||||
|
if (config->files_from) {
|
||||||
|
char err[256];
|
||||||
|
FileListSet* set = file_list_load(config->files_from, config->from0, err, sizeof(err));
|
||||||
|
if (!set) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--files-from: %s", err);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
file_list_destroy((FileListSet*)config->files_from_set);
|
||||||
|
config->files_from_set = set;
|
||||||
|
}
|
||||||
|
|
||||||
/* Incremental and delta transfers need metadata unless the user disabled it. */
|
/* Incremental and delta transfers need metadata unless the user disabled it. */
|
||||||
if ((config->use_incremental || config->use_delta) && !config->use_metadata &&
|
if ((config->use_incremental || config->use_delta) && !config->use_metadata &&
|
||||||
!config->metadata_explicitly_disabled) {
|
!config->metadata_explicitly_disabled) {
|
||||||
|
|||||||
+154
-23
@@ -7,6 +7,8 @@
|
|||||||
#include "data.h"
|
#include "data.h"
|
||||||
#include "delta.h"
|
#include "delta.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
|
#include "file_list.h"
|
||||||
|
#include "filter.h"
|
||||||
#include "metadata.h"
|
#include "metadata.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "multiprocessing.h"
|
#include "multiprocessing.h"
|
||||||
@@ -40,16 +42,112 @@ static const char* display_bytes(unsigned long long bytes, bool human_readable,
|
|||||||
return buffer;
|
return buffer;
|
||||||
}
|
}
|
||||||
|
|
||||||
static ScannerOptions scanner_options_from_config(const Config* config, int num_threads) {
|
/* Compiled scanner inputs that are shared read-only across scanner instances
|
||||||
ScannerOptions options = {config->use_metadata, config->chunk_size,
|
* and, in -m mode, across worker threads. `base_filters` owns the compiled
|
||||||
config->exclude_patterns, config->exclude_count,
|
* command-line + -C rules; the FileListSet allow-set lives in the Config. */
|
||||||
config->include_patterns, config->include_count,
|
typedef struct {
|
||||||
config->max_size, config->min_size,
|
ScannerOptions options;
|
||||||
config->max_depth, num_threads,
|
FilterRuleList* base_filters; /* owned; may be NULL */
|
||||||
config->follow_symlinks, config->copy_links,
|
} PreparedScanner;
|
||||||
config->safe_links, config->copy_unsafe_links,
|
|
||||||
config->checksum, config->one_file_system};
|
/* Build the scanner options for one scan. Returns false and logs on failure. */
|
||||||
return options;
|
static bool prepare_scanner(const Config* config, int num_threads, PreparedScanner* out) {
|
||||||
|
if (!out)
|
||||||
|
return false;
|
||||||
|
out->base_filters = NULL;
|
||||||
|
memset(&out->options, 0, sizeof(out->options));
|
||||||
|
|
||||||
|
int rule_count = config->filters ? config->filters->size : 0;
|
||||||
|
const char** texts = NULL;
|
||||||
|
if (rule_count > 0) {
|
||||||
|
texts = malloc((size_t)rule_count * sizeof(char*));
|
||||||
|
if (!texts) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed for filter rules");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < rule_count; i++)
|
||||||
|
texts[i] = (const char*)config->filters->items[i];
|
||||||
|
}
|
||||||
|
if (rule_count > 0 || config->cvs_exclude) {
|
||||||
|
char err[160];
|
||||||
|
out->base_filters = filter_base_build(texts, rule_count, config->cvs_exclude, err, sizeof(err));
|
||||||
|
free(texts);
|
||||||
|
if (!out->base_filters) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "invalid filter rule: %s", err);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
free(texts);
|
||||||
|
}
|
||||||
|
|
||||||
|
ScannerOptions* options = &out->options;
|
||||||
|
options->use_metadata = config->use_metadata;
|
||||||
|
options->chunk_size = config->chunk_size;
|
||||||
|
options->exclude_patterns = config->exclude_patterns;
|
||||||
|
options->exclude_count = config->exclude_count;
|
||||||
|
options->include_patterns = config->include_patterns;
|
||||||
|
options->include_count = config->include_count;
|
||||||
|
options->max_size = config->max_size;
|
||||||
|
options->min_size = config->min_size;
|
||||||
|
options->max_depth = config->max_depth;
|
||||||
|
options->num_threads = num_threads;
|
||||||
|
options->follow_symlinks = config->follow_symlinks;
|
||||||
|
options->copy_links = config->copy_links;
|
||||||
|
options->safe_links = config->safe_links;
|
||||||
|
options->copy_unsafe_links = config->copy_unsafe_links;
|
||||||
|
options->checksum = config->checksum;
|
||||||
|
options->one_file_system = config->one_file_system;
|
||||||
|
options->file_list = (const FileListSet*)config->files_from_set;
|
||||||
|
options->base_filters = out->base_filters;
|
||||||
|
options->per_dir_filters = config->per_dir_filter;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void prepared_scanner_destroy(PreparedScanner* prepared) {
|
||||||
|
if (!prepared)
|
||||||
|
return;
|
||||||
|
filter_rule_list_free(prepared->base_filters);
|
||||||
|
prepared->base_filters = NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --files-from semantics: every listed entry must resolve under the source
|
||||||
|
* root, otherwise rsync reports a hard error instead of silently transferring
|
||||||
|
* nothing. An empty list is also an error. An entry of "." (the whole tree)
|
||||||
|
* and listed-but-empty directories are valid. Runs before any transfer so the
|
||||||
|
* failure is surfaced uniformly in the single-threaded, -m, dry-run and
|
||||||
|
* --list-only paths. */
|
||||||
|
static bool files_from_list_valid(const Config* config) {
|
||||||
|
const FileListSet* set = (const FileListSet*)config->files_from_set;
|
||||||
|
if (!set)
|
||||||
|
return true;
|
||||||
|
if (!config->send_directory) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--files-from requires a source directory");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (set->count == 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--files-from file '%s' contains no entries; nothing to transfer",
|
||||||
|
config->files_from ? config->files_from : "");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < set->count; i++) {
|
||||||
|
const char* entry = set->entries[i];
|
||||||
|
if (entry[0] == '\0')
|
||||||
|
continue; /* "." == list the whole tree */
|
||||||
|
char* full = path_cat(config->send_directory, entry);
|
||||||
|
if (!full) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed while validating --files-from");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
struct stat st;
|
||||||
|
if (lstat(full, &st) != 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--files-from entry '%s' not found in source '%s'", entry,
|
||||||
|
config->send_directory);
|
||||||
|
free(full);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
free(full);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Select the configured transport for both transfer execution paths. */
|
/* Select the configured transport for both transfer execution paths. */
|
||||||
@@ -249,11 +347,17 @@ static void pipeline_cancel(PipelineContextSender* context) {
|
|||||||
|
|
||||||
/* Print dry-run manifest showing files that would be transferred. Returns 0 on success. */
|
/* Print dry-run manifest showing files that would be transferred. Returns 0 on success. */
|
||||||
static int send_dry_run_manifest(const Config* config) {
|
static int send_dry_run_manifest(const Config* config) {
|
||||||
ScannerOptions options = scanner_options_from_config(config, 0);
|
if (!files_from_list_valid(config))
|
||||||
DirectoryScanner* scanner =
|
|
||||||
directory_scanner_create_with_options(config->send_directory, &options);
|
|
||||||
if (!scanner)
|
|
||||||
return -1;
|
return -1;
|
||||||
|
PreparedScanner prepared;
|
||||||
|
if (!prepare_scanner(config, 0, &prepared))
|
||||||
|
return -1;
|
||||||
|
DirectoryScanner* scanner =
|
||||||
|
directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
||||||
|
if (!scanner) {
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
Chunk* chunk;
|
Chunk* chunk;
|
||||||
int file_count = 0;
|
int file_count = 0;
|
||||||
unsigned long long total_bytes = 0;
|
unsigned long long total_bytes = 0;
|
||||||
@@ -267,6 +371,7 @@ static int send_dry_run_manifest(const Config* config) {
|
|||||||
if (!escaped_path) {
|
if (!escaped_path) {
|
||||||
chunk_destroy(chunk);
|
chunk_destroy(chunk);
|
||||||
directory_scanner_destroy(scanner);
|
directory_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
if (config->human_readable)
|
if (config->human_readable)
|
||||||
@@ -283,6 +388,7 @@ static int send_dry_run_manifest(const Config* config) {
|
|||||||
chunk_destroy(chunk);
|
chunk_destroy(chunk);
|
||||||
}
|
}
|
||||||
directory_scanner_destroy(scanner);
|
directory_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
if (!config->quiet) {
|
if (!config->quiet) {
|
||||||
if (config->human_readable)
|
if (config->human_readable)
|
||||||
printf("Total: %d files, %s\n", file_count,
|
printf("Total: %d files, %s\n", file_count,
|
||||||
@@ -319,12 +425,18 @@ static int compare_list_entries(const void* left, const void* right) {
|
|||||||
* Directory lines are not printed because the scanner only yields regular
|
* Directory lines are not printed because the scanner only yields regular
|
||||||
* transfer candidates. Returns 0 on success, 1 on error. */
|
* transfer candidates. Returns 0 on success, 1 on error. */
|
||||||
static int send_list_only(const Config* config) {
|
static int send_list_only(const Config* config) {
|
||||||
ScannerOptions options = scanner_options_from_config(config, 0);
|
if (!files_from_list_valid(config))
|
||||||
options.use_metadata = true; /* capture mode + mtime for the listing */
|
|
||||||
DirectoryScanner* scanner =
|
|
||||||
directory_scanner_create_with_options(config->send_directory, &options);
|
|
||||||
if (!scanner)
|
|
||||||
return 1;
|
return 1;
|
||||||
|
PreparedScanner prepared;
|
||||||
|
if (!prepare_scanner(config, 0, &prepared))
|
||||||
|
return 1;
|
||||||
|
prepared.options.use_metadata = true; /* capture mode + mtime for the listing */
|
||||||
|
DirectoryScanner* scanner =
|
||||||
|
directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
||||||
|
if (!scanner) {
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
ListEntry* entries = NULL;
|
ListEntry* entries = NULL;
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
size_t capacity = 0;
|
size_t capacity = 0;
|
||||||
@@ -378,6 +490,7 @@ static int send_list_only(const Config* config) {
|
|||||||
}
|
}
|
||||||
bool failed = oom || directory_scanner_failed(scanner);
|
bool failed = oom || directory_scanner_failed(scanner);
|
||||||
directory_scanner_destroy(scanner);
|
directory_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
if (failed) {
|
if (failed) {
|
||||||
list_entries_destroy(entries, count);
|
list_entries_destroy(entries, count);
|
||||||
if (oom)
|
if (oom)
|
||||||
@@ -770,14 +883,20 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
static int scan_directory_multithreaded(void* pipeline_context) {
|
static int scan_directory_multithreaded(void* pipeline_context) {
|
||||||
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
||||||
protocol_session_bind(&context->allocation_session);
|
protocol_session_bind(&context->allocation_session);
|
||||||
ScannerOptions options = scanner_options_from_config(context->config, 4);
|
PreparedScanner prepared;
|
||||||
|
if (!prepare_scanner(context->config, 4, &prepared)) {
|
||||||
|
pipeline_cancel(context);
|
||||||
|
protocol_session_unbind();
|
||||||
|
return thrd_error;
|
||||||
|
}
|
||||||
ParallelScanner* scanner = parallel_scanner_create_with_options(
|
ParallelScanner* scanner = parallel_scanner_create_with_options(
|
||||||
context->config->send_directory, &options, &context->allocation_session);
|
context->config->send_directory, &prepared.options, &context->allocation_session);
|
||||||
|
|
||||||
Chunk* current_chunk;
|
Chunk* current_chunk;
|
||||||
if (scanner == NULL) {
|
if (scanner == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to create parallel scanner");
|
log_message(LOG_LEVEL_ERROR, "Failed to create parallel scanner");
|
||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
@@ -790,6 +909,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
|||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
chunk_destroy(current_chunk);
|
chunk_destroy(current_chunk);
|
||||||
parallel_scanner_destroy(scanner);
|
parallel_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
@@ -801,12 +921,14 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
|||||||
chunk_destroy(current_chunk);
|
chunk_destroy(current_chunk);
|
||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
parallel_scanner_destroy(scanner);
|
parallel_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (parallel_scanner_failed(scanner)) {
|
if (parallel_scanner_failed(scanner)) {
|
||||||
parallel_scanner_destroy(scanner);
|
parallel_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
mtx_lock(&context->mutex_scanner);
|
mtx_lock(&context->mutex_scanner);
|
||||||
context->scanner_done = true;
|
context->scanner_done = true;
|
||||||
cnd_broadcast(&context->condition_not_empty_scanner);
|
cnd_broadcast(&context->condition_not_empty_scanner);
|
||||||
@@ -822,6 +944,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
|||||||
mtx_unlock(&context->mutex_scanner);
|
mtx_unlock(&context->mutex_scanner);
|
||||||
|
|
||||||
parallel_scanner_destroy(scanner);
|
parallel_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
return thrd_success;
|
return thrd_success;
|
||||||
}
|
}
|
||||||
@@ -923,6 +1046,8 @@ int send_files(Config* config) {
|
|||||||
return send_list_only(config);
|
return send_list_only(config);
|
||||||
if (config->dry_run)
|
if (config->dry_run)
|
||||||
return send_dry_run_manifest(config);
|
return send_dry_run_manifest(config);
|
||||||
|
if (!files_from_list_valid(config))
|
||||||
|
return 1;
|
||||||
|
|
||||||
Client* client = connect_transfer_client(config);
|
Client* client = connect_transfer_client(config);
|
||||||
if (!client) {
|
if (!client) {
|
||||||
@@ -939,10 +1064,13 @@ int send_files(Config* config) {
|
|||||||
DirectoryScanner* scanner = NULL;
|
DirectoryScanner* scanner = NULL;
|
||||||
ArrayList* manifest = NULL;
|
ArrayList* manifest = NULL;
|
||||||
ArrayList* remove_sources = NULL;
|
ArrayList* remove_sources = NULL;
|
||||||
|
PreparedScanner prepared;
|
||||||
|
memset(&prepared, 0, sizeof(prepared));
|
||||||
if (!config_send(client->file_descriptor, config))
|
if (!config_send(client->file_descriptor, config))
|
||||||
goto send_fail;
|
goto send_fail;
|
||||||
ScannerOptions scanner_options = scanner_options_from_config(config, 0);
|
if (!prepare_scanner(config, 0, &prepared))
|
||||||
scanner = directory_scanner_create_with_options(config->send_directory, &scanner_options);
|
goto send_fail;
|
||||||
|
scanner = directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
||||||
manifest = create_transfer_manifest(config);
|
manifest = create_transfer_manifest(config);
|
||||||
if (config->remove_source_files)
|
if (config->remove_source_files)
|
||||||
remove_sources = array_list_create(source_file_destroy);
|
remove_sources = array_list_create(source_file_destroy);
|
||||||
@@ -1043,6 +1171,7 @@ send_fail:
|
|||||||
array_list_delete(remove_sources);
|
array_list_delete(remove_sources);
|
||||||
if (scanner)
|
if (scanner)
|
||||||
directory_scanner_destroy(scanner);
|
directory_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
disconnect_transfer_client(client);
|
disconnect_transfer_client(client);
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
return ret;
|
return ret;
|
||||||
@@ -1056,6 +1185,8 @@ int send_files_multithreaded(Config** config_ptr) {
|
|||||||
return send_list_only(config);
|
return send_list_only(config);
|
||||||
if (config->dry_run)
|
if (config->dry_run)
|
||||||
return send_dry_run_manifest(config);
|
return send_dry_run_manifest(config);
|
||||||
|
if (!files_from_list_valid(config))
|
||||||
|
return 1;
|
||||||
|
|
||||||
long pages = sysconf(_SC_AVPHYS_PAGES);
|
long pages = sysconf(_SC_AVPHYS_PAGES);
|
||||||
long page_size = sysconf(_SC_PAGE_SIZE);
|
long page_size = sysconf(_SC_PAGE_SIZE);
|
||||||
|
|||||||
+296
-24
@@ -17,8 +17,62 @@
|
|||||||
typedef struct {
|
typedef struct {
|
||||||
char* path;
|
char* path;
|
||||||
int depth;
|
int depth;
|
||||||
|
FilterNode* context; /* inherited per-directory filter context */
|
||||||
} DirEntry;
|
} DirEntry;
|
||||||
|
|
||||||
|
/* A chain node: `own` holds the .rsync-filter rules of one directory, `parent`
|
||||||
|
* the context that directory inherited (nearest ancestor with a filter file).
|
||||||
|
* The chain for a directory's contents runs from that directory's own node up
|
||||||
|
* to the root; the command-line base rules are evaluated after the whole
|
||||||
|
* chain. */
|
||||||
|
struct FilterNode {
|
||||||
|
FilterNode* parent;
|
||||||
|
FilterRuleList* own;
|
||||||
|
};
|
||||||
|
|
||||||
|
static void filter_node_destroy(void* item) {
|
||||||
|
if (item) {
|
||||||
|
FilterNode* node = (FilterNode*)item;
|
||||||
|
if (node->own)
|
||||||
|
filter_rule_list_free(node->own);
|
||||||
|
free(node);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static FilterNode* filter_node_alloc(FilterNode* parent, FilterRuleList* own) {
|
||||||
|
FilterNode* node = malloc(sizeof(FilterNode));
|
||||||
|
if (!node)
|
||||||
|
return NULL;
|
||||||
|
node->parent = parent;
|
||||||
|
node->own = own;
|
||||||
|
return node;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Evaluate a rule chain for an entry inside the directory whose content
|
||||||
|
* context is `node`. rsync precedence, highest first: the innermost (current)
|
||||||
|
* directory's .rsync-filter rules, then each ancestor's, then the root's, and
|
||||||
|
* finally the command-line base rules (--filter/-C). A deeper per-directory
|
||||||
|
* file therefore overrides a shallower one, and per-directory files override
|
||||||
|
* the base rules by default. Returns FILTER_ACTION_NONE when nothing matched. */
|
||||||
|
static FilterAction chain_rules_apply(const FilterRuleList* base, const FilterNode* node,
|
||||||
|
const char* rel, const char* leaf, bool is_dir) {
|
||||||
|
if (node) {
|
||||||
|
FilterAction own_action = filter_rules_apply(node->own, rel, leaf, is_dir);
|
||||||
|
if (own_action != FILTER_ACTION_NONE)
|
||||||
|
return own_action;
|
||||||
|
return chain_rules_apply(base, node->parent, rel, leaf, is_dir);
|
||||||
|
}
|
||||||
|
return base ? filter_rules_apply(base, rel, leaf, is_dir) : FILTER_ACTION_NONE;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool entry_allowed(const FilterRuleList* base, const FilterNode* node, const char* rel,
|
||||||
|
const char* leaf, bool is_dir, bool per_dir_filters) {
|
||||||
|
/* -F: per-directory .rsync-filter files are never transferred. */
|
||||||
|
if (per_dir_filters && !is_dir && strcmp(leaf, ".rsync-filter") == 0)
|
||||||
|
return false;
|
||||||
|
return chain_rules_apply(base, node, rel, leaf, is_dir) != FILTER_ACTION_EXCLUDE;
|
||||||
|
}
|
||||||
|
|
||||||
static void dir_entry_destroy(void* item) {
|
static void dir_entry_destroy(void* item) {
|
||||||
if (item) {
|
if (item) {
|
||||||
DirEntry* de = (DirEntry*)item;
|
DirEntry* de = (DirEntry*)item;
|
||||||
@@ -27,7 +81,7 @@ static void dir_entry_destroy(void* item) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static DirEntry* dir_entry_create(const char* path, int depth) {
|
static DirEntry* dir_entry_create(const char* path, int depth, FilterNode* context) {
|
||||||
DirEntry* de = malloc(sizeof(DirEntry));
|
DirEntry* de = malloc(sizeof(DirEntry));
|
||||||
if (!de)
|
if (!de)
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -37,6 +91,7 @@ static DirEntry* dir_entry_create(const char* path, int depth) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
de->depth = depth;
|
de->depth = depth;
|
||||||
|
de->context = context;
|
||||||
return de;
|
return de;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -66,6 +121,79 @@ bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entr
|
|||||||
return !one_file_system || entry_device == root_device;
|
return !one_file_system || entry_device == root_device;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Relative path of an on-disk path below `root`. The transfer root may be
|
||||||
|
* given with a trailing slash; the returned rel path never has one and is ""
|
||||||
|
* for the root itself. A root of "/" is handled (its children start at "/").
|
||||||
|
* Exposed so tests can exercise the mapping directly. */
|
||||||
|
char* scanner_path_relative(const char* root, const char* fs_path) {
|
||||||
|
size_t root_len = strlen(root);
|
||||||
|
while (root_len > 1 && root[root_len - 1] == '/')
|
||||||
|
root_len--;
|
||||||
|
if (strncmp(root, fs_path, root_len) != 0)
|
||||||
|
return NULL;
|
||||||
|
if (root_len == 1 && root[0] == '/') {
|
||||||
|
if (fs_path[1] == '\0')
|
||||||
|
return str_dup("");
|
||||||
|
return str_dup(fs_path + 1);
|
||||||
|
}
|
||||||
|
if (fs_path[root_len] == '\0')
|
||||||
|
return str_dup("");
|
||||||
|
if (fs_path[root_len] != '/')
|
||||||
|
return NULL;
|
||||||
|
return str_dup(fs_path + root_len + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Relative path of a child entry below the current directory. */
|
||||||
|
static char* child_rel_path(const char* parent_rel, const char* name) {
|
||||||
|
if (!parent_rel || parent_rel[0] == '\0')
|
||||||
|
return str_dup(name);
|
||||||
|
return path_cat(parent_rel, name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Apply the --files-from allow-set and the filter layer to one entry. */
|
||||||
|
static bool entry_passes_selection(const FileListSet* file_list, const FilterRuleList* base,
|
||||||
|
const FilterNode* node, const char* rel, const char* leaf,
|
||||||
|
bool is_dir, bool per_dir_filters) {
|
||||||
|
if (file_list && !file_list_affects(file_list, rel))
|
||||||
|
return false;
|
||||||
|
if (base || per_dir_filters)
|
||||||
|
return entry_allowed(base, node, rel, leaf, is_dir, per_dir_filters);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Merge the open directory's own .rsync-filter rules into the inherited
|
||||||
|
* context, returning the context used for this directory's entries. On a parse
|
||||||
|
* error the scanner is marked failed. Returns 0 on success, -1 on failure. */
|
||||||
|
static int open_directory_filter_context(DirectoryScanner* scanner, const FilterNode* inherited) {
|
||||||
|
if (!scanner->per_dir_filters) {
|
||||||
|
scanner->current_node = (FilterNode*)inherited;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
char err[256];
|
||||||
|
bool exists = false;
|
||||||
|
FilterRuleList* own =
|
||||||
|
filter_file_read(scanner->current_path, scanner->current_rel ? scanner->current_rel : "",
|
||||||
|
&exists, err, sizeof(err));
|
||||||
|
if (!own) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "invalid .rsync-filter in %s: %s", scanner->current_path, err);
|
||||||
|
scanner->failed = true;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (exists && own->count > 0) {
|
||||||
|
FilterNode* node = filter_node_alloc((FilterNode*)inherited, own);
|
||||||
|
if (!node || !array_list_add(scanner->filter_nodes, node)) {
|
||||||
|
filter_node_destroy(node);
|
||||||
|
scanner->failed = true;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
scanner->current_node = node;
|
||||||
|
} else {
|
||||||
|
filter_rule_list_free(own);
|
||||||
|
scanner->current_node = (FilterNode*)inherited;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
/* Inspect symlinks, resolve the entry type, and apply file filters once for both scanners. */
|
/* Inspect symlinks, resolve the entry type, and apply file filters once for both scanners. */
|
||||||
static int scanner_inspect_entry(const ScannerOptions* options, const char* source_root,
|
static int scanner_inspect_entry(const ScannerOptions* options, const char* source_root,
|
||||||
const char* containing_dir, const char* name,
|
const char* containing_dir, const char* name,
|
||||||
@@ -165,25 +293,54 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
|||||||
scanner->checksum = options->checksum;
|
scanner->checksum = options->checksum;
|
||||||
scanner->one_file_system = options->one_file_system;
|
scanner->one_file_system = options->one_file_system;
|
||||||
scanner->failed = false;
|
scanner->failed = false;
|
||||||
|
scanner->root_path = str_dup(root_directory);
|
||||||
|
if (!scanner->root_path) {
|
||||||
|
queue_destroy(scanner->directories);
|
||||||
|
free(scanner);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
scanner->current_rel = NULL;
|
||||||
|
scanner->at_seed_dir = true;
|
||||||
|
scanner->seed_node = NULL;
|
||||||
|
scanner->current_node = NULL;
|
||||||
|
scanner->file_list = options->file_list;
|
||||||
|
scanner->base_filters = options->base_filters;
|
||||||
|
scanner->per_dir_filters = options->per_dir_filters;
|
||||||
|
scanner->filter_nodes = NULL;
|
||||||
|
if (scanner->base_filters || scanner->per_dir_filters) {
|
||||||
|
scanner->filter_nodes = array_list_create(filter_node_destroy);
|
||||||
|
if (!scanner->filter_nodes) {
|
||||||
|
free(scanner->root_path);
|
||||||
|
queue_destroy(scanner->directories);
|
||||||
|
free(scanner);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
if (scanner->one_file_system) {
|
if (scanner->one_file_system) {
|
||||||
struct stat root_stats;
|
struct stat root_stats;
|
||||||
if (stat(root_directory, &root_stats) != 0) {
|
if (stat(root_directory, &root_stats) != 0) {
|
||||||
log_perror("Could not stat source directory");
|
log_perror("Could not stat source directory");
|
||||||
|
free(scanner->root_path);
|
||||||
queue_destroy(scanner->directories);
|
queue_destroy(scanner->directories);
|
||||||
|
array_list_delete(scanner->filter_nodes);
|
||||||
free(scanner);
|
free(scanner);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
scanner->root_dev = root_stats.st_dev;
|
scanner->root_dev = root_stats.st_dev;
|
||||||
}
|
}
|
||||||
DirEntry* root = dir_entry_create(root_directory, 0);
|
DirEntry* root = dir_entry_create(root_directory, 0, NULL);
|
||||||
if (!root) {
|
if (!root) {
|
||||||
|
free(scanner->root_path);
|
||||||
queue_destroy(scanner->directories);
|
queue_destroy(scanner->directories);
|
||||||
|
array_list_delete(scanner->filter_nodes);
|
||||||
free(scanner);
|
free(scanner);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
if (!queue_enqueue(scanner->directories, root)) {
|
if (!queue_enqueue(scanner->directories, root)) {
|
||||||
dir_entry_destroy(root);
|
dir_entry_destroy(root);
|
||||||
|
free(scanner->root_path);
|
||||||
queue_destroy(scanner->directories);
|
queue_destroy(scanner->directories);
|
||||||
|
array_list_delete(scanner->filter_nodes);
|
||||||
free(scanner);
|
free(scanner);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
@@ -197,14 +354,25 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
|
|||||||
unsigned long long min_size, int max_depth,
|
unsigned long long min_size, int max_depth,
|
||||||
bool follow_symlinks, bool copy_links, bool safe_links,
|
bool follow_symlinks, bool copy_links, bool safe_links,
|
||||||
bool copy_unsafe_links, bool checksum) {
|
bool copy_unsafe_links, bool checksum) {
|
||||||
ScannerOptions options = {use_metadata, chunk_size,
|
ScannerOptions options = {use_metadata,
|
||||||
exclude_patterns, exclude_count,
|
chunk_size,
|
||||||
include_patterns, include_count,
|
exclude_patterns,
|
||||||
max_size, min_size,
|
exclude_count,
|
||||||
max_depth, 0,
|
include_patterns,
|
||||||
follow_symlinks, copy_links,
|
include_count,
|
||||||
safe_links, copy_unsafe_links,
|
max_size,
|
||||||
checksum, false};
|
min_size,
|
||||||
|
max_depth,
|
||||||
|
0,
|
||||||
|
follow_symlinks,
|
||||||
|
copy_links,
|
||||||
|
safe_links,
|
||||||
|
copy_unsafe_links,
|
||||||
|
checksum,
|
||||||
|
false,
|
||||||
|
NULL,
|
||||||
|
NULL,
|
||||||
|
false};
|
||||||
return directory_scanner_create_with_options(root_directory, &options);
|
return directory_scanner_create_with_options(root_directory, &options);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -216,6 +384,9 @@ void directory_scanner_destroy(DirectoryScanner* scanner) {
|
|||||||
scanner->current_dir = NULL;
|
scanner->current_dir = NULL;
|
||||||
}
|
}
|
||||||
free(scanner->current_path);
|
free(scanner->current_path);
|
||||||
|
free(scanner->current_rel);
|
||||||
|
free(scanner->root_path);
|
||||||
|
array_list_delete(scanner->filter_nodes);
|
||||||
queue_destroy(scanner->directories);
|
queue_destroy(scanner->directories);
|
||||||
free(scanner);
|
free(scanner);
|
||||||
}
|
}
|
||||||
@@ -246,7 +417,21 @@ static int open_next_directory(DirectoryScanner* scanner) {
|
|||||||
DirEntry* de = (DirEntry*)queue_dequeue(scanner->directories);
|
DirEntry* de = (DirEntry*)queue_dequeue(scanner->directories);
|
||||||
scanner->current_path = de->path;
|
scanner->current_path = de->path;
|
||||||
scanner->current_depth = de->depth;
|
scanner->current_depth = de->depth;
|
||||||
|
/* The seed directory inherits the scanner's configured context (the root
|
||||||
|
* .rsync-filter context in parallel mode); other dirs inherit the context of
|
||||||
|
* the directory that enqueued them. */
|
||||||
|
const FilterNode* inherited = scanner->at_seed_dir ? scanner->seed_node : de->context;
|
||||||
|
scanner->at_seed_dir = false;
|
||||||
free(de);
|
free(de);
|
||||||
|
|
||||||
|
free(scanner->current_rel);
|
||||||
|
scanner->current_rel = scanner_path_relative(scanner->root_path, scanner->current_path);
|
||||||
|
if (!scanner->current_rel) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Could not compute relative path under %s", scanner->root_path);
|
||||||
|
scanner->failed = true;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
scanner->current_dir = opendir(scanner->current_path);
|
scanner->current_dir = opendir(scanner->current_path);
|
||||||
if (scanner->current_dir == NULL) {
|
if (scanner->current_dir == NULL) {
|
||||||
log_perror("Could not open directory");
|
log_perror("Could not open directory");
|
||||||
@@ -255,6 +440,11 @@ static int open_next_directory(DirectoryScanner* scanner) {
|
|||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
if (open_directory_filter_context(scanner, inherited) != 0) {
|
||||||
|
closedir(scanner->current_dir);
|
||||||
|
scanner->current_dir = NULL;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -294,7 +484,9 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
scanner->max_depth, 0,
|
scanner->max_depth, 0,
|
||||||
scanner->follow_symlinks, scanner->copy_links,
|
scanner->follow_symlinks, scanner->copy_links,
|
||||||
scanner->safe_links, scanner->copy_unsafe_links,
|
scanner->safe_links, scanner->copy_unsafe_links,
|
||||||
scanner->checksum, scanner->one_file_system};
|
scanner->checksum, scanner->one_file_system,
|
||||||
|
scanner->file_list, scanner->base_filters,
|
||||||
|
scanner->per_dir_filters};
|
||||||
ScannerEntry inspected;
|
ScannerEntry inspected;
|
||||||
int inspection = scanner_inspect_entry(&options, scanner->current_path, scanner->current_path,
|
int inspection = scanner_inspect_entry(&options, scanner->current_path, scanner->current_path,
|
||||||
entry->d_name, &inspected);
|
entry->d_name, &inspected);
|
||||||
@@ -307,14 +499,32 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
char* cur_path = inspected.path;
|
char* cur_path = inspected.path;
|
||||||
struct stat stats = inspected.stats;
|
struct stat stats = inspected.stats;
|
||||||
|
|
||||||
if (inspected.is_directory) {
|
/* --files-from allow-set and the filter layer apply to files and to
|
||||||
|
* directories (an excluded directory is not descended into). */
|
||||||
|
bool is_dir = inspected.is_directory;
|
||||||
|
char* rel = child_rel_path(scanner->current_rel, entry->d_name);
|
||||||
|
if (!rel) {
|
||||||
|
free(cur_path);
|
||||||
|
scanner->failed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
bool passes_selection =
|
||||||
|
entry_passes_selection(scanner->file_list, scanner->base_filters, scanner->current_node,
|
||||||
|
rel, entry->d_name, is_dir, scanner->per_dir_filters);
|
||||||
|
free(rel);
|
||||||
|
if (!passes_selection) {
|
||||||
|
free(cur_path);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (is_dir) {
|
||||||
if (!scanner_same_filesystem(scanner->one_file_system, scanner->root_dev, stats.st_dev)) {
|
if (!scanner_same_filesystem(scanner->one_file_system, scanner->root_dev, stats.st_dev)) {
|
||||||
free(cur_path);
|
free(cur_path);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
int next_depth = scanner->current_depth + 1;
|
int next_depth = scanner->current_depth + 1;
|
||||||
if (scanner->max_depth <= 0 || next_depth < scanner->max_depth) {
|
if (scanner->max_depth <= 0 || next_depth < scanner->max_depth) {
|
||||||
DirEntry* de = dir_entry_create(cur_path, next_depth);
|
DirEntry* de = dir_entry_create(cur_path, next_depth, scanner->current_node);
|
||||||
if (!de || !queue_enqueue(scanner->directories, de)) {
|
if (!de || !queue_enqueue(scanner->directories, de)) {
|
||||||
dir_entry_destroy(de);
|
dir_entry_destroy(de);
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
@@ -376,6 +586,7 @@ typedef struct {
|
|||||||
ParallelScanner* ps;
|
ParallelScanner* ps;
|
||||||
char** dirs;
|
char** dirs;
|
||||||
int dir_count;
|
int dir_count;
|
||||||
|
char* root_dir; /* the transfer root, for relative-path computation */
|
||||||
ScannerOptions options;
|
ScannerOptions options;
|
||||||
ProtocolSession* allocation_session;
|
ProtocolSession* allocation_session;
|
||||||
} ParallelWorkerArg;
|
} ParallelWorkerArg;
|
||||||
@@ -398,6 +609,13 @@ static int parallel_worker_thread(void* arg) {
|
|||||||
free(wa->dirs[j]);
|
free(wa->dirs[j]);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
/* Root .rsync-filter rules (parsed by the parallel scanner) apply to the
|
||||||
|
* contents of every assigned subdirectory. Relative paths (used by the
|
||||||
|
* allow-set and per-directory rules) are computed against the transfer
|
||||||
|
* root, not the subdirectory the worker is seeded with. */
|
||||||
|
free(ds->root_path);
|
||||||
|
ds->root_path = str_dup(wa->root_dir);
|
||||||
|
ds->seed_node = wa->ps->root_filter_node;
|
||||||
Chunk* chunk;
|
Chunk* chunk;
|
||||||
while ((chunk = directory_scanner_next(ds)) != NULL) {
|
while ((chunk = directory_scanner_next(ds)) != NULL) {
|
||||||
if (!queue_enqueue_multithreaded_cancel(wa->ps->result_queue, chunk, &wa->ps->result_mutex,
|
if (!queue_enqueue_multithreaded_cancel(wa->ps->result_queue, chunk, &wa->ps->result_mutex,
|
||||||
@@ -419,6 +637,7 @@ static int parallel_worker_thread(void* arg) {
|
|||||||
free(wa->dirs[i]);
|
free(wa->dirs[i]);
|
||||||
}
|
}
|
||||||
ParallelScanner* ps = wa->ps;
|
ParallelScanner* ps = wa->ps;
|
||||||
|
free(wa->root_dir);
|
||||||
free(wa->dirs);
|
free(wa->dirs);
|
||||||
free(wa);
|
free(wa);
|
||||||
mtx_lock(&ps->result_mutex);
|
mtx_lock(&ps->result_mutex);
|
||||||
@@ -549,9 +768,10 @@ static Chunk* batch_files(ArrayList* files, unsigned long long chunk_size, Queue
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* Scan one root-directory entry into either the subdirs or files list. */
|
/* Scan one root-directory entry into either the subdirs or files list. */
|
||||||
static void scan_root_entry(const ScannerOptions* options, const char* root_directory,
|
static void scan_root_entry(const ScannerOptions* options, const FilterNode* root_node,
|
||||||
const struct dirent* entry, ArrayList* root_files, ArrayList* subdirs,
|
const char* root_directory, const struct dirent* entry,
|
||||||
dev_t root_dev, ParallelScanner* ps) {
|
ArrayList* root_files, ArrayList* subdirs, dev_t root_dev,
|
||||||
|
ParallelScanner* ps) {
|
||||||
ScannerEntry inspected;
|
ScannerEntry inspected;
|
||||||
int inspection =
|
int inspection =
|
||||||
scanner_inspect_entry(options, root_directory, root_directory, entry->d_name, &inspected);
|
scanner_inspect_entry(options, root_directory, root_directory, entry->d_name, &inspected);
|
||||||
@@ -563,7 +783,21 @@ static void scan_root_entry(const ScannerOptions* options, const char* root_dire
|
|||||||
return;
|
return;
|
||||||
char* cur_path = inspected.path;
|
char* cur_path = inspected.path;
|
||||||
struct stat st = inspected.stats;
|
struct stat st = inspected.stats;
|
||||||
if (inspected.is_directory) {
|
bool is_dir = inspected.is_directory;
|
||||||
|
char* rel = str_dup(entry->d_name);
|
||||||
|
if (!rel) {
|
||||||
|
free(cur_path);
|
||||||
|
ps->failed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
bool passes = entry_passes_selection(options->file_list, options->base_filters, root_node, rel,
|
||||||
|
entry->d_name, is_dir, options->per_dir_filters);
|
||||||
|
free(rel);
|
||||||
|
if (!passes) {
|
||||||
|
free(cur_path);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (is_dir) {
|
||||||
if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) {
|
if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) {
|
||||||
free(cur_path);
|
free(cur_path);
|
||||||
return;
|
return;
|
||||||
@@ -597,8 +831,8 @@ static void scan_root_entry(const ScannerOptions* options, const char* root_dire
|
|||||||
/* Scan the root directory itself, collecting root files and subdirectories.
|
/* Scan the root directory itself, collecting root files and subdirectories.
|
||||||
* Returns false if the root directory could not be opened. */
|
* Returns false if the root directory could not be opened. */
|
||||||
static bool scan_root_directory(ParallelScanner* ps, const char* root_directory,
|
static bool scan_root_directory(ParallelScanner* ps, const char* root_directory,
|
||||||
const ScannerOptions* options, dev_t root_dev,
|
const ScannerOptions* options, const FilterNode* root_node,
|
||||||
ArrayList* root_files, ArrayList* subdirs) {
|
dev_t root_dev, ArrayList* root_files, ArrayList* subdirs) {
|
||||||
DIR* dir = opendir(root_directory);
|
DIR* dir = opendir(root_directory);
|
||||||
if (!dir) {
|
if (!dir) {
|
||||||
log_perror("Could not open root directory for parallel scan");
|
log_perror("Could not open root directory for parallel scan");
|
||||||
@@ -608,7 +842,7 @@ static bool scan_root_directory(ParallelScanner* ps, const char* root_directory,
|
|||||||
while ((entry = readdir(dir)) != NULL) {
|
while ((entry = readdir(dir)) != NULL) {
|
||||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||||
continue;
|
continue;
|
||||||
scan_root_entry(options, root_directory, entry, root_files, subdirs, root_dev, ps);
|
scan_root_entry(options, root_node, root_directory, entry, root_files, subdirs, root_dev, ps);
|
||||||
}
|
}
|
||||||
closedir(dir);
|
closedir(dir);
|
||||||
return true;
|
return true;
|
||||||
@@ -616,7 +850,8 @@ static bool scan_root_directory(ParallelScanner* ps, const char* root_directory,
|
|||||||
|
|
||||||
/* Spawn worker threads, one per group of subdirectories. */
|
/* Spawn worker threads, one per group of subdirectories. */
|
||||||
static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs,
|
static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs,
|
||||||
const ScannerOptions* options, unsigned long long cs) {
|
const ScannerOptions* options, const char* root_directory,
|
||||||
|
unsigned long long cs) {
|
||||||
if (subdirs->size <= 0)
|
if (subdirs->size <= 0)
|
||||||
return;
|
return;
|
||||||
int n = options->num_threads > 0 ? options->num_threads : 4;
|
int n = options->num_threads > 0 ? options->num_threads : 4;
|
||||||
@@ -647,7 +882,10 @@ static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs,
|
|||||||
}
|
}
|
||||||
wa->ps = ps;
|
wa->ps = ps;
|
||||||
wa->dirs = calloc(count, sizeof(char*));
|
wa->dirs = calloc(count, sizeof(char*));
|
||||||
if (!wa->dirs) {
|
wa->root_dir = str_dup(root_directory);
|
||||||
|
if (!wa->dirs || !wa->root_dir) {
|
||||||
|
free(wa->root_dir);
|
||||||
|
free(wa->dirs);
|
||||||
free(wa);
|
free(wa);
|
||||||
parallel_scanner_creation_failed(ps);
|
parallel_scanner_creation_failed(ps);
|
||||||
break;
|
break;
|
||||||
@@ -661,6 +899,7 @@ static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs,
|
|||||||
if (!dup_ok) {
|
if (!dup_ok) {
|
||||||
for (int j = 0; j < count; j++)
|
for (int j = 0; j < count; j++)
|
||||||
free(wa->dirs[j]);
|
free(wa->dirs[j]);
|
||||||
|
free(wa->root_dir);
|
||||||
free(wa->dirs);
|
free(wa->dirs);
|
||||||
free(wa);
|
free(wa);
|
||||||
parallel_scanner_creation_failed(ps);
|
parallel_scanner_creation_failed(ps);
|
||||||
@@ -674,6 +913,7 @@ static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs,
|
|||||||
if (thrd_create(&ps->threads[t], parallel_worker_thread, wa) != thrd_success) {
|
if (thrd_create(&ps->threads[t], parallel_worker_thread, wa) != thrd_success) {
|
||||||
for (int j = 0; j < count; j++)
|
for (int j = 0; j < count; j++)
|
||||||
free(wa->dirs[j]);
|
free(wa->dirs[j]);
|
||||||
|
free(wa->root_dir);
|
||||||
free(wa->dirs);
|
free(wa->dirs);
|
||||||
free(wa);
|
free(wa);
|
||||||
parallel_scanner_creation_failed(ps);
|
parallel_scanner_creation_failed(ps);
|
||||||
@@ -720,7 +960,37 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
|
|||||||
root_dev = root_stats.st_dev;
|
root_dev = root_stats.st_dev;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!scan_root_directory(ps, root_directory, options, root_dev, root_files, subdirs)) {
|
/* Build the root directory's .rsync-filter context once; workers seed their
|
||||||
|
* scanners with it so per-dir rules behave identically to the sequential
|
||||||
|
* scanner. */
|
||||||
|
FilterNode* root_node = NULL;
|
||||||
|
if (options->per_dir_filters) {
|
||||||
|
char err[256];
|
||||||
|
bool exists = false;
|
||||||
|
FilterRuleList* own = filter_file_read(root_directory, "", &exists, err, sizeof(err));
|
||||||
|
if (!own) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "invalid .rsync-filter in %s: %s", root_directory, err);
|
||||||
|
array_list_delete(root_files);
|
||||||
|
array_list_delete(subdirs);
|
||||||
|
parallel_scanner_destroy(ps);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (exists && own->count > 0) {
|
||||||
|
root_node = filter_node_alloc(NULL, own);
|
||||||
|
if (!root_node) {
|
||||||
|
filter_rule_list_free(own);
|
||||||
|
array_list_delete(root_files);
|
||||||
|
array_list_delete(subdirs);
|
||||||
|
parallel_scanner_destroy(ps);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
filter_rule_list_free(own);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ps->root_filter_node = root_node;
|
||||||
|
|
||||||
|
if (!scan_root_directory(ps, root_directory, options, root_node, root_dev, root_files, subdirs)) {
|
||||||
array_list_delete(root_files);
|
array_list_delete(root_files);
|
||||||
array_list_delete(subdirs);
|
array_list_delete(subdirs);
|
||||||
parallel_scanner_destroy(ps);
|
parallel_scanner_destroy(ps);
|
||||||
@@ -731,7 +1001,7 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
|
|||||||
ps->initial_chunk = batch_files(root_files, cs, ps->result_queue, &ps->failed);
|
ps->initial_chunk = batch_files(root_files, cs, ps->result_queue, &ps->failed);
|
||||||
array_list_delete(root_files);
|
array_list_delete(root_files);
|
||||||
|
|
||||||
spawn_parallel_workers(ps, subdirs, options, cs);
|
spawn_parallel_workers(ps, subdirs, options, root_directory, cs);
|
||||||
array_list_delete(subdirs);
|
array_list_delete(subdirs);
|
||||||
return ps;
|
return ps;
|
||||||
}
|
}
|
||||||
@@ -774,6 +1044,8 @@ void parallel_scanner_destroy(ParallelScanner* ps) {
|
|||||||
for (int i = 0; i < ps->num_threads; i++)
|
for (int i = 0; i < ps->num_threads; i++)
|
||||||
thrd_join(ps->threads[i], NULL);
|
thrd_join(ps->threads[i], NULL);
|
||||||
free(ps->threads);
|
free(ps->threads);
|
||||||
|
if (ps->root_filter_node)
|
||||||
|
filter_node_destroy(ps->root_filter_node);
|
||||||
if (ps->initial_chunk)
|
if (ps->initial_chunk)
|
||||||
chunk_destroy(ps->initial_chunk);
|
chunk_destroy(ps->initial_chunk);
|
||||||
queue_destroy(ps->result_queue);
|
queue_destroy(ps->result_queue);
|
||||||
|
|||||||
@@ -2,6 +2,8 @@
|
|||||||
#define SCANNER_H
|
#define SCANNER_H
|
||||||
|
|
||||||
#include "chunk.h"
|
#include "chunk.h"
|
||||||
|
#include "file_list.h"
|
||||||
|
#include "filter.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "queue.h"
|
#include "queue.h"
|
||||||
#include <dirent.h>
|
#include <dirent.h>
|
||||||
@@ -27,8 +29,18 @@ typedef struct {
|
|||||||
bool copy_unsafe_links;
|
bool copy_unsafe_links;
|
||||||
bool checksum;
|
bool checksum;
|
||||||
bool one_file_system;
|
bool one_file_system;
|
||||||
|
/* Phase 2 (files-from / filter layer). All pointers are shared read-only
|
||||||
|
* across scanner instances and worker threads; ownership stays with the
|
||||||
|
* caller (client_send). */
|
||||||
|
const FileListSet* file_list; /* --files-from allow-set, or NULL */
|
||||||
|
const FilterRuleList* base_filters; /* command-line + -C rules, or NULL */
|
||||||
|
bool per_dir_filters; /* -F: read .rsync-filter per directory */
|
||||||
} ScannerOptions;
|
} ScannerOptions;
|
||||||
|
|
||||||
|
/* Internal per-scanner filter state. FilterNode chains represent the ordered
|
||||||
|
* per-directory .rsync-filter rules that apply below a directory. */
|
||||||
|
typedef struct FilterNode FilterNode;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
Queue* directories;
|
Queue* directories;
|
||||||
DIR* current_dir;
|
DIR* current_dir;
|
||||||
@@ -51,6 +63,16 @@ typedef struct {
|
|||||||
bool one_file_system;
|
bool one_file_system;
|
||||||
dev_t root_dev;
|
dev_t root_dev;
|
||||||
bool failed;
|
bool failed;
|
||||||
|
/* Phase 2 (files-from / filter layer). */
|
||||||
|
char* root_path; /* transfer root (fs path) for rel computation */
|
||||||
|
char* current_rel; /* rel path of the open directory ("" == root) */
|
||||||
|
bool at_seed_dir; /* next open is the seed directory */
|
||||||
|
FilterNode* seed_node; /* inherited context of the seed dir, or NULL */
|
||||||
|
FilterNode* current_node; /* filter context of the open directory */
|
||||||
|
ArrayList* filter_nodes; /* owned FilterNode arena (may be NULL) */
|
||||||
|
const FileListSet* file_list;
|
||||||
|
const FilterRuleList* base_filters;
|
||||||
|
bool per_dir_filters;
|
||||||
} DirectoryScanner;
|
} DirectoryScanner;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
@@ -68,6 +90,7 @@ typedef struct {
|
|||||||
int completed;
|
int completed;
|
||||||
Chunk* initial_chunk;
|
Chunk* initial_chunk;
|
||||||
ProtocolSession* allocation_session;
|
ProtocolSession* allocation_session;
|
||||||
|
FilterNode* root_filter_node; /* root .rsync-filter context (owned by ps) */
|
||||||
} ParallelScanner;
|
} ParallelScanner;
|
||||||
|
|
||||||
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
|
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
|
||||||
@@ -88,6 +111,11 @@ void directory_scanner_destroy(DirectoryScanner* scanner);
|
|||||||
* the transfer root. Exposed so tests can exercise the rule directly. */
|
* the transfer root. Exposed so tests can exercise the rule directly. */
|
||||||
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device);
|
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device);
|
||||||
|
|
||||||
|
/* Relative path of an on-disk path below `root` ("" == the root itself, NULL
|
||||||
|
* when `fs_path` is not under `root`). Handles trailing slashes and a root of
|
||||||
|
* "/". Exposed so tests can exercise the mapping directly. */
|
||||||
|
char* scanner_path_relative(const char* root, const char* fs_path);
|
||||||
|
|
||||||
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
||||||
const ScannerOptions* options,
|
const ScannerOptions* options,
|
||||||
ProtocolSession* allocation_session);
|
ProtocolSession* allocation_session);
|
||||||
|
|||||||
@@ -33,6 +33,13 @@ void print_usage(void) {
|
|||||||
printf(" --include <pattern> Only include files matching pattern\n");
|
printf(" --include <pattern> Only include files matching pattern\n");
|
||||||
printf(" --exclude-from <file> Read exclude patterns from file\n");
|
printf(" --exclude-from <file> Read exclude patterns from file\n");
|
||||||
printf(" --include-from <file> Read include patterns from file\n");
|
printf(" --include-from <file> Read include patterns from file\n");
|
||||||
|
printf(" --files-from <file> Read the source file list from FILE (paths relative to the "
|
||||||
|
"source root)\n");
|
||||||
|
printf(" -0, --from0 Entries in --files-from are NUL-delimited\n");
|
||||||
|
printf(" --filter=RULE rsync-style filter rule (+/- include/exclude; repeatable; the\n");
|
||||||
|
printf(" rsync -f short form conflicts with FastSync sendfile -f)\n");
|
||||||
|
printf(" -C, --cvs-exclude Auto-ignore common CVS/SCM files (.git/, .svn/, *.o, *~, ...)\n");
|
||||||
|
printf(" -F Apply per-directory .rsync-filter files during the scan\n");
|
||||||
printf(" --max-size <n> Skip files larger than n bytes\n");
|
printf(" --max-size <n> Skip files larger than n bytes\n");
|
||||||
printf(" --min-size <n> Skip files smaller than n bytes\n");
|
printf(" --min-size <n> Skip files smaller than n bytes\n");
|
||||||
printf(" --max-alloc <SIZE> Maximum single allocation (default: 1G)\n");
|
printf(" --max-alloc <SIZE> Maximum single allocation (default: 1G)\n");
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
#include "chmod.h"
|
#include "chmod.h"
|
||||||
#include "delay_updates.h"
|
#include "delay_updates.h"
|
||||||
#include "delta.h"
|
#include "delta.h"
|
||||||
|
#include "file_list.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
@@ -94,7 +95,10 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->max_delete = 0;
|
config->max_delete = 0;
|
||||||
config->filters = NULL;
|
config->filters = NULL;
|
||||||
config->files_from = NULL;
|
config->files_from = NULL;
|
||||||
|
config->files_from_set = NULL;
|
||||||
|
config->from0 = false;
|
||||||
config->cvs_exclude = false;
|
config->cvs_exclude = false;
|
||||||
|
config->per_dir_filter = false;
|
||||||
config->prune_empty_dirs = false;
|
config->prune_empty_dirs = false;
|
||||||
config->one_file_system = false;
|
config->one_file_system = false;
|
||||||
config->relative = false;
|
config->relative = false;
|
||||||
@@ -232,6 +236,7 @@ void config_delete(Config* config) {
|
|||||||
free(config->out_format);
|
free(config->out_format);
|
||||||
free(config->log_file_format);
|
free(config->log_file_format);
|
||||||
free(config->files_from);
|
free(config->files_from);
|
||||||
|
file_list_destroy((FileListSet*)config->files_from_set);
|
||||||
free(config->rsh_command);
|
free(config->rsh_command);
|
||||||
free(config->rsync_path);
|
free(config->rsync_path);
|
||||||
free(config->temp_dir);
|
free(config->temp_dir);
|
||||||
|
|||||||
+8
-4
@@ -105,10 +105,14 @@ typedef struct Config {
|
|||||||
bool delete_after;
|
bool delete_after;
|
||||||
int max_delete;
|
int max_delete;
|
||||||
|
|
||||||
// Issue #129: Advanced file selection
|
// Issue #129: Advanced file selection. These fields are CLIENT-ONLY: they are
|
||||||
ArrayList* filters;
|
// never serialized to the wire (the receiver must not learn them).
|
||||||
char* files_from;
|
ArrayList* filters; /* --filter=RULE rule strings, in order */
|
||||||
bool cvs_exclude;
|
char* files_from; /* --files-from path (may be NULL) */
|
||||||
|
void* files_from_set; /* parsed FileListSet* allow-set, or NULL */
|
||||||
|
bool from0; /* -0/--from0: NUL-delimited *-from files */
|
||||||
|
bool cvs_exclude; /* -C/--cvs-exclude: standard CVS ignore set */
|
||||||
|
bool per_dir_filter; /* -F: apply per-directory .rsync-filter files */
|
||||||
bool prune_empty_dirs;
|
bool prune_empty_dirs;
|
||||||
bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */
|
bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */
|
||||||
bool relative;
|
bool relative;
|
||||||
|
|||||||
@@ -0,0 +1,191 @@
|
|||||||
|
#include "file_list.h"
|
||||||
|
#include "log.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include <errno.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
char** items;
|
||||||
|
int count;
|
||||||
|
int capacity;
|
||||||
|
} StringList;
|
||||||
|
|
||||||
|
static void string_list_destroy(StringList* list) {
|
||||||
|
if (!list)
|
||||||
|
return;
|
||||||
|
for (int i = 0; i < list->count; i++)
|
||||||
|
free(list->items[i]);
|
||||||
|
free(list->items);
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool string_list_add(StringList* list, const char* text) {
|
||||||
|
if (list->count == list->capacity) {
|
||||||
|
int new_cap = list->capacity > 0 ? list->capacity * 2 : 16;
|
||||||
|
char** grown = realloc(list->items, (size_t)new_cap * sizeof(char*));
|
||||||
|
if (!grown)
|
||||||
|
return false;
|
||||||
|
list->items = grown;
|
||||||
|
list->capacity = new_cap;
|
||||||
|
}
|
||||||
|
list->items[list->count] = str_dup(text);
|
||||||
|
if (!list->items[list->count])
|
||||||
|
return false;
|
||||||
|
list->count++;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Validate and normalize one entry. Returns:
|
||||||
|
* 1 -> added to `out`
|
||||||
|
* 0 -> blank entry, skip
|
||||||
|
* -1 -> invalid (message set in `err`)
|
||||||
|
* `strip_line_endings` trims a trailing CR/LF (line mode only); NUL mode keeps
|
||||||
|
* the entry bytes verbatim so names ending in CR/LF survive. */
|
||||||
|
static int normalize_entry(const char* raw, size_t len, bool strip_line_endings, StringList* out,
|
||||||
|
char* err, size_t err_size) {
|
||||||
|
if (strip_line_endings) {
|
||||||
|
while (len > 0 && (raw[len - 1] == '\n' || raw[len - 1] == '\r'))
|
||||||
|
len--;
|
||||||
|
}
|
||||||
|
if (len == 0)
|
||||||
|
return 0;
|
||||||
|
if (raw[0] == '/') {
|
||||||
|
snprintf(err, err_size, "absolute path entries are not allowed: '%.*s'", (int)len, raw);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
/* Reject NUL bytes inside a token defensively (NUL-delimited mode splits on
|
||||||
|
* them, so this only guards against embedded garbage). */
|
||||||
|
char* dup = malloc(len + 1);
|
||||||
|
if (!dup) {
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
memcpy(dup, raw, len);
|
||||||
|
dup[len] = '\0';
|
||||||
|
|
||||||
|
/* Rebuild the path token-by-token: skip '.' and empty segments, reject '..'. */
|
||||||
|
size_t out_len = 0;
|
||||||
|
for (const char* part = dup;;) {
|
||||||
|
const char* slash = strchr(part, '/');
|
||||||
|
size_t part_len = slash ? (size_t)(slash - part) : strlen(part);
|
||||||
|
if (part_len == 1 && part[0] == '.') {
|
||||||
|
/* skip "." segment */
|
||||||
|
} else if (part_len == 2 && part[0] == '.' && part[1] == '.') {
|
||||||
|
snprintf(err, err_size, "path traversal entry is not allowed: '%s'", dup);
|
||||||
|
free(dup);
|
||||||
|
return -1;
|
||||||
|
} else if (part_len > 0) {
|
||||||
|
if (out_len > 0)
|
||||||
|
dup[out_len++] = '/';
|
||||||
|
memmove(dup + out_len, part, part_len);
|
||||||
|
out_len += part_len;
|
||||||
|
}
|
||||||
|
if (!slash)
|
||||||
|
break;
|
||||||
|
part = slash + 1;
|
||||||
|
}
|
||||||
|
dup[out_len] = '\0';
|
||||||
|
|
||||||
|
int result;
|
||||||
|
if (out_len == 0) {
|
||||||
|
/* "." / "./" lists the source root: the whole tree is transferred. */
|
||||||
|
result = string_list_add(out, "") ? 1 : -1;
|
||||||
|
if (result < 0)
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
} else {
|
||||||
|
result = string_list_add(out, dup) ? 1 : -1;
|
||||||
|
if (result < 0)
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
}
|
||||||
|
free(dup);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
static FileListSet* string_list_to_set(StringList* raw, char* err, size_t err_size) {
|
||||||
|
FileListSet* set = malloc(sizeof(FileListSet));
|
||||||
|
if (!set) {
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
set->count = raw->count;
|
||||||
|
set->entries = raw->items;
|
||||||
|
raw->items = NULL;
|
||||||
|
raw->count = 0;
|
||||||
|
return set;
|
||||||
|
}
|
||||||
|
|
||||||
|
FileListSet* file_list_load(const char* path, bool null_separated, char* err, size_t err_size) {
|
||||||
|
if (err && err_size > 0)
|
||||||
|
err[0] = '\0';
|
||||||
|
if (!path || !*path) {
|
||||||
|
snprintf(err, err_size, "no file given");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
FILE* fp = fopen(path, "r");
|
||||||
|
if (!fp) {
|
||||||
|
char* escaped = output_escape(path, false);
|
||||||
|
snprintf(err, err_size, "could not open '%s': %s", escaped ? escaped : path, strerror(errno));
|
||||||
|
free(escaped);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
StringList raw = {0};
|
||||||
|
char* line = NULL;
|
||||||
|
size_t line_cap = 0;
|
||||||
|
ssize_t n;
|
||||||
|
bool ok = true;
|
||||||
|
char delim = null_separated ? '\0' : '\n';
|
||||||
|
while (ok && (n = getdelim(&line, &line_cap, delim, fp)) != -1) {
|
||||||
|
int r = normalize_entry(line, (size_t)n, !null_separated, &raw, err, err_size);
|
||||||
|
if (r < 0) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
free(line);
|
||||||
|
fclose(fp);
|
||||||
|
if (!ok) {
|
||||||
|
string_list_destroy(&raw);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
FileListSet* set = string_list_to_set(&raw, err, err_size);
|
||||||
|
if (!set)
|
||||||
|
string_list_destroy(&raw);
|
||||||
|
return set;
|
||||||
|
}
|
||||||
|
|
||||||
|
void file_list_destroy(FileListSet* set) {
|
||||||
|
if (!set)
|
||||||
|
return;
|
||||||
|
for (int i = 0; i < set->count; i++)
|
||||||
|
free(set->entries[i]);
|
||||||
|
free(set->entries);
|
||||||
|
free(set);
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool path_has_prefix(const char* path, const char* prefix) {
|
||||||
|
size_t plen = strlen(prefix);
|
||||||
|
if (strncmp(path, prefix, plen) != 0)
|
||||||
|
return false;
|
||||||
|
return path[plen] == '/' || path[plen] == '\0';
|
||||||
|
}
|
||||||
|
|
||||||
|
bool file_list_affects(const FileListSet* set, const char* rel) {
|
||||||
|
if (!set)
|
||||||
|
return true;
|
||||||
|
if (!rel)
|
||||||
|
return false;
|
||||||
|
for (int i = 0; i < set->count; i++) {
|
||||||
|
const char* entry = set->entries[i];
|
||||||
|
if (entry[0] == '\0')
|
||||||
|
return true; /* whole tree listed */
|
||||||
|
if (strcmp(rel, entry) == 0)
|
||||||
|
return true; /* the entry itself is listed */
|
||||||
|
if (path_has_prefix(rel, entry))
|
||||||
|
return true; /* rel lives under a listed directory */
|
||||||
|
if (path_has_prefix(entry, rel))
|
||||||
|
return true; /* rel is an ancestor directory of a listed entry */
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
#ifndef FILE_LIST_H
|
||||||
|
#define FILE_LIST_H
|
||||||
|
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <stddef.h>
|
||||||
|
|
||||||
|
/* --files-from allow-set. The file lists source paths RELATIVE to the source
|
||||||
|
* root. A listed regular file is transferred; a listed directory transfers its
|
||||||
|
* whole subtree (FastSync's recursion is always on). Blank lines are ignored.
|
||||||
|
*
|
||||||
|
* Entries are normalized: leading "./" and duplicate "/" are removed, an entry
|
||||||
|
* of "." means the whole tree, absolute entries and ".." traversal are
|
||||||
|
* rejected at parse time. The set is immutable and shared read-only across
|
||||||
|
* scanner worker threads.
|
||||||
|
*/
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
char** entries; /* normalized rel paths; "" means the whole tree */
|
||||||
|
int count;
|
||||||
|
} FileListSet;
|
||||||
|
|
||||||
|
/* Load and validate a --files-from file. When `null_separated` (-0/--from0)
|
||||||
|
* entries are delimited by NUL instead of newlines. Returns NULL with a message
|
||||||
|
* in `err` on open/validation failure. An empty file yields an empty set
|
||||||
|
* (nothing is transferred). */
|
||||||
|
FileListSet* file_list_load(const char* path, bool null_separated, char* err, size_t err_size);
|
||||||
|
void file_list_destroy(FileListSet* set);
|
||||||
|
|
||||||
|
/* True when `rel` (path relative to the source root, "" == root) is a listed
|
||||||
|
* entry, lives under a listed directory, or is an ancestor directory of a
|
||||||
|
* listed entry. Used to prune scanning: directories are descended only when
|
||||||
|
* this returns true, files are transferred only when it returns true. */
|
||||||
|
bool file_list_affects(const FileListSet* set, const char* rel);
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -0,0 +1,427 @@
|
|||||||
|
#include "filter.h"
|
||||||
|
#include "log.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include <errno.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
/* ---- Single rule parsing ---- */
|
||||||
|
|
||||||
|
static bool rule_text_is_unsupported_word(const char* p, size_t len) {
|
||||||
|
static const char* const words[] = {"merge", "dir-merge", "hide", "show",
|
||||||
|
"protect", "risk", "clear"};
|
||||||
|
for (size_t i = 0; i < sizeof(words) / sizeof(words[0]); i++) {
|
||||||
|
size_t wl = strlen(words[i]);
|
||||||
|
if (len == wl && strncmp(p, words[i], wl) == 0)
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync include/exclude rule modifiers we do NOT implement. A rule whose +/- is
|
||||||
|
* immediately followed by one of these is rejected instead of being silently
|
||||||
|
* parsed as a literal pattern. */
|
||||||
|
static bool is_unsupported_rule_modifier(char c) {
|
||||||
|
return c == '!' || c == 'C' || c == 's' || c == 'r' || c == 'p' || c == 'x';
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterRule* filter_rule_parse(const char* line, char* err, size_t err_size) {
|
||||||
|
if (err && err_size > 0)
|
||||||
|
err[0] = '\0';
|
||||||
|
if (!line)
|
||||||
|
return NULL;
|
||||||
|
char* text = str_dup(line);
|
||||||
|
if (!text) {
|
||||||
|
if (err)
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
size_t len = strlen(text);
|
||||||
|
while (len > 0 && (text[len - 1] == '\n' || text[len - 1] == '\r'))
|
||||||
|
text[--len] = '\0';
|
||||||
|
|
||||||
|
const char* p = text;
|
||||||
|
while (*p == ' ' || *p == '\t')
|
||||||
|
p++;
|
||||||
|
if (*p == '\0') {
|
||||||
|
snprintf(err, err_size, "empty filter rule");
|
||||||
|
free(text);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterAction action = FILTER_ACTION_EXCLUDE;
|
||||||
|
if (*p == '+' || *p == '-') {
|
||||||
|
action = *p == '+' ? FILTER_ACTION_INCLUDE : FILTER_ACTION_EXCLUDE;
|
||||||
|
p++;
|
||||||
|
/* rsync attaches rule modifiers directly to the +/- (e.g. "-s foo"). Only
|
||||||
|
* the '/' anchor modifier is supported; anything else is a clear error
|
||||||
|
* rather than a silently-ignored literal. */
|
||||||
|
if (*p != ' ' && *p != '\t' && *p != '\0' && is_unsupported_rule_modifier(*p)) {
|
||||||
|
snprintf(err, err_size,
|
||||||
|
"filter rule modifier '%c' is not supported (only the '/' anchor after +/- "
|
||||||
|
"is implemented; put a space between +/- and the pattern)",
|
||||||
|
*p);
|
||||||
|
free(text);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
while (*p == ' ' || *p == '\t')
|
||||||
|
p++;
|
||||||
|
} else {
|
||||||
|
/* ':' (dir-merge) and '.' (merge) are rsync filter-rule shorthands. At the
|
||||||
|
* start of a rule they mean "merge this file", so reject them instead of
|
||||||
|
* silently turning them into inert exclude patterns. */
|
||||||
|
if (*p == ':' || *p == '.' || *p == '!') {
|
||||||
|
snprintf(err, err_size,
|
||||||
|
"filter rule starting with '%c' is not supported (merge/dir-merge/list-clear "
|
||||||
|
"shorthands are not implemented; use +/- include/exclude rules)",
|
||||||
|
*p);
|
||||||
|
free(text);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
const char* sp = p;
|
||||||
|
while (*sp != '\0' && *sp != ' ' && *sp != '\t')
|
||||||
|
sp++;
|
||||||
|
size_t word_len = (size_t)(sp - p);
|
||||||
|
if (rule_text_is_unsupported_word(p, word_len)) {
|
||||||
|
snprintf(err, err_size,
|
||||||
|
"'%.*s' filter directives are not supported (only +/- include/exclude rules "
|
||||||
|
"with an optional '/' anchor and trailing '/' dir marker)",
|
||||||
|
(int)word_len, p);
|
||||||
|
free(text);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (word_len == strlen("include") && strncmp(p, "include", word_len) == 0) {
|
||||||
|
action = FILTER_ACTION_INCLUDE;
|
||||||
|
p = sp;
|
||||||
|
} else if (word_len == strlen("exclude") && strncmp(p, "exclude", word_len) == 0) {
|
||||||
|
action = FILTER_ACTION_EXCLUDE;
|
||||||
|
p = sp;
|
||||||
|
}
|
||||||
|
while (*p == ' ' || *p == '\t')
|
||||||
|
p++;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (*p == '\0') {
|
||||||
|
snprintf(err, err_size, "filter rule has no pattern");
|
||||||
|
free(text);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A pattern beginning with '/' is anchored (either as "-/foo" or "- /foo"). */
|
||||||
|
bool anchored = false;
|
||||||
|
if (*p == '/') {
|
||||||
|
anchored = true;
|
||||||
|
p++;
|
||||||
|
while (*p == ' ' || *p == '\t')
|
||||||
|
p++;
|
||||||
|
}
|
||||||
|
if (*p == '\0') {
|
||||||
|
snprintf(err, err_size, "filter rule has no pattern after '/' anchor");
|
||||||
|
free(text);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Pattern runs to the end of the rule; a single trailing '/' marks dir-only. */
|
||||||
|
size_t pat_len = strlen(p);
|
||||||
|
bool dir_only = false;
|
||||||
|
if (pat_len > 1 && p[pat_len - 1] == '/') {
|
||||||
|
dir_only = true;
|
||||||
|
pat_len--;
|
||||||
|
} else if (pat_len == 1 && p[0] == '/') {
|
||||||
|
/* "//" anchored with nothing after: meaningless. */
|
||||||
|
snprintf(err, err_size, "filter rule has no pattern");
|
||||||
|
free(text);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
||||||
|
if (!rule) {
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
free(text);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
rule->pattern = malloc(pat_len + 1);
|
||||||
|
if (!rule->pattern) {
|
||||||
|
free(rule);
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
free(text);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
memcpy(rule->pattern, p, pat_len);
|
||||||
|
rule->pattern[pat_len] = '\0';
|
||||||
|
rule->action = action;
|
||||||
|
rule->anchored = anchored;
|
||||||
|
rule->dir_only = dir_only;
|
||||||
|
rule->owner = NULL;
|
||||||
|
free(text);
|
||||||
|
return rule;
|
||||||
|
}
|
||||||
|
|
||||||
|
void filter_rule_free(FilterRule* rule) {
|
||||||
|
if (!rule)
|
||||||
|
return;
|
||||||
|
free(rule->pattern);
|
||||||
|
free(rule->owner);
|
||||||
|
free(rule);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- Ordered rule lists ---- */
|
||||||
|
|
||||||
|
FilterRuleList* filter_rule_list_create(void) {
|
||||||
|
return calloc(1, sizeof(FilterRuleList));
|
||||||
|
}
|
||||||
|
|
||||||
|
bool filter_rule_list_add(FilterRuleList* list, FilterRule* rule) {
|
||||||
|
if (!list || !rule)
|
||||||
|
return false;
|
||||||
|
if (list->count == list->capacity) {
|
||||||
|
int new_cap = list->capacity > 0 ? list->capacity * 2 : 8;
|
||||||
|
FilterRule** grown = realloc(list->items, (size_t)new_cap * sizeof(FilterRule*));
|
||||||
|
if (!grown)
|
||||||
|
return false;
|
||||||
|
list->items = grown;
|
||||||
|
list->capacity = new_cap;
|
||||||
|
}
|
||||||
|
list->items[list->count++] = rule;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool filter_rule_list_parse_append(FilterRuleList* list, const char* line, char* err,
|
||||||
|
size_t err_size) {
|
||||||
|
FilterRule* rule = filter_rule_parse(line, err, err_size);
|
||||||
|
if (!rule)
|
||||||
|
return false;
|
||||||
|
if (!filter_rule_list_add(list, rule)) {
|
||||||
|
filter_rule_free(rule);
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void filter_rule_list_free(FilterRuleList* list) {
|
||||||
|
if (!list)
|
||||||
|
return;
|
||||||
|
for (int i = 0; i < list->count; i++)
|
||||||
|
filter_rule_free(list->items[i]);
|
||||||
|
free(list->items);
|
||||||
|
free(list);
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool set_rule_owner(FilterRule* rule, const char* owner) {
|
||||||
|
char* dup = str_dup(owner ? owner : "");
|
||||||
|
if (!dup)
|
||||||
|
return false;
|
||||||
|
free(rule->owner);
|
||||||
|
rule->owner = dup;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- CVS default excludes (-C) ---- */
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
const char* pattern;
|
||||||
|
bool dir_only;
|
||||||
|
} CvsDefaultRule;
|
||||||
|
|
||||||
|
static const CvsDefaultRule CVS_DEFAULTS[] = {
|
||||||
|
{"RCS", false}, {"SCCS", false}, {"CVS", false}, {"CVS.adm", false},
|
||||||
|
{"RCSLOG", false}, {"cvslog.*", false}, {"tags", false}, {"TAGS", false},
|
||||||
|
{".make.state", false}, {".nse_depinfo", false}, {"*~", false}, {"#*", false},
|
||||||
|
{".#*", false}, {",*", false}, {"_$*", false}, {"*$", false},
|
||||||
|
{"*.old", false}, {"*.bak", false}, {"*.BAK", false}, {"*.orig", false},
|
||||||
|
{"*.rej", false}, {".del-*", false}, {"*.a", false}, {"*.olb", false},
|
||||||
|
{"*.o", false}, {"*.obj", false}, {"*.so", false}, {"*.exe", false},
|
||||||
|
{"*.Z", false}, {"*.elc", false}, {"*.ln", false}, {"core", false},
|
||||||
|
{".svn/", true}, {".git/", true}, {".hg/", true}, {".bzr/", true},
|
||||||
|
};
|
||||||
|
|
||||||
|
static bool cvs_rule_list_append(FilterRuleList* list) {
|
||||||
|
for (size_t i = 0; i < sizeof(CVS_DEFAULTS) / sizeof(CVS_DEFAULTS[0]); i++) {
|
||||||
|
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
||||||
|
if (!rule)
|
||||||
|
return false;
|
||||||
|
rule->action = FILTER_ACTION_EXCLUDE;
|
||||||
|
rule->dir_only = CVS_DEFAULTS[i].dir_only;
|
||||||
|
size_t plen = strlen(CVS_DEFAULTS[i].pattern);
|
||||||
|
if (rule->dir_only && plen > 0 && CVS_DEFAULTS[i].pattern[plen - 1] == '/')
|
||||||
|
plen--; /* keep the cleaned pattern, matching filter_rule_parse */
|
||||||
|
rule->pattern = malloc(plen + 1);
|
||||||
|
if (!rule->pattern) {
|
||||||
|
free(rule);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
memcpy(rule->pattern, CVS_DEFAULTS[i].pattern, plen);
|
||||||
|
rule->pattern[plen] = '\0';
|
||||||
|
if (!set_rule_owner(rule, "")) {
|
||||||
|
filter_rule_free(rule);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!filter_rule_list_add(list, rule)) {
|
||||||
|
filter_rule_free(rule);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterRuleList* filter_base_build(const char* const* rule_texts, int rule_count, bool cvs_exclude,
|
||||||
|
char* err, size_t err_size) {
|
||||||
|
if (err && err_size > 0)
|
||||||
|
err[0] = '\0';
|
||||||
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
|
if (!list) {
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < rule_count; i++) {
|
||||||
|
if (!rule_texts || !rule_texts[i])
|
||||||
|
continue;
|
||||||
|
FilterRule* rule = filter_rule_parse(rule_texts[i], err, err_size);
|
||||||
|
if (!rule) {
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (!set_rule_owner(rule, "")) {
|
||||||
|
filter_rule_free(rule);
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (!filter_rule_list_add(list, rule)) {
|
||||||
|
filter_rule_free(rule);
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (cvs_exclude && !cvs_rule_list_append(list)) {
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- Per-directory .rsync-filter files ---- */
|
||||||
|
|
||||||
|
FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bool* exists,
|
||||||
|
char* err, size_t err_size) {
|
||||||
|
if (err && err_size > 0)
|
||||||
|
err[0] = '\0';
|
||||||
|
if (exists)
|
||||||
|
*exists = false;
|
||||||
|
char* filter_path = path_cat(dir_path, ".rsync-filter");
|
||||||
|
if (!filter_path) {
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
FILE* fp = fopen(filter_path, "r");
|
||||||
|
free(filter_path);
|
||||||
|
if (!fp) {
|
||||||
|
if (errno == ENOENT || errno == ENOTDIR)
|
||||||
|
return filter_rule_list_create();
|
||||||
|
log_message(LOG_LEVEL_WARNING, "Could not read .rsync-filter in %s: %s", dir_path,
|
||||||
|
strerror(errno));
|
||||||
|
return filter_rule_list_create();
|
||||||
|
}
|
||||||
|
if (exists)
|
||||||
|
*exists = true;
|
||||||
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
|
if (!list) {
|
||||||
|
fclose(fp);
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
char* line = NULL;
|
||||||
|
size_t line_cap = 0;
|
||||||
|
ssize_t n;
|
||||||
|
bool ok = true;
|
||||||
|
while ((n = getline(&line, &line_cap, fp)) != -1) {
|
||||||
|
const char* p = line;
|
||||||
|
while (*p == ' ' || *p == '\t')
|
||||||
|
p++;
|
||||||
|
if (*p == '\0' || *p == '\n' || *p == '\r' || *p == '#')
|
||||||
|
continue;
|
||||||
|
FilterRule* rule = filter_rule_parse(p, err, err_size);
|
||||||
|
if (!rule) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (!set_rule_owner(rule, owner_rel)) {
|
||||||
|
filter_rule_free(rule);
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (!filter_rule_list_add(list, rule)) {
|
||||||
|
filter_rule_free(rule);
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
free(line);
|
||||||
|
fclose(fp);
|
||||||
|
if (!ok) {
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- Rule matching ---- */
|
||||||
|
|
||||||
|
/* Match a pattern that contains '/' (non-anchored) against the end of the
|
||||||
|
* relative path, starting at any path-component boundary. */
|
||||||
|
static bool glob_suffix_match(const char* pattern, const char* str) {
|
||||||
|
if (glob_match(pattern, str))
|
||||||
|
return true;
|
||||||
|
for (const char* slash = strchr(str, '/'); slash; slash = strchr(slash + 1, '/')) {
|
||||||
|
if (glob_match(pattern, slash + 1))
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
static FilterAction rule_matches(const FilterRule* rule, const char* rel_path, const char* leaf,
|
||||||
|
bool is_dir) {
|
||||||
|
if (!rule || !rule->pattern)
|
||||||
|
return FILTER_ACTION_NONE;
|
||||||
|
if (rule->dir_only && !is_dir)
|
||||||
|
return FILTER_ACTION_NONE;
|
||||||
|
/* A rule applies only to entries below its owner directory. */
|
||||||
|
const char* rel2 = rel_path;
|
||||||
|
if (rule->owner && rule->owner[0] != '\0') {
|
||||||
|
size_t owner_len = strlen(rule->owner);
|
||||||
|
if (strncmp(rule->owner, rel_path, owner_len) != 0)
|
||||||
|
return FILTER_ACTION_NONE;
|
||||||
|
if (rel_path[owner_len] != '/')
|
||||||
|
return FILTER_ACTION_NONE;
|
||||||
|
rel2 = rel_path + owner_len + 1;
|
||||||
|
}
|
||||||
|
if (rel2[0] == '\0')
|
||||||
|
return FILTER_ACTION_NONE;
|
||||||
|
bool matched;
|
||||||
|
if (rule->anchored) {
|
||||||
|
matched = glob_match(rule->pattern, rel2);
|
||||||
|
} else if (strchr(rule->pattern, '/') != NULL) {
|
||||||
|
matched = glob_suffix_match(rule->pattern, rel2);
|
||||||
|
} else {
|
||||||
|
matched = glob_match(rule->pattern, leaf);
|
||||||
|
}
|
||||||
|
return matched ? rule->action : FILTER_ACTION_NONE;
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf,
|
||||||
|
bool is_dir) {
|
||||||
|
if (!list)
|
||||||
|
return FILTER_ACTION_NONE;
|
||||||
|
for (int i = 0; i < list->count; i++) {
|
||||||
|
FilterAction action = rule_matches(list->items[i], rel_path, leaf, is_dir);
|
||||||
|
if (action != FILTER_ACTION_NONE)
|
||||||
|
return action;
|
||||||
|
}
|
||||||
|
return FILTER_ACTION_NONE;
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
#ifndef FILTER_H
|
||||||
|
#define FILTER_H
|
||||||
|
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <stddef.h>
|
||||||
|
|
||||||
|
/* rsync-style filter rule engine (client-side file selection).
|
||||||
|
*
|
||||||
|
* Supported rule syntax (documented subset):
|
||||||
|
* [+|-] [anchored '/' prefix] pattern [trailing '/' for dir-only]
|
||||||
|
*
|
||||||
|
* "+ PATTERN" include rule (first match wins)
|
||||||
|
* "- PATTERN" exclude rule
|
||||||
|
* "PATTERN" implicit exclude rule (rsync default)
|
||||||
|
* "include PATTERN" / "exclude PATTERN" word forms
|
||||||
|
* leading '/' after the +/- anchors the pattern to its owner directory
|
||||||
|
* (the transfer root for command-line/-C rules, the directory that
|
||||||
|
* contains a .rsync-filter file for per-directory rules)
|
||||||
|
* a trailing '/' makes the rule match directories only
|
||||||
|
*
|
||||||
|
* Rejected explicitly (no silent no-ops): the rsync merge/dir-merge/list-clear
|
||||||
|
* shorthands written as a rule that starts with ':' or '.' or '!', the
|
||||||
|
* merge/dir-merge/hide/show/protect/risk/clear words, and every include/exclude
|
||||||
|
* rule modifier other than '/' (! C s r p x). The pattern must be separated
|
||||||
|
* from +/- by a space (or a single '/' anchor), exactly like rsync's
|
||||||
|
* "-s foo"/"-p ..." modifier syntax is refused.
|
||||||
|
*/
|
||||||
|
|
||||||
|
typedef enum {
|
||||||
|
FILTER_ACTION_NONE = 0, /* no rule matched */
|
||||||
|
FILTER_ACTION_EXCLUDE = -1,
|
||||||
|
FILTER_ACTION_INCLUDE = 1
|
||||||
|
} FilterAction;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
FilterAction action;
|
||||||
|
bool anchored; /* pattern anchored to the rule's owner directory */
|
||||||
|
bool dir_only; /* pattern had a trailing '/': matches directories only */
|
||||||
|
char* owner; /* owning directory rel path ("" == transfer root) */
|
||||||
|
char* pattern; /* cleaned glob pattern (no leading '/', no trailing '/') */
|
||||||
|
} FilterRule;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
FilterRule** items; /* owned array of rule pointers */
|
||||||
|
int count;
|
||||||
|
int capacity;
|
||||||
|
} FilterRuleList;
|
||||||
|
|
||||||
|
/* Parse a single filter-rule line (no trailing newline required). Returns an
|
||||||
|
* owned rule, or NULL on unsupported/invalid syntax with a message in `err`. */
|
||||||
|
FilterRule* filter_rule_parse(const char* line, char* err, size_t err_size);
|
||||||
|
void filter_rule_free(FilterRule* rule);
|
||||||
|
|
||||||
|
FilterRuleList* filter_rule_list_create(void);
|
||||||
|
/* Append a fully-parsed rule (takes ownership). Returns false on OOM. */
|
||||||
|
bool filter_rule_list_add(FilterRuleList* list, FilterRule* rule);
|
||||||
|
/* Parse `line` and append it. Returns false and fills `err` on bad syntax. */
|
||||||
|
bool filter_rule_list_parse_append(FilterRuleList* list, const char* line, char* err,
|
||||||
|
size_t err_size);
|
||||||
|
void filter_rule_list_free(FilterRuleList* list);
|
||||||
|
|
||||||
|
/* Build the command-line filter set: `rule_texts` (--filter=RULE in the order
|
||||||
|
* given, 0..rule_count) followed by the -C CVS default excludes when
|
||||||
|
* cvs_exclude is true. All rules are owned by "" (the transfer root).
|
||||||
|
* Returns NULL on unsupported rule text (message in `err`). */
|
||||||
|
FilterRuleList* filter_base_build(const char* const* rule_texts, int rule_count, bool cvs_exclude,
|
||||||
|
char* err, size_t err_size);
|
||||||
|
|
||||||
|
/* Read "<dir_path>/.rsync-filter" and return its rules, each owned by
|
||||||
|
* `owner_rel`. A missing file yields an empty list with *exists=false; an
|
||||||
|
* unreadable file is treated as missing. Returns NULL only on parse or
|
||||||
|
* allocation failure (message in `err`). */
|
||||||
|
FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bool* exists,
|
||||||
|
char* err, size_t err_size);
|
||||||
|
|
||||||
|
/* Evaluate an entry against one ordered rule list. Returns FILTER_ACTION_NONE
|
||||||
|
* when no rule matched, otherwise the first matching rule's action.
|
||||||
|
* `rel_path` is the entry's path relative to the transfer root ("" == root),
|
||||||
|
* `leaf` its final name, `is_dir` whether it is a directory. */
|
||||||
|
FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf,
|
||||||
|
bool is_dir);
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -1629,3 +1629,13 @@ class TestDelayUpdates:
|
|||||||
result, _ = run_client(source, dest, flags=["--delay-updates", "--inplace"])
|
result, _ = run_client(source, dest, flags=["--delay-updates", "--inplace"])
|
||||||
assert result.returncode != 0, "--inplace with --delay-updates was accepted"
|
assert result.returncode != 0, "--inplace with --delay-updates was accepted"
|
||||||
assert not os.path.isdir(os.path.join(dest, self.STAGING))
|
assert not os.path.isdir(os.path.join(dest, self.STAGING))
|
||||||
|
|
||||||
|
class TestFilesFrom:
|
||||||
|
"""--files-from transfers exactly the listed files; a listed directory
|
||||||
|
transfers its whole subtree. The manifest (and thus --delete) derives from
|
||||||
|
what was actually sent."""
|
||||||
|
|
||||||
|
class TestFilters:
|
||||||
|
"""--filter/-C/-F rule layer: excludes prune, ordering is first-match-wins,
|
||||||
|
the default with no matching rule is include, and legacy --exclude remains
|
||||||
|
an independent layer."""
|
||||||
|
|||||||
+187
-3
@@ -2,6 +2,7 @@
|
|||||||
#include "client_validation.h"
|
#include "client_validation.h"
|
||||||
#include "chmod.h"
|
#include "chmod.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
|
#include "file_list.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "test_utils.h"
|
#include "test_utils.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
@@ -607,9 +608,6 @@ static void test_parse_args_rejects_unimplemented_options() {
|
|||||||
"--delete-excluded",
|
"--delete-excluded",
|
||||||
"--delete-after",
|
"--delete-after",
|
||||||
"--max-delete",
|
"--max-delete",
|
||||||
"--filter",
|
|
||||||
"--files-from",
|
|
||||||
"--cvs-exclude",
|
|
||||||
"--prune-empty-dirs",
|
"--prune-empty-dirs",
|
||||||
"-R",
|
"-R",
|
||||||
"--relative",
|
"--relative",
|
||||||
@@ -1308,6 +1306,189 @@ static void test_validate_config_delay_updates_rejects_reserved_backup_dir() {
|
|||||||
config_delete(ok);
|
config_delete(ok);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_parse_args_filter_rules() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
char* argv[] = {"fastsync", "--filter", "- *.tmp", "--filter=+ /keep.txt", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 6, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_NOT_NULL(cfg->filters);
|
||||||
|
EXPECT_EQ_INT(cfg->filters->size, 2);
|
||||||
|
EXPECT_EQ_STR((char*)cfg->filters->items[0], "- *.tmp");
|
||||||
|
EXPECT_EQ_STR((char*)cfg->filters->items[1], "+ /keep.txt");
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
/* An unsupported rsync rule type is rejected with a clear error. */
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* bad_argv[] = {"fastsync", "--filter=merge /tmp/excludes", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, bad_argv, positional_args, &positional_count), -1);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
/* A trailing --filter with no rule is a missing-argument error. */
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* missing_argv[] = {"fastsync", "/src", "/dst", "--filter"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, missing_argv, positional_args, &positional_count), -1);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
/* rsync shorthands/modifiers we do not support are rejected instead of being
|
||||||
|
* silently parsed as literal patterns. */
|
||||||
|
static const char* const unsupported[] = {
|
||||||
|
": .rsync-filter", ". /tmp/rules", "-s foo", "-p bar", "-C", "-! *.o", "!",
|
||||||
|
};
|
||||||
|
for (size_t i = 0; i < sizeof(unsupported) / sizeof(unsupported[0]); i++) {
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* rule_argv[] = {"fastsync", "--filter", (char*)unsupported[i], "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, rule_argv, positional_args, &positional_count), -1);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Supported spellings still parse: space- or slash-separated, attached
|
||||||
|
* wildcards, and anchored rules. */
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* ok_argv[] = {"fastsync", "--filter=-*.o", "--filter=- /foo",
|
||||||
|
"--filter=+ /bar/", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 6, ok_argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_EQ_INT(cfg->filters->size, 3);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static void test_parse_args_from0_cvs_filter_file_flags() {
|
||||||
|
static const struct {
|
||||||
|
const char* arg;
|
||||||
|
bool from0;
|
||||||
|
bool cvs;
|
||||||
|
bool per_dir;
|
||||||
|
} cases[] = {
|
||||||
|
{"--from0", true, false, false},
|
||||||
|
{"-0", true, false, false},
|
||||||
|
{"--cvs-exclude", false, true, false},
|
||||||
|
{"-C", false, true, false},
|
||||||
|
{"-F", false, false, true},
|
||||||
|
};
|
||||||
|
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", (char*)cases[i].arg, "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_EQ_INT(cfg->from0, cases[i].from0);
|
||||||
|
EXPECT_EQ_INT(cfg->cvs_exclude, cases[i].cvs);
|
||||||
|
EXPECT_EQ_INT(cfg->per_dir_filter, cases[i].per_dir);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The plain booleans are negatable (--no-* simply clears the flag). */
|
||||||
|
static const char* const on[][2] = {{"--from0", "--no-from0"}, {"-C", "--no-cvs-exclude"}};
|
||||||
|
for (size_t i = 0; i < sizeof(on) / sizeof(on[0]); i++) {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
char* argv[] = {"fastsync", (char*)on[i][0], (char*)on[i][1], "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_FALSE(cfg->from0);
|
||||||
|
EXPECT_FALSE(cfg->cvs_exclude);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static void write_file_bytes(const char* path, const char* bytes, size_t len) {
|
||||||
|
FILE* fp = fopen(path, "wb");
|
||||||
|
EXPECT_NOT_NULL(fp);
|
||||||
|
EXPECT_EQ_INT((int)fwrite(bytes, 1, len, fp), (int)len);
|
||||||
|
fclose(fp);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static void test_parse_args_files_from() {
|
||||||
|
const char* list_path = "cli_files_from_list.txt";
|
||||||
|
write_file_bytes(list_path, "a.txt\nsub/b.bin\n\n./c.txt\n", 25);
|
||||||
|
Config* cfg = config_create();
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
char* argv[] = {"fastsync", "--files-from", (char*)list_path, "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_EQ_STR(cfg->files_from, list_path);
|
||||||
|
EXPECT_NOT_NULL(cfg->files_from_set);
|
||||||
|
FileListSet* set = (FileListSet*)cfg->files_from_set;
|
||||||
|
EXPECT_TRUE(file_list_affects(set, "a.txt"));
|
||||||
|
EXPECT_TRUE(file_list_affects(set, "sub/b.bin"));
|
||||||
|
EXPECT_TRUE(file_list_affects(set, "sub/b.bin/x"));
|
||||||
|
EXPECT_TRUE(file_list_affects(set, "sub"));
|
||||||
|
EXPECT_TRUE(file_list_affects(set, "c.txt"));
|
||||||
|
EXPECT_FALSE(file_list_affects(set, "other.txt"));
|
||||||
|
config_delete(cfg);
|
||||||
|
remove(list_path);
|
||||||
|
|
||||||
|
/* -0 switches the separator to NUL regardless of argument order, and NUL
|
||||||
|
* mode preserves entry bytes exactly (a trailing CR/LF is part of the name). */
|
||||||
|
write_file_bytes(list_path, "x.txt\0y/z.bin\0", 14);
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* nul_argv[] = {"fastsync",
|
||||||
|
"--files-from="
|
||||||
|
"cli_files_from_list.txt",
|
||||||
|
"-0", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, nul_argv, positional_args, &positional_count), 0);
|
||||||
|
set = (FileListSet*)cfg->files_from_set;
|
||||||
|
EXPECT_NOT_NULL(set);
|
||||||
|
EXPECT_TRUE(file_list_affects(set, "x.txt"));
|
||||||
|
EXPECT_TRUE(file_list_affects(set, "y/z.bin"));
|
||||||
|
EXPECT_TRUE(file_list_affects(set, "y"));
|
||||||
|
EXPECT_FALSE(file_list_affects(set, "z.txt"));
|
||||||
|
config_delete(cfg);
|
||||||
|
remove(list_path);
|
||||||
|
|
||||||
|
write_file_bytes(list_path, "crlf\n\0tail\0", 11);
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* nul_nl_argv[] = {"fastsync",
|
||||||
|
"--files-from="
|
||||||
|
"cli_files_from_list.txt",
|
||||||
|
"-0", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, nul_nl_argv, positional_args, &positional_count), 0);
|
||||||
|
set = (FileListSet*)cfg->files_from_set;
|
||||||
|
EXPECT_NOT_NULL(set);
|
||||||
|
EXPECT_TRUE(file_list_affects(set, "crlf\n"));
|
||||||
|
EXPECT_TRUE(file_list_affects(set, "tail"));
|
||||||
|
config_delete(cfg);
|
||||||
|
remove(list_path);
|
||||||
|
|
||||||
|
/* A missing list file is a hard parse-time error. */
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* missing_argv[] = {"fastsync", "--files-from", "does_not_exist_ff.txt", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, missing_argv, positional_args, &positional_count), -1);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
/* Absolute and traversal entries are rejected. */
|
||||||
|
write_file_bytes(list_path, "/abs/path\n", 10);
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* abs_argv[] = {"fastsync",
|
||||||
|
"--files-from="
|
||||||
|
"cli_files_from_list.txt",
|
||||||
|
"/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, abs_argv, positional_args, &positional_count), -1);
|
||||||
|
config_delete(cfg);
|
||||||
|
write_file_bytes(list_path, "../escape\n", 10);
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* trav_argv[] = {"fastsync",
|
||||||
|
"--files-from="
|
||||||
|
"cli_files_from_list.txt",
|
||||||
|
"/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, trav_argv, positional_args, &positional_count), -1);
|
||||||
|
config_delete(cfg);
|
||||||
|
remove(list_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
void test_client_cli() {
|
void test_client_cli() {
|
||||||
test_validate_config_required_paths();
|
test_validate_config_required_paths();
|
||||||
test_validate_config_incompatible_options();
|
test_validate_config_incompatible_options();
|
||||||
@@ -1390,4 +1571,7 @@ void test_client_cli() {
|
|||||||
test_parse_args_delay_updates();
|
test_parse_args_delay_updates();
|
||||||
test_validate_config_delay_updates_rejects_inplace();
|
test_validate_config_delay_updates_rejects_inplace();
|
||||||
test_validate_config_delay_updates_rejects_reserved_backup_dir();
|
test_validate_config_delay_updates_rejects_reserved_backup_dir();
|
||||||
|
test_parse_args_files_from();
|
||||||
|
test_parse_args_filter_rules();
|
||||||
|
test_parse_args_from0_cvs_filter_file_flags();
|
||||||
}
|
}
|
||||||
|
|||||||
+440
-2
@@ -1,7 +1,10 @@
|
|||||||
#include "test_utils.h"
|
#include "test_utils.h"
|
||||||
#include "scanner.h"
|
#include "scanner.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
|
#include "file_list.h"
|
||||||
|
#include "filter.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
@@ -394,8 +397,8 @@ static void test_parallel_scanner_root_chunks_without_workers() {
|
|||||||
create_test_file(file1, "a");
|
create_test_file(file1, "a");
|
||||||
create_test_file(file2, "b");
|
create_test_file(file2, "b");
|
||||||
|
|
||||||
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0,
|
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0, 0, 0,
|
||||||
0, 0, false, false, false, false, false, false};
|
false, false, false, false, false, false, NULL, NULL, false};
|
||||||
ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options, NULL);
|
ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options, NULL);
|
||||||
EXPECT_NOT_NULL(scanner);
|
EXPECT_NOT_NULL(scanner);
|
||||||
|
|
||||||
@@ -653,6 +656,428 @@ static void test_scanner_one_file_system_cross_device() {
|
|||||||
EXPECT_EQ_INT(par_on_total, 1);
|
EXPECT_EQ_INT(par_on_total, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Collect emitted file paths (relative to `root`) from a sequential scan.
|
||||||
|
* Returns 0 on success with *out and *count set (caller frees *out). */
|
||||||
|
static int collect_files(const char* root, const ScannerOptions* options, char*** out,
|
||||||
|
int* out_count) {
|
||||||
|
DirectoryScanner* scanner = directory_scanner_create_with_options(root, options);
|
||||||
|
if (!scanner)
|
||||||
|
return -1;
|
||||||
|
size_t root_len = strlen(root);
|
||||||
|
while (root_len > 0 && root[root_len - 1] == '/')
|
||||||
|
root_len--;
|
||||||
|
int cap = 16;
|
||||||
|
int count = 0;
|
||||||
|
char** paths = malloc((size_t)cap * sizeof(char*));
|
||||||
|
if (!paths) {
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
Chunk* chunk;
|
||||||
|
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
const char* rel = chunk->items[i]->path + root_len;
|
||||||
|
if (*rel == '/')
|
||||||
|
rel++;
|
||||||
|
if (count == cap) {
|
||||||
|
cap *= 2;
|
||||||
|
char** grown = realloc(paths, (size_t)cap * sizeof(char*));
|
||||||
|
if (!grown) {
|
||||||
|
for (int k = 0; k < count; k++)
|
||||||
|
free(paths[k]);
|
||||||
|
free(paths);
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
paths = grown;
|
||||||
|
}
|
||||||
|
paths[count++] = str_dup(rel);
|
||||||
|
}
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
}
|
||||||
|
bool failed = directory_scanner_failed(scanner);
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
if (failed) {
|
||||||
|
for (int k = 0; k < count; k++)
|
||||||
|
free(paths[k]);
|
||||||
|
free(paths);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
*out = paths;
|
||||||
|
*out_count = count;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int collect_files_parallel(const char* root, const ScannerOptions* options, char*** out,
|
||||||
|
int* out_count) {
|
||||||
|
ParallelScanner* scanner = parallel_scanner_create_with_options(root, options, NULL);
|
||||||
|
if (!scanner)
|
||||||
|
return -1;
|
||||||
|
size_t root_len = strlen(root);
|
||||||
|
while (root_len > 0 && root[root_len - 1] == '/')
|
||||||
|
root_len--;
|
||||||
|
int cap = 16;
|
||||||
|
int count = 0;
|
||||||
|
char** paths = malloc((size_t)cap * sizeof(char*));
|
||||||
|
if (!paths) {
|
||||||
|
parallel_scanner_destroy(scanner);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
Chunk* chunk;
|
||||||
|
while ((chunk = parallel_scanner_next(scanner)) != NULL) {
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
const char* rel = chunk->items[i]->path + root_len;
|
||||||
|
if (*rel == '/')
|
||||||
|
rel++;
|
||||||
|
if (count == cap) {
|
||||||
|
cap *= 2;
|
||||||
|
char** grown = realloc(paths, (size_t)cap * sizeof(char*));
|
||||||
|
if (!grown) {
|
||||||
|
for (int k = 0; k < count; k++)
|
||||||
|
free(paths[k]);
|
||||||
|
free(paths);
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
parallel_scanner_destroy(scanner);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
paths = grown;
|
||||||
|
}
|
||||||
|
paths[count++] = str_dup(rel);
|
||||||
|
}
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
}
|
||||||
|
bool failed = parallel_scanner_failed(scanner);
|
||||||
|
parallel_scanner_destroy(scanner);
|
||||||
|
if (failed) {
|
||||||
|
for (int k = 0; k < count; k++)
|
||||||
|
free(paths[k]);
|
||||||
|
free(paths);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
*out = paths;
|
||||||
|
*out_count = count;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool has_path(char** paths, int count, const char* rel) {
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
if (strcmp(paths[i], rel) == 0)
|
||||||
|
return true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void free_paths(char** paths, int count) {
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
free(paths[i]);
|
||||||
|
free(paths);
|
||||||
|
}
|
||||||
|
|
||||||
|
static const char* FILE_LIST_PATH = "test_scan_files_from.txt";
|
||||||
|
|
||||||
|
/* --files-from: only the listed files (and the subtree of a listed directory)
|
||||||
|
* are emitted; unrelated files and directories are pruned. */
|
||||||
|
static void test_files_from_subset(bool parallel) {
|
||||||
|
const char* root = "test_scan_ff";
|
||||||
|
const char* sub = "test_scan_ff/sub";
|
||||||
|
const char* other = "test_scan_ff/other";
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||||
|
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
|
||||||
|
EXPECT_EQ_INT(mkdir(other, 0755), 0);
|
||||||
|
create_test_file("test_scan_ff/root.txt", "root");
|
||||||
|
create_test_file("test_scan_ff/sub/keep.txt", "keep");
|
||||||
|
create_test_file("test_scan_ff/sub/skip.bin", "skip");
|
||||||
|
create_test_file("test_scan_ff/other/unrelated.txt", "unrelated");
|
||||||
|
|
||||||
|
/* List a root file and a file under sub: sub is descended but its other file
|
||||||
|
* is not listed, and the whole `other` directory is pruned. */
|
||||||
|
create_test_file(FILE_LIST_PATH, "root.txt\nsub/keep.txt\n");
|
||||||
|
char err[160];
|
||||||
|
FileListSet* set = file_list_load(FILE_LIST_PATH, false, err, sizeof(err));
|
||||||
|
EXPECT_NOT_NULL(set);
|
||||||
|
|
||||||
|
ScannerOptions options = {0};
|
||||||
|
options.file_list = set;
|
||||||
|
if (parallel)
|
||||||
|
options.num_threads = 2;
|
||||||
|
char** paths = NULL;
|
||||||
|
int count = 0;
|
||||||
|
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||||
|
: collect_files(root, &options, &paths, &count);
|
||||||
|
EXPECT_EQ_INT(rc, 0);
|
||||||
|
EXPECT_EQ_INT(count, 2);
|
||||||
|
EXPECT_TRUE(has_path(paths, count, "root.txt"));
|
||||||
|
EXPECT_TRUE(has_path(paths, count, "sub/keep.txt"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, "sub/skip.bin"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, "other/unrelated.txt"));
|
||||||
|
free_paths(paths, count);
|
||||||
|
file_list_destroy(set);
|
||||||
|
remove(FILE_LIST_PATH);
|
||||||
|
|
||||||
|
/* Listing a directory transfers its whole subtree. */
|
||||||
|
create_test_file(FILE_LIST_PATH, "sub\n");
|
||||||
|
set = file_list_load(FILE_LIST_PATH, false, err, sizeof(err));
|
||||||
|
EXPECT_NOT_NULL(set);
|
||||||
|
options.file_list = set;
|
||||||
|
rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||||
|
: collect_files(root, &options, &paths, &count);
|
||||||
|
EXPECT_EQ_INT(rc, 0);
|
||||||
|
EXPECT_EQ_INT(count, 2);
|
||||||
|
EXPECT_TRUE(has_path(paths, count, "sub/keep.txt"));
|
||||||
|
EXPECT_TRUE(has_path(paths, count, "sub/skip.bin"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, "root.txt"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, "other/unrelated.txt"));
|
||||||
|
free_paths(paths, count);
|
||||||
|
file_list_destroy(set);
|
||||||
|
remove(FILE_LIST_PATH);
|
||||||
|
|
||||||
|
unlink("test_scan_ff/root.txt");
|
||||||
|
unlink("test_scan_ff/sub/keep.txt");
|
||||||
|
unlink("test_scan_ff/sub/skip.bin");
|
||||||
|
unlink("test_scan_ff/other/unrelated.txt");
|
||||||
|
rmdir(other);
|
||||||
|
rmdir(sub);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Filter layer: '-' excludes, first-match-wins ordering with '+', anchored
|
||||||
|
* rules, and dir-only rules all prune during the scan. */
|
||||||
|
static void test_filter_rules(bool parallel) {
|
||||||
|
const char* root = "test_scan_filter";
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||||
|
create_test_file("test_scan_filter/a.txt", "a");
|
||||||
|
create_test_file("test_scan_filter/b.tmp", "b");
|
||||||
|
create_test_file("test_scan_filter/c.txt", "c");
|
||||||
|
|
||||||
|
/* - *.tmp excludes only the tmp file; other files remain (default include). */
|
||||||
|
const char* exclude_only[] = {"- *.tmp"};
|
||||||
|
char err[160];
|
||||||
|
FilterRuleList* base = filter_base_build(exclude_only, 1, false, err, sizeof(err));
|
||||||
|
EXPECT_NOT_NULL(base);
|
||||||
|
ScannerOptions options = {0};
|
||||||
|
options.base_filters = base;
|
||||||
|
if (parallel)
|
||||||
|
options.num_threads = 2;
|
||||||
|
char** paths = NULL;
|
||||||
|
int count = 0;
|
||||||
|
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||||
|
: collect_files(root, &options, &paths, &count);
|
||||||
|
EXPECT_EQ_INT(rc, 0);
|
||||||
|
EXPECT_EQ_INT(count, 2);
|
||||||
|
EXPECT_TRUE(has_path(paths, count, "a.txt"));
|
||||||
|
EXPECT_TRUE(has_path(paths, count, "c.txt"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, "b.tmp"));
|
||||||
|
free_paths(paths, count);
|
||||||
|
filter_rule_list_free(base);
|
||||||
|
|
||||||
|
/* Anchored include then exclude-all: only root-level keep* survives. */
|
||||||
|
const char* anchored[] = {"+ /a.txt", "- *"};
|
||||||
|
base = filter_base_build(anchored, 2, false, err, sizeof(err));
|
||||||
|
EXPECT_NOT_NULL(base);
|
||||||
|
options.base_filters = base;
|
||||||
|
rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||||
|
: collect_files(root, &options, &paths, &count);
|
||||||
|
EXPECT_EQ_INT(rc, 0);
|
||||||
|
EXPECT_EQ_INT(count, 1);
|
||||||
|
EXPECT_TRUE(has_path(paths, count, "a.txt"));
|
||||||
|
free_paths(paths, count);
|
||||||
|
filter_rule_list_free(base);
|
||||||
|
|
||||||
|
unlink("test_scan_filter/a.txt");
|
||||||
|
unlink("test_scan_filter/b.tmp");
|
||||||
|
unlink("test_scan_filter/c.txt");
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Anchored dir-only rules prune a whole subtree. */
|
||||||
|
static void test_filter_dir_only_and_anchored(bool parallel) {
|
||||||
|
const char* root = "test_scan_filter_dir";
|
||||||
|
const char* sub = "test_scan_filter_dir/sub";
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||||
|
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
|
||||||
|
create_test_file("test_scan_filter_dir/sub/inner.txt", "x");
|
||||||
|
create_test_file("test_scan_filter_dir/keep.txt", "keep");
|
||||||
|
|
||||||
|
const char* rules[] = {"- /sub/"};
|
||||||
|
char err[160];
|
||||||
|
FilterRuleList* base = filter_base_build(rules, 1, false, err, sizeof(err));
|
||||||
|
EXPECT_NOT_NULL(base);
|
||||||
|
ScannerOptions options = {0};
|
||||||
|
options.base_filters = base;
|
||||||
|
if (parallel)
|
||||||
|
options.num_threads = 2;
|
||||||
|
char** paths = NULL;
|
||||||
|
int count = 0;
|
||||||
|
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||||
|
: collect_files(root, &options, &paths, &count);
|
||||||
|
EXPECT_EQ_INT(rc, 0);
|
||||||
|
EXPECT_EQ_INT(count, 1);
|
||||||
|
EXPECT_TRUE(has_path(paths, count, "keep.txt"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, "sub/inner.txt"));
|
||||||
|
free_paths(paths, count);
|
||||||
|
filter_rule_list_free(base);
|
||||||
|
|
||||||
|
unlink("test_scan_filter_dir/sub/inner.txt");
|
||||||
|
unlink("test_scan_filter_dir/keep.txt");
|
||||||
|
rmdir(sub);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* -C default CVS excludes prune .git/ directories and *.o files. */
|
||||||
|
static void test_cvs_defaults(bool parallel) {
|
||||||
|
const char* root = "test_scan_cvs";
|
||||||
|
const char* git = "test_scan_cvs/.git";
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||||
|
EXPECT_EQ_INT(mkdir(git, 0755), 0);
|
||||||
|
create_test_file("test_scan_cvs/.git/config", "cfg");
|
||||||
|
create_test_file("test_scan_cvs/object.o", "o");
|
||||||
|
create_test_file("test_scan_cvs/keep.txt", "keep");
|
||||||
|
|
||||||
|
char err[160];
|
||||||
|
FilterRuleList* base = filter_base_build(NULL, 0, true, err, sizeof(err));
|
||||||
|
EXPECT_NOT_NULL(base);
|
||||||
|
ScannerOptions options = {0};
|
||||||
|
options.base_filters = base;
|
||||||
|
if (parallel)
|
||||||
|
options.num_threads = 2;
|
||||||
|
char** paths = NULL;
|
||||||
|
int count = 0;
|
||||||
|
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||||
|
: collect_files(root, &options, &paths, &count);
|
||||||
|
EXPECT_EQ_INT(rc, 0);
|
||||||
|
EXPECT_EQ_INT(count, 1);
|
||||||
|
EXPECT_TRUE(has_path(paths, count, "keep.txt"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, ".git/config"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, "object.o"));
|
||||||
|
free_paths(paths, count);
|
||||||
|
filter_rule_list_free(base);
|
||||||
|
|
||||||
|
unlink("test_scan_cvs/.git/config");
|
||||||
|
unlink("test_scan_cvs/object.o");
|
||||||
|
unlink("test_scan_cvs/keep.txt");
|
||||||
|
rmdir(git);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* -F: a .rsync-filter placed in a directory governs its subtree and the file
|
||||||
|
* itself is never transferred. */
|
||||||
|
static void test_per_dir_filter(bool parallel) {
|
||||||
|
const char* root = "test_scan_perdir";
|
||||||
|
const char* sub = "test_scan_perdir/sub";
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||||
|
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
|
||||||
|
create_test_file("test_scan_perdir/drop.tmp", "tmp");
|
||||||
|
create_test_file("test_scan_perdir/keep.txt", "keep");
|
||||||
|
create_test_file("test_scan_perdir/sub/nested.tmp", "tmp");
|
||||||
|
create_test_file("test_scan_perdir/.rsync-filter", "- *.tmp\n");
|
||||||
|
|
||||||
|
ScannerOptions options = {0};
|
||||||
|
options.per_dir_filters = true;
|
||||||
|
if (parallel)
|
||||||
|
options.num_threads = 2;
|
||||||
|
char** paths = NULL;
|
||||||
|
int count = 0;
|
||||||
|
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||||
|
: collect_files(root, &options, &paths, &count);
|
||||||
|
EXPECT_EQ_INT(rc, 0);
|
||||||
|
EXPECT_EQ_INT(count, 1);
|
||||||
|
EXPECT_TRUE(has_path(paths, count, "keep.txt"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, "drop.tmp"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, "sub/nested.tmp"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, ".rsync-filter"));
|
||||||
|
free_paths(paths, count);
|
||||||
|
|
||||||
|
unlink("test_scan_perdir/drop.tmp");
|
||||||
|
unlink("test_scan_perdir/keep.txt");
|
||||||
|
unlink("test_scan_perdir/sub/nested.tmp");
|
||||||
|
unlink("test_scan_perdir/.rsync-filter");
|
||||||
|
rmdir(sub);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* scanner_path_relative maps an on-disk path to its transfer-relative path,
|
||||||
|
* including the "/" transfer-root edge case (regression: children of "/" used
|
||||||
|
* to abort the scan because the suffix was mis-read). */
|
||||||
|
static void test_scanner_path_relative() {
|
||||||
|
char* rel = NULL;
|
||||||
|
|
||||||
|
rel = scanner_path_relative("/", "/");
|
||||||
|
EXPECT_NOT_NULL(rel);
|
||||||
|
EXPECT_EQ_STR(rel, "");
|
||||||
|
free(rel);
|
||||||
|
|
||||||
|
rel = scanner_path_relative("/", "/etc");
|
||||||
|
EXPECT_NOT_NULL(rel);
|
||||||
|
EXPECT_EQ_STR(rel, "etc");
|
||||||
|
free(rel);
|
||||||
|
|
||||||
|
rel = scanner_path_relative("/", "/etc/passwd");
|
||||||
|
EXPECT_NOT_NULL(rel);
|
||||||
|
EXPECT_EQ_STR(rel, "etc/passwd");
|
||||||
|
free(rel);
|
||||||
|
|
||||||
|
/* Normal roots: with and without a trailing slash on the root. */
|
||||||
|
rel = scanner_path_relative("/tmp/foo", "/tmp/foo");
|
||||||
|
EXPECT_NOT_NULL(rel);
|
||||||
|
EXPECT_EQ_STR(rel, "");
|
||||||
|
free(rel);
|
||||||
|
|
||||||
|
rel = scanner_path_relative("/tmp/foo", "/tmp/foo/bar");
|
||||||
|
EXPECT_NOT_NULL(rel);
|
||||||
|
EXPECT_EQ_STR(rel, "bar");
|
||||||
|
free(rel);
|
||||||
|
|
||||||
|
rel = scanner_path_relative("/tmp/foo/", "/tmp/foo/bar/baz.txt");
|
||||||
|
EXPECT_NOT_NULL(rel);
|
||||||
|
EXPECT_EQ_STR(rel, "bar/baz.txt");
|
||||||
|
free(rel);
|
||||||
|
|
||||||
|
/* A path outside the root maps to NULL. */
|
||||||
|
EXPECT_NULL(scanner_path_relative("/tmp/foo", "/tmp"));
|
||||||
|
EXPECT_NULL(scanner_path_relative("/tmp/foo", "/tmp/foobar"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync precedence: a deeper .rsync-filter overrides a shallower one, so an
|
||||||
|
* inner "+ *.tmp" re-includes what the outer "- *.tmp" excluded. */
|
||||||
|
static void test_per_dir_filter_override(bool parallel) {
|
||||||
|
const char* root = "test_scan_perdir_ovr";
|
||||||
|
const char* sub = "test_scan_perdir_ovr/sub";
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||||
|
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
|
||||||
|
create_test_file("test_scan_perdir_ovr/.rsync-filter", "- *.tmp\n");
|
||||||
|
create_test_file("test_scan_perdir_ovr/sub/.rsync-filter", "+ *.tmp\n");
|
||||||
|
create_test_file("test_scan_perdir_ovr/top.tmp", "x");
|
||||||
|
create_test_file("test_scan_perdir_ovr/keep.txt", "keep");
|
||||||
|
create_test_file("test_scan_perdir_ovr/sub/inside.tmp", "x");
|
||||||
|
|
||||||
|
ScannerOptions options = {0};
|
||||||
|
options.per_dir_filters = true;
|
||||||
|
if (parallel)
|
||||||
|
options.num_threads = 2;
|
||||||
|
char** paths = NULL;
|
||||||
|
int count = 0;
|
||||||
|
int rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||||
|
: collect_files(root, &options, &paths, &count);
|
||||||
|
EXPECT_EQ_INT(rc, 0);
|
||||||
|
/* top.tmp is still excluded by the root file; inside.tmp is re-included by
|
||||||
|
* the subdir file; .rsync-filter files are never transferred. */
|
||||||
|
EXPECT_EQ_INT(count, 2);
|
||||||
|
EXPECT_TRUE(has_path(paths, count, "keep.txt"));
|
||||||
|
EXPECT_TRUE(has_path(paths, count, "sub/inside.tmp"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, "top.tmp"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, ".rsync-filter"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, "sub/.rsync-filter"));
|
||||||
|
free_paths(paths, count);
|
||||||
|
|
||||||
|
unlink("test_scan_perdir_ovr/top.tmp");
|
||||||
|
unlink("test_scan_perdir_ovr/keep.txt");
|
||||||
|
unlink("test_scan_perdir_ovr/sub/inside.tmp");
|
||||||
|
unlink("test_scan_perdir_ovr/.rsync-filter");
|
||||||
|
unlink("test_scan_perdir_ovr/sub/.rsync-filter");
|
||||||
|
rmdir(sub);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
void test_scanner() {
|
void test_scanner() {
|
||||||
test_scanner_single_file();
|
test_scanner_single_file();
|
||||||
test_scanner_multiple_files();
|
test_scanner_multiple_files();
|
||||||
@@ -672,4 +1097,17 @@ void test_scanner() {
|
|||||||
test_scanner_one_file_system_same_device();
|
test_scanner_one_file_system_same_device();
|
||||||
test_parallel_scanner_one_file_system_same_device();
|
test_parallel_scanner_one_file_system_same_device();
|
||||||
test_scanner_one_file_system_cross_device();
|
test_scanner_one_file_system_cross_device();
|
||||||
|
test_files_from_subset(false);
|
||||||
|
test_files_from_subset(true);
|
||||||
|
test_filter_rules(false);
|
||||||
|
test_filter_rules(true);
|
||||||
|
test_filter_dir_only_and_anchored(false);
|
||||||
|
test_filter_dir_only_and_anchored(true);
|
||||||
|
test_cvs_defaults(false);
|
||||||
|
test_cvs_defaults(true);
|
||||||
|
test_per_dir_filter(false);
|
||||||
|
test_per_dir_filter(true);
|
||||||
|
test_scanner_path_relative();
|
||||||
|
test_per_dir_filter_override(false);
|
||||||
|
test_per_dir_filter_override(true);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user