TapTap
82a1d5e240
- Scope the --delete extras walk to directories synchronized by the transfer: add a synchronized-directory section to the delete manifest (protocol 2.23.0) so --files-from subsets no longer delete untransmitted paths outside listed directory subtrees (data-loss fix). - Separate --max-size/--min-size prune protection from --delete-excluded so size-pruned source mirrors survive (rsync parity). - Unlink extraneous destination symlinks instead of skipping them. - Make --max-delete partial (delete up to N, skip the rest) and exit 25; accept negative values as unlimited. - Draw --delete-missing-args deletions from the shared --max-delete budget. - Honor --force during --delay-updates publication. Add unit and integration regression tests; update the pinned config wire golden and version strings for the 2.23.0 manifest/status additions.
225 lines
11 KiB
C
225 lines
11 KiB
C
#ifndef SCANNER_H
|
|
#define SCANNER_H
|
|
|
|
#include "chunk.h"
|
|
#include "file_list.h"
|
|
#include "filter.h"
|
|
#include "hardlink.h"
|
|
#include "protocol.h"
|
|
#include "queue.h"
|
|
#include "stop_condition.h"
|
|
#include <dirent.h>
|
|
#include <stdbool.h>
|
|
#include <stdatomic.h>
|
|
#include <sys/types.h>
|
|
#include <threads.h>
|
|
|
|
/* Upper bound on the configurable parallel scanner worker count (--threads=N):
|
|
* keeps one transfer from spawning an unbounded pool on a very large machine. */
|
|
#define MAX_SCANNER_THREADS 256
|
|
|
|
typedef struct {
|
|
bool use_metadata;
|
|
/* Phase 4 metadata capture: -U/--atimes and -N/--crtimes tell the scanner to
|
|
* capture the source access / birth time into each entry's FileMetadata. */
|
|
bool preserve_atimes;
|
|
bool preserve_crtimes;
|
|
/* Phase 4 xattrs: when preserve_xattrs || preserve_acls is set the scanner
|
|
* captures each regular file's whitelisted xattr set onto the File. */
|
|
bool preserve_xattrs;
|
|
bool preserve_acls;
|
|
unsigned long long chunk_size;
|
|
char** exclude_patterns;
|
|
int exclude_count;
|
|
char** include_patterns;
|
|
int include_count;
|
|
unsigned long long max_size;
|
|
unsigned long long min_size;
|
|
int max_depth;
|
|
int num_threads;
|
|
bool follow_symlinks;
|
|
bool copy_links;
|
|
bool safe_links;
|
|
bool copy_unsafe_links;
|
|
/* Phase 4 symlink-trust sender options: -k/--copy-dirlinks (dereference a
|
|
* symlink to a directory as a directory, keeping symlinks-to-files as
|
|
* symlinks) and --munge-links (rewrite each transmitted symlink target with a
|
|
* marker; escaping targets are never transmitted). Both are client/sender
|
|
* side only and never serialized to the wire (keep_dirlinks is the
|
|
* receiver-side counterpart). */
|
|
bool copy_dirlinks;
|
|
bool munge_links;
|
|
bool checksum;
|
|
bool one_file_system;
|
|
/* Phase 4 special/devices: whether device nodes (--devices) and special files
|
|
* (--specials) are preserved via recreation, and whether --copy-devices
|
|
* copies a device's content as an ordinary regular file. */
|
|
bool preserve_devices;
|
|
bool preserve_specials;
|
|
bool copy_devices;
|
|
/* Phase 2 (files-from / filter layer). All pointers are shared read-only
|
|
* across scanner instances and worker threads; ownership stays with the
|
|
* caller (client_send). */
|
|
const FileListSet* file_list; /* --files-from allow-set, or NULL */
|
|
const FilterRuleList* base_filters; /* command-line + -C rules, or NULL */
|
|
bool per_dir_filters; /* -F: read .rsync-filter per directory */
|
|
bool dirs; /* -d/--dirs: transfer dir entries, no recursion */
|
|
bool relative; /* -R/--relative (dest rel paths, with --files-from) */
|
|
/* --prune-empty-dirs (long only): in --dirs mode an empty source directory's
|
|
explicit entry is omitted from the transfer file list (so nothing is
|
|
created at the destination and it can be pruned by --delete); explicitly
|
|
--files-from-listed directories always pass through. Recursive transfers
|
|
never emit empty directories, so the flag has no additional effect there. */
|
|
bool prune_empty_dirs;
|
|
/* Delete-excluded protection sink (optional): when non-NULL the scanner
|
|
* appends the destination-relative path of every entry it prunes because a
|
|
* USER SELECTION rule excluded it (--filter/-C/per-dir rules and the legacy
|
|
* --exclude/--include layer). The sender turns this list into the manifest's
|
|
* protected prefixes so `--delete` leaves the destination mirror of excluded
|
|
* source paths alone (rsync's default), and drops it when --delete-excluded
|
|
* opts back into deleting them. NOT recorded for --files-from subset pruning
|
|
* (whose delete semantics derive from the synchronized-directory set) or for
|
|
* -R/--files-from relative wire paths. When `excluded_mutex` is non-NULL it
|
|
* is taken around every append (the parallel scanner shares one list across
|
|
* its worker threads). */
|
|
ArrayList* excluded_paths;
|
|
mtx_t* excluded_mutex;
|
|
/* Size-prune protection sink (optional): when non-NULL the scanner appends
|
|
* the destination-relative path of every entry it skipped because of
|
|
* --max-size/--min-size. rsync never deletes a size-skipped source mirror,
|
|
* even under --delete-excluded, so the sender always transmits this list as
|
|
* protected prefixes (unlike excluded_paths, which --delete-excluded drops).
|
|
* Guarded by `excluded_mutex` like excluded_paths. */
|
|
ArrayList* size_skipped_paths;
|
|
/* Synchronized-directory sink (optional): when non-NULL the scanner appends
|
|
* the destination-relative path of every directory it is about to traverse
|
|
* that lies inside a --files-from listed directory (or of every traversed
|
|
* directory when there is no list). The sender sends this set with the delete
|
|
* manifest so the receiver confines its extras walk to synchronized
|
|
* directories, exactly like rsync; the receive root is the "." sentinel.
|
|
* Guarded by `excluded_mutex`. */
|
|
ArrayList* synced_dirs;
|
|
/* --ignore-errors: an unreadable directory during the scan is recorded as an
|
|
* I/O error and skipped instead of aborting the scan. Client-only. */
|
|
bool ignore_io_errors;
|
|
/* --ignore-missing-args (implied by --delete-missing-args): an explicitly
|
|
* --files-from-listed entry that does not exist under the source is skipped
|
|
* instead of failing (the --dirs generator is the only scanner path that
|
|
* observes a listed-but-missing entry). */
|
|
bool ignore_missing_args;
|
|
/* --hard-links (-H): shared, mutable (mutex-guarded) link-group detection
|
|
* table, NULL when -H is off. Owned by the caller (client_send), shared
|
|
* read-only here; the parallel scanner passes it unchanged to every worker so
|
|
* one table detects every group across all subdirectories. */
|
|
HardLinkTable* hardlinks;
|
|
/* Phase 6: optional sender stop deadline. When non-NULL the scanner checks
|
|
* it at natural loop boundaries and stops emitting chunks once reached
|
|
* (without marking the scan as failed), so a busy scan itself stops early.
|
|
* Client-only, never serialized to the wire. */
|
|
const StopCondition* stop_condition;
|
|
/* P7 Wave D (protocol 2.17.0): directory-time capture sink. When
|
|
* `capture_dir_times` is true the recursive scan appends one is_dir File
|
|
* (with metadata, no payload) per source directory it traverses to
|
|
* `dir_entries`, so the sender can transmit trailing STATUS_DIR_TIMES
|
|
* frame(s) and the receiver can apply directory mtimes AFTER all children
|
|
* are written. `dir_entries_mutex` (optional) guards the list
|
|
* for the parallel scanner's shared worker threads; the caller owns both.
|
|
* The --dirs generator does not use this (its directory entries carry their
|
|
* metadata inline through STATUS_MKDIR). */
|
|
bool capture_dir_times;
|
|
ArrayList* dir_entries;
|
|
mtx_t* dir_entries_mutex;
|
|
} ScannerOptions;
|
|
|
|
/* Internal per-scanner filter state. FilterNode chains represent the ordered
|
|
* per-directory .rsync-filter rules that apply below a directory. */
|
|
typedef struct FilterNode FilterNode;
|
|
|
|
typedef struct {
|
|
/* Scan inputs, copied once at create time. Everything that is also a
|
|
ScannerOptions field lives here (with the normalized chunk_size); only
|
|
scanner-owned bookkeeping stays as direct members below. */
|
|
ScannerOptions options;
|
|
Queue* directories;
|
|
DIR* current_dir;
|
|
char* current_path;
|
|
int current_depth;
|
|
dev_t root_dev;
|
|
bool failed;
|
|
/* Phase 2 (files-from / filter layer). */
|
|
char* root_path; /* transfer root (fs path) for rel computation */
|
|
char* current_rel; /* rel path of the open directory ("" == root) */
|
|
bool at_seed_dir; /* next open is the seed directory */
|
|
FilterNode* seed_node; /* inherited context of the seed dir, or NULL */
|
|
FilterNode* current_node; /* filter context of the open directory */
|
|
ArrayList* filter_nodes; /* owned FilterNode arena (may be NULL) */
|
|
/* --dirs / -R state for the directory-entry generator (options.dirs replaces
|
|
the recursive scan). */
|
|
bool relative_mode; /* file_list && relative: send bare relative wire paths */
|
|
bool dirs_root_emitted;
|
|
int list_index;
|
|
ArrayList* dirs_batch; /* owned when non-NULL */
|
|
unsigned long long dirs_batch_size;
|
|
/* A directory could not be opened (I/O error, e.g. EACCES). With
|
|
--ignore-errors the scan continues past it and the caller decides what to
|
|
do; `failed` is reserved for fatal errors that always abort the scan. */
|
|
bool io_error;
|
|
} DirectoryScanner;
|
|
|
|
typedef struct {
|
|
Queue* result_queue;
|
|
mtx_t result_mutex;
|
|
cnd_t result_not_empty;
|
|
cnd_t result_not_full;
|
|
int num_threads;
|
|
int expected_threads;
|
|
int created_threads;
|
|
thrd_t* threads;
|
|
bool done;
|
|
bool failed;
|
|
/* A worker skipped an unreadable directory under --ignore-errors (non-fatal). */
|
|
bool io_error;
|
|
atomic_bool cancelled;
|
|
int completed;
|
|
Chunk* initial_chunk;
|
|
ProtocolSession* allocation_session;
|
|
FilterNode* root_filter_node; /* root .rsync-filter context (owned by ps) */
|
|
} ParallelScanner;
|
|
|
|
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
|
|
unsigned long long chunk_size, char** exclude_patterns,
|
|
int exclude_count, char** include_patterns,
|
|
int include_count, unsigned long long max_size,
|
|
unsigned long long min_size, int max_depth,
|
|
bool follow_symlinks, bool copy_links, bool safe_links,
|
|
bool copy_unsafe_links, bool checksum);
|
|
DirectoryScanner* directory_scanner_create_with_options(const char* root_directory,
|
|
const ScannerOptions* options);
|
|
Chunk* directory_scanner_next(DirectoryScanner* scanner);
|
|
bool directory_scanner_failed(const DirectoryScanner* scanner);
|
|
void directory_scanner_destroy(DirectoryScanner* scanner);
|
|
|
|
/* --one-file-system (-x) decision: a directory entry may be descended into
|
|
* only when the option is disabled or the entry lives on the same device as
|
|
* the transfer root. Exposed so tests can exercise the rule directly. */
|
|
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device);
|
|
|
|
/* Relative path of an on-disk path below `root` ("" == the root itself, NULL
|
|
* when `fs_path` is not under `root`). Handles trailing slashes and a root of
|
|
* "/". Exposed so tests can exercise the mapping directly. */
|
|
char* scanner_path_relative(const char* root, const char* fs_path);
|
|
|
|
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
|
const ScannerOptions* options,
|
|
ProtocolSession* allocation_session);
|
|
Chunk* parallel_scanner_next(ParallelScanner* scanner);
|
|
bool parallel_scanner_failed(const ParallelScanner* scanner);
|
|
bool parallel_scanner_had_io_error(const ParallelScanner* scanner);
|
|
void parallel_scanner_destroy(ParallelScanner* scanner);
|
|
|
|
/* True when a directory could not be opened during the scan (an I/O error,
|
|
recorded even when --ignore-errors keeps the scan going past it). */
|
|
bool directory_scanner_had_io_error(const DirectoryScanner* scanner);
|
|
|
|
#endif
|