TapTap
79a28cdb96
Buffer and sort each directory's inspected entries (non-directories ascending, then directories ascending) and walk them depth-first via a LIFO directory stack, so the sequential scanner's stream matches rsync 3.4.1's flist order. This makes the --info=name transfer order and the --delete-during/--delete-delay deletion sequence byte-identical to rsync (differential tests in test_parity_order.py); --threads stays unordered (no rsync analogue) and is documented as such. Adds LIFO queue_push/queue_pop over the existing ring buffer.
296 lines
15 KiB
C
296 lines
15 KiB
C
#ifndef SCANNER_H
|
|
#define SCANNER_H
|
|
|
|
#include "chunk.h"
|
|
#include "file_list.h"
|
|
#include "filter.h"
|
|
#include "hardlink.h"
|
|
#include "protocol.h"
|
|
#include "queue.h"
|
|
#include "stop_condition.h"
|
|
#include <dirent.h>
|
|
#include <stdbool.h>
|
|
#include <stddef.h>
|
|
#include <stdatomic.h>
|
|
#include <sys/types.h>
|
|
#include <threads.h>
|
|
|
|
/* Upper bound on the configurable parallel scanner worker count (--threads=N):
|
|
* keeps one transfer from spawning an unbounded pool on a very large machine. */
|
|
#define MAX_SCANNER_THREADS 256
|
|
|
|
typedef struct {
|
|
bool use_metadata;
|
|
/* Phase 4 metadata capture: -U/--atimes and -N/--crtimes tell the scanner to
|
|
* capture the source access / birth time into each entry's FileMetadata. */
|
|
bool preserve_atimes;
|
|
bool preserve_crtimes;
|
|
/* Phase 4 xattrs: when preserve_xattrs || preserve_acls is set the scanner
|
|
* captures each regular file's whitelisted xattr set onto the File. */
|
|
bool preserve_xattrs;
|
|
bool preserve_acls;
|
|
unsigned long long chunk_size;
|
|
char** exclude_patterns;
|
|
int exclude_count;
|
|
char** include_patterns;
|
|
int include_count;
|
|
unsigned long long max_size;
|
|
unsigned long long min_size;
|
|
int max_depth;
|
|
int num_threads;
|
|
bool follow_symlinks;
|
|
bool copy_links;
|
|
bool safe_links;
|
|
bool copy_unsafe_links;
|
|
/* Phase 4 symlink-trust sender options: -k/--copy-dirlinks (dereference a
|
|
* symlink to a directory as a directory, keeping symlinks-to-files as
|
|
* symlinks) and --munge-links (rewrite each transmitted symlink target with a
|
|
* marker; escaping targets are never transmitted). Both are client/sender
|
|
* side only and never serialized to the wire (keep_dirlinks is the
|
|
* receiver-side counterpart). */
|
|
bool copy_dirlinks;
|
|
bool munge_links;
|
|
bool checksum;
|
|
bool one_file_system;
|
|
/* Phase 4 special/devices: whether device nodes (--devices) and special files
|
|
* (--specials) are preserved via recreation, and whether --copy-devices
|
|
* copies a device's content as an ordinary regular file. */
|
|
bool preserve_devices;
|
|
bool preserve_specials;
|
|
bool copy_devices;
|
|
/* Phase 2 (files-from / filter layer). All pointers are shared read-only
|
|
* across scanner instances and worker threads; ownership stays with the
|
|
* caller (client_send). */
|
|
const FileListSet* file_list; /* --files-from allow-set, or NULL */
|
|
const FilterRuleList* base_filters; /* command-line + -C rules, or NULL */
|
|
bool per_dir_filters; /* -F: read .rsync-filter per directory */
|
|
/* --delete-excluded: per-directory plain rules become sender-only, so they no
|
|
longer protect the receiver from deletion. */
|
|
bool delete_excluded;
|
|
/* -FF: also exclude the per-directory filter files themselves from the
|
|
transfer (single -F transfers them). */
|
|
bool exclude_per_dir_filter_files;
|
|
bool dirs; /* -d/--dirs: transfer dir entries, no recursion */
|
|
bool relative; /* -R/--relative (dest rel paths, with --files-from) */
|
|
/* -R/--relative outside --files-from: the destination-relative path prefix
|
|
* reconstructed from the source spec (rsync's '/./' cut point), or NULL when
|
|
* -R is off or --files-from is in use (the bare-relative path then comes from
|
|
* the listed entry). Borrowed read-only; owned by client_send. */
|
|
const char* relative_prefix;
|
|
/* --list-only: emit an is_dir File for every traversed directory (the listing
|
|
* includes directory entries, matching rsync). Client-only; never set on a
|
|
* real transfer, which relies on implicit parent creation. */
|
|
bool list_dirs;
|
|
/* --prune-empty-dirs (long only): in --dirs mode an empty source directory's
|
|
explicit entry is omitted from the transfer file list (so nothing is
|
|
created at the destination and it can be pruned by --delete); explicitly
|
|
--files-from-listed directories always pass through. Recursive transfers
|
|
never emit empty directories, so the flag has no additional effect there. */
|
|
bool prune_empty_dirs;
|
|
/* Delete-excluded protection sink (optional): when non-NULL the scanner
|
|
* appends the destination-relative path of every entry it prunes because a
|
|
* USER SELECTION rule excluded it (--filter/-C/per-dir rules and the legacy
|
|
* --exclude/--include layer). The sender turns this list into the manifest's
|
|
* protected prefixes so `--delete` leaves the destination mirror of excluded
|
|
* source paths alone (rsync's default), and drops it when --delete-excluded
|
|
* opts back into deleting them. NOT recorded for --files-from subset pruning
|
|
* (whose delete semantics derive from the synchronized-directory set) or for
|
|
* -R/--files-from relative wire paths. When `excluded_mutex` is non-NULL it
|
|
* is taken around every append (the parallel scanner shares one list across
|
|
* its worker threads). */
|
|
ArrayList* excluded_paths;
|
|
mtx_t* excluded_mutex;
|
|
/* Size-prune protection sink (optional): when non-NULL the scanner appends
|
|
* the destination-relative path of every entry it skipped because of
|
|
* --max-size/--min-size. rsync never deletes a size-skipped source mirror,
|
|
* even under --delete-excluded, so the sender always transmits this list as
|
|
* protected prefixes (unlike excluded_paths, which --delete-excluded drops).
|
|
* Guarded by `excluded_mutex` like excluded_paths. */
|
|
ArrayList* size_skipped_paths;
|
|
/* Synchronized-directory sink (optional): when non-NULL the scanner appends
|
|
* the destination-relative path of every directory it is about to traverse
|
|
* that lies inside a --files-from listed directory (or of every traversed
|
|
* directory when there is no list). The sender sends this set with the delete
|
|
* manifest so the receiver confines its extras walk to synchronized
|
|
* directories, exactly like rsync; the receive root is the "." sentinel.
|
|
* Guarded by `excluded_mutex`. */
|
|
ArrayList* synced_dirs;
|
|
/* Delete-plan directory sink (optional): when non-NULL the scanner appends
|
|
* the destination-relative path of every directory it traverses (except the
|
|
* receive root). The per-directory --delete-during/--delete-delay plan
|
|
* builder uses this to keep an empty in-scope source directory (rsync keeps
|
|
* it) and to emit its plan after the data stream, when no file frame would
|
|
* otherwise trigger it. Guarded by `excluded_mutex`. */
|
|
ArrayList* plan_dirs;
|
|
/* --ignore-errors: an unreadable subdirectory no longer aborts the scan (it
|
|
* is always skipped so the rest of the tree transfers); this flag is kept so
|
|
* the client can distinguish the option state when deciding deletion policy.
|
|
* Client-only. */
|
|
bool ignore_io_errors;
|
|
/* --info=nonreg: print rsync's `skipping non-regular file "NAME"` line for a
|
|
* non-regular entry that is not being preserved. Client-only. */
|
|
bool note_nonreg;
|
|
/* Source root and 8-bit-output policy used to render a `--info=nonreg` name
|
|
* relative to the transfer root. Borrowed read-only. */
|
|
const char* send_directory;
|
|
bool eight_bit_output;
|
|
/* --ignore-missing-args (implied by --delete-missing-args): an explicitly
|
|
* --files-from-listed entry that does not exist under the source is skipped
|
|
* instead of failing (the --dirs generator is the only scanner path that
|
|
* observes a listed-but-missing entry). */
|
|
bool ignore_missing_args;
|
|
/* --hard-links (-H): shared, mutable (mutex-guarded) link-group detection
|
|
* table, NULL when -H is off. Owned by the caller (client_send), shared
|
|
* read-only here; the parallel scanner passes it unchanged to every worker so
|
|
* one table detects every group across all subdirectories. */
|
|
HardLinkTable* hardlinks;
|
|
/* Phase 6: optional sender stop deadline. When non-NULL the scanner checks
|
|
* it at natural loop boundaries and stops emitting chunks once reached
|
|
* (without marking the scan as failed), so a busy scan itself stops early.
|
|
* Client-only, never serialized to the wire. */
|
|
const StopCondition* stop_condition;
|
|
/* P7 Wave D (protocol 2.17.0): directory-time capture sink. When
|
|
* `capture_dir_times` is true the recursive scan appends one is_dir File
|
|
* (with metadata, no payload) per source directory it traverses to
|
|
* `dir_entries`, so the sender can transmit trailing STATUS_DIR_TIMES
|
|
* frame(s) and the receiver can apply directory mtimes AFTER all children
|
|
* are written. `dir_entries_mutex` (optional) guards the list
|
|
* for the parallel scanner's shared worker threads; the caller owns both.
|
|
* The --dirs generator does not use this (its directory entries carry their
|
|
* metadata inline through STATUS_MKDIR). */
|
|
bool capture_dir_times;
|
|
ArrayList* dir_entries;
|
|
mtx_t* dir_entries_mutex;
|
|
/* Recreate empty source directories on a recursive transfer: emit a
|
|
* payload-less directory entry for every traversed directory that produced
|
|
* no transferred/descended child. Off by default so low-level scanner users
|
|
* (unit helpers, --list-only) see only the historical file list; the real
|
|
* sender sets it in prepare_scanner. */
|
|
bool emit_empty_dirs;
|
|
/* --no-implied-dirs with -R + --files-from: a directory that is only an
|
|
* implied parent of a listed entry (not itself listed, nor below a listed
|
|
* directory) must not carry source metadata; it is created with default
|
|
* attributes at the destination, matching rsync. */
|
|
bool no_implied_dirs;
|
|
} ScannerOptions;
|
|
|
|
/* Internal per-scanner filter state. FilterNode chains represent the ordered
|
|
* per-directory .rsync-filter rules that apply below a directory. */
|
|
typedef struct FilterNode FilterNode;
|
|
|
|
typedef struct {
|
|
/* Scan inputs, copied once at create time. Everything that is also a
|
|
ScannerOptions field lives here (with the normalized chunk_size); only
|
|
scanner-owned bookkeeping stays as direct members below. */
|
|
ScannerOptions options;
|
|
Queue* directories;
|
|
DIR* current_dir;
|
|
char* current_path;
|
|
int current_depth;
|
|
dev_t root_dev;
|
|
bool failed;
|
|
/* rsync-order traversal: each opened directory's entries are inspected once
|
|
and buffered (an internal SortedEntry[] owned here) sorted as rsync's flist
|
|
orders them -- non-directories ascending, then directories ascending. The
|
|
entries are walked in order and child directories are collected in
|
|
`pending_dirs` (an ArrayList of DirEntry*, owned here) and pushed onto the
|
|
LIFO `directories` stack in reverse at directory exhaustion, so the emitted
|
|
stream is depth-first like rsync. `sorted_*` are reset per directory. */
|
|
void* sorted_entries;
|
|
size_t sorted_count;
|
|
size_t sorted_index;
|
|
void* pending_dirs;
|
|
/* Recursive scan: whether the open directory yielded any transferred or
|
|
descended entry. When it did not, closing it emits a directory entry so
|
|
the empty source directory is recreated at the destination (rsync
|
|
parity). */
|
|
bool current_dir_produced;
|
|
/* Phase 2 (files-from / filter layer). */
|
|
char* root_path; /* transfer root (fs path) for rel computation */
|
|
char* current_rel; /* rel path of the open directory ("" == root) */
|
|
bool at_seed_dir; /* next open is the seed directory */
|
|
FilterNode* seed_node; /* inherited context of the seed dir, or NULL */
|
|
FilterNode* current_node; /* filter context of the open directory */
|
|
ArrayList* filter_nodes; /* owned FilterNode arena (may be NULL) */
|
|
/* --dirs / -R state for the directory-entry generator (options.dirs replaces
|
|
the recursive scan). */
|
|
bool relative_mode; /* file_list && relative: send bare relative wire paths */
|
|
bool dirs_root_emitted;
|
|
int list_index;
|
|
ArrayList* dirs_batch; /* owned when non-NULL */
|
|
unsigned long long dirs_batch_size;
|
|
/* A directory could not be opened (I/O error, e.g. EACCES). With
|
|
--ignore-errors the scan continues past it and the caller decides what to
|
|
do; `failed` is reserved for fatal errors that always abort the scan. */
|
|
bool io_error;
|
|
/* The transfer ROOT could not be opened. It is always fatal, even under
|
|
--ignore-errors, but the client still maps it to rsync's partial-transfer
|
|
exit (23) rather than a generic failure. */
|
|
bool root_io_error;
|
|
} DirectoryScanner;
|
|
|
|
typedef struct {
|
|
Queue* result_queue;
|
|
mtx_t result_mutex;
|
|
cnd_t result_not_empty;
|
|
cnd_t result_not_full;
|
|
int num_threads;
|
|
int expected_threads;
|
|
int created_threads;
|
|
thrd_t* threads;
|
|
bool done;
|
|
bool failed;
|
|
/* A worker skipped an unreadable directory under --ignore-errors (non-fatal). */
|
|
bool io_error;
|
|
atomic_bool cancelled;
|
|
int completed;
|
|
Chunk* initial_chunk;
|
|
ProtocolSession* allocation_session;
|
|
FilterNode* root_filter_node; /* root .rsync-filter context (owned by ps) */
|
|
const ScannerOptions* options; /* borrowed scan options (--info=nonreg output) */
|
|
} ParallelScanner;
|
|
|
|
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
|
|
unsigned long long chunk_size, char** exclude_patterns,
|
|
int exclude_count, char** include_patterns,
|
|
int include_count, unsigned long long max_size,
|
|
unsigned long long min_size, int max_depth,
|
|
bool follow_symlinks, bool copy_links, bool safe_links,
|
|
bool copy_unsafe_links, bool checksum);
|
|
DirectoryScanner* directory_scanner_create_with_options(const char* root_directory,
|
|
const ScannerOptions* options);
|
|
Chunk* directory_scanner_next(DirectoryScanner* scanner);
|
|
bool directory_scanner_failed(const DirectoryScanner* scanner);
|
|
void directory_scanner_destroy(DirectoryScanner* scanner);
|
|
|
|
/* --one-file-system (-x) decision: a directory entry may be descended into
|
|
* only when the option is disabled or the entry lives on the same device as
|
|
* the transfer root. Exposed so tests can exercise the rule directly. */
|
|
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device);
|
|
|
|
/* Relative path of an on-disk path below `root` ("" == the root itself, NULL
|
|
* when `fs_path` is not under `root`). Handles trailing slashes and a root of
|
|
* "/". Exposed so tests can exercise the mapping directly. */
|
|
char* scanner_path_relative(const char* root, const char* fs_path);
|
|
|
|
/* -R/--relative destination-relative prefix reconstructed from a source spec:
|
|
* the path after rsync's first '.' path component (the '/./' cut point), with
|
|
* leading/trailing slashes removed, or the whole spec (normalized) when there
|
|
* is no cut. Returns "" for the receive root, or NULL when `spec` is NULL or
|
|
* allocation fails. Exposed so tests can exercise the mapping directly. */
|
|
char* scanner_relative_prefix(const char* spec);
|
|
|
|
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
|
const ScannerOptions* options,
|
|
ProtocolSession* allocation_session);
|
|
Chunk* parallel_scanner_next(ParallelScanner* scanner);
|
|
bool parallel_scanner_failed(const ParallelScanner* scanner);
|
|
bool parallel_scanner_had_io_error(const ParallelScanner* scanner);
|
|
void parallel_scanner_destroy(ParallelScanner* scanner);
|
|
|
|
/* True when a directory could not be opened during the scan (an I/O error,
|
|
recorded even when --ignore-errors keeps the scan going past it). */
|
|
bool directory_scanner_had_io_error(const DirectoryScanner* scanner);
|
|
|
|
#endif
|