TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat(parity): general -R/--relative path semantics and --no-implied-dirs

Browse Source 
Reconstruct the destination-relative prefix from the source spec outside
--files-from: cut at rsync's first '/./' (or a leading './'), normalize
later '.' components and trailing slashes.  Apply it as each File's
send_path in the sequential and parallel scanners (root and worker paths,
files, one-file-system mount entries and directory-time capture).

Transmit the metadata of implied parent directories (prefix components
above the source root), suppressed by --no-implied-dirs, so parent attrs
match rsync in both the single-threaded and -m pipelines.
This commit is contained in:
TapTap
2026-09-16 22:41:28 +02:00
parent 9883757190
commit 583d3c8edb
4 changed files with 317 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/client_send.c
+101
View File
@@ -136,6 +136,7 @@ typedef struct {
ScannerOptions options;
FilterRuleList* base_filters; /* owned; may be NULL */
HardLinkTable* hardlinks; /* owned; may be NULL */
char* relative_prefix; /* owned -R prefix; may be NULL */
} PreparedScanner;
/* Build the scanner options for one scan. Returns false and logs on failure. */
@@ -144,6 +145,7 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann
return false;
out->base_filters = NULL;
out->hardlinks = NULL;
out->relative_prefix = NULL;
memset(&out->options, 0, sizeof(out->options));
int rule_count = config->filters ? config->filters->size : 0;
@@ -206,6 +208,19 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann
options->per_dir_filters = config->per_dir_filter;
options->dirs = config->dirs;
options->relative = config->relative;
/* -R/--relative outside --files-from reconstructs every destination path from
* the source spec (rsync's '/./' cut point). With --files-from the listed
* entry already supplies the bare relative path, so no prefix is built. */
if (config->relative && config->files_from_set == NULL && config->send_directory) {
out->relative_prefix = scanner_relative_prefix(config->send_directory);
if (!out->relative_prefix) {
log_message(LOG_LEVEL_ERROR, "memory allocation failed building --relative path prefix");
filter_rule_list_free(out->base_filters);
out->base_filters = NULL;
return false;
}
options->relative_prefix = out->relative_prefix;
}
options->prune_empty_dirs = config->prune_empty_dirs;
options->ignore_io_errors = config->ignore_errors;
options->ignore_missing_args = config->ignore_missing_args || config->delete_missing_args;
@@ -239,6 +254,85 @@ static void prepared_scanner_destroy(PreparedScanner* prepared) {
prepared->base_filters = NULL;
hardlink_table_destroy(prepared->hardlinks);
prepared->hardlinks = NULL;
free(prepared->relative_prefix);
prepared->relative_prefix = NULL;
}
/* -R/--relative implied directories: rsync transmits the metadata of the
* parent directories implied by the source path (every prefix component above
* the source root) so the receiver applies their attributes to the created
* parents. FastSync's scan only covers the source root and below, so append
* one metadata-only directory entry per implied ancestor. --no-implied-dirs
* suppresses this exactly like rsync. A missing ancestor is never fatal. */
static bool append_implied_dir_times(const Config* config, ArrayList* dir_entries) {
if (!dir_entries || !config->relative || config->files_from_set != NULL ||
config->no_implied_dirs || !config->send_directory)
return true;
char* prefix = scanner_relative_prefix(config->send_directory);
if (!prefix)
return true;
int ncomp = 0;
for (const char* s = prefix; *s;) {
while (*s == '/')
s++;
if (!*s)
break;
while (*s && *s != '/')
s++;
ncomp++;
}
if (ncomp <= 1) {
free(prefix);
return true;
}
char* fs = str_dup(config->send_directory);
if (!fs) {
free(prefix);
return true;
}
size_t flen = strlen(fs);
while (flen > 1 && fs[flen - 1] == '/')
fs[--flen] = '\0';
bool ok = true;
/* Walk the source path upwards one component at a time; the previous
iteration's truncation is restored so every ancestor is stat'ed in full. */
for (int depth = ncomp - 2; depth >= 0 && ok; depth--) {
char* slash = strrchr(fs, '/');
if (!slash || slash == fs)
break;
*slash = '\0';
char* p = prefix;
int c = 0;
while (c <= depth) {
while (*p == '/')
p++;
while (*p && *p != '/')
p++;
c++;
}
char saved = *p;
*p = '\0';
struct stat st;
if (stat(fs, &st) == 0 && S_ISDIR(st.st_mode)) {
File* file = file_create(fs);
if (!file) {
ok = false;
} else {
file->is_dir = true;
file->metadata =
file_metadata_create(fs, &st, config->preserve_atimes, config->preserve_crtimes);
file->send_path = str_dup(prefix);
if (!file->metadata || !file->send_path || !array_list_add(dir_entries, file)) {
file_destroy(file);
ok = false;
}
}
}
*p = saved;
}
free(fs);
free(prefix);
return ok;
}
/* True when some --files-from entry is an ancestor-or-equal directory of
@@ -2098,6 +2192,11 @@ static int scan_directory_multithreaded(void* pipeline_context) {
parallel workers append under the context's dedicated mutex. */
prepared.options.dir_entries = context->dir_entries;
prepared.options.dir_entries_mutex = &context->dir_entries_mutex;
if (!append_implied_dir_times(context->config, context->dir_entries)) {
pipeline_cancel(context);
protocol_session_unbind();
return thrd_error;
}
/* The keep-set manifest for the late modes is built from this data pass, so
the parallel scanner records the protected excluded prefixes and the
synchronized directories here (the size-prune protection is collected in
@@ -2434,6 +2533,8 @@ int send_files(Config* config) {
dir_entries = array_list_create(file_destroy);
if (!dir_entries)
goto send_fail;
if (!append_implied_dir_times(config, dir_entries))
goto send_fail;
}
if (config->remove_source_files)
remove_sources = array_list_create(source_file_destroy);
src/client/scanner.c
+149 -14
View File
@@ -220,6 +220,44 @@ char* scanner_path_relative(const char* root, const char* fs_path) {
return str_dup(fs_path + root_len + 1);
}
/* -R/--relative destination-relative prefix reconstructed from a source spec:
* everything after the first '.' path component (rsync's '/./' cut point),
* with leading/trailing slashes removed; or the whole spec (normalized) when
* there is no cut. Returns "" for the receive root. Exposed for tests. */
char* scanner_relative_prefix(const char* spec) {
if (!spec || spec[0] == '\0')
return NULL;
const char* after = spec;
if (spec[0] == '.' && spec[1] == '/') {
after = spec + 2;
} else {
const char* cut = strstr(spec, "/./");
if (cut)
after = cut + 3;
}
size_t cap = strlen(spec) + 1;
char* out = malloc(cap);
if (!out)
return NULL;
size_t len = 0;
for (const char* s = after; *s;) {
while (*s == '/')
s++;
const char* comp = s;
while (*s && *s != '/')
s++;
size_t clen = (size_t)(s - comp);
if (clen == 0 || (clen == 1 && comp[0] == '.'))
continue;
if (len)
out[len++] = '/';
memcpy(out + len, comp, clen);
len += clen;
}
out[len] = '\0';
return out;
}
/* Relative path of a child entry below the current directory. */
static char* child_rel_path(const char* parent_rel, const char* name) {
if (!parent_rel || parent_rel[0] == '\0')
@@ -227,6 +265,15 @@ static char* child_rel_path(const char* parent_rel, const char* name) {
return path_cat(parent_rel, name);
}
/* Destination-relative wire path for an entry under an -R prefix. */
static char* scanner_prefix_send_path(const char* prefix, const char* rel) {
if (prefix[0] == '\0')
return str_dup(rel);
if (rel[0] == '\0')
return str_dup(prefix);
return path_cat(prefix, rel);
}
/* Apply the --files-from allow-set and the filter layer to one entry. */
static bool entry_passes_selection(const FileListSet* file_list, const FilterRuleList* base,
const FilterNode* node, const char* rel, const char* leaf,
@@ -367,12 +414,25 @@ static bool scanner_record_synced_dir(const ScannerOptions* options, const char*
return true;
if (!file_list_dir_in_scope(options->file_list, rel))
return true;
const char* dest = relative_mode ? rel : fs_path;
char* prefixed = NULL;
const char* dest;
if (relative_mode) {
dest = rel;
} else if (options->relative_prefix) {
prefixed = scanner_prefix_send_path(options->relative_prefix, rel);
if (!prefixed)
return false;
dest = prefixed;
} else {
dest = fs_path;
}
if (dest[0] == '/')
dest++;
if (dest[0] == '\0')
dest = ".";
return excluded_sink_append(options->synced_dirs, options->excluded_mutex, dest);
bool ok = excluded_sink_append(options->synced_dirs, options->excluded_mutex, dest);
free(prefixed);
return ok;
}
/* Merge the open directory's own .rsync-filter rules into the inherited
@@ -672,7 +732,8 @@ static Chunk* chunk_data_to_chunk(ArrayList* chunk_data) {
* non-directory path is silently skipped (the transfer is unaffected); an
* allocation failure is fatal and reported to the caller. */
static bool scanner_capture_dir_time(ArrayList* dir_entries, mtx_t* mutex, const char* root_path,
const char* fs_path, bool relative_mode, bool preserve_atimes,
const char* fs_path, bool relative_mode,
const char* relative_prefix, bool preserve_atimes,
bool preserve_crtimes, bool preserve_xattrs,
bool preserve_acls) {
if (!dir_entries || !root_path || !fs_path)
@@ -689,14 +750,30 @@ static bool scanner_capture_dir_time(ArrayList* dir_entries, mtx_t* mutex, const
free(rel);
return true;
}
char* prefixed = NULL;
if (relative_prefix) {
prefixed = scanner_prefix_send_path(relative_prefix, rel);
if (!prefixed) {
free(rel);
return false;
}
if (prefixed[0] == '\0') {
/* -R with a cut at the receive root: the root itself has no wire path. */
free(prefixed);
free(rel);
return true;
}
}
File* file = file_create(fs_path);
if (!file) {
free(prefixed);
free(rel);
return false;
}
file->is_dir = true;
file->metadata = file_metadata_create(fs_path, &st, preserve_atimes, preserve_crtimes);
if (!file->metadata) {
free(prefixed);
free(rel);
file_destroy(file);
return false;
@@ -709,7 +786,11 @@ static bool scanner_capture_dir_time(ArrayList* dir_entries, mtx_t* mutex, const
if (relative_mode) {
file->send_path = rel;
rel = NULL;
} else if (prefixed) {
file->send_path = prefixed;
prefixed = NULL;
}
free(prefixed);
free(rel);
bool added;
if (mutex) {
@@ -807,9 +888,9 @@ static int open_next_directory(DirectoryScanner* scanner) {
if (scanner->options.capture_dir_times &&
!scanner_capture_dir_time(
scanner->options.dir_entries, scanner->options.dir_entries_mutex, scanner->root_path,
scanner->current_path, scanner->relative_mode, scanner->options.preserve_atimes,
scanner->options.preserve_crtimes, scanner->options.preserve_xattrs,
scanner->options.preserve_acls)) {
scanner->current_path, scanner->relative_mode, scanner->options.relative_prefix,
scanner->options.preserve_atimes, scanner->options.preserve_crtimes,
scanner->options.preserve_xattrs, scanner->options.preserve_acls)) {
closedir(scanner->current_dir);
scanner->current_dir = NULL;
free(scanner->current_path);
@@ -1175,14 +1256,28 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
wire paths are never recorded (see ScannerOptions.excluded_paths). */
bool files_from_prune =
scanner->options.file_list && !file_list_affects(scanner->options.file_list, rel);
if (!files_from_prune && !scanner->relative_mode)
if (!files_from_prune && !scanner->relative_mode) {
if (scanner->options.relative_prefix) {
char* wrel = scanner_prefix_send_path(scanner->options.relative_prefix, rel);
if (!wrel) {
free(rel);
free(cur_path);
scanner->failed = true;
break;
}
scanner_record_excluded(scanner, wrel);
free(wrel);
} else {
scanner_record_excluded(scanner, cur_path);
}
/* With -R + --files-from the wire/destination path is the entry's bare
relative path; keep `rel` alive to attach it to a transferred file. */
char* rel_copy = scanner->relative_mode ? str_dup(rel) : NULL;
}
}
/* With -R the wire/destination path is a reconstructed relative path, not
the source path; keep `rel` alive to build it for a transferred file. */
bool needs_rel = scanner->relative_mode || scanner->options.relative_prefix != NULL;
char* rel_copy = needs_rel ? str_dup(rel) : NULL;
free(rel);
if (rel_copy == NULL && scanner->relative_mode) {
if (rel_copy == NULL && needs_rel) {
free(cur_path);
scanner->failed = true;
break;
@@ -1257,6 +1352,15 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
if (scanner->relative_mode) {
file->send_path = rel_copy;
rel_copy = NULL;
} else if (scanner->options.relative_prefix) {
file->send_path = scanner_prefix_send_path(scanner->options.relative_prefix, rel_copy);
free(rel_copy);
rel_copy = NULL;
if (!file->send_path) {
file_destroy(file);
scanner->failed = true;
break;
}
}
/* --devices/--specials: a device/FIFO/socket entry marked for preservation
becomes a node to recreate (is_special, no data, rdev captured); an
@@ -1536,6 +1640,15 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
if (options->relative && options->file_list != NULL) {
if (!excluded_sink_append(sink, options->excluded_mutex, entry->d_name))
ps->failed = true;
} else if (options->relative_prefix) {
char* wrel = scanner_prefix_send_path(options->relative_prefix, entry->d_name);
if (!wrel) {
ps->failed = true;
} else {
if (!excluded_sink_append(sink, options->excluded_mutex, wrel))
ps->failed = true;
free(wrel);
}
} else {
char* abs_path = path_cat(root_directory, entry->d_name);
if (!abs_path) {
@@ -1577,13 +1690,13 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
return;
}
if (is_dir) {
free(rel);
if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) {
/* -x/--one-file-system: emit the mount-point directory entry (empty) but
do not descend into it (see the sequential scanner for the same rule). */
File* mount = file_create(cur_path);
free(cur_path);
if (mount == NULL) {
free(rel);
ps->failed = true;
return;
}
@@ -1592,17 +1705,29 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
mount->metadata = file_metadata_create(mount->path, &st, options->preserve_atimes,
options->preserve_crtimes);
if (!mount->metadata) {
free(rel);
file_destroy(mount);
ps->failed = true;
return;
}
}
if (options->relative_prefix) {
mount->send_path = scanner_prefix_send_path(options->relative_prefix, rel);
if (!mount->send_path) {
free(rel);
file_destroy(mount);
ps->failed = true;
return;
}
}
free(rel);
if (!array_list_add(root_files, mount)) {
file_destroy(mount);
ps->failed = true;
}
return;
}
free(rel);
if (!array_list_add(subdirs, cur_path)) {
free(cur_path);
ps->failed = true;
@@ -1628,6 +1753,15 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
if (use_rel) {
file->send_path = rel;
rel = NULL;
} else if (options->relative_prefix) {
file->send_path = scanner_prefix_send_path(options->relative_prefix, rel);
free(rel);
rel = NULL;
if (!file->send_path) {
file_destroy(file);
ps->failed = true;
return;
}
}
ScannerSpecial special = scanner_prepare_special(
options->preserve_devices, options->preserve_specials, options->copy_devices, file, &st);
@@ -1857,8 +1991,9 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
if (options->capture_dir_times &&
!scanner_capture_dir_time(options->dir_entries, options->dir_entries_mutex, root_directory,
root_directory, options->relative && options->file_list != NULL,
options->preserve_atimes, options->preserve_crtimes,
options->preserve_xattrs, options->preserve_acls)) {
options->relative_prefix, options->preserve_atimes,
options->preserve_crtimes, options->preserve_xattrs,
options->preserve_acls)) {
array_list_delete(root_files);
array_list_delete(subdirs);
parallel_scanner_destroy(ps);
src/client/scanner.h
+12
View File
@@ -65,6 +65,11 @@ typedef struct {
bool per_dir_filters; /* -F: read .rsync-filter per directory */
bool dirs; /* -d/--dirs: transfer dir entries, no recursion */
bool relative; /* -R/--relative (dest rel paths, with --files-from) */
/* -R/--relative outside --files-from: the destination-relative path prefix
* reconstructed from the source spec (rsync's '/./' cut point), or NULL when
* -R is off or --files-from is in use (the bare-relative path then comes from
* the listed entry). Borrowed read-only; owned by client_send. */
const char* relative_prefix;
/* --list-only: emit an is_dir File for every traversed directory (the listing
* includes directory entries, matching rsync). Client-only; never set on a
* real transfer, which relies on implicit parent creation. */
@@ -213,6 +218,13 @@ bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entr
* "/". Exposed so tests can exercise the mapping directly. */
char* scanner_path_relative(const char* root, const char* fs_path);
/* -R/--relative destination-relative prefix reconstructed from a source spec:
* the path after rsync's first '.' path component (the '/./' cut point), with
* leading/trailing slashes removed, or the whole spec (normalized) when there
* is no cut. Returns "" for the receive root, or NULL when `spec` is NULL or
* allocation fails. Exposed so tests can exercise the mapping directly. */
char* scanner_relative_prefix(const char* spec);
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
const ScannerOptions* options,
ProtocolSession* allocation_session);
tests/test_scanner.c
+54
View File
@@ -1069,6 +1069,59 @@ static void test_scanner_path_relative() {
EXPECT_NULL(scanner_path_relative("/tmp/foo", "/tmp/foobar"));
}
/* -R/--relative destination prefix: the '/./' cut point and normalization. */
static void test_scanner_relative_prefix() {
char* p = NULL;
/* No cut: the whole spec with leading/trailing slashes removed. */
p = scanner_relative_prefix("/tmp/src/foo/");
EXPECT_NOT_NULL(p);
EXPECT_EQ_STR(p, "tmp/src/foo");
free(p);
p = scanner_relative_prefix("src/foo");
EXPECT_NOT_NULL(p);
EXPECT_EQ_STR(p, "src/foo");
free(p);
/* Trailing "/." is the directory itself, not a cut. */
p = scanner_relative_prefix("src/foo/.");
EXPECT_NOT_NULL(p);
EXPECT_EQ_STR(p, "src/foo");
free(p);
/* The first "/./" cuts everything before it. */
p = scanner_relative_prefix("/a/./b/c");
EXPECT_NOT_NULL(p);
EXPECT_EQ_STR(p, "b/c");
free(p);
p = scanner_relative_prefix("src/./");
EXPECT_NOT_NULL(p);
EXPECT_EQ_STR(p, "");
free(p);
/* A later "." component is normalized away. */
p = scanner_relative_prefix("a/./b/./c");
EXPECT_NOT_NULL(p);
EXPECT_EQ_STR(p, "b/c");
free(p);
/* A leading "./" is the cut at the start. */
p = scanner_relative_prefix("./s2");
EXPECT_NOT_NULL(p);
EXPECT_EQ_STR(p, "s2");
free(p);
p = scanner_relative_prefix(".");
EXPECT_NOT_NULL(p);
EXPECT_EQ_STR(p, "");
free(p);
EXPECT_NULL(scanner_relative_prefix(NULL));
EXPECT_NULL(scanner_relative_prefix(""));
}
/* rsync precedence: a deeper .rsync-filter overrides a shallower one, so an
* inner "+ *.tmp" re-includes what the outer "- *.tmp" excluded. */
static void test_per_dir_filter_override(bool parallel) {
@@ -1607,6 +1660,7 @@ void test_scanner() {
test_per_dir_filter(false);
test_per_dir_filter(true);
test_scanner_path_relative();
test_scanner_relative_prefix();
test_per_dir_filter_override(false);
test_per_dir_filter_override(true);
test_dirs_no_descent();