TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

scanner: emit entries in rsync's sorted depth-first flist order

Browse Source 
Buffer and sort each directory's inspected entries (non-directories
ascending, then directories ascending) and walk them depth-first via a
LIFO directory stack, so the sequential scanner's stream matches rsync
3.4.1's flist order.  This makes the --info=name transfer order and the
--delete-during/--delete-delay deletion sequence byte-identical to rsync
(differential tests in test_parity_order.py); --threads stays unordered
(no rsync analogue) and is documented as such.

Adds LIFO queue_push/queue_pop over the existing ring buffer.
This commit is contained in:
TapTap
2026-09-20 13:49:04 +02:00
parent 402cae80ad
commit 79a28cdb96
5 changed files with 345 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/scanner.c
+174 -47
View File
@@ -205,6 +205,17 @@ typedef struct {
bool referent_error;
} ScannerEntry;
/* One inspected directory entry buffered so the sequential scanner can emit the
stream in rsync's flist order. `name` is the raw dirent name (owned here);
`entry` is the scanner_inspect_entry() result whose path/link_target are owned
when `inspection == 1`; `inspection` is that call's return code (1 keep,
0 skip, <0 fatal). */
typedef struct {
char* name;
ScannerEntry entry;
int inspection;
} SortedEntry;
/* --one-file-system (-x) decision. Only directories can carry a different
* device than their parent (mount points), so this is checked when a child
* directory is about to be descended into. */
@@ -687,6 +698,114 @@ skip:
return 0;
}
static void sorted_entry_destroy(void* item) {
SortedEntry* se = (SortedEntry*)item;
if (!se)
return;
free(se->name);
free(se->entry.path);
free(se->entry.link_target);
}
/* rsync flist order within one directory: non-directories first, then
directories, each group by ascending name. strcmp() compares as unsigned
char, matching rsync's f_name_cmp(). */
static int sorted_entry_cmp(const void* a, const void* b) {
const SortedEntry* x = (const SortedEntry*)a;
const SortedEntry* y = (const SortedEntry*)b;
bool x_dir = x->inspection > 0 && x->entry.is_directory;
bool y_dir = y->inspection > 0 && y->entry.is_directory;
if (x_dir != y_dir)
return x_dir ? 1 : -1;
return strcmp(x->name, y->name);
}
static void scanner_free_sorted(DirectoryScanner* scanner) {
SortedEntry* entries = (SortedEntry*)scanner->sorted_entries;
for (size_t i = 0; i < scanner->sorted_count; i++)
sorted_entry_destroy(&entries[i]);
free(entries);
scanner->sorted_entries = NULL;
scanner->sorted_count = 0;
scanner->sorted_index = 0;
}
/* Read every entry of the open directory, inspect it once and store it sorted in
rsync's flist order. Returns 0 on success, -1 on a fatal error (the caller
aborts the scan). */
static int scanner_buffer_current_directory(DirectoryScanner* scanner) {
size_t capacity = 64;
size_t count = 0;
SortedEntry* entries = malloc(capacity * sizeof(*entries));
if (!entries) {
scanner->failed = true;
return -1;
}
const struct dirent* dirent;
while ((dirent = readdir(scanner->current_dir)) != NULL) {
if (strcmp(dirent->d_name, ".") == 0 || strcmp(dirent->d_name, "..") == 0)
continue;
if (count == capacity) {
size_t next = capacity * 2;
SortedEntry* grown = realloc(entries, next * sizeof(*entries));
if (!grown) {
scanner->failed = true;
break;
}
entries = grown;
capacity = next;
}
char* name = str_dup(dirent->d_name);
if (!name) {
scanner->failed = true;
break;
}
char* link_rel = child_rel_path(scanner->current_rel, dirent->d_name);
if (!link_rel) {
free(name);
scanner->failed = true;
break;
}
int inspection = scanner_inspect_entry(&scanner->options, scanner->current_path, link_rel,
dirent->d_name, &entries[count].entry);
free(link_rel);
if (inspection < 0) {
free(name);
scanner->failed = true;
break;
}
entries[count].name = name;
entries[count].inspection = inspection;
count++;
}
if (scanner->failed) {
for (size_t i = 0; i < count; i++)
sorted_entry_destroy(&entries[i]);
free(entries);
return -1;
}
qsort(entries, count, sizeof(*entries), sorted_entry_cmp);
scanner->sorted_entries = entries;
scanner->sorted_count = count;
scanner->sorted_index = 0;
return 0;
}
/* Push this directory's collected child directories onto the LIFO stack in
reverse so the first (ascending) child is popped first (depth-first). */
static void scanner_push_pending_dirs(DirectoryScanner* scanner) {
ArrayList* pending = (ArrayList*)scanner->pending_dirs;
if (!pending)
return;
for (int i = pending->size - 1; i >= 0; i--) {
if (!queue_push(scanner->directories, pending->items[i])) {
dir_entry_destroy(pending->items[i]);
scanner->failed = true;
}
}
pending->size = 0;
}
DirectoryScanner* directory_scanner_create_with_options(const char* root_directory,
const ScannerOptions* options) {
if (!root_directory || !options)
@@ -704,12 +823,22 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
free(scanner);
return NULL;
}
scanner->pending_dirs = array_list_create(NULL);
if (!scanner->pending_dirs) {
queue_destroy(scanner->directories);
free(scanner);
return NULL;
}
scanner->current_dir = NULL;
scanner->current_path = NULL;
scanner->current_depth = 0;
scanner->failed = false;
scanner->sorted_entries = NULL;
scanner->sorted_count = 0;
scanner->sorted_index = 0;
scanner->root_path = str_dup(root_directory);
if (!scanner->root_path) {
array_list_delete(scanner->pending_dirs);
queue_destroy(scanner->directories);
free(scanner);
return NULL;
@@ -729,6 +858,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
scanner->filter_nodes = array_list_create(filter_node_destroy);
if (!scanner->filter_nodes) {
free(scanner->root_path);
array_list_delete(scanner->pending_dirs);
queue_destroy(scanner->directories);
free(scanner);
return NULL;
@@ -739,6 +869,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
if (stat(root_directory, &root_stats) != 0) {
log_perror("Could not stat source directory");
free(scanner->root_path);
array_list_delete(scanner->pending_dirs);
queue_destroy(scanner->directories);
array_list_delete(scanner->filter_nodes);
free(scanner);
@@ -757,6 +888,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
if (!queue_enqueue(scanner->directories, root)) {
dir_entry_destroy(root);
free(scanner->root_path);
array_list_delete(scanner->pending_dirs);
queue_destroy(scanner->directories);
array_list_delete(scanner->filter_nodes);
free(scanner);
@@ -808,6 +940,13 @@ void directory_scanner_destroy(DirectoryScanner* scanner) {
free(scanner->current_path);
free(scanner->current_rel);
free(scanner->root_path);
scanner_free_sorted(scanner);
ArrayList* pending = (ArrayList*)scanner->pending_dirs;
if (pending) {
for (int i = 0; i < pending->size; i++)
dir_entry_destroy(pending->items[i]);
array_list_delete(pending);
}
array_list_delete(scanner->filter_nodes);
array_list_delete(scanner->dirs_batch);
queue_destroy(scanner->directories);
@@ -975,7 +1114,7 @@ static int open_next_directory(DirectoryScanner* scanner) {
scanner->current_path = NULL;
while (!queue_is_empty(scanner->directories)) {
DirEntry* de = (DirEntry*)queue_dequeue(scanner->directories);
DirEntry* de = (DirEntry*)queue_pop(scanner->directories);
scanner->current_path = de->path;
scanner->current_depth = de->depth;
/* The seed directory inherits the scanner's configured context (the root
@@ -1060,6 +1199,14 @@ static int open_next_directory(DirectoryScanner* scanner) {
scanner->failed = true;
return -1;
}
/* Buffer and sort this directory's entries in rsync's flist order. */
if (scanner_buffer_current_directory(scanner) != 0) {
closedir(scanner->current_dir);
scanner->current_dir = NULL;
free(scanner->current_path);
scanner->current_path = NULL;
return -1;
}
return 1;
}
return 0;
@@ -1415,8 +1562,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
break;
}
const struct dirent* entry = readdir(scanner->current_dir);
if (entry == NULL) {
if (scanner->sorted_index >= scanner->sorted_count) {
/* The directory is exhausted: if nothing was transferred or descended
from it, recreate it at the destination as an explicit entry. */
if (scanner->options.emit_empty_dirs && !scanner->current_dir_produced &&
@@ -1425,10 +1571,12 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
if (!scanner_emit_empty_dir(scanner, chunk_data))
scanner->failed = true;
}
scanner_push_pending_dirs(scanner);
closedir(scanner->current_dir);
scanner->current_dir = NULL;
free(scanner->current_path);
scanner->current_path = NULL;
scanner_free_sorted(scanner);
if (scanner->failed) {
array_list_delete(chunk_data);
return NULL;
@@ -1436,26 +1584,15 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
continue;
}
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
continue;
SortedEntry* sorted = &((SortedEntry*)scanner->sorted_entries)[scanner->sorted_index++];
const char* name = sorted->name;
ScannerEntry* inspected = &sorted->entry;
int inspection = sorted->inspection;
ScannerEntry inspected;
char* link_rel = child_rel_path(scanner->current_rel, entry->d_name);
if (!link_rel) {
scanner->failed = true;
break;
}
int inspection = scanner_inspect_entry(&scanner->options, scanner->current_path, link_rel,
entry->d_name, &inspected);
free(link_rel);
if (inspection < 0) {
scanner->failed = true;
break;
}
if (inspection == 0) {
/* A dereferenced symlink with no referent is a partial-transfer error
(rsync exit 23): record it as a non-fatal scan I/O error. */
if (inspected.referent_error)
if (inspected->referent_error)
scanner->io_error = true;
/* A user-selection exclude protects its destination mirror from --delete
unless --delete-excluded; a size prune is always protected. Other
@@ -1463,23 +1600,23 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
--files-from the protected prefix must be the entry's bare relative
wire path, not its source path (which would not match the destination
layout and would leave the mirror deletable). */
if (inspected.excluded) {
if (inspected->excluded) {
char* protected_path;
if (scanner->relative_mode) {
protected_path = child_rel_path(scanner->current_rel, entry->d_name);
protected_path = child_rel_path(scanner->current_rel, name);
} else if (scanner->options.relative_prefix) {
char* relc = child_rel_path(scanner->current_rel, entry->d_name);
char* relc = child_rel_path(scanner->current_rel, name);
protected_path =
relc ? scanner_prefix_send_path(scanner->options.relative_prefix, relc) : NULL;
free(relc);
} else {
protected_path = path_cat(scanner->current_path, entry->d_name);
protected_path = path_cat(scanner->current_path, name);
}
if (!protected_path) {
scanner->failed = true;
break;
}
if (inspected.size_excluded)
if (inspected->size_excluded)
scanner_record_size_skipped(scanner, protected_path);
else
scanner_record_excluded(scanner, protected_path);
@@ -1487,23 +1624,22 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
}
continue;
}
char* cur_path = inspected.path;
struct stat stats = inspected.stats;
char* cur_path = inspected->path;
struct stat stats = inspected->stats;
/* --files-from allow-set and the filter layer apply to files and to
* directories (an excluded directory is not descended into). */
bool is_dir = inspected.is_directory;
char* rel = child_rel_path(scanner->current_rel, entry->d_name);
bool is_dir = inspected->is_directory;
char* rel = child_rel_path(scanner->current_rel, name);
if (!rel) {
free(cur_path);
scanner->failed = true;
break;
}
bool protect = false;
bool passes_selection = entry_passes_selection(
scanner->options.file_list, scanner->options.base_filters, scanner->current_node, rel,
entry->d_name, is_dir, scanner->options.per_dir_filters,
scanner->options.exclude_per_dir_filter_files, &protect);
scanner->options.file_list, scanner->options.base_filters, scanner->current_node, rel, name,
is_dir, scanner->options.per_dir_filters, scanner->options.exclude_per_dir_filter_files,
&protect);
/* A sender-side hide leaves the entry out of the transfer; an independent
receiver-side protect rule keeps a transferred entry's destination mirror
from being deleted. Both are recorded in the same protection set. */
@@ -1525,7 +1661,6 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
char* wrel = scanner_prefix_send_path(scanner->options.relative_prefix, rel);
if (!wrel) {
free(rel);
free(cur_path);
scanner->failed = true;
break;
}
@@ -1542,13 +1677,11 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
char* rel_copy = needs_rel ? str_dup(rel) : NULL;
free(rel);
if (rel_copy == NULL && needs_rel) {
free(cur_path);
scanner->failed = true;
break;
}
if (!passes_selection) {
free(rel_copy);
free(cur_path);
continue;
}
@@ -1563,12 +1696,10 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
File* mount = scanner_build_dir_file(cur_path, &stats, &scanner->options);
if (mount == NULL || !array_list_add(chunk_data, mount)) {
file_destroy(mount);
free(cur_path);
scanner->failed = true;
break;
}
scanner->current_dir_produced = true;
free(cur_path);
continue;
}
/* --list-only: list directory entries too (rsync prints them), even
@@ -1577,7 +1708,6 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
File* dir = scanner_build_dir_file(cur_path, &stats, &scanner->options);
if (dir == NULL || !array_list_add(chunk_data, dir)) {
file_destroy(dir);
free(cur_path);
scanner->failed = true;
break;
}
@@ -1586,32 +1716,29 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
int next_depth = scanner->current_depth + 1;
if (scanner->options.max_depth <= 0 || next_depth < scanner->options.max_depth) {
DirEntry* de = dir_entry_create(cur_path, next_depth, scanner->current_node);
if (!de || !queue_enqueue(scanner->directories, de)) {
if (!de || !array_list_add((ArrayList*)scanner->pending_dirs, de)) {
dir_entry_destroy(de);
scanner->failed = true;
}
}
free(cur_path);
} else {
if (scanner->options.max_depth > 0 &&
scanner->current_depth + 1 > scanner->options.max_depth) {
free(rel_copy);
free(cur_path);
continue;
}
File* file = file_create(cur_path);
free(cur_path);
if (file == NULL) {
free(rel_copy);
free(inspected.link_target);
inspected.link_target = NULL;
free(inspected->link_target);
inspected->link_target = NULL;
scanner->failed = true;
continue;
}
if (inspected.is_symlink) {
if (inspected->is_symlink) {
file->is_symlink = true;
file->symlink_target = inspected.link_target;
inspected.link_target = NULL;
file->symlink_target = inspected->link_target;
inspected->link_target = NULL;
} else {
file->data->size = stats.st_size;
}