Release v2.26.0 #284

Merged
TapTap merged 210 commits from dev into main 2026-09-18 19:05:52 +02:00
9 changed files with 127 additions and 190 deletions
Showing only changes of commit 99f8045105 - Show all commits
+38 -29
View File
@@ -44,6 +44,10 @@
receiver's RECEIVER_QUEUE_MAX_BYTES). */
#define SENDER_QUEUE_MAX_BYTES (MAX_CONNECTION_MEMORY - 2 * MAX_CHUNK_SIZE)
/* One mebibyte in bytes; the unit used by the --stats/--progress lines.
Always cast to double when dividing so the output stays fractional. */
#define BYTES_PER_MIB (1024ULL * 1024ULL)
/* Forward declaration for progress-reporting thread used in multithreaded send. */
static int progress_thread_fn(void* arg);
@@ -51,10 +55,32 @@ static const char* display_bytes(unsigned long long bytes, bool human_readable,
size_t buffer_size) {
if (human_readable && format_human_bytes(bytes, buffer, buffer_size))
return buffer;
snprintf(buffer, buffer_size, "%.1f MB", bytes / 1048576.0);
snprintf(buffer, buffer_size, "%.1f MB", (double)bytes / (double)BYTES_PER_MIB);
return buffer;
}
/* Print the canonical `--stats` line. Shared by the single-threaded and
multithreaded send paths so both honor --stats, --human-readable and --quiet
identically; `start` marks the beginning of the transfer for the rate. */
static void report_transfer_stats(const Config* config, int total_files,
unsigned long long total_bytes, time_t start) {
if (!config->stats || config->quiet)
return;
double elapsed = difftime(time(NULL), start);
double rate = elapsed > 0.0 ? (double)total_bytes / ((double)BYTES_PER_MIB * elapsed) : 0.0;
if (config->human_readable) {
char total_buffer[32];
char rate_buffer[32];
fprintf(stderr, "Stats: %d files, %s, %s/s\n", total_files,
display_bytes(total_bytes, true, total_buffer, sizeof(total_buffer)),
display_bytes((unsigned long long)(rate * (double)BYTES_PER_MIB), true, rate_buffer,
sizeof(rate_buffer)));
} else {
fprintf(stderr, "Stats: %d files, %.1f MB, %.1f MB/s\n", total_files,
(double)total_bytes / (double)BYTES_PER_MIB, rate);
}
}
/* Compiled scanner inputs that are shared read-only across scanner instances
* and, in -m mode, across worker threads. `base_filters` owns the compiled
* command-line + -C rules; the FileListSet allow-set lives in the Config.
@@ -688,7 +714,7 @@ static int send_dry_run_manifest(const Config* config) {
printf("Total: %d files, %s\n", file_count,
display_bytes(total_bytes, true, size_buffer, sizeof(size_buffer)));
else
printf("Total: %d files, %.1f MB\n", file_count, total_bytes / 1048576.0);
printf("Total: %d files, %.1f MB\n", file_count, (double)total_bytes / (double)BYTES_PER_MIB);
}
return 0;
}
@@ -1425,12 +1451,9 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
return 0;
}
int send_chunk(Client* client, Chunk* chunk, Config* config) {
return send_chunk_with_removal(client, chunk, config, NULL);
}
static int send_chunks_multithreaded(void* pipeline_context) {
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
time_t start = time(NULL);
Client* client = connect_transfer_client(context->config);
if (!client) {
if (context->config->transport == TRANSPORT_TCP)
@@ -1578,10 +1601,9 @@ static int send_chunks_multithreaded(void* pipeline_context) {
int total_files = context->total_files;
unsigned long long total_bytes = context->total_bytes;
mtx_unlock(&context->mutex_progress);
if (context->config->stats)
fprintf(stderr, "Stats: %d files, %.1f MB\n", total_files, total_bytes / 1048576.0);
report_transfer_stats(context->config, total_files, total_bytes, start);
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
total_bytes / 1048576.0);
(double)total_bytes / (double)BYTES_PER_MIB);
disconnect_transfer_client(client);
mark_sender_done(context);
protocol_session_unbind();
@@ -1754,17 +1776,18 @@ static int load_files_multithreaded(void* pipeline_context) {
static void print_transfer_progress(unsigned long long total_bytes, time_t start,
const char* suffix, bool human_readable) {
double elapsed = difftime(time(NULL), start);
double rate = elapsed > 0.0 ? total_bytes / (1048576.0 * elapsed) : 0.0;
double rate = elapsed > 0.0 ? (double)total_bytes / ((double)BYTES_PER_MIB * elapsed) : 0.0;
if (human_readable) {
char total_buffer[32];
char rate_buffer[32];
fprintf(stderr, "\rSent %s (%s/s) %s",
display_bytes(total_bytes, true, total_buffer, sizeof(total_buffer)),
display_bytes((unsigned long long)(rate * 1048576.0), true, rate_buffer,
display_bytes((unsigned long long)(rate * (double)BYTES_PER_MIB), true, rate_buffer,
sizeof(rate_buffer)),
suffix);
} else {
fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) %s", total_bytes / 1048576.0, rate, suffix);
fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) %s", (double)total_bytes / (double)BYTES_PER_MIB,
rate, suffix);
}
fflush(stderr);
}
@@ -2132,23 +2155,9 @@ int send_files(Config* config) {
remove_transferred_sources(config, remove_sources);
if (config->show_progress && !config->quiet)
print_transfer_progress(total_bytes, start, "Done.\n", config->human_readable);
if (config->stats && !config->quiet) {
double elapsed_total = difftime(time(NULL), start);
double rate = elapsed_total > 0 ? total_bytes / (1048576.0 * elapsed_total) : 0;
if (config->human_readable) {
char total_buffer[32];
char rate_buffer[32];
fprintf(stderr, "Stats: %d files, %s, %s/s\n", total_files,
display_bytes(total_bytes, true, total_buffer, sizeof(total_buffer)),
display_bytes((unsigned long long)(rate * 1048576.0), true, rate_buffer,
sizeof(rate_buffer)));
} else {
fprintf(stderr, "Stats: %d files, %.1f MB, %.1f MB/s\n", total_files, total_bytes / 1048576.0,
rate);
}
}
report_transfer_stats(config, total_files, total_bytes, start);
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
total_bytes / 1048576.0);
(double)total_bytes / (double)BYTES_PER_MIB);
/* --ignore-errors: an unreadable source directory was skipped but the run
still completed (and deleted); report the run as errored like rsync does. */
ret = (ok && !had_scan_io) ? 0 : 1;
@@ -2237,7 +2246,7 @@ int send_files_multithreaded(Config** config_ptr) {
context->missing_args = missing_args;
missing_args = NULL; /* owned by the context from here on */
pipeline_context_sender_set_queue_byte_limit(context, SENDER_QUEUE_MAX_BYTES);
*config_ptr = NULL; /* context now owns config through all remaining paths */
/* The context borrows `config`; the caller (main) still owns and frees it. */
struct timespec now_mono;
if (clock_gettime(CLOCK_MONOTONIC, &now_mono) != 0) {
now_mono.tv_sec = 0;
+3 -2
View File
@@ -5,9 +5,10 @@
#include "config.h"
#include "transport_tcp.h"
int send_chunk(Client* client, Chunk* chunk, Config* config);
/* Both sender entry points BORROW `config` for the duration of the call; they
* never free it, and the caller retains ownership (freeing it with
* config_delete() once the call returns). */
int send_files(Config* config);
/* Takes ownership only when *config is set to NULL on return. */
int send_files_multithreaded(Config** config);
/* Phase 6 residual-batch (client-only). See client_send.c. */
int write_batch_from_source(const Config* config, const char* batch_path);
+57 -108
View File
@@ -177,7 +177,7 @@ static bool entry_passes_selection(const FileListSet* file_list, const FilterRul
/* Best-effort capture of the file's whitelisted xattrs (-X/-A). A failure to
* read xattrs is non-fatal: the file is transferred without them. */
static void scanner_capture_xattrs(const DirectoryScanner* scanner, File* file) {
if (!scanner || !file || !(scanner->preserve_xattrs || scanner->preserve_acls))
if (!scanner || !file || !(scanner->options.preserve_xattrs || scanner->options.preserve_acls))
return;
file->xattrs = xattr_capture_path(file->path);
}
@@ -263,10 +263,10 @@ static bool excluded_sink_append(ArrayList* list, mtx_t* mtx, const char* rel) {
how manifest keep entries are stored), so the receiver's walker prefixes
match the destination layout. An allocation failure is a fatal scan error. */
static void scanner_record_excluded(DirectoryScanner* scanner, const char* fs_path) {
if (!scanner->excluded_paths || !fs_path)
if (!scanner->options.excluded_paths || !fs_path)
return;
const char* rel = *fs_path == '/' ? fs_path + 1 : fs_path;
if (!excluded_sink_append(scanner->excluded_paths, scanner->excluded_mutex, rel))
if (!excluded_sink_append(scanner->options.excluded_paths, scanner->options.excluded_mutex, rel))
scanner->failed = true;
}
@@ -274,7 +274,7 @@ static void scanner_record_excluded(DirectoryScanner* scanner, const char* fs_pa
* context, returning the context used for this directory's entries. On a parse
* error the scanner is marked failed. Returns 0 on success, -1 on failure. */
static int open_directory_filter_context(DirectoryScanner* scanner, const FilterNode* inherited) {
if (!scanner->per_dir_filters) {
if (!scanner->options.per_dir_filters) {
scanner->current_node = (FilterNode*)inherited;
return 0;
}
@@ -436,6 +436,11 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
DirectoryScanner* scanner = calloc(1, sizeof(DirectoryScanner));
if (scanner == NULL)
return NULL;
/* One copy of the scan inputs; normalize chunk_size as the old field-by-field
copy did. */
scanner->options = *options;
if (scanner->options.chunk_size == 0)
scanner->options.chunk_size = DESIRED_CHUNK_SIZE;
scanner->directories = queue_create(100, dir_entry_destroy);
if (!scanner->directories) {
free(scanner);
@@ -443,31 +448,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
}
scanner->current_dir = NULL;
scanner->current_path = NULL;
scanner->use_metadata = options->use_metadata;
scanner->preserve_atimes = options->preserve_atimes;
scanner->preserve_crtimes = options->preserve_crtimes;
scanner->preserve_xattrs = options->preserve_xattrs;
scanner->preserve_acls = options->preserve_acls;
scanner->chunk_size = options->chunk_size > 0 ? options->chunk_size : DESIRED_CHUNK_SIZE;
scanner->exclude_patterns = options->exclude_patterns;
scanner->exclude_count = options->exclude_count;
scanner->include_patterns = options->include_patterns;
scanner->include_count = options->include_count;
scanner->max_size = options->max_size;
scanner->min_size = options->min_size;
scanner->max_depth = options->max_depth;
scanner->current_depth = 0;
scanner->follow_symlinks = options->follow_symlinks;
scanner->copy_links = options->copy_links;
scanner->safe_links = options->safe_links;
scanner->copy_unsafe_links = options->copy_unsafe_links;
scanner->copy_dirlinks = options->copy_dirlinks;
scanner->munge_links = options->munge_links;
scanner->checksum = options->checksum;
scanner->one_file_system = options->one_file_system;
scanner->preserve_devices = options->preserve_devices;
scanner->preserve_specials = options->preserve_specials;
scanner->copy_devices = options->copy_devices;
scanner->failed = false;
scanner->root_path = str_dup(root_directory);
if (!scanner->root_path) {
@@ -479,28 +460,14 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
scanner->at_seed_dir = true;
scanner->seed_node = NULL;
scanner->current_node = NULL;
scanner->file_list = options->file_list;
scanner->base_filters = options->base_filters;
scanner->per_dir_filters = options->per_dir_filters;
scanner->excluded_paths = options->excluded_paths;
scanner->excluded_mutex = options->excluded_mutex;
scanner->ignore_io_errors = options->ignore_io_errors;
scanner->ignore_missing_args = options->ignore_missing_args;
scanner->io_error = false;
scanner->dirs_mode = options->dirs;
scanner->relative_mode = options->relative && options->file_list != NULL;
scanner->hardlinks = options->hardlinks;
scanner->prune_empty_dirs = options->prune_empty_dirs;
scanner->stop_condition = options->stop_condition;
scanner->capture_dir_times = options->capture_dir_times;
scanner->dir_entries = options->dir_entries;
scanner->dir_entries_mutex = options->dir_entries_mutex;
scanner->dirs_root_emitted = false;
scanner->list_index = 0;
scanner->dirs_batch = NULL;
scanner->dirs_batch_size = 0;
scanner->filter_nodes = NULL;
if (scanner->base_filters || scanner->per_dir_filters) {
if (scanner->options.base_filters || scanner->options.per_dir_filters) {
scanner->filter_nodes = array_list_create(filter_node_destroy);
if (!scanner->filter_nodes) {
free(scanner->root_path);
@@ -509,7 +476,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
return NULL;
}
}
if (scanner->one_file_system) {
if (scanner->options.one_file_system) {
struct stat root_stats;
if (stat(root_directory, &root_stats) != 0) {
log_perror("Could not stat source directory");
@@ -715,7 +682,7 @@ static int open_next_directory(DirectoryScanner* scanner) {
scanner->current_rel = NULL;
free(scanner->current_path);
scanner->current_path = NULL;
if (!scanner->ignore_io_errors || is_root_seed) {
if (!scanner->options.ignore_io_errors || is_root_seed) {
scanner->failed = true;
return -1;
}
@@ -729,10 +696,11 @@ static int open_next_directory(DirectoryScanner* scanner) {
scanner->current_path = NULL;
return -1;
}
if (scanner->capture_dir_times &&
!scanner_capture_dir_time(scanner->dir_entries, scanner->dir_entries_mutex,
if (scanner->options.capture_dir_times &&
!scanner_capture_dir_time(scanner->options.dir_entries, scanner->options.dir_entries_mutex,
scanner->root_path, scanner->current_path, scanner->relative_mode,
scanner->preserve_atimes, scanner->preserve_crtimes)) {
scanner->options.preserve_atimes,
scanner->options.preserve_crtimes)) {
closedir(scanner->current_dir);
scanner->current_dir = NULL;
free(scanner->current_path);
@@ -773,9 +741,9 @@ static File* dirs_root_dir_file(DirectoryScanner* scanner) {
return NULL;
}
file->is_dir = true;
if (scanner->use_metadata) {
file->metadata = file_metadata_create(scanner->root_path, &st, scanner->preserve_atimes,
scanner->preserve_crtimes);
if (scanner->options.use_metadata) {
file->metadata = file_metadata_create(scanner->root_path, &st, scanner->options.preserve_atimes,
scanner->options.preserve_crtimes);
if (!file->metadata) {
file_destroy(file);
scanner->failed = true;
@@ -808,7 +776,7 @@ static File* dirs_file_for_entry(DirectoryScanner* scanner, const char* entry) {
missing argument and is skipped here, exactly as the recursive scan skips
nothing (missing entries never appear there). Without the flags it stays
a hard pre-transfer error. */
if (scanner->ignore_missing_args) {
if (scanner->options.ignore_missing_args) {
log_info_message(LOG_INFO_MISC, "skipping missing --files-from entry '%s'", entry);
free(abs_path);
return NULL;
@@ -822,8 +790,8 @@ static File* dirs_file_for_entry(DirectoryScanner* scanner, const char* entry) {
if (S_ISLNK(link_stats.st_mode)) {
/* A symlink is transferred (following its referent) only when a link
resolution option is active, mirroring the regular scanner. */
bool resolve = scanner->follow_symlinks || scanner->copy_links || scanner->safe_links ||
scanner->copy_unsafe_links;
bool resolve = scanner->options.follow_symlinks || scanner->options.copy_links ||
scanner->options.safe_links || scanner->options.copy_unsafe_links;
if (!resolve || stat(abs_path, &effective) != 0) {
free(abs_path);
return NULL;
@@ -851,9 +819,9 @@ static File* dirs_file_for_entry(DirectoryScanner* scanner, const char* entry) {
return NULL;
}
}
if (scanner->use_metadata) {
file->metadata = file_metadata_create(file->path, &effective, scanner->preserve_atimes,
scanner->preserve_crtimes);
if (scanner->options.use_metadata) {
file->metadata = file_metadata_create(file->path, &effective, scanner->options.preserve_atimes,
scanner->options.preserve_crtimes);
if (!file->metadata) {
file_destroy(file);
scanner->failed = true;
@@ -884,18 +852,18 @@ static bool dirs_source_dir_is_empty(const char* path) {
/* The next File from the --dirs generator, or NULL when exhausted. */
static File* dirs_next_file(DirectoryScanner* scanner) {
if (!scanner->file_list) {
if (!scanner->options.file_list) {
if (scanner->dirs_root_emitted)
return NULL;
scanner->dirs_root_emitted = true;
/* --prune-empty-dirs: a physically empty source directory's explicit entry
would only create an empty destination directory, so it is omitted. */
if (scanner->prune_empty_dirs && dirs_source_dir_is_empty(scanner->root_path))
if (scanner->options.prune_empty_dirs && dirs_source_dir_is_empty(scanner->root_path))
return NULL;
return dirs_root_dir_file(scanner);
}
while (scanner->list_index < scanner->file_list->count) {
const char* entry = scanner->file_list->entries[scanner->list_index++];
while (scanner->list_index < scanner->options.file_list->count) {
const char* entry = scanner->options.file_list->entries[scanner->list_index++];
File* file = dirs_file_for_entry(scanner, entry);
if (scanner->failed)
return NULL;
@@ -922,8 +890,9 @@ static Chunk* dirs_flush_batch(DirectoryScanner* scanner) {
}
static Chunk* directory_scanner_next_dirs(DirectoryScanner* scanner) {
while (scanner->dirs_batch == NULL || scanner->dirs_batch_size <= scanner->chunk_size) {
if (scanner->stop_condition && stop_condition_reached(scanner->stop_condition)) {
while (scanner->dirs_batch == NULL || scanner->dirs_batch_size <= scanner->options.chunk_size) {
if (scanner->options.stop_condition &&
stop_condition_reached(scanner->options.stop_condition)) {
Chunk* leftover = dirs_flush_batch(scanner);
if (leftover)
chunk_destroy(leftover);
@@ -962,7 +931,7 @@ static Chunk* directory_scanner_next_dirs(DirectoryScanner* scanner) {
}
Chunk* directory_scanner_next(DirectoryScanner* scanner) {
if (scanner && scanner->dirs_mode)
if (scanner && scanner->options.dirs)
return directory_scanner_next_dirs(scanner);
ArrayList* chunk_data = array_list_create(file_destroy);
if (!chunk_data) {
@@ -972,7 +941,8 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
unsigned long long chunk_data_size = 0;
while (1) {
if (scanner->stop_condition && stop_condition_reached(scanner->stop_condition)) {
if (scanner->options.stop_condition &&
stop_condition_reached(scanner->options.stop_condition)) {
array_list_delete(chunk_data);
return NULL;
}
@@ -996,34 +966,9 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
continue;
ScannerOptions options = {
.use_metadata = scanner->use_metadata,
.chunk_size = scanner->chunk_size,
.exclude_patterns = scanner->exclude_patterns,
.exclude_count = scanner->exclude_count,
.include_patterns = scanner->include_patterns,
.include_count = scanner->include_count,
.max_size = scanner->max_size,
.min_size = scanner->min_size,
.max_depth = scanner->max_depth,
.num_threads = 0,
.follow_symlinks = scanner->follow_symlinks,
.copy_links = scanner->copy_links,
.safe_links = scanner->safe_links,
.copy_unsafe_links = scanner->copy_unsafe_links,
.copy_dirlinks = scanner->copy_dirlinks,
.munge_links = scanner->munge_links,
.checksum = scanner->checksum,
.one_file_system = scanner->one_file_system,
.file_list = scanner->file_list,
.base_filters = scanner->base_filters,
.per_dir_filters = scanner->per_dir_filters,
.dirs = false,
.relative = false,
};
ScannerEntry inspected;
int inspection = scanner_inspect_entry(&options, scanner->current_path, scanner->current_path,
entry->d_name, &inspected);
int inspection = scanner_inspect_entry(&scanner->options, scanner->current_path,
scanner->current_path, entry->d_name, &inspected);
if (inspection < 0) {
scanner->failed = true;
break;
@@ -1055,16 +1000,17 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
scanner->failed = true;
break;
}
bool passes_selection =
entry_passes_selection(scanner->file_list, scanner->base_filters, scanner->current_node,
rel, entry->d_name, is_dir, scanner->per_dir_filters);
bool passes_selection = entry_passes_selection(
scanner->options.file_list, scanner->options.base_filters, scanner->current_node, rel,
entry->d_name, is_dir, scanner->options.per_dir_filters);
if (!passes_selection) {
/* --files-from subset pruning is not a filter exclusion: its delete
semantics stay keep-set-only (an unlisted source path is treated as
absent, so its destination mirror is a deletable extra). A rule-based
exclusion is recorded as a protected prefix. -R + --files-from bare
wire paths are never recorded (see ScannerOptions.excluded_paths). */
bool files_from_prune = scanner->file_list && !file_list_affects(scanner->file_list, rel);
bool files_from_prune =
scanner->options.file_list && !file_list_affects(scanner->options.file_list, rel);
if (!files_from_prune && !scanner->relative_mode)
scanner_record_excluded(scanner, cur_path);
}
@@ -1085,12 +1031,13 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
if (is_dir) {
free(rel_copy);
if (!scanner_same_filesystem(scanner->one_file_system, scanner->root_dev, stats.st_dev)) {
if (!scanner_same_filesystem(scanner->options.one_file_system, scanner->root_dev,
stats.st_dev)) {
free(cur_path);
continue;
}
int next_depth = scanner->current_depth + 1;
if (scanner->max_depth <= 0 || next_depth < scanner->max_depth) {
if (scanner->options.max_depth <= 0 || next_depth < scanner->options.max_depth) {
DirEntry* de = dir_entry_create(cur_path, next_depth, scanner->current_node);
if (!de || !queue_enqueue(scanner->directories, de)) {
dir_entry_destroy(de);
@@ -1099,7 +1046,8 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
}
free(cur_path);
} else {
if (scanner->max_depth > 0 && scanner->current_depth + 1 > scanner->max_depth) {
if (scanner->options.max_depth > 0 &&
scanner->current_depth + 1 > scanner->options.max_depth) {
free(rel_copy);
free(cur_path);
continue;
@@ -1126,13 +1074,14 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
}
/* --devices/--specials: a device/FIFO/socket entry marked for preservation
becomes a node to recreate (is_special, no data, rdev captured). */
scanner_prepare_special(scanner->preserve_devices, scanner->preserve_specials, file, &stats);
if (scanner->hardlinks && S_ISREG(stats.st_mode))
scanner_assign_hardlink(scanner, scanner->hardlinks, file, &stats);
if (scanner->use_metadata)
file->metadata = file_metadata_create(file->path, &stats, scanner->preserve_atimes,
scanner->preserve_crtimes);
if (scanner->use_metadata && !file->metadata) {
scanner_prepare_special(scanner->options.preserve_devices, scanner->options.preserve_specials,
file, &stats);
if (scanner->options.hardlinks && S_ISREG(stats.st_mode))
scanner_assign_hardlink(scanner, scanner->options.hardlinks, file, &stats);
if (scanner->options.use_metadata)
file->metadata = file_metadata_create(file->path, &stats, scanner->options.preserve_atimes,
scanner->options.preserve_crtimes);
if (scanner->options.use_metadata && !file->metadata) {
free(rel_copy);
file_destroy(file);
scanner->failed = true;
@@ -1147,7 +1096,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
break;
}
chunk_data_size += file->data->size;
if (chunk_data_size > scanner->chunk_size) {
if (chunk_data_size > scanner->options.chunk_size) {
free(rel_copy);
Chunk* result = chunk_data_to_chunk(chunk_data);
if (!result)
@@ -1211,7 +1160,7 @@ static int parallel_worker_thread(void* arg) {
free(ds->root_path);
ds->root_path = str_dup(wa->root_dir);
ds->seed_node = wa->ps->root_filter_node;
ds->excluded_mutex = &wa->ps->result_mutex;
ds->options.excluded_mutex = &wa->ps->result_mutex;
Chunk* chunk;
while ((chunk = directory_scanner_next(ds)) != NULL) {
if (!queue_enqueue_multithreaded_cancel(wa->ps->result_queue, chunk, &wa->ps->result_mutex,
+5 -49
View File
@@ -117,37 +117,16 @@ typedef struct {
typedef struct FilterNode FilterNode;
typedef struct {
/* Scan inputs, copied once at create time. Everything that is also a
ScannerOptions field lives here (with the normalized chunk_size); only
scanner-owned bookkeeping stays as direct members below. */
ScannerOptions options;
Queue* directories;
DIR* current_dir;
char* current_path;
bool use_metadata;
bool preserve_atimes;
bool preserve_crtimes;
bool preserve_xattrs;
bool preserve_acls;
unsigned long long chunk_size;
char** exclude_patterns;
int exclude_count;
char** include_patterns;
int include_count;
unsigned long long max_size;
unsigned long long min_size;
int max_depth;
int current_depth;
bool follow_symlinks;
bool copy_links;
bool safe_links;
bool copy_unsafe_links;
bool copy_dirlinks;
bool munge_links;
bool checksum;
bool one_file_system;
dev_t root_dev;
bool failed;
/* Phase 4 special/devices (see ScannerOptions). */
bool preserve_devices;
bool preserve_specials;
bool copy_devices;
/* Phase 2 (files-from / filter layer). */
char* root_path; /* transfer root (fs path) for rel computation */
char* current_rel; /* rel path of the open directory ("" == root) */
@@ -155,40 +134,17 @@ typedef struct {
FilterNode* seed_node; /* inherited context of the seed dir, or NULL */
FilterNode* current_node; /* filter context of the open directory */
ArrayList* filter_nodes; /* owned FilterNode arena (may be NULL) */
const FileListSet* file_list;
const FilterRuleList* base_filters;
bool per_dir_filters;
/* --dirs / -R state for the directory-entry generator (dirs_mode replaces
/* --dirs / -R state for the directory-entry generator (options.dirs replaces
the recursive scan). */
bool dirs_mode;
bool relative_mode; /* file_list && relative: send bare relative wire paths */
bool prune_empty_dirs;
bool dirs_root_emitted;
int list_index;
ArrayList* dirs_batch; /* owned when non-NULL */
unsigned long long dirs_batch_size;
/* Excluded-path sink (see ScannerOptions). `excluded_mutex` is shared across
parallel worker threads. */
ArrayList* excluded_paths;
mtx_t* excluded_mutex;
/* --ignore-errors: continue past unreadable directories (records io_error). */
bool ignore_io_errors;
/* --ignore-missing-args: --dirs listed-but-missing entries are skipped, not
fatal (see ScannerOptions.ignore_missing_args). */
bool ignore_missing_args;
/* A directory could not be opened (I/O error, e.g. EACCES). With
--ignore-errors the scan continues past it and the caller decides what to
do; `failed` is reserved for fatal errors that always abort the scan. */
bool io_error;
/* --hard-links (-H): shared link-group detection table (see ScannerOptions).
NULL when -H is off. */
HardLinkTable* hardlinks;
/* Phase 6: sender stop deadline (from ScannerOptions). */
const StopCondition* stop_condition;
/* P7 Wave D directory-time capture (see ScannerOptions). */
bool capture_dir_times;
ArrayList* dir_entries;
mtx_t* dir_entries_mutex;
} DirectoryScanner;
typedef struct {
+3 -1
View File
@@ -179,6 +179,9 @@ bool pipeline_context_sender_enqueue_chunk(PipelineContextSender* context, Chunk
}
void pipeline_context_sender_destroy(PipelineContextSender* context) {
/* `config` is borrowed: the caller retains ownership and frees it after the
pipeline has been destroyed (the worker threads are already joined, so no
config access can outlive this call). */
if (context->manifest) {
array_list_delete(context->manifest);
}
@@ -192,7 +195,6 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) {
array_list_delete(context->dir_entries);
if (context->dir_entries_mutex_init)
mtx_destroy(&context->dir_entries_mutex);
config_delete(context->config);
queue_destroy(context->queue_scanner);
queue_destroy(context->queue_loader);
mtx_destroy(&context->mutex_scanner);
+2
View File
@@ -120,6 +120,8 @@ typedef struct PipelineContextReceiver {
DirTimeList dir_times;
} PipelineContextReceiver;
/* `config` is borrowed and must outlive the context: destroy does NOT free it,
so the caller owns it and frees it with config_delete() afterwards. */
PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* queue_scanner,
Queue* queue_loader);
void pipeline_context_sender_destroy(PipelineContextSender* context);
+14
View File
@@ -1239,6 +1239,20 @@ class TestProgress:
assert "Stats:" in result.stderr
assert "KB" in result.stderr
def test_human_readable_stats_multithreaded(self, shared_server):
# The multithreaded sender shares the single-threaded --stats format,
# including --human-readable and the rate suffix.
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--threads", "-h", "--stats"],
port=shared_server.port,
)
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
assert "Stats:" in result.stderr
assert "KB" in result.stderr
assert "/s" in result.stderr
def test_human_readable_progress_multithreaded(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
+1
View File
@@ -383,6 +383,7 @@ static void test_pipeline_sender_lifecycle() {
EXPECT_EQ_INT((int)pcs->allocation_session.max_alloc, (int)cfg->max_alloc);
pipeline_context_sender_destroy(pcs);
config_delete(cfg); /* the context borrows cfg; the caller owns it */
}
static void test_pipeline_receiver_lifecycle() {
+4 -1
View File
@@ -37,6 +37,7 @@ static void test_sender_create_destroy() {
EXPECT_NULL(ctx->manifest);
pipeline_context_sender_destroy(ctx);
config_delete(cfg); /* the context borrows cfg; the caller owns it */
}
/* Test pipeline_context_receiver_create/destroy with valid arguments */
@@ -80,6 +81,7 @@ static void test_sender_queue_capacities() {
EXPECT_EQ_INT(ctx->queue_scanner->capacity, 1);
EXPECT_EQ_INT(ctx->queue_loader->capacity, 1);
pipeline_context_sender_destroy(ctx);
config_delete(cfg); /* the context borrows cfg; the caller owns it */
}
/* Invalid queue capacities must not create unusable pipeline queues. */
@@ -445,8 +447,9 @@ static void test_sender_enqueue_byte_budget() {
EXPECT_EQ_INT((int)ctx->queued_bytes, 2000); /* second payload now in flight */
/* pipeline_context_sender_destroy frees the still-queued second chunk and
owns cfg/q_scanner/q_loader from here on. */
owns q_scanner/q_loader; cfg stays borrowed and is freed by the caller. */
pipeline_context_sender_destroy(ctx);
config_delete(cfg);
}
void test_multiprocessing() {