TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Merge remote-tracking branch 'origin/feat/max-alloc' into dev

Browse Source
This commit is contained in:
TapTap
2026-09-05 02:13:35 +02:00
parent 3304541337 638d953b1d
commit 90112a7585
26 changed files with 554 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/client_cli.c
+67 -1
View File
@@ -191,6 +191,56 @@ static int parse_ull_arg(const char* val, unsigned long long* out, const char* o
return 0;
}
static int parse_size_arg(const char* value, unsigned long long* out) {
if (!value || *value < '0' || *value > '9')
return -1;
char* end;
errno = 0;
unsigned long long number = strtoull(value, &end, 10);
if (errno != 0 || end == value)
return -1;
unsigned long long multiplier = 1;
if (*end != '\0') {
if (end[1] != '\0')
return -1;
switch (*end) {
case 'b':
case 'B':
break;
case 'k':
case 'K':
multiplier = 1024ULL;
break;
case 'm':
case 'M':
multiplier = 1024ULL * 1024;
break;
case 'g':
case 'G':
multiplier = 1024ULL * 1024 * 1024;
break;
case 't':
case 'T':
multiplier = 1024ULL * 1024 * 1024 * 1024;
break;
case 'p':
case 'P':
multiplier = 1024ULL * 1024 * 1024 * 1024 * 1024;
break;
case 'e':
case 'E':
multiplier = 1024ULL * 1024 * 1024 * 1024 * 1024 * 1024;
break;
default:
return -1;
}
}
if (number == 0 || number > ULLONG_MAX / multiplier)
return -1;
*out = number * multiplier;
return 0;
}
/* Append a duplicated pattern to a growable pattern array. Returns 0 on success, -1 on error. */
static int config_add_pattern(char*** patterns, int* count, const char* value,
const char* optname) {
@@ -369,7 +419,7 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
bool verbose = false;
protocol_set_8_bit_output(config->eight_bit_output);
for (int i = 1; i < argc; i++) {
const char* modify_window_prefix = "--modify-window=";
const char* modify_window_prefix = "--modify-window=";
if (strncmp(argv[i], modify_window_prefix, strlen(modify_window_prefix)) == 0) {
if (set_nonneg_int_option(&config->modify_window, argv[i] + strlen(modify_window_prefix),
"--modify-window") != 0)
@@ -388,6 +438,22 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
return -1;
continue;
}
if (strncmp(argv[i], "--max-alloc=", 12) == 0 || strcmp(argv[i], "--max-alloc") == 0) {
const char* value = strcmp(argv[i], "--max-alloc") == 0 ? "" : argv[i] + 12;
if (*value == '\0') {
if (i + 1 >= argc) {
log_message(LOG_LEVEL_ERROR, "missing argument for --max-alloc");
return -1;
}
value = argv[++i];
}
if (parse_size_arg(value, &config->max_alloc) != 0) {
log_message(LOG_LEVEL_ERROR,
"--max-alloc must be a positive size (B, K, M, G, T, P, or E)");
return -1;
}
continue;
}
const OptionEntry* entry = find_table_option(argv[i]);
const char* inline_value = NULL;
if (!entry)
src/client/client_send.c
+12 -2
View File
@@ -612,14 +612,16 @@ static int send_chunks_multithreaded(void* pipeline_context) {
static int scan_directory_multithreaded(void* pipeline_context) {
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
protocol_session_bind(&context->allocation_session);
ScannerOptions options = scanner_options_from_config(context->config, 4);
ParallelScanner* scanner =
parallel_scanner_create_with_options(context->config->send_directory, &options);
ParallelScanner* scanner = parallel_scanner_create_with_options(
context->config->send_directory, &options, &context->allocation_session);
Chunk* current_chunk;
if (scanner == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to create parallel scanner");
pipeline_cancel(context);
protocol_session_unbind();
return thrd_error;
}
while ((current_chunk = parallel_scanner_next(scanner)) != NULL) {
@@ -631,6 +633,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
pipeline_cancel(context);
chunk_destroy(current_chunk);
parallel_scanner_destroy(scanner);
protocol_session_unbind();
return thrd_error;
}
}
@@ -641,6 +644,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
chunk_destroy(current_chunk);
pipeline_cancel(context);
parallel_scanner_destroy(scanner);
protocol_session_unbind();
return thrd_error;
}
}
@@ -652,6 +656,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
cnd_broadcast(&context->condition_not_full_scanner);
mtx_unlock(&context->mutex_scanner);
pipeline_cancel(context);
protocol_session_unbind();
return thrd_error;
}
mtx_lock(&context->mutex_scanner);
@@ -660,11 +665,13 @@ static int scan_directory_multithreaded(void* pipeline_context) {
mtx_unlock(&context->mutex_scanner);
parallel_scanner_destroy(scanner);
protocol_session_unbind();
return thrd_success;
}
static int load_files_multithreaded(void* pipeline_context) {
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
protocol_session_bind(&context->allocation_session);
while (true) {
Chunk* chunk = queue_dequeue_multithreaded(
context->queue_scanner, &context->mutex_scanner, &context->condition_not_empty_scanner,
@@ -674,6 +681,7 @@ static int load_files_multithreaded(void* pipeline_context) {
context->loader_done = true;
cnd_signal(&context->condition_not_empty_loader);
mtx_unlock(&context->mutex_loader);
protocol_session_unbind();
return thrd_success;
}
if (!context->config->use_sendfile) {
@@ -685,6 +693,7 @@ static int load_files_multithreaded(void* pipeline_context) {
log_message(LOG_LEVEL_ERROR, "Failed to load file data");
chunk_destroy(chunk);
pipeline_cancel(context);
protocol_session_unbind();
return thrd_error;
}
}
@@ -695,6 +704,7 @@ static int load_files_multithreaded(void* pipeline_context) {
&context->cancelled)) {
chunk_destroy(chunk);
pipeline_cancel(context);
protocol_session_unbind();
return thrd_error;
}
}
src/client/scanner.c
+10 -1
View File
@@ -351,10 +351,14 @@ typedef struct {
char** dirs;
int dir_count;
ScannerOptions options;
ProtocolSession* allocation_session;
} ParallelWorkerArg;
static int parallel_worker_thread(void* arg) {
ParallelWorkerArg* wa = (ParallelWorkerArg*)arg;
ProtocolSession* allocation_session = wa->allocation_session;
if (allocation_session)
protocol_session_bind(allocation_session);
for (int i = 0; i < wa->dir_count; i++) {
DirectoryScanner* ds = directory_scanner_create_with_options(wa->dirs[i], &wa->options);
if (!ds) {
@@ -398,6 +402,8 @@ static int parallel_worker_thread(void* arg) {
cnd_signal(&ps->result_not_empty);
}
mtx_unlock(&ps->result_mutex);
if (allocation_session)
protocol_session_unbind();
return thrd_success;
}
@@ -633,6 +639,7 @@ static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs,
wa->dir_count = count;
wa->options = *options;
wa->options.chunk_size = cs;
wa->allocation_session = ps->allocation_session;
start += count;
if (thrd_create(&ps->threads[t], parallel_worker_thread, wa) != thrd_success) {
for (int j = 0; j < count; j++)
@@ -648,7 +655,8 @@ static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs,
}
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
const ScannerOptions* options) {
const ScannerOptions* options,
ProtocolSession* allocation_session) {
if (!root_directory || !options)
return NULL;
ParallelScanner* ps = calloc(1, sizeof(ParallelScanner));
@@ -658,6 +666,7 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
free(ps);
return NULL;
}
ps->allocation_session = allocation_session;
ArrayList* root_files = array_list_create(file_destroy);
ArrayList* subdirs = array_list_create(free);
src/client/scanner.h
+4 -1
View File
@@ -2,6 +2,7 @@
#define SCANNER_H
#include "chunk.h"
#include "protocol.h"
#include "queue.h"
#include <dirent.h>
#include <stdbool.h>
@@ -62,6 +63,7 @@ typedef struct {
atomic_bool cancelled;
int completed;
Chunk* initial_chunk;
ProtocolSession* allocation_session;
} ParallelScanner;
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
@@ -78,7 +80,8 @@ bool directory_scanner_failed(const DirectoryScanner* scanner);
void directory_scanner_destroy(DirectoryScanner* scanner);
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
const ScannerOptions* options);
const ScannerOptions* options,
ProtocolSession* allocation_session);
Chunk* parallel_scanner_next(ParallelScanner* scanner);
bool parallel_scanner_failed(const ParallelScanner* scanner);
void parallel_scanner_destroy(ParallelScanner* scanner);
src/client/usage.c
+1
View File
@@ -30,6 +30,7 @@ void print_usage(void) {
printf(" --include-from <file> Read include patterns from file\n");
printf(" --max-size <n> Skip files larger than n bytes\n");
printf(" --min-size <n> Skip files smaller than n bytes\n");
printf(" --max-alloc <SIZE> Maximum single allocation (default: 1G)\n");
printf(" --incremental Skip files unchanged since last transfer\n");
printf(" --size-only Skip incremental files matching in size, ignoring mtime\n");
printf(" -I, --ignore-times Transfer files even when size and mtime match\n");