TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge: resolve conflict with dev, apply helpers to new options
CI / lint (pull_request) Successful in 23s
Details
CI / sanitizers (address) (pull_request) Successful in 35s
Details
CI / sanitizers (undefined) (pull_request) Successful in 36s
Details
CI / fuzz-build (pull_request) Successful in 14s
Details
CI / coverage (pull_request) Successful in 31s
Details
CI / build-and-test (pull_request) Successful in 1m15s
Details
CI / valgrind (pull_request) Successful in 33s
Details

Browse Source
This commit is contained in:
TapTap
2026-08-01 18:14:55 +02:00
parent 5eaea9d92e 9d67fcbf92
commit bf91c5b588
27 changed files with 576 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/client_cli.c
+50
View File
@@ -373,6 +373,42 @@ static int parse_args(Config* config, int argc, char* argv[], int* positional_ar
} else if (strcmp(argv[i], "--link-dest") == 0 && i + 1 < argc) {
if (set_string_option(&config->link_dest, argv[++i], "--link-dest") != 0)
return -1;
} else if (strcmp(argv[i], "--partial-dir") == 0 && i + 1 < argc) {
if (set_string_option(&config->partial_dir, argv[++i], "--partial-dir") != 0)
return -1;
} else if (strcmp(argv[i], "--suffix") == 0 && i + 1 < argc) {
if (set_string_option(&config->suffix, argv[++i], "--suffix") != 0)
return -1;
} else if (strcmp(argv[i], "--delete-before") == 0) {
config->delete_before = true;
} else if (strcmp(argv[i], "-T") == 0 && i + 1 < argc) {
if (set_positive_int_option(&config->timeout, argv[++i], "-T") != 0)
return -1;
} else if (strcmp(argv[i], "--address") == 0 && i + 1 < argc) {
if (set_string_option(&config->address, argv[++i], "--address") != 0)
return -1;
} else if (strcmp(argv[i], "--bind-address") == 0 && i + 1 < argc) {
if (set_string_option(&config->bind_address, argv[++i], "--bind-address") != 0)
return -1;
} else if (strcmp(argv[i], "--ipv6") == 0) {
config->ipv6 = true;
} else if (strcmp(argv[i], "--ipv4") == 0) {
config->ipv4 = true;
} else if (strcmp(argv[i], "--daemon") == 0) {
config->daemon = true;
} else if (strcmp(argv[i], "--config") == 0 && i + 1 < argc) {
if (set_string_option(&config->daemon_config, argv[++i], "--config") != 0)
return -1;
} else if (strcmp(argv[i], "--server") == 0) {
config->server_mode = true;
} else if (strcmp(argv[i], "--checksum") == 0) {
config->checksum = true;
} else if (strcmp(argv[i], "--compress-choice") == 0 && i + 1 < argc) {
if (set_string_option(&config->compress_choice, argv[++i], "--compress-choice") != 0)
return -1;
} else if (strcmp(argv[i], "--compress-level") == 0 && i + 1 < argc) {
if (set_positive_int_option(&config->compression_level, argv[++i], "--compress-level") != 0)
return -1;
} else if (argv[i][0] == '-') {
fprintf(stderr, "Unknown option: %s\n", argv[i]);
print_usage();
@@ -478,16 +514,23 @@ static void print_usage(void) {
printf(" --key <path> TLS private key file (PEM)\n");
printf(" --ca <path> TLS CA certificate file (PEM)\n");
printf(" --timeout <sec> I/O timeout in seconds (default: 30)\n");
printf(" -T <sec> Alias for --timeout\n");
printf(" --contimeout <sec> Connection timeout in seconds (default: 10)\n");
printf(" --address <host> Server hostname/IP to connect to\n");
printf(" --bind-address <ip> Bind to specific local address\n");
printf(" --ipv6 Prefer IPv6 connections\n");
printf(" --ipv4 Prefer IPv4 connections\n");
printf(" -q, --quiet Suppress non-error output\n");
printf(" --silent Alias for --quiet\n");
printf(" --backup Backup existing files before overwriting\n");
printf(" --backup-dir <dir> Directory for backups (requires --backup)\n");
printf(" --suffix <str> Backup suffix (default: ~)\n");
printf(" --stats Print transfer statistics at end\n");
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
printf(" --log-file <path> Write log messages to file\n");
printf(" --queue-size <n> Queue capacity for multithreaded mode (default: 100)\n");
printf(" --partial Keep partial files on interrupted transfer\n");
printf(" --partial-dir <dir> Directory for partial files\n");
printf(" --fastsync-server-path <path>\n");
printf(" Path to fastsync-server on remote (default: fastsync-server)\n");
printf(" -l, --links Copy symlinks as symlinks\n");
@@ -509,6 +552,7 @@ static void print_usage(void) {
printf(" --inplace Update files in-place (no temp+rename)\n");
printf(" --append Append data to shorter files\n");
printf(" --append-verify Append with verify\n");
printf(" --delete-before Delete before transfer\n");
printf(" --delete-excluded Also delete excluded files\n");
printf(" --delete-after Delete after transfer, not before\n");
printf(" --max-delete <n> Maximum number of files to delete\n");
@@ -519,6 +563,12 @@ static void print_usage(void) {
printf(" -R, --relative Use relative paths\n");
printf(" -e, --rsh <cmd> Specify remote shell\n");
printf(" --rsync-path <path> Path to remote binary\n");
printf(" --daemon Run in daemon mode\n");
printf(" --config <path> Path to configuration file\n");
printf(" --server Run in server mode\n");
printf(" --checksum Skip files based on checksum, not mod-time/size\n");
printf(" --compress-choice <alg> Compression algorithm (default: zstd)\n");
printf(" --compress-level <n> Compression level (default: 5)\n");
printf(" --temp-dir <dir> Temporary directory for files\n");
printf(" --compare-dest <dir> Compare destination\n");
printf(" --copy-dest <dir> Copy destination\n");
src/client/client_send.c
+99 -6
View File
@@ -25,13 +25,16 @@
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
/* Forward declaration for progress-reporting thread used in multithreaded send. */
static int progress_thread_fn(void* arg);
/* Print dry-run manifest showing files that would be transferred. Returns 0 on success. */
static int send_dry_run_manifest(Config* config) {
DirectoryScanner* scanner = directory_scanner_create(
config->send_directory, config->use_metadata, config->chunk_size, config->exclude_patterns,
config->exclude_count, config->include_patterns, config->include_count, config->max_size,
config->min_size, config->max_depth, config->follow_symlinks, config->copy_links,
config->safe_links, config->copy_unsafe_links);
config->safe_links, config->copy_unsafe_links, config->checksum);
if (!scanner)
return -1;
Chunk* chunk;
@@ -271,6 +274,9 @@ static int send_chunks_multithreaded(void* pipeline_context) {
if (context->config->transport == TRANSPORT_SSH) {
if (context->config->use_sendfile) {
fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
return 1;
}
client = client_connect_ssh(context->config->ssh_destination, context->config->ssh_port,
@@ -283,6 +289,9 @@ static int send_chunks_multithreaded(void* pipeline_context) {
if (client)
client_delete(client);
fprintf(stderr, "Error: could not connect to server via TLS\n");
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
return thrd_error;
}
} else {
@@ -292,12 +301,18 @@ static int send_chunks_multithreaded(void* pipeline_context) {
if (client)
client_delete(client);
fprintf(stderr, "Error: could not connect to server\n");
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
return thrd_error;
}
}
if (!config_send(client->file_descriptor, context->config)) {
client_disconnect(client);
client_delete(client);
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
return thrd_error;
}
@@ -316,19 +331,38 @@ static int send_chunks_multithreaded(void* pipeline_context) {
int ok = receive_status(client->file_descriptor, &s) && s == STATUS_OK;
client_disconnect(client);
client_delete(client);
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
return ok ? thrd_success : thrd_error;
send_fail:
client_disconnect(client);
client_delete(client);
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
return thrd_error;
}
if (send_chunk(client, current_chunk, context->config) != 0) {
fprintf(stderr, "Error: unexpected error while sending chunk\n");
client_disconnect(client);
client_delete(client);
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
return thrd_error;
}
if (context->config->show_progress) {
unsigned long long chunk_bytes = 0;
for (int i = 0; i < current_chunk->element_count; i++) {
if (current_chunk->items[i] && current_chunk->items[i]->data)
chunk_bytes += current_chunk->items[i]->data->size;
}
mtx_lock(&context->mutex_progress);
context->progress_bytes += chunk_bytes;
mtx_unlock(&context->mutex_progress);
}
chunk_destroy(current_chunk);
}
}
@@ -340,7 +374,8 @@ static int scan_directory_multithreaded(void* pipeline_context) {
context->config->exclude_patterns, context->config->exclude_count,
context->config->include_patterns, context->config->include_count, context->config->max_size,
context->config->min_size, context->config->max_depth, 4, context->config->follow_symlinks,
context->config->copy_links, context->config->safe_links, context->config->copy_unsafe_links);
context->config->copy_links, context->config->safe_links, context->config->copy_unsafe_links,
context->config->checksum);
Chunk* current_chunk;
while ((current_chunk = parallel_scanner_next(scanner)) != NULL) {
@@ -398,6 +433,42 @@ static int load_files_multithreaded(void* pipeline_context) {
}
}
/* Progress-reporting thread for multithreaded send. Runs in parallel with
the scanner/loader/sender threads and prints periodic progress to stderr. */
static int progress_thread_fn(void* arg) {
PipelineContextSender* context = (PipelineContextSender*)arg;
time_t last_progress = 0;
time_t start = time(NULL);
while (true) {
mtx_lock(&context->mutex_progress);
bool done = context->sender_done;
unsigned long long total = context->progress_bytes;
mtx_unlock(&context->mutex_progress);
if (done) {
time_t now = time(NULL);
double elapsed = difftime(now, start);
double rate = elapsed > 0.0 ? total / (1048576.0 * elapsed) : 0.0;
fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) Done.\n", total / 1048576.0, rate);
break;
}
time_t now = time(NULL);
if (now - last_progress >= 1) {
last_progress = now;
double elapsed = difftime(now, start);
double rate = elapsed > 0.0 ? total / (1048576.0 * elapsed) : 0.0;
fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) ", total / 1048576.0, rate);
fflush(stderr);
}
struct timespec ts = {0, 100 * 1000000L}; /* 100 ms */
thrd_sleep(&ts, NULL);
}
return thrd_success;
}
int send_files(Config* config) {
if (config->dry_run)
return send_dry_run_manifest(config);
@@ -439,9 +510,10 @@ int send_files(Config* config) {
config->send_directory, config->use_metadata, config->chunk_size, config->exclude_patterns,
config->exclude_count, config->include_patterns, config->include_count, config->max_size,
config->min_size, config->max_depth, config->follow_symlinks, config->copy_links,
config->safe_links, config->copy_unsafe_links);
config->safe_links, config->copy_unsafe_links, config->checksum);
Chunk* current_chunk;
unsigned long long total_bytes = 0;
int total_files = 0;
time_t last_progress = 0;
time_t start = time(NULL);
ArrayList* manifest = config->use_delete ? array_list_create(free) : NULL;
@@ -449,6 +521,7 @@ int send_files(Config* config) {
unsigned long long chunk_bytes = 0;
for (int i = 0; i < current_chunk->element_count; i++) {
chunk_bytes += current_chunk->items[i]->data->size;
total_files++;
if (manifest) {
const char* p = current_chunk->items[i]->path;
if (*p == '/')
@@ -496,11 +569,16 @@ int send_files(Config* config) {
goto send_fail;
Status s;
int ok = receive_status(client->file_descriptor, &s) && s == STATUS_OK;
double elapsed_total = difftime(time(NULL), start);
if (config->show_progress) {
double elapsed = difftime(time(NULL), start);
double rate = elapsed > 0 ? total_bytes / (1048576.0 * elapsed) : 0;
double rate = elapsed_total > 0 ? total_bytes / (1048576.0 * elapsed_total) : 0;
fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) Done.\n", total_bytes / 1048576.0, rate);
}
if (config->stats) {
double rate = elapsed_total > 0 ? total_bytes / (1048576.0 * elapsed_total) : 0;
fprintf(stderr, "Stats: %d files, %.1f MB, %.1f MB/s\n", total_files, total_bytes / 1048576.0,
rate);
}
directory_scanner_destroy(scanner);
client_disconnect(client);
client_delete(client);
@@ -547,7 +625,7 @@ int send_files_multithreaded(Config* config) {
if (config->use_delete)
context->manifest = array_list_create(free);
thrd_t scanner, loader, sender;
thrd_t scanner, loader, sender, progress;
if (thrd_create(&scanner, scan_directory_multithreaded, context) != thrd_success ||
thrd_create(&loader, load_files_multithreaded, context) != thrd_success ||
thrd_create(&sender, send_chunks_multithreaded, context) != thrd_success) {
@@ -556,11 +634,26 @@ int send_files_multithreaded(Config* config) {
return 1;
}
if (config->show_progress) {
if (thrd_create(&progress, progress_thread_fn, context) != thrd_success) {
perror("Error creating progress thread.\n");
/* Non-fatal; continue without progress reporting */
}
}
int sender_result;
thrd_join(scanner, NULL);
thrd_join(loader, NULL);
thrd_join(sender, &sender_result);
if (config->show_progress) {
/* Signal progress thread to exit if it hasn't already */
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
thrd_join(progress, NULL);
}
pipeline_context_sender_destroy(context);
return sender_result == thrd_success ? 0 : 1;
}
src/client/scanner.c
+59 -5
View File
@@ -40,7 +40,7 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
int include_count, unsigned long long max_size,
unsigned long long min_size, int max_depth,
bool follow_symlinks, bool copy_links, bool safe_links,
bool copy_unsafe_links) {
bool copy_unsafe_links, bool checksum) {
DirectoryScanner* scanner = malloc(sizeof(DirectoryScanner));
if (scanner == NULL)
return NULL;
@@ -61,6 +61,7 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
scanner->copy_links = copy_links;
scanner->safe_links = safe_links;
scanner->copy_unsafe_links = copy_unsafe_links;
scanner->checksum = checksum;
queue_enqueue(scanner->directories, dir_entry_create(root_directory, 0));
return scanner;
}
@@ -276,6 +277,7 @@ typedef struct {
bool copy_links;
bool safe_links;
bool copy_unsafe_links;
bool checksum;
} ParallelWorkerArg;
static int parallel_worker_thread(void* arg) {
@@ -284,7 +286,7 @@ static int parallel_worker_thread(void* arg) {
DirectoryScanner* ds = directory_scanner_create(
wa->dirs[i], wa->use_metadata, wa->chunk_size, wa->exclude_patterns, wa->exclude_count,
wa->include_patterns, wa->include_count, wa->max_size, wa->min_size, wa->max_depth,
wa->follow_symlinks, wa->copy_links, wa->safe_links, wa->copy_unsafe_links);
wa->follow_symlinks, wa->copy_links, wa->safe_links, wa->copy_unsafe_links, wa->checksum);
Chunk* chunk;
while ((chunk = directory_scanner_next(ds)) != NULL) {
queue_enqueue_multithreaded(wa->ps->result_queue, chunk, &wa->ps->result_mutex,
@@ -312,7 +314,7 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
int include_count, unsigned long long max_size,
unsigned long long min_size, int max_depth,
int num_threads, bool follow_symlinks, bool copy_links,
bool safe_links, bool copy_unsafe_links) {
bool safe_links, bool copy_unsafe_links, bool checksum) {
ParallelScanner* ps = calloc(1, sizeof(ParallelScanner));
if (!ps)
return NULL;
@@ -345,11 +347,62 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
char* cur_path = path_cat(root_directory, entry->d_name);
if (!cur_path)
continue;
struct stat st;
if (stat(cur_path, &st) != 0) {
struct stat lstats;
if (lstat(cur_path, &lstats) != 0) {
free(cur_path);
continue;
}
bool is_symlink = S_ISLNK(lstats.st_mode);
// Skip symlinks unless the user explicitly enabled following/copying them.
if (is_symlink && !follow_symlinks && !copy_links && !safe_links && !copy_unsafe_links) {
free(cur_path);
continue;
}
// --safe-links: reject symlinks pointing outside the source tree.
if (is_symlink && safe_links) {
char link_target[4096];
ssize_t len = readlink(cur_path, link_target, sizeof(link_target) - 1);
if (len < 0) {
free(cur_path);
continue;
}
link_target[len] = 0;
if (link_target[0] == '/') {
free(cur_path);
continue;
}
}
// --copy-unsafe-links (without --copy-links): only copy absolute symlinks.
if (is_symlink && copy_unsafe_links && !copy_links) {
char link_target[4096];
ssize_t len = readlink(cur_path, link_target, sizeof(link_target) - 1);
if (len < 0) {
free(cur_path);
continue;
}
link_target[len] = 0;
bool unsafe = (link_target[0] == '/');
if (!unsafe) {
free(cur_path);
continue;
}
}
// Determine whether to use lstat or stat results for the entry.
struct stat st;
bool use_lstat_res = is_symlink && follow_symlinks && !copy_links;
if (use_lstat_res) {
st = lstats;
} else {
if (stat(cur_path, &st) != 0) {
free(cur_path);
continue;
}
}
if (S_ISDIR(st.st_mode)) {
array_list_add(subdirs, cur_path);
} else {
@@ -475,6 +528,7 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
wa->copy_links = copy_links;
wa->safe_links = safe_links;
wa->copy_unsafe_links = copy_unsafe_links;
wa->checksum = checksum;
start += count;
if (thrd_create(&ps->threads[t], parallel_worker_thread, wa) != thrd_success) {
for (int j = 0; j < count; j++)
src/client/scanner.h
+3 -2
View File
@@ -25,6 +25,7 @@ typedef struct {
bool copy_links;
bool safe_links;
bool copy_unsafe_links;
bool checksum;
} DirectoryScanner;
typedef struct {
@@ -45,7 +46,7 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
int include_count, unsigned long long max_size,
unsigned long long min_size, int max_depth,
bool follow_symlinks, bool copy_links, bool safe_links,
bool copy_unsafe_links);
bool copy_unsafe_links, bool checksum);
Chunk* directory_scanner_next(DirectoryScanner* scanner);
void directory_scanner_destroy(DirectoryScanner* scanner);
@@ -55,7 +56,7 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
int include_count, unsigned long long max_size,
unsigned long long min_size, int max_depth,
int num_threads, bool follow_symlinks, bool copy_links,
bool safe_links, bool copy_unsafe_links);
bool safe_links, bool copy_unsafe_links, bool checksum);
Chunk* parallel_scanner_next(ParallelScanner* scanner);
void parallel_scanner_destroy(ParallelScanner* scanner);