TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

merge: resolve origin dev refactor conflicts
CI / lint (pull_request) Successful in 28s
Details
CI / sanitizers (address) (pull_request) Successful in 36s
Details
CI / sanitizers (undefined) (pull_request) Successful in 37s
Details
CI / fuzz-build (pull_request) Successful in 14s
Details
CI / coverage (pull_request) Successful in 32s
Details
CI / build-and-test (pull_request) Successful in 1m15s
Details
CI / valgrind (pull_request) Successful in 33s
Details

Browse Source
This commit is contained in:
TapTap
2026-08-16 09:56:15 +02:00
parent 1391564ce7 1abc17142f
commit 6c4d0246bc
33 changed files with 1782 additions and 1193 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/client_cli.c
+2 -50
View File
@@ -1,10 +1,12 @@
#include "client_send.h"
#include "client_validation.h"
#include "config.h"
#include "delta.h"
#include "log.h"
#include "protocol.h"
#include "transport_tcp.h"
#include "transport_tls.h"
#include "usage.h"
#include "utils.h"
#include <errno.h>
#include <limits.h>
@@ -12,7 +14,6 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "usage.c"
#ifndef FASTSYNC_TEST_BUILD
/* Parse environment variables for source/destination directories and save-to-disk flag. */
@@ -360,55 +361,6 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
return 0;
}
#ifndef FASTSYNC_TEST_BUILD
/* Validate config after parsing. Returns true if valid. */
static bool validate_config(const Config* config) {
if (!config->send_directory || !config->receive_root_directory) {
fprintf(stderr, "Error: source and destination directories are required\n");
print_usage();
return false;
}
if (config->use_sendfile && (config->use_chunk_serialization || config->use_compression)) {
fprintf(stderr, "Error: -f/--sendfile cannot be combined with -c (compression) or -s (chunk "
"serialization)\n");
return false;
}
if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
return false;
}
if (config->use_incremental && config->use_chunk_serialization) {
fprintf(stderr, "Error: --incremental is not supported with -s (chunk serialization)\n");
return false;
}
if (config->use_delta && !config->use_incremental) {
fprintf(stderr, "Error: --delta requires --incremental\n");
return false;
}
if (config->use_delta && config->use_chunk_serialization) {
fprintf(stderr, "Error: --delta cannot be combined with -s (chunk serialization)\n");
return false;
}
if (config->use_delta && config->use_sendfile) {
fprintf(stderr, "Error: --delta cannot be combined with -f (sendfile)\n");
return false;
}
if (config->append || config->append_verify) {
fprintf(
stderr,
"Error: --append and --append-verify are not supported yet; refusing to ignore option\n");
return false;
}
if (config->use_tls) {
if (!config->tls_cert || !config->tls_key || !config->tls_ca) {
fprintf(stderr, "Error: --tls requires --cert, --key, and --ca\n");
return false;
}
}
return true;
}
#endif /* FASTSYNC_TEST_BUILD */
static int read_patterns_from_file(const char* filepath, char*** patterns, int* count) {
FILE* fp = fopen(filepath, "r");
if (!fp) {
src/client/client_send.c
+165 -176
View File
@@ -28,6 +28,88 @@
/* Forward declaration for progress-reporting thread used in multithreaded send. */
static int progress_thread_fn(void* arg);
static ScannerOptions scanner_options_from_config(const Config* config, int num_threads) {
ScannerOptions options = {
config->use_metadata, config->chunk_size, config->exclude_patterns,
config->exclude_count, config->include_patterns, config->include_count,
config->max_size, config->min_size, config->max_depth,
num_threads, config->follow_symlinks, config->copy_links,
config->safe_links, config->copy_unsafe_links, config->checksum};
return options;
}
/* Select the configured transport for both transfer execution paths. */
static Client* connect_transfer_client(const Config* config) {
if (config->transport == TRANSPORT_SSH) {
if (config->use_sendfile) {
fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
return NULL;
}
return client_connect_ssh(config->ssh_destination, config->ssh_port,
config->fastsync_server_path);
}
Client* client = client_create();
if (!client)
return NULL;
bool connected;
if (config->use_tls) {
connected = client_connect_tls(client, config->server_host, config->server_port,
config->tls_cert, config->tls_key, config->tls_ca);
} else {
connected = client_connect(client, config->server_host, config->server_port);
}
if (!connected) {
client_disconnect(client);
client_delete(client);
return NULL;
}
return client;
}
static void disconnect_transfer_client(Client* client) {
if (!client)
return;
client_disconnect(client);
client_delete(client);
}
static ArrayList* create_transfer_manifest(const Config* config) {
return config->use_delete ? array_list_create(free) : NULL;
}
static bool add_chunk_to_manifest(ArrayList* manifest, const Chunk* chunk) {
if (!manifest)
return true;
for (int i = 0; i < chunk->element_count; i++) {
const char* path = chunk->items[i]->path;
if (*path == '/')
path++;
char* entry = str_dup(path);
if (!entry) {
log_message(LOG_LEVEL_ERROR, "Failed to allocate manifest entry");
return false;
}
if (!array_list_add(manifest, entry)) {
free(entry);
return false;
}
}
return true;
}
static bool finalize_transfer(Client* client) {
Status status;
return send_status(client->file_descriptor, STATUS_FINISHED) &&
receive_status(client->file_descriptor, &status) && status == STATUS_OK;
}
static void mark_sender_done(PipelineContextSender* context) {
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
}
static void pipeline_cancel(PipelineContextSender* context) {
mtx_lock(&context->mutex_scanner);
mtx_lock(&context->mutex_loader);
@@ -43,12 +125,10 @@ static void pipeline_cancel(PipelineContextSender* context) {
}
/* Print dry-run manifest showing files that would be transferred. Returns 0 on success. */
static int send_dry_run_manifest(Config* config) {
DirectoryScanner* scanner = directory_scanner_create(
config->send_directory, config->use_metadata, config->chunk_size, config->exclude_patterns,
config->exclude_count, config->include_patterns, config->include_count, config->max_size,
config->min_size, config->max_depth, config->follow_symlinks, config->copy_links,
config->safe_links, config->copy_unsafe_links, config->checksum);
static int send_dry_run_manifest(const Config* config) {
ScannerOptions options = scanner_options_from_config(config, 0);
DirectoryScanner* scanner =
directory_scanner_create_with_options(config->send_directory, &options);
if (!scanner)
return -1;
Chunk* chunk;
@@ -301,49 +381,24 @@ int send_chunk(Client* client, Chunk* chunk, Config* config) {
static int send_chunks_multithreaded(void* pipeline_context) {
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
Client* client;
if (context->config->transport == TRANSPORT_SSH) {
if (context->config->use_sendfile) {
fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
return 1;
}
client = client_connect_ssh(context->config->ssh_destination, context->config->ssh_port,
context->config->fastsync_server_path);
} else if (context->config->use_tls) {
client = client_create();
if (!client || !client_connect_tls(client, context->config->server_host,
context->config->server_port, context->config->tls_cert,
context->config->tls_key, context->config->tls_ca)) {
if (client)
client_delete(client);
fprintf(stderr, "Error: could not connect to server via TLS\n");
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
return thrd_error;
}
} else {
client = client_create();
if (!client ||
!client_connect(client, context->config->server_host, context->config->server_port)) {
if (client)
client_delete(client);
fprintf(stderr, "Error: could not connect to server\n");
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
return thrd_error;
}
Client* client = connect_transfer_client(context->config);
if (!client) {
if (context->config->transport == TRANSPORT_TCP)
fprintf(stderr, "Error: could not connect to server%s\n",
context->config->use_tls ? " via TLS" : "");
pipeline_cancel(context);
mark_sender_done(context);
return thrd_error;
}
ProtocolSession session;
protocol_session_init(&session, client->file_descriptor, client->file_descriptor);
protocol_session_set_ssl(&session, (SSL*)client->ssl);
protocol_session_bind(&session);
if (!config_send(client->file_descriptor, context->config)) {
client_disconnect(client);
client_delete(client);
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
pipeline_cancel(context);
disconnect_transfer_client(client);
mark_sender_done(context);
protocol_session_unbind();
return thrd_error;
}
@@ -352,39 +407,37 @@ static int send_chunks_multithreaded(void* pipeline_context) {
context->queue_loader, &context->mutex_loader, &context->condition_not_empty_loader,
&context->condition_not_full_loader, &context->loader_done);
if (current_chunk == NULL) {
if (atomic_load(&context->cancelled)) {
pipeline_cancel(context);
disconnect_transfer_client(client);
mark_sender_done(context);
protocol_session_unbind();
return thrd_error;
}
if (context->config->use_delete) {
if (send_delete_manifest(client->file_descriptor, context->manifest) != 0)
goto send_fail;
}
if (!send_status(client->file_descriptor, STATUS_FINISHED))
goto send_fail;
Status s;
int ok = receive_status(client->file_descriptor, &s) && s == STATUS_OK;
client_disconnect(client);
client_delete(client);
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
bool ok = finalize_transfer(client);
disconnect_transfer_client(client);
mark_sender_done(context);
protocol_session_unbind();
return ok ? thrd_success : thrd_error;
send_fail:
pipeline_cancel(context);
client_disconnect(client);
client_delete(client);
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
disconnect_transfer_client(client);
mark_sender_done(context);
protocol_session_unbind();
return thrd_error;
}
if (send_chunk(client, current_chunk, context->config) != 0) {
fprintf(stderr, "Error: unexpected error while sending chunk\n");
chunk_destroy(current_chunk);
pipeline_cancel(context);
client_disconnect(client);
client_delete(client);
mtx_lock(&context->mutex_progress);
context->sender_done = true;
mtx_unlock(&context->mutex_progress);
disconnect_transfer_client(client);
mark_sender_done(context);
protocol_session_unbind();
return thrd_error;
}
if (context->config->show_progress) {
@@ -403,13 +456,9 @@ static int send_chunks_multithreaded(void* pipeline_context) {
static int scan_directory_multithreaded(void* pipeline_context) {
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
ParallelScanner* scanner = parallel_scanner_create(
context->config->send_directory, context->config->use_metadata, context->config->chunk_size,
context->config->exclude_patterns, context->config->exclude_count,
context->config->include_patterns, context->config->include_count, context->config->max_size,
context->config->min_size, context->config->max_depth, 4, context->config->follow_symlinks,
context->config->copy_links, context->config->safe_links, context->config->copy_unsafe_links,
context->config->checksum);
ScannerOptions options = scanner_options_from_config(context->config, 4);
ParallelScanner* scanner =
parallel_scanner_create_with_options(context->config->send_directory, &options);
Chunk* current_chunk;
if (scanner == NULL) {
@@ -420,28 +469,14 @@ static int scan_directory_multithreaded(void* pipeline_context) {
while ((current_chunk = parallel_scanner_next(scanner)) != NULL) {
if (context->config->use_delete) {
mtx_lock(&context->mutex_scanner);
for (int i = 0; i < current_chunk->element_count; i++) {
const char* p = current_chunk->items[i]->path;
if (*p == '/')
p++;
char* manifest_entry = str_dup(p);
if (!manifest_entry) {
log_message(LOG_LEVEL_ERROR, "Failed to allocate manifest entry");
mtx_unlock(&context->mutex_scanner);
pipeline_cancel(context);
parallel_scanner_destroy(scanner);
return thrd_error;
}
if (!array_list_add(context->manifest, manifest_entry)) {
free(manifest_entry);
mtx_unlock(&context->mutex_scanner);
pipeline_cancel(context);
chunk_destroy(current_chunk);
parallel_scanner_destroy(scanner);
return thrd_error;
}
}
bool manifest_ok = add_chunk_to_manifest(context->manifest, current_chunk);
mtx_unlock(&context->mutex_scanner);
if (!manifest_ok) {
pipeline_cancel(context);
chunk_destroy(current_chunk);
parallel_scanner_destroy(scanner);
return thrd_error;
}
}
if (!queue_enqueue_multithreaded_cancel(
context->queue_scanner, current_chunk, &context->mutex_scanner,
@@ -491,9 +526,10 @@ static int load_files_multithreaded(void* pipeline_context) {
if (f->data->size > STREAM_THRESHOLD && !context->config->use_compression)
continue;
if (!file_load_data(f)) {
log_message(LOG_LEVEL_ERROR, "Failed to load file data, skipping");
file_destroy(f);
chunk->items[i] = NULL;
log_message(LOG_LEVEL_ERROR, "Failed to load file data");
chunk_destroy(chunk);
pipeline_cancel(context);
return thrd_error;
}
}
}
@@ -502,9 +538,7 @@ static int load_files_multithreaded(void* pipeline_context) {
&context->condition_not_full_loader,
&context->cancelled)) {
chunk_destroy(chunk);
atomic_store(&context->cancelled, true);
cnd_broadcast(&context->condition_not_full_loader);
cnd_broadcast(&context->condition_not_empty_loader);
pipeline_cancel(context);
return thrd_error;
}
}
@@ -550,61 +584,37 @@ int send_files(Config* config) {
if (config->dry_run)
return send_dry_run_manifest(config);
Client* client;
if (config->transport == TRANSPORT_SSH) {
if (config->use_sendfile) {
fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
return 1;
}
client =
client_connect_ssh(config->ssh_destination, config->ssh_port, config->fastsync_server_path);
if (!client)
return 1;
} else if (config->use_tls) {
client = client_create();
if (!client || !client_connect_tls(client, config->server_host, config->server_port,
config->tls_cert, config->tls_key, config->tls_ca)) {
if (client) {
client_disconnect(client);
client_delete(client);
}
fprintf(stderr, "Error: could not connect to server via TLS\n");
return 1;
}
} else {
client = client_create();
if (!client || !client_connect(client, config->server_host, config->server_port)) {
if (client) {
client_disconnect(client);
client_delete(client);
}
fprintf(stderr, "Error: could not connect to server\n");
return 1;
}
}
if (!config_send(client->file_descriptor, config)) {
client_disconnect(client);
client_delete(client);
Client* client = connect_transfer_client(config);
if (!client) {
if (config->transport == TRANSPORT_TCP)
fprintf(stderr, "Error: could not connect to server%s\n", config->use_tls ? " via TLS" : "");
return 1;
}
DirectoryScanner* scanner = directory_scanner_create(
config->send_directory, config->use_metadata, config->chunk_size, config->exclude_patterns,
config->exclude_count, config->include_patterns, config->include_count, config->max_size,
config->min_size, config->max_depth, config->follow_symlinks, config->copy_links,
config->safe_links, config->copy_unsafe_links, config->checksum);
ProtocolSession session;
protocol_session_init(&session, client->file_descriptor, client->file_descriptor);
protocol_session_set_ssl(&session, (SSL*)client->ssl);
protocol_session_bind(&session);
if (!config_send(client->file_descriptor, config)) {
disconnect_transfer_client(client);
protocol_session_unbind();
return 1;
}
ScannerOptions scanner_options = scanner_options_from_config(config, 0);
DirectoryScanner* scanner =
directory_scanner_create_with_options(config->send_directory, &scanner_options);
Chunk* current_chunk;
unsigned long long total_bytes = 0;
int total_files = 0;
time_t last_progress = 0;
time_t start = time(NULL);
ArrayList* manifest = config->use_delete ? array_list_create(free) : NULL;
ArrayList* manifest = create_transfer_manifest(config);
if (!scanner || (config->use_delete && !manifest)) {
if (scanner)
directory_scanner_destroy(scanner);
if (manifest)
array_list_delete(manifest);
client_disconnect(client);
client_delete(client);
disconnect_transfer_client(client);
protocol_session_unbind();
return 1;
}
while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
@@ -612,30 +622,10 @@ int send_files(Config* config) {
for (int i = 0; i < current_chunk->element_count; i++) {
chunk_bytes += current_chunk->items[i]->data->size;
total_files++;
if (manifest) {
const char* p = current_chunk->items[i]->path;
if (*p == '/')
p++;
char* manifest_entry = str_dup(p);
if (!manifest_entry) {
log_message(LOG_LEVEL_ERROR, "Failed to allocate manifest entry");
chunk_destroy(current_chunk);
array_list_delete(manifest);
directory_scanner_destroy(scanner);
client_disconnect(client);
client_delete(client);
return 1;
}
if (!array_list_add(manifest, manifest_entry)) {
free(manifest_entry);
chunk_destroy(current_chunk);
array_list_delete(manifest);
directory_scanner_destroy(scanner);
client_disconnect(client);
client_delete(client);
return 1;
}
}
}
if (!add_chunk_to_manifest(manifest, current_chunk)) {
chunk_destroy(current_chunk);
goto send_fail;
}
if (!config->use_sendfile) {
for (int i = 0; i < current_chunk->element_count; i++) {
@@ -644,7 +634,8 @@ int send_files(Config* config) {
continue;
if (!file_load_data(f)) {
log_message(LOG_LEVEL_ERROR, "Failed to load file data");
continue;
chunk_destroy(current_chunk);
goto send_fail;
}
}
}
@@ -674,15 +665,13 @@ int send_files(Config* config) {
if (config->use_delete) {
if (send_delete_manifest(client->file_descriptor, manifest) != 0) {
array_list_delete(manifest);
manifest = NULL;
goto send_fail;
}
array_list_delete(manifest);
manifest = NULL;
}
if (!send_status(client->file_descriptor, STATUS_FINISHED))
goto send_fail;
Status s;
int ok = receive_status(client->file_descriptor, &s) && s == STATUS_OK;
bool ok = finalize_transfer(client);
double elapsed_total = difftime(time(NULL), start);
if (config->show_progress) {
double rate = elapsed_total > 0 ? total_bytes / (1048576.0 * elapsed_total) : 0;
@@ -694,16 +683,16 @@ int send_files(Config* config) {
rate);
}
directory_scanner_destroy(scanner);
client_disconnect(client);
client_delete(client);
disconnect_transfer_client(client);
protocol_session_unbind();
return ok ? 0 : 1;
send_fail:
if (manifest)
array_list_delete(manifest);
directory_scanner_destroy(scanner);
client_disconnect(client);
client_delete(client);
disconnect_transfer_client(client);
protocol_session_unbind();
return 1;
}
src/client/client_validation.c
+50
View File
@@ -0,0 +1,50 @@
#include "client_validation.h"
#include "usage.h"
#include <stdio.h>
/* Validate config after parsing. Returns true if valid. */
bool validate_config(const Config* config) {
if (!config->send_directory || !config->receive_root_directory) {
fprintf(stderr, "Error: source and destination directories are required\n");
print_usage();
return false;
}
if (config->use_sendfile && (config->use_chunk_serialization || config->use_compression)) {
fprintf(stderr, "Error: -f/--sendfile cannot be combined with -c (compression) or -s (chunk "
"serialization)\n");
return false;
}
if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
return false;
}
if (config->use_incremental && config->use_chunk_serialization) {
fprintf(stderr, "Error: --incremental is not supported with -s (chunk serialization)\n");
return false;
}
if (config->use_delta && !config->use_incremental) {
fprintf(stderr, "Error: --delta requires --incremental\n");
return false;
}
if (config->use_delta && config->use_chunk_serialization) {
fprintf(stderr, "Error: --delta cannot be combined with -s (chunk serialization)\n");
return false;
}
if (config->use_delta && config->use_sendfile) {
fprintf(stderr, "Error: --delta cannot be combined with -f (sendfile)\n");
return false;
}
if (config->append || config->append_verify) {
fprintf(
stderr,
"Error: --append and --append-verify are not supported yet; refusing to ignore option\n");
return false;
}
if (config->use_tls) {
if (!config->tls_cert || !config->tls_key || !config->tls_ca) {
fprintf(stderr, "Error: --tls requires --cert, --key, and --ca\n");
return false;
}
}
return true;
}
src/client/client_validation.h
+9
View File
@@ -0,0 +1,9 @@
#ifndef CLIENT_VALIDATION_H
#define CLIENT_VALIDATION_H
#include "config.h"
#include <stdbool.h>
bool validate_config(const Config* config);
#endif
src/client/scanner.c
+184 -255
View File
@@ -52,13 +52,84 @@ static bool safe_relative_link(const char* source_root, const char* containing_d
return safe;
}
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
unsigned long long chunk_size, char** exclude_patterns,
int exclude_count, char** include_patterns,
int include_count, unsigned long long max_size,
unsigned long long min_size, int max_depth,
bool follow_symlinks, bool copy_links, bool safe_links,
bool copy_unsafe_links, bool checksum) {
typedef struct {
char* path;
struct stat stats;
bool is_directory;
} ScannerEntry;
/* Inspect symlinks, resolve the entry type, and apply file filters once for both scanners. */
static int scanner_inspect_entry(const ScannerOptions* options, const char* source_root,
const char* containing_dir, const char* name,
ScannerEntry* entry) {
entry->path = path_cat(containing_dir, name);
if (!entry->path)
return -1;
struct stat link_stats;
if (lstat(entry->path, &link_stats) != 0) {
free(entry->path);
return 0;
}
bool is_symlink = S_ISLNK(link_stats.st_mode);
if (is_symlink && !options->follow_symlinks && !options->copy_links && !options->safe_links &&
!options->copy_unsafe_links)
goto skip;
if (is_symlink && options->safe_links) {
char link_target[4096];
ssize_t length = readlink(entry->path, link_target, sizeof(link_target) - 1);
if (length < 0)
goto skip;
link_target[length] = '\0';
if (link_target[0] == '/' || !safe_relative_link(source_root, containing_dir, link_target))
goto skip;
}
if (is_symlink && options->copy_unsafe_links && !options->copy_links) {
char link_target[4096];
ssize_t length = readlink(entry->path, link_target, sizeof(link_target) - 1);
if (length < 0)
goto skip;
link_target[length] = '\0';
if (link_target[0] != '/')
goto skip;
}
if (is_symlink && options->follow_symlinks && !options->copy_links)
entry->stats = link_stats;
else if (stat(entry->path, &entry->stats) != 0)
goto skip;
entry->is_directory = S_ISDIR(entry->stats.st_mode);
if (entry->is_directory)
return 1;
for (int i = 0; i < options->exclude_count; i++)
if (glob_match(options->exclude_patterns[i], name))
goto skip;
if (options->include_count > 0) {
bool included = false;
for (int i = 0; i < options->include_count; i++)
if (glob_match(options->include_patterns[i], name))
included = true;
if (!included)
goto skip;
}
if ((options->max_size > 0 && (unsigned long long)entry->stats.st_size > options->max_size) ||
(options->min_size > 0 && (unsigned long long)entry->stats.st_size < options->min_size))
goto skip;
return 1;
skip:
free(entry->path);
entry->path = NULL;
return 0;
}
DirectoryScanner* directory_scanner_create_with_options(const char* root_directory,
const ScannerOptions* options) {
if (!root_directory || !options)
return NULL;
DirectoryScanner* scanner = calloc(1, sizeof(DirectoryScanner));
if (scanner == NULL)
return NULL;
@@ -69,21 +140,21 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
}
scanner->current_dir = NULL;
scanner->current_path = NULL;
scanner->use_metadata = use_metadata;
scanner->chunk_size = chunk_size > 0 ? chunk_size : DESIRED_CHUNK_SIZE;
scanner->exclude_patterns = exclude_patterns;
scanner->exclude_count = exclude_count;
scanner->include_patterns = include_patterns;
scanner->include_count = include_count;
scanner->max_size = max_size;
scanner->min_size = min_size;
scanner->max_depth = max_depth;
scanner->use_metadata = options->use_metadata;
scanner->chunk_size = options->chunk_size > 0 ? options->chunk_size : DESIRED_CHUNK_SIZE;
scanner->exclude_patterns = options->exclude_patterns;
scanner->exclude_count = options->exclude_count;
scanner->include_patterns = options->include_patterns;
scanner->include_count = options->include_count;
scanner->max_size = options->max_size;
scanner->min_size = options->min_size;
scanner->max_depth = options->max_depth;
scanner->current_depth = 0;
scanner->follow_symlinks = follow_symlinks;
scanner->copy_links = copy_links;
scanner->safe_links = safe_links;
scanner->copy_unsafe_links = copy_unsafe_links;
scanner->checksum = checksum;
scanner->follow_symlinks = options->follow_symlinks;
scanner->copy_links = options->copy_links;
scanner->safe_links = options->safe_links;
scanner->copy_unsafe_links = options->copy_unsafe_links;
scanner->checksum = options->checksum;
scanner->failed = false;
DirEntry* root = dir_entry_create(root_directory, 0);
if (!root) {
@@ -100,6 +171,20 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
return scanner;
}
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
unsigned long long chunk_size, char** exclude_patterns,
int exclude_count, char** include_patterns,
int include_count, unsigned long long max_size,
unsigned long long min_size, int max_depth,
bool follow_symlinks, bool copy_links, bool safe_links,
bool copy_unsafe_links, bool checksum) {
ScannerOptions options = {
use_metadata, chunk_size, exclude_patterns, exclude_count, include_patterns,
include_count, max_size, min_size, max_depth, 0,
follow_symlinks, copy_links, safe_links, copy_unsafe_links, checksum};
return directory_scanner_create_with_options(root_directory, &options);
}
void directory_scanner_destroy(DirectoryScanner* scanner) {
if (scanner == NULL)
return;
@@ -167,7 +252,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
break;
}
struct dirent* entry = readdir(scanner->current_dir);
const struct dirent* entry = readdir(scanner->current_dir);
if (entry == NULL) {
closedir(scanner->current_dir);
scanner->current_dir = NULL;
@@ -179,67 +264,27 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
continue;
char* cur_path = path_cat(scanner->current_path, entry->d_name);
if (!cur_path) {
ScannerOptions options = {scanner->use_metadata, scanner->chunk_size,
scanner->exclude_patterns, scanner->exclude_count,
scanner->include_patterns, scanner->include_count,
scanner->max_size, scanner->min_size,
scanner->max_depth, 0,
scanner->follow_symlinks, scanner->copy_links,
scanner->safe_links, scanner->copy_unsafe_links,
scanner->checksum};
ScannerEntry inspected;
int inspection = scanner_inspect_entry(&options, scanner->current_path, scanner->current_path,
entry->d_name, &inspected);
if (inspection < 0) {
scanner->failed = true;
break;
}
struct stat stats;
struct stat lstats;
bool is_symlink = false;
if (lstat(cur_path, &lstats) != 0) {
free(cur_path);
if (inspection == 0)
continue;
}
is_symlink = S_ISLNK(lstats.st_mode);
char* cur_path = inspected.path;
struct stat stats = inspected.stats;
if (is_symlink && !scanner->follow_symlinks && !scanner->copy_links && !scanner->safe_links &&
!scanner->copy_unsafe_links) {
free(cur_path);
continue;
}
if (is_symlink && scanner->safe_links) {
char link_target[4096];
ssize_t len = readlink(cur_path, link_target, sizeof(link_target) - 1);
if (len < 0) {
free(cur_path);
continue;
}
link_target[len] = '\0';
if (link_target[0] == '/' ||
!safe_relative_link(scanner->current_path, scanner->current_path, link_target)) {
free(cur_path);
continue;
}
}
if (is_symlink && scanner->copy_unsafe_links && !scanner->copy_links) {
char link_target[4096];
ssize_t len = readlink(cur_path, link_target, sizeof(link_target) - 1);
if (len < 0) {
free(cur_path);
continue;
}
link_target[len] = '\0';
bool unsafe = (link_target[0] == '/');
if (!unsafe) {
free(cur_path);
continue;
}
}
bool use_lstat = is_symlink && scanner->follow_symlinks && !scanner->copy_links;
if (use_lstat) {
stats = lstats;
} else {
if (stat(cur_path, &stats) != 0) {
free(cur_path);
continue;
}
}
if (S_ISDIR(stats.st_mode)) {
if (inspected.is_directory) {
int next_depth = scanner->current_depth + 1;
if (scanner->max_depth <= 0 || next_depth < scanner->max_depth) {
DirEntry* de = dir_entry_create(cur_path, next_depth);
@@ -254,38 +299,6 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
free(cur_path);
continue;
}
bool excluded = false;
for (int i = 0; i < scanner->exclude_count; i++) {
if (glob_match(scanner->exclude_patterns[i], entry->d_name)) {
excluded = true;
break;
}
}
if (excluded) {
free(cur_path);
continue;
}
if (scanner->include_count > 0) {
bool included = false;
for (int i = 0; i < scanner->include_count; i++) {
if (glob_match(scanner->include_patterns[i], entry->d_name)) {
included = true;
break;
}
}
if (!included) {
free(cur_path);
continue;
}
}
if ((scanner->max_size > 0 && (unsigned long long)stats.st_size > scanner->max_size) ||
(scanner->min_size > 0 && (unsigned long long)stats.st_size < scanner->min_size)) {
free(cur_path);
continue;
}
File* file = file_create(cur_path);
if (file == NULL) {
free(cur_path);
@@ -336,29 +349,13 @@ typedef struct {
ParallelScanner* ps;
char** dirs;
int dir_count;
bool use_metadata;
unsigned long long chunk_size;
char** exclude_patterns;
int exclude_count;
char** include_patterns;
int include_count;
unsigned long long max_size;
unsigned long long min_size;
int max_depth;
bool follow_symlinks;
bool copy_links;
bool safe_links;
bool copy_unsafe_links;
bool checksum;
ScannerOptions options;
} ParallelWorkerArg;
static int parallel_worker_thread(void* arg) {
ParallelWorkerArg* wa = (ParallelWorkerArg*)arg;
for (int i = 0; i < wa->dir_count; i++) {
DirectoryScanner* ds = directory_scanner_create(
wa->dirs[i], wa->use_metadata, wa->chunk_size, wa->exclude_patterns, wa->exclude_count,
wa->include_patterns, wa->include_count, wa->max_size, wa->min_size, wa->max_depth,
wa->follow_symlinks, wa->copy_links, wa->safe_links, wa->copy_unsafe_links, wa->checksum);
DirectoryScanner* ds = directory_scanner_create_with_options(wa->dirs[i], &wa->options);
if (!ds) {
mtx_lock(&wa->ps->result_mutex);
wa->ps->failed = true;
@@ -366,6 +363,8 @@ static int parallel_worker_thread(void* arg) {
cnd_broadcast(&wa->ps->result_not_empty);
cnd_broadcast(&wa->ps->result_not_full);
mtx_unlock(&wa->ps->result_mutex);
for (int j = i; j < wa->dir_count; j++)
free(wa->dirs[j]);
break;
}
Chunk* chunk;
@@ -401,13 +400,22 @@ static int parallel_worker_thread(void* arg) {
return thrd_success;
}
ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata,
unsigned long long chunk_size, char** exclude_patterns,
int exclude_count, char** include_patterns,
int include_count, unsigned long long max_size,
unsigned long long min_size, int max_depth,
int num_threads, bool follow_symlinks, bool copy_links,
bool safe_links, bool copy_unsafe_links, bool checksum) {
static void parallel_scanner_creation_failed(ParallelScanner* ps) {
mtx_lock(&ps->result_mutex);
ps->failed = true;
atomic_store(&ps->cancelled, true);
ps->expected_threads = ps->created_threads;
if (ps->completed >= ps->expected_threads)
ps->done = true;
cnd_broadcast(&ps->result_not_empty);
cnd_broadcast(&ps->result_not_full);
mtx_unlock(&ps->result_mutex);
}
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
const ScannerOptions* options) {
if (!root_directory || !options)
return NULL;
ParallelScanner* ps = calloc(1, sizeof(ParallelScanner));
if (!ps)
return NULL;
@@ -460,107 +468,27 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
parallel_scanner_destroy(ps);
return NULL;
}
struct dirent* entry;
const struct dirent* entry;
while ((entry = readdir(dir)) != NULL) {
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
continue;
char* cur_path = path_cat(root_directory, entry->d_name);
if (!cur_path) {
ScannerEntry inspected;
int inspection =
scanner_inspect_entry(options, root_directory, root_directory, entry->d_name, &inspected);
if (inspection < 0) {
ps->failed = true;
continue;
}
struct stat lstats;
if (lstat(cur_path, &lstats) != 0) {
free(cur_path);
if (inspection == 0)
continue;
}
bool is_symlink = S_ISLNK(lstats.st_mode);
// Skip symlinks unless the user explicitly enabled following/copying them.
if (is_symlink && !follow_symlinks && !copy_links && !safe_links && !copy_unsafe_links) {
free(cur_path);
continue;
}
// --safe-links: reject symlinks pointing outside the source tree.
if (is_symlink && safe_links) {
char link_target[4096];
ssize_t len = readlink(cur_path, link_target, sizeof(link_target) - 1);
if (len < 0) {
free(cur_path);
continue;
}
link_target[len] = 0;
if (link_target[0] == '/' ||
!safe_relative_link(root_directory, root_directory, link_target)) {
free(cur_path);
continue;
}
}
// --copy-unsafe-links (without --copy-links): only copy absolute symlinks.
if (is_symlink && copy_unsafe_links && !copy_links) {
char link_target[4096];
ssize_t len = readlink(cur_path, link_target, sizeof(link_target) - 1);
if (len < 0) {
free(cur_path);
continue;
}
link_target[len] = 0;
bool unsafe = (link_target[0] == '/');
if (!unsafe) {
free(cur_path);
continue;
}
}
// Determine whether to use lstat or stat results for the entry.
struct stat st;
bool use_lstat_res = is_symlink && follow_symlinks && !copy_links;
if (use_lstat_res) {
st = lstats;
} else {
if (stat(cur_path, &st) != 0) {
free(cur_path);
continue;
}
}
if (S_ISDIR(st.st_mode)) {
char* cur_path = inspected.path;
struct stat st = inspected.stats;
if (inspected.is_directory) {
if (!array_list_add(subdirs, cur_path)) {
free(cur_path);
ps->failed = true;
}
} else {
bool excluded = false;
for (int i = 0; i < exclude_count; i++) {
if (glob_match(exclude_patterns[i], entry->d_name)) {
excluded = true;
break;
}
}
if (excluded) {
free(cur_path);
continue;
}
if (include_count > 0) {
bool included = false;
for (int i = 0; i < include_count; i++) {
if (glob_match(include_patterns[i], entry->d_name)) {
included = true;
break;
}
}
if (!included) {
free(cur_path);
continue;
}
}
if ((max_size > 0 && (unsigned long long)st.st_size > max_size) ||
(min_size > 0 && (unsigned long long)st.st_size < min_size)) {
free(cur_path);
continue;
}
File* file = file_create(cur_path);
free(cur_path);
if (!file) {
@@ -568,9 +496,9 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
continue;
}
file->data->size = st.st_size;
if (use_metadata)
if (options->use_metadata)
file->metadata = file_metadata_create(&st);
if (use_metadata && !file->metadata) {
if (options->use_metadata && !file->metadata) {
file_destroy(file);
ps->failed = true;
continue;
@@ -583,7 +511,7 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
}
closedir(dir);
unsigned long long cs = chunk_size > 0 ? chunk_size : DESIRED_CHUNK_SIZE;
unsigned long long cs = options->chunk_size > 0 ? options->chunk_size : DESIRED_CHUNK_SIZE;
if (root_files->size > 0) {
ArrayList* batch = array_list_create(NULL);
if (!batch) {
@@ -606,7 +534,6 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
void** items = array_list_to_array(batch);
if (!items) {
ps->failed = true;
batch->item_destroyer = file_destroy;
array_list_delete(batch);
batch = NULL;
break;
@@ -615,11 +542,13 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
free(items);
if (!c) {
ps->failed = true;
batch->item_destroyer = file_destroy;
array_list_delete(batch);
batch = NULL;
break;
}
int batch_start = i - batch->size + 1;
for (int j = batch_start; j <= i; j++)
root_files->items[j] = NULL;
batch->item_destroyer = NULL;
array_list_delete(batch);
batch = NULL;
@@ -646,11 +575,10 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
array_list_delete(batch);
}
ps->initial_chunk = first;
root_files->item_destroyer = NULL;
}
array_list_delete(root_files);
int n = num_threads > 0 ? num_threads : 4;
int n = options->num_threads > 0 ? options->num_threads : 4;
if (n > subdirs->size)
n = subdirs->size > 0 ? subdirs->size : 1;
@@ -673,14 +601,14 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
break;
ParallelWorkerArg* wa = calloc(1, sizeof(ParallelWorkerArg));
if (!wa) {
ps->failed = true;
parallel_scanner_creation_failed(ps);
break;
}
wa->ps = ps;
wa->dirs = calloc(count, sizeof(char*));
if (!wa->dirs) {
free(wa);
ps->failed = true;
parallel_scanner_creation_failed(ps);
break;
}
bool dup_ok = true;
@@ -694,37 +622,19 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
free(wa->dirs[j]);
free(wa->dirs);
free(wa);
ps->failed = true;
parallel_scanner_creation_failed(ps);
break;
}
wa->dir_count = count;
wa->use_metadata = use_metadata;
wa->chunk_size = cs;
wa->exclude_patterns = exclude_patterns;
wa->exclude_count = exclude_count;
wa->include_patterns = include_patterns;
wa->include_count = include_count;
wa->max_size = max_size;
wa->min_size = min_size;
wa->max_depth = max_depth;
wa->follow_symlinks = follow_symlinks;
wa->copy_links = copy_links;
wa->safe_links = safe_links;
wa->copy_unsafe_links = copy_unsafe_links;
wa->checksum = checksum;
wa->options = *options;
wa->options.chunk_size = cs;
start += count;
if (thrd_create(&ps->threads[t], parallel_worker_thread, wa) != thrd_success) {
for (int j = 0; j < count; j++)
free(wa->dirs[j]);
free(wa->dirs);
free(wa);
ps->failed = true;
atomic_store(&ps->cancelled, true);
ps->expected_threads = ps->created_threads;
mtx_lock(&ps->result_mutex);
cnd_broadcast(&ps->result_not_empty);
cnd_broadcast(&ps->result_not_full);
mtx_unlock(&ps->result_mutex);
parallel_scanner_creation_failed(ps);
break;
}
ps->num_threads++;
@@ -735,6 +645,20 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
return ps;
}
ParallelScanner* parallel_scanner_create(const char* root_directory, bool use_metadata,
unsigned long long chunk_size, char** exclude_patterns,
int exclude_count, char** include_patterns,
int include_count, unsigned long long max_size,
unsigned long long min_size, int max_depth,
int num_threads, bool follow_symlinks, bool copy_links,
bool safe_links, bool copy_unsafe_links, bool checksum) {
ScannerOptions options = {use_metadata, chunk_size, exclude_patterns, exclude_count,
include_patterns, include_count, max_size, min_size,
max_depth, num_threads, follow_symlinks, copy_links,
safe_links, copy_unsafe_links, checksum};
return parallel_scanner_create_with_options(root_directory, &options);
}
Chunk* parallel_scanner_next(ParallelScanner* ps) {
if (ps->initial_chunk) {
Chunk* c = ps->initial_chunk;
@@ -743,6 +667,11 @@ Chunk* parallel_scanner_next(ParallelScanner* ps) {
}
if (ps->num_threads == 0) {
mtx_lock(&ps->result_mutex);
if (!queue_is_empty(ps->result_queue)) {
Chunk* chunk = queue_dequeue(ps->result_queue);
mtx_unlock(&ps->result_mutex);
return chunk;
}
ps->done = true;
mtx_unlock(&ps->result_mutex);
return NULL;
src/client/scanner.h
+23 -1
View File
@@ -8,6 +8,24 @@
#include <threads.h>
#include <stdatomic.h>
typedef struct {
bool use_metadata;
unsigned long long chunk_size;
char** exclude_patterns;
int exclude_count;
char** include_patterns;
int include_count;
unsigned long long max_size;
unsigned long long min_size;
int max_depth;
int num_threads;
bool follow_symlinks;
bool copy_links;
bool safe_links;
bool copy_unsafe_links;
bool checksum;
} ScannerOptions;
typedef struct {
Queue* directories;
DIR* current_dir;
@@ -53,17 +71,21 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
unsigned long long min_size, int max_depth,
bool follow_symlinks, bool copy_links, bool safe_links,
bool copy_unsafe_links, bool checksum);
DirectoryScanner* directory_scanner_create_with_options(const char* root_directory,
const ScannerOptions* options);
Chunk* directory_scanner_next(DirectoryScanner* scanner);
bool directory_scanner_failed(const DirectoryScanner* scanner);
void directory_scanner_destroy(DirectoryScanner* scanner);
ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata,
ParallelScanner* parallel_scanner_create(const char* root_directory, bool use_metadata,
unsigned long long chunk_size, char** exclude_patterns,
int exclude_count, char** include_patterns,
int include_count, unsigned long long max_size,
unsigned long long min_size, int max_depth,
int num_threads, bool follow_symlinks, bool copy_links,
bool safe_links, bool copy_unsafe_links, bool checksum);
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
const ScannerOptions* options);
Chunk* parallel_scanner_next(ParallelScanner* scanner);
bool parallel_scanner_failed(const ParallelScanner* scanner);
void parallel_scanner_destroy(ParallelScanner* scanner);
src/client/usage.c
+3 -2
View File
@@ -1,8 +1,9 @@
#include "stdio.h"
#include "usage.h"
#include <stdio.h>
#include <delta.h>
#include <chunk.h>
static __attribute__((unused)) void print_usage() {
void print_usage(void) {
printf("Usage:\n");
printf(" fastsync [options] <source> <destination>\n");
printf(" fastsync [options] --source-dir <src> --dest-dir <dst>\n");
src/client/usage.h
+6
View File
@@ -0,0 +1,6 @@
#ifndef USAGE_H
#define USAGE_H
void print_usage(void);
#endif