TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix exit() in library code, add --include/--max-size/--min-size

Browse Source
This commit is contained in:
TapTap
2026-07-16 14:15:38 +02:00
parent 07207cd71d
commit bbdf027be8
27 changed files with 568 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/client/client_cli.c
+11
View File
@@ -28,6 +28,9 @@ static void print_usage(void) {
printf(" --progress Show transfer progress\n");
printf(" --delete Delete files on receiver not in source\n");
printf(" --exclude <pattern> Exclude files matching pattern\n");
printf(" --include <pattern> Only include files matching pattern\n");
printf(" --max-size <n> Skip files larger than n bytes\n");
printf(" --min-size <n> Skip files smaller than n bytes\n");
printf(" -m Enable multithreading\n");
printf(" -s Enable chunk serialization\n");
printf(" -f Enable sendfile (TCP only, not with -c or -s)\n");
@@ -78,6 +81,14 @@ int main(int argc, char *argv[]) {
int idx = config->exclude_count++;
config->exclude_patterns = realloc(config->exclude_patterns, config->exclude_count * sizeof(char *));
config->exclude_patterns[idx] = str_dup(argv[++i]);
} else if (strcmp(argv[i], "--include") == 0 && i + 1 < argc) {
int idx = config->include_count++;
config->include_patterns = realloc(config->include_patterns, config->include_count * sizeof(char *));
config->include_patterns[idx] = str_dup(argv[++i]);
} else if (strcmp(argv[i], "--max-size") == 0 && i + 1 < argc) {
config->max_size = strtoull(argv[++i], NULL, 10);
} else if (strcmp(argv[i], "--min-size") == 0 && i + 1 < argc) {
config->min_size = strtoull(argv[++i], NULL, 10);
} else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) {
config->use_compression = true;
log_message(LOG_LEVEL_INFO, "Enabled Compression");
src/client/client_send.c
+83 -26
View File
@@ -20,26 +20,29 @@
int send_chunk(Client *client, Chunk *chunk, Config *config) {
if (config->use_chunk_serialization) {
send_status(client->file_descriptor, STATUS_CHUNK);
if (!send_status(client->file_descriptor, STATUS_CHUNK)) return -1;
Data *data;
if (config->use_compression) {
data = chunk_compress(chunk, config->compression_level, config->use_metadata);
} else {
data = chunk_serialize(chunk, config->use_metadata);
}
send_data(client->file_descriptor, data);
if (data == NULL) return -1;
if (!send_data(client->file_descriptor, data)) { data_destroy(data); return -1; }
data_destroy(data);
} else if (config->use_sendfile && !config->use_compression) {
for (int i = 0; i < chunk->element_count; i++) {
send_status(client->file_descriptor, STATUS_NEXT);
file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata);
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata))
return -1;
}
} else {
for (int i = 0; i < chunk->element_count; i++) {
send_status(client->file_descriptor, STATUS_NEXT);
file_send_single_calls(chunk->items[i], client->file_descriptor,
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
if (!file_send_single_calls(chunk->items[i], client->file_descriptor,
config->use_metadata,
config->use_compression ? config->compression_level : 0);
config->use_compression ? config->compression_level : 0))
return -1;
}
}
return 0;
@@ -56,9 +59,17 @@ static int send_chunks_multithreaded(void *pipeline_context) {
client = client_connect_ssh(context->config->ssh_destination, context->config->ssh_port);
} else {
client = client_create();
client_connect(client, server_host, server_port);
if (!client || !client_connect(client, server_host, server_port)) {
if (client) client_delete(client);
fprintf(stderr, "Error: could not connect to server\n");
return thrd_error;
}
}
if (!config_send(client->file_descriptor, context->config)) {
client_disconnect(client);
client_delete(client);
return thrd_error;
}
config_send(client->file_descriptor, context->config);
while (true) {
Chunk *current_chunk = queue_dequeue_multithreaded(
@@ -74,14 +85,17 @@ static int send_chunks_multithreaded(void *pipeline_context) {
(char *)context->manifest->items[i]);
}
send_status(client->file_descriptor, STATUS_FINISHED);
int ok = receive_status(client->file_descriptor) == STATUS_OK;
Status s;
int ok = receive_status(client->file_descriptor, &s) && s == STATUS_OK;
client_disconnect(client);
client_delete(client);
return ok ? thrd_success : thrd_error;
}
if (send_chunk(client, current_chunk, context->config) != 0) {
perror("Something unexpected happend while sending the chunk");
exit(EXIT_FAILURE);
fprintf(stderr, "Error: unexpected error while sending chunk\n");
client_disconnect(client);
client_delete(client);
return thrd_error;
}
chunk_destroy(current_chunk);
}
@@ -93,7 +107,9 @@ static int scan_directory_multithreaded(void *pipeline_context) {
DirectoryScanner *scanner = directory_scanner_create(
context->config->send_directory, context->config->use_metadata,
context->config->chunk_size, context->config->exclude_patterns,
context->config->exclude_count);
context->config->exclude_count, context->config->include_patterns,
context->config->include_count, context->config->max_size,
context->config->min_size);
mtx_unlock(&context->mutex_scanner);
Chunk *current_chunk;
@@ -136,8 +152,13 @@ static int load_files_multithreaded(void *pipeline_context) {
return thrd_success;
}
if (!context->config->use_sendfile) {
for (int i = 0; i < chunk->element_count; i++)
file_load_data(chunk->items[i]);
for (int i = 0; i < chunk->element_count; i++) {
if (!file_load_data(chunk->items[i])) {
log_message(LOG_LEVEL_ERROR, "Failed to load file data, skipping");
file_destroy(chunk->items[i]);
chunk->items[i] = NULL;
}
}
}
queue_enqueue_multithreaded(context->queue_loader, chunk,
&context->mutex_loader,
@@ -150,7 +171,9 @@ int send_files(Config *config) {
if (config->dry_run) {
DirectoryScanner *scanner = directory_scanner_create(
config->send_directory, config->use_metadata, config->chunk_size,
config->exclude_patterns, config->exclude_count);
config->exclude_patterns, config->exclude_count,
config->include_patterns, config->include_count,
config->max_size, config->min_size);
Chunk *chunk;
int file_count = 0;
unsigned long long total_bytes = 0;
@@ -177,14 +200,25 @@ int send_files(Config *config) {
return 1;
}
client = client_connect_ssh(config->ssh_destination, config->ssh_port);
if (!client) return 1;
} else {
client = client_create();
client_connect(client, server_host, server_port);
if (!client || !client_connect(client, server_host, server_port)) {
if (client) client_delete(client);
fprintf(stderr, "Error: could not connect to server\n");
return 1;
}
}
if (!config_send(client->file_descriptor, config)) {
client_disconnect(client);
client_delete(client);
return 1;
}
config_send(client->file_descriptor, config);
DirectoryScanner *scanner = directory_scanner_create(
config->send_directory, config->use_metadata, config->chunk_size,
config->exclude_patterns, config->exclude_count);
config->exclude_patterns, config->exclude_count,
config->include_patterns, config->include_count,
config->max_size, config->min_size);
Chunk *current_chunk;
unsigned long long total_bytes = 0;
time_t last_progress = 0;
@@ -201,10 +235,18 @@ int send_files(Config *config) {
}
}
if (!config->use_sendfile) {
for (int i = 0; i < current_chunk->element_count; i++)
file_load_data(current_chunk->items[i]);
for (int i = 0; i < current_chunk->element_count; i++) {
if (!file_load_data(current_chunk->items[i])) {
log_message(LOG_LEVEL_ERROR, "Failed to load file data");
continue;
}
}
}
if (send_chunk(client, current_chunk, config) != 0) {
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
chunk_destroy(current_chunk);
break;
}
send_chunk(client, current_chunk, config);
if (config->show_progress) {
total_bytes += chunk_bytes;
time_t now = time(NULL);
@@ -226,7 +268,8 @@ int send_files(Config *config) {
array_list_delete(manifest);
}
send_status(client->file_descriptor, STATUS_FINISHED);
int ok = receive_status(client->file_descriptor) == STATUS_OK;
Status s;
int ok = receive_status(client->file_descriptor, &s) && s == STATUS_OK;
if (config->show_progress) {
double elapsed = difftime(time(NULL), start);
double rate = elapsed > 0 ? total_bytes / (1048576.0 * elapsed) : 0;
@@ -242,7 +285,9 @@ int send_files_multithreaded(Config *config) {
if (config->dry_run) {
DirectoryScanner *scanner = directory_scanner_create(
config->send_directory, config->use_metadata, config->chunk_size,
config->exclude_patterns, config->exclude_count);
config->exclude_patterns, config->exclude_count,
config->include_patterns, config->include_count,
config->max_size, config->min_size);
Chunk *chunk;
int file_count = 0;
unsigned long long total_bytes = 0;
@@ -262,9 +307,20 @@ int send_files_multithreaded(Config *config) {
return 0;
}
Queue *q1 = queue_create(100, chunk_destroy);
Queue *q2 = queue_create(100, chunk_destroy);
if (!q1 || !q2) {
if (q1) queue_destroy(q1);
if (q2) queue_destroy(q2);
return 1;
}
PipelineContextSender *context =
pipeline_context_sender_create(config, queue_create(100, chunk_destroy),
queue_create(100, chunk_destroy));
pipeline_context_sender_create(config, q1, q2);
if (!context) {
queue_destroy(q1);
queue_destroy(q2);
return 1;
}
if (config->use_delete)
context->manifest = array_list_create(free);
@@ -275,6 +331,7 @@ int send_files_multithreaded(Config *config) {
thrd_create(&sender, send_chunks_multithreaded, context) !=
thrd_success) {
perror("Error creating threads.\n");
pipeline_context_sender_destroy(context);
return 1;
}
src/client/scanner.c
+34 -3
View File
@@ -11,7 +11,7 @@
#include <sys/stat.h>
#include <unistd.h>
DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size, char **exclude_patterns, int exclude_count) {
DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size, char **exclude_patterns, int exclude_count, char **include_patterns, int include_count, unsigned long long max_size, unsigned long long min_size) {
DirectoryScanner *scanner = malloc(sizeof(DirectoryScanner));
scanner->directories = queue_create(100, free);
scanner->current_dir = NULL;
@@ -20,6 +20,10 @@ DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metada
scanner->chunk_size = chunk_size > 0 ? chunk_size : DESIRED_CHUNK_SIZE;
scanner->exclude_patterns = exclude_patterns;
scanner->exclude_count = exclude_count;
scanner->include_patterns = include_patterns;
scanner->include_count = include_count;
scanner->max_size = max_size;
scanner->min_size = min_size;
queue_enqueue(scanner->directories, str_dup(root_directory));
return scanner;
}
@@ -58,8 +62,10 @@ static int open_next_directory(DirectoryScanner *scanner) {
scanner->current_path = (char *)queue_dequeue(scanner->directories);
scanner->current_dir = opendir(scanner->current_path);
if (scanner->current_dir == NULL) {
perror("Could not open directory!");
exit(EXIT_FAILURE);
perror("Could not open directory");
free(scanner->current_path);
scanner->current_path = NULL;
return 0;
}
return 1;
}
@@ -107,7 +113,32 @@ Chunk *directory_scanner_next(DirectoryScanner *scanner) {
free(cur_path);
continue;
}
if (scanner->include_count > 0) {
bool included = false;
for (int i = 0; i < scanner->include_count; i++) {
if (glob_match(scanner->include_patterns[i], entry->d_name)) {
included = true;
break;
}
}
if (!included) {
free(cur_path);
continue;
}
}
if ((scanner->max_size > 0 && (unsigned long long)stats.st_size > scanner->max_size) ||
(scanner->min_size > 0 && (unsigned long long)stats.st_size < scanner->min_size)) {
free(cur_path);
continue;
}
File *file = file_create(cur_path);
if (file == NULL) {
free(cur_path);
continue;
}
file->data->size = stats.st_size;
if (scanner->use_metadata)
file->metadata = file_metadata_create(&stats);
src/client/scanner.h
+5 -1
View File
@@ -14,9 +14,13 @@ typedef struct {
unsigned long long chunk_size;
char **exclude_patterns;
int exclude_count;
char **include_patterns;
int include_count;
unsigned long long max_size;
unsigned long long min_size;
} DirectoryScanner;
DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size, char **exclude_patterns, int exclude_count);
DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size, char **exclude_patterns, int exclude_count, char **include_patterns, int include_count, unsigned long long max_size, unsigned long long min_size);
Chunk *directory_scanner_next(DirectoryScanner *scanner);
void directory_scanner_destroy(DirectoryScanner *scanner);