rsync flags: -a, -n, -p, --exclude, --delete
Feature changes across 15 files: - -a/--archive: enables -c -m -M (no -s, which slows transfers) - -n/--dry-run: scan + print without connecting or transferring - -p <port>: custom SSH port (passed as -p to ssh via execvp) - --exclude <pattern>: glob-based filename filtering in scanner - --delete: sender collects file manifest; receiver deletes unlisted files Protocol: added STATUS_MANIFEST, use_delete field in Config wire format. New utilities: glob_match(), delete_extras() with recursive directory walk. Scanner: accepts exclude patterns; skips matching entries. SSH transport: switched from execlp to execvp for dynamic port arg. Server + multiprocessing: handle STATUS_MANIFEST in both single and multithreaded receive paths. Builds clean, all 7 tests pass.
This commit is contained in:
@@ -22,7 +22,12 @@ static void print_usage(void) {
|
||||
printf("Options:\n");
|
||||
printf(" -c [level] Enable compression (level 1-22, default 5)\n");
|
||||
printf(" -z [level] Alias for -c\n");
|
||||
printf(" -a, --archive Archive mode (-c -m -M)\n");
|
||||
printf(" -n, --dry-run Show what would be transferred\n");
|
||||
printf(" -p <port> SSH port (default: 22)\n");
|
||||
printf(" --progress Show transfer progress\n");
|
||||
printf(" --delete Delete files on receiver not in source\n");
|
||||
printf(" --exclude <pattern> Exclude files matching pattern\n");
|
||||
printf(" -m Enable multithreading\n");
|
||||
printf(" -s Enable chunk serialization\n");
|
||||
printf(" -f Enable sendfile (TCP only, not with -c or -s)\n");
|
||||
@@ -58,6 +63,21 @@ int main(int argc, char *argv[]) {
|
||||
if (strcmp(argv[i], "--help") == 0) {
|
||||
print_usage();
|
||||
return 0;
|
||||
} else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--archive") == 0) {
|
||||
config->use_compression = true;
|
||||
config->use_multithreading = true;
|
||||
config->use_metadata = true;
|
||||
log_message(LOG_LEVEL_INFO, "Enabled archive mode (-c -m -M)");
|
||||
} else if (strcmp(argv[i], "-n") == 0 || strcmp(argv[i], "--dry-run") == 0) {
|
||||
config->dry_run = true;
|
||||
} else if (strcmp(argv[i], "-p") == 0 && i + 1 < argc) {
|
||||
config->ssh_port = atoi(argv[++i]);
|
||||
} else if (strcmp(argv[i], "--delete") == 0) {
|
||||
config->use_delete = true;
|
||||
} else if (strcmp(argv[i], "--exclude") == 0 && i + 1 < argc) {
|
||||
int idx = config->exclude_count++;
|
||||
config->exclude_patterns = realloc(config->exclude_patterns, config->exclude_count * sizeof(char *));
|
||||
config->exclude_patterns[idx] = str_dup(argv[++i]);
|
||||
} else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) {
|
||||
config->use_compression = true;
|
||||
log_message(LOG_LEVEL_INFO, "Enabled Compression");
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "client_send.h"
|
||||
#include "array_list.h"
|
||||
#include "chunk.h"
|
||||
#include "config.h"
|
||||
#include "data.h"
|
||||
@@ -52,7 +53,7 @@ static int send_chunks_multithreaded(void *pipeline_context) {
|
||||
fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
|
||||
return 1;
|
||||
}
|
||||
client = client_connect_ssh(context->config->ssh_destination);
|
||||
client = client_connect_ssh(context->config->ssh_destination, context->config->ssh_port);
|
||||
} else {
|
||||
client = client_create();
|
||||
client_connect(client, server_host, server_port);
|
||||
@@ -65,6 +66,13 @@ static int send_chunks_multithreaded(void *pipeline_context) {
|
||||
&context->condition_not_empty_loader,
|
||||
&context->condition_not_full_loader, &context->loader_done);
|
||||
if (current_chunk == NULL) {
|
||||
if (context->config->use_delete) {
|
||||
send_status(client->file_descriptor, STATUS_MANIFEST);
|
||||
send_int(client->file_descriptor, context->manifest->size);
|
||||
for (int i = 0; i < context->manifest->size; i++)
|
||||
send_str(client->file_descriptor,
|
||||
(char *)context->manifest->items[i]);
|
||||
}
|
||||
send_status(client->file_descriptor, STATUS_FINISHED);
|
||||
int ok = receive_status(client->file_descriptor) == STATUS_OK;
|
||||
client_disconnect(client);
|
||||
@@ -82,16 +90,26 @@ static int send_chunks_multithreaded(void *pipeline_context) {
|
||||
static int scan_directory_multithreaded(void *pipeline_context) {
|
||||
PipelineContextSender *context = (PipelineContextSender *)pipeline_context;
|
||||
mtx_lock(&context->mutex_scanner);
|
||||
DirectoryScanner *scanner =
|
||||
directory_scanner_create(context->config->send_directory, context->config->use_metadata, context->config->chunk_size);
|
||||
DirectoryScanner *scanner = directory_scanner_create(
|
||||
context->config->send_directory, context->config->use_metadata,
|
||||
context->config->chunk_size, context->config->exclude_patterns,
|
||||
context->config->exclude_count);
|
||||
mtx_unlock(&context->mutex_scanner);
|
||||
|
||||
Chunk *current_chunk;
|
||||
while ((current_chunk = directory_scanner_next(scanner)) != NULL)
|
||||
while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
|
||||
if (context->config->use_delete) {
|
||||
mtx_lock(&context->mutex_scanner);
|
||||
for (int i = 0; i < current_chunk->element_count; i++)
|
||||
array_list_add(context->manifest,
|
||||
str_dup(current_chunk->items[i]->path));
|
||||
mtx_unlock(&context->mutex_scanner);
|
||||
}
|
||||
queue_enqueue_multithreaded(context->queue_scanner, current_chunk,
|
||||
&context->mutex_scanner,
|
||||
&context->condition_not_empty_scanner,
|
||||
&context->condition_not_full_scanner);
|
||||
}
|
||||
mtx_lock(&context->mutex_scanner);
|
||||
context->scanner_done = true;
|
||||
cnd_signal(&context->condition_not_empty_scanner);
|
||||
@@ -127,27 +145,56 @@ static int load_files_multithreaded(void *pipeline_context) {
|
||||
}
|
||||
|
||||
int send_files(Config *config) {
|
||||
if (config->dry_run) {
|
||||
DirectoryScanner *scanner = directory_scanner_create(
|
||||
config->send_directory, config->use_metadata, config->chunk_size,
|
||||
config->exclude_patterns, config->exclude_count);
|
||||
Chunk *chunk;
|
||||
int file_count = 0;
|
||||
unsigned long long total_bytes = 0;
|
||||
printf("Dry run: files to be transferred\n");
|
||||
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
printf(" %s (%zu bytes)\n", chunk->items[i]->path,
|
||||
chunk->items[i]->data->size);
|
||||
total_bytes += chunk->items[i]->data->size;
|
||||
file_count++;
|
||||
}
|
||||
chunk_destroy(chunk);
|
||||
}
|
||||
directory_scanner_destroy(scanner);
|
||||
printf("Total: %d files, %.1f MB\n", file_count,
|
||||
total_bytes / 1048576.0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
Client *client;
|
||||
if (config->transport == TRANSPORT_SSH) {
|
||||
if (config->use_sendfile) {
|
||||
fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
|
||||
return 1;
|
||||
}
|
||||
client = client_connect_ssh(config->ssh_destination);
|
||||
client = client_connect_ssh(config->ssh_destination, config->ssh_port);
|
||||
} else {
|
||||
client = client_create();
|
||||
client_connect(client, server_host, server_port);
|
||||
}
|
||||
config_send(client->file_descriptor, config);
|
||||
DirectoryScanner *scanner = directory_scanner_create(config->send_directory, config->use_metadata, config->chunk_size);
|
||||
DirectoryScanner *scanner = directory_scanner_create(
|
||||
config->send_directory, config->use_metadata, config->chunk_size,
|
||||
config->exclude_patterns, config->exclude_count);
|
||||
Chunk *current_chunk;
|
||||
unsigned long long total_bytes = 0;
|
||||
time_t last_progress = 0;
|
||||
time_t start = time(NULL);
|
||||
ArrayList *manifest = config->use_delete ? array_list_create(free) : NULL;
|
||||
while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
|
||||
unsigned long long chunk_bytes = 0;
|
||||
for (int i = 0; i < current_chunk->element_count; i++)
|
||||
for (int i = 0; i < current_chunk->element_count; i++) {
|
||||
chunk_bytes += current_chunk->items[i]->data->size;
|
||||
if (manifest)
|
||||
array_list_add(manifest, str_dup(current_chunk->items[i]->path));
|
||||
}
|
||||
if (!config->use_sendfile) {
|
||||
for (int i = 0; i < current_chunk->element_count; i++)
|
||||
file_load_data(current_chunk->items[i]);
|
||||
@@ -166,6 +213,15 @@ int send_files(Config *config) {
|
||||
}
|
||||
chunk_destroy(current_chunk);
|
||||
}
|
||||
if (config->use_delete) {
|
||||
send_status(client->file_descriptor, STATUS_MANIFEST);
|
||||
send_int(client->file_descriptor, manifest->size);
|
||||
for (int i = 0; i < manifest->size; i++)
|
||||
send_str(client->file_descriptor, (char *)manifest->items[i]);
|
||||
for (int i = 0; i < manifest->size; i++)
|
||||
free(manifest->items[i]);
|
||||
array_list_delete(manifest);
|
||||
}
|
||||
send_status(client->file_descriptor, STATUS_FINISHED);
|
||||
int ok = receive_status(client->file_descriptor) == STATUS_OK;
|
||||
if (config->show_progress) {
|
||||
@@ -180,9 +236,34 @@ int send_files(Config *config) {
|
||||
}
|
||||
|
||||
int send_files_multithreaded(Config *config) {
|
||||
if (config->dry_run) {
|
||||
DirectoryScanner *scanner = directory_scanner_create(
|
||||
config->send_directory, config->use_metadata, config->chunk_size,
|
||||
config->exclude_patterns, config->exclude_count);
|
||||
Chunk *chunk;
|
||||
int file_count = 0;
|
||||
unsigned long long total_bytes = 0;
|
||||
printf("Dry run: files to be transferred\n");
|
||||
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
printf(" %s (%zu bytes)\n", chunk->items[i]->path,
|
||||
chunk->items[i]->data->size);
|
||||
total_bytes += chunk->items[i]->data->size;
|
||||
file_count++;
|
||||
}
|
||||
chunk_destroy(chunk);
|
||||
}
|
||||
directory_scanner_destroy(scanner);
|
||||
printf("Total: %d files, %.1f MB\n", file_count,
|
||||
total_bytes / 1048576.0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
PipelineContextSender *context =
|
||||
pipeline_context_sender_create(config, queue_create(100, chunk_destroy),
|
||||
queue_create(100, chunk_destroy));
|
||||
if (config->use_delete)
|
||||
context->manifest = array_list_create(free);
|
||||
|
||||
thrd_t scanner, loader, sender;
|
||||
if (thrd_create(&scanner, scan_directory_multithreaded, context) !=
|
||||
|
||||
+14
-1
@@ -11,13 +11,15 @@
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size) {
|
||||
DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size, char **exclude_patterns, int exclude_count) {
|
||||
DirectoryScanner *scanner = malloc(sizeof(DirectoryScanner));
|
||||
scanner->directories = queue_create(100, free);
|
||||
scanner->current_dir = NULL;
|
||||
scanner->current_path = NULL;
|
||||
scanner->use_metadata = use_metadata;
|
||||
scanner->chunk_size = chunk_size > 0 ? chunk_size : DESIRED_CHUNK_SIZE;
|
||||
scanner->exclude_patterns = exclude_patterns;
|
||||
scanner->exclude_count = exclude_count;
|
||||
queue_enqueue(scanner->directories, str_dup(root_directory));
|
||||
return scanner;
|
||||
}
|
||||
@@ -94,6 +96,17 @@ Chunk *directory_scanner_next(DirectoryScanner *scanner) {
|
||||
if (S_ISDIR(stats.st_mode)) {
|
||||
queue_enqueue(scanner->directories, (void *)cur_path);
|
||||
} else {
|
||||
bool excluded = false;
|
||||
for (int i = 0; i < scanner->exclude_count; i++) {
|
||||
if (glob_match(scanner->exclude_patterns[i], entry->d_name)) {
|
||||
excluded = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (excluded) {
|
||||
free(cur_path);
|
||||
continue;
|
||||
}
|
||||
File *file = file_create(cur_path);
|
||||
file->data->size = stats.st_size;
|
||||
if (scanner->use_metadata)
|
||||
|
||||
@@ -12,9 +12,11 @@ typedef struct {
|
||||
char *current_path;
|
||||
bool use_metadata;
|
||||
unsigned long long chunk_size;
|
||||
char **exclude_patterns;
|
||||
int exclude_count;
|
||||
} DirectoryScanner;
|
||||
|
||||
DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size);
|
||||
DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size, char **exclude_patterns, int exclude_count);
|
||||
Chunk *directory_scanner_next(DirectoryScanner *scanner);
|
||||
void directory_scanner_destroy(DirectoryScanner *scanner);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user