diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 2965061..9ff7bf1 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -22,7 +22,12 @@ static void print_usage(void) { printf("Options:\n"); printf(" -c [level] Enable compression (level 1-22, default 5)\n"); printf(" -z [level] Alias for -c\n"); + printf(" -a, --archive Archive mode (-c -m -M)\n"); + printf(" -n, --dry-run Show what would be transferred\n"); + printf(" -p SSH port (default: 22)\n"); printf(" --progress Show transfer progress\n"); + printf(" --delete Delete files on receiver not in source\n"); + printf(" --exclude Exclude files matching pattern\n"); printf(" -m Enable multithreading\n"); printf(" -s Enable chunk serialization\n"); printf(" -f Enable sendfile (TCP only, not with -c or -s)\n"); @@ -58,6 +63,21 @@ int main(int argc, char *argv[]) { if (strcmp(argv[i], "--help") == 0) { print_usage(); return 0; + } else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--archive") == 0) { + config->use_compression = true; + config->use_multithreading = true; + config->use_metadata = true; + log_message(LOG_LEVEL_INFO, "Enabled archive mode (-c -m -M)"); + } else if (strcmp(argv[i], "-n") == 0 || strcmp(argv[i], "--dry-run") == 0) { + config->dry_run = true; + } else if (strcmp(argv[i], "-p") == 0 && i + 1 < argc) { + config->ssh_port = atoi(argv[++i]); + } else if (strcmp(argv[i], "--delete") == 0) { + config->use_delete = true; + } else if (strcmp(argv[i], "--exclude") == 0 && i + 1 < argc) { + int idx = config->exclude_count++; + config->exclude_patterns = realloc(config->exclude_patterns, config->exclude_count * sizeof(char *)); + config->exclude_patterns[idx] = str_dup(argv[++i]); } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) { config->use_compression = true; log_message(LOG_LEVEL_INFO, "Enabled Compression"); diff --git a/src/client/client_send.c b/src/client/client_send.c index 8e6a2d3..2187b33 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -1,4 +1,5 @@ #include "client_send.h" +#include "array_list.h" #include "chunk.h" #include "config.h" #include "data.h" @@ -52,7 +53,7 @@ static int send_chunks_multithreaded(void *pipeline_context) { fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n"); return 1; } - client = client_connect_ssh(context->config->ssh_destination); + client = client_connect_ssh(context->config->ssh_destination, context->config->ssh_port); } else { client = client_create(); client_connect(client, server_host, server_port); @@ -65,6 +66,13 @@ static int send_chunks_multithreaded(void *pipeline_context) { &context->condition_not_empty_loader, &context->condition_not_full_loader, &context->loader_done); if (current_chunk == NULL) { + if (context->config->use_delete) { + send_status(client->file_descriptor, STATUS_MANIFEST); + send_int(client->file_descriptor, context->manifest->size); + for (int i = 0; i < context->manifest->size; i++) + send_str(client->file_descriptor, + (char *)context->manifest->items[i]); + } send_status(client->file_descriptor, STATUS_FINISHED); int ok = receive_status(client->file_descriptor) == STATUS_OK; client_disconnect(client); @@ -82,16 +90,26 @@ static int send_chunks_multithreaded(void *pipeline_context) { static int scan_directory_multithreaded(void *pipeline_context) { PipelineContextSender *context = (PipelineContextSender *)pipeline_context; mtx_lock(&context->mutex_scanner); - DirectoryScanner *scanner = - directory_scanner_create(context->config->send_directory, context->config->use_metadata, context->config->chunk_size); + DirectoryScanner *scanner = directory_scanner_create( + context->config->send_directory, context->config->use_metadata, + context->config->chunk_size, context->config->exclude_patterns, + context->config->exclude_count); mtx_unlock(&context->mutex_scanner); Chunk *current_chunk; - while ((current_chunk = directory_scanner_next(scanner)) != NULL) + while ((current_chunk = directory_scanner_next(scanner)) != NULL) { + if (context->config->use_delete) { + mtx_lock(&context->mutex_scanner); + for (int i = 0; i < current_chunk->element_count; i++) + array_list_add(context->manifest, + str_dup(current_chunk->items[i]->path)); + mtx_unlock(&context->mutex_scanner); + } queue_enqueue_multithreaded(context->queue_scanner, current_chunk, &context->mutex_scanner, &context->condition_not_empty_scanner, &context->condition_not_full_scanner); + } mtx_lock(&context->mutex_scanner); context->scanner_done = true; cnd_signal(&context->condition_not_empty_scanner); @@ -127,27 +145,56 @@ static int load_files_multithreaded(void *pipeline_context) { } int send_files(Config *config) { + if (config->dry_run) { + DirectoryScanner *scanner = directory_scanner_create( + config->send_directory, config->use_metadata, config->chunk_size, + config->exclude_patterns, config->exclude_count); + Chunk *chunk; + int file_count = 0; + unsigned long long total_bytes = 0; + printf("Dry run: files to be transferred\n"); + while ((chunk = directory_scanner_next(scanner)) != NULL) { + for (int i = 0; i < chunk->element_count; i++) { + printf(" %s (%zu bytes)\n", chunk->items[i]->path, + chunk->items[i]->data->size); + total_bytes += chunk->items[i]->data->size; + file_count++; + } + chunk_destroy(chunk); + } + directory_scanner_destroy(scanner); + printf("Total: %d files, %.1f MB\n", file_count, + total_bytes / 1048576.0); + return 0; + } + Client *client; if (config->transport == TRANSPORT_SSH) { if (config->use_sendfile) { fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n"); return 1; } - client = client_connect_ssh(config->ssh_destination); + client = client_connect_ssh(config->ssh_destination, config->ssh_port); } else { client = client_create(); client_connect(client, server_host, server_port); } config_send(client->file_descriptor, config); - DirectoryScanner *scanner = directory_scanner_create(config->send_directory, config->use_metadata, config->chunk_size); + DirectoryScanner *scanner = directory_scanner_create( + config->send_directory, config->use_metadata, config->chunk_size, + config->exclude_patterns, config->exclude_count); Chunk *current_chunk; unsigned long long total_bytes = 0; time_t last_progress = 0; time_t start = time(NULL); + ArrayList *manifest = config->use_delete ? array_list_create(free) : NULL; while ((current_chunk = directory_scanner_next(scanner)) != NULL) { unsigned long long chunk_bytes = 0; - for (int i = 0; i < current_chunk->element_count; i++) + for (int i = 0; i < current_chunk->element_count; i++) { chunk_bytes += current_chunk->items[i]->data->size; + if (manifest) + array_list_add(manifest, str_dup(current_chunk->items[i]->path)); + } if (!config->use_sendfile) { for (int i = 0; i < current_chunk->element_count; i++) file_load_data(current_chunk->items[i]); @@ -166,6 +213,15 @@ int send_files(Config *config) { } chunk_destroy(current_chunk); } + if (config->use_delete) { + send_status(client->file_descriptor, STATUS_MANIFEST); + send_int(client->file_descriptor, manifest->size); + for (int i = 0; i < manifest->size; i++) + send_str(client->file_descriptor, (char *)manifest->items[i]); + for (int i = 0; i < manifest->size; i++) + free(manifest->items[i]); + array_list_delete(manifest); + } send_status(client->file_descriptor, STATUS_FINISHED); int ok = receive_status(client->file_descriptor) == STATUS_OK; if (config->show_progress) { @@ -180,9 +236,34 @@ int send_files(Config *config) { } int send_files_multithreaded(Config *config) { + if (config->dry_run) { + DirectoryScanner *scanner = directory_scanner_create( + config->send_directory, config->use_metadata, config->chunk_size, + config->exclude_patterns, config->exclude_count); + Chunk *chunk; + int file_count = 0; + unsigned long long total_bytes = 0; + printf("Dry run: files to be transferred\n"); + while ((chunk = directory_scanner_next(scanner)) != NULL) { + for (int i = 0; i < chunk->element_count; i++) { + printf(" %s (%zu bytes)\n", chunk->items[i]->path, + chunk->items[i]->data->size); + total_bytes += chunk->items[i]->data->size; + file_count++; + } + chunk_destroy(chunk); + } + directory_scanner_destroy(scanner); + printf("Total: %d files, %.1f MB\n", file_count, + total_bytes / 1048576.0); + return 0; + } + PipelineContextSender *context = pipeline_context_sender_create(config, queue_create(100, chunk_destroy), queue_create(100, chunk_destroy)); + if (config->use_delete) + context->manifest = array_list_create(free); thrd_t scanner, loader, sender; if (thrd_create(&scanner, scan_directory_multithreaded, context) != diff --git a/src/client/scanner.c b/src/client/scanner.c index 854f112..175e238 100644 --- a/src/client/scanner.c +++ b/src/client/scanner.c @@ -11,13 +11,15 @@ #include #include -DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size) { +DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size, char **exclude_patterns, int exclude_count) { DirectoryScanner *scanner = malloc(sizeof(DirectoryScanner)); scanner->directories = queue_create(100, free); scanner->current_dir = NULL; scanner->current_path = NULL; scanner->use_metadata = use_metadata; scanner->chunk_size = chunk_size > 0 ? chunk_size : DESIRED_CHUNK_SIZE; + scanner->exclude_patterns = exclude_patterns; + scanner->exclude_count = exclude_count; queue_enqueue(scanner->directories, str_dup(root_directory)); return scanner; } @@ -94,6 +96,17 @@ Chunk *directory_scanner_next(DirectoryScanner *scanner) { if (S_ISDIR(stats.st_mode)) { queue_enqueue(scanner->directories, (void *)cur_path); } else { + bool excluded = false; + for (int i = 0; i < scanner->exclude_count; i++) { + if (glob_match(scanner->exclude_patterns[i], entry->d_name)) { + excluded = true; + break; + } + } + if (excluded) { + free(cur_path); + continue; + } File *file = file_create(cur_path); file->data->size = stats.st_size; if (scanner->use_metadata) diff --git a/src/client/scanner.h b/src/client/scanner.h index d2a53bd..76cf437 100644 --- a/src/client/scanner.h +++ b/src/client/scanner.h @@ -12,9 +12,11 @@ typedef struct { char *current_path; bool use_metadata; unsigned long long chunk_size; + char **exclude_patterns; + int exclude_count; } DirectoryScanner; -DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size); +DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size, char **exclude_patterns, int exclude_count); Chunk *directory_scanner_next(DirectoryScanner *scanner); void directory_scanner_destroy(DirectoryScanner *scanner); diff --git a/src/server/server.c b/src/server/server.c index db35a07..1be205c 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -1,3 +1,4 @@ +#include "array_list.h" #include "chunk.h" #include "compression.h" #include "config.h" @@ -55,6 +56,18 @@ int receive_files(Config *config, int file_descriptor) { } status = receive_status(file_descriptor); } + if (status == STATUS_MANIFEST) { + int count = receive_int(file_descriptor); + ArrayList *manifest = array_list_create(free); + for (int i = 0; i < count; i++) + array_list_add(manifest, receive_str(file_descriptor)); + fprintf(stderr, "Deleting files not in manifest...\n"); + delete_extras(config->receive_root_directory, manifest); + for (int i = 0; i < manifest->size; i++) + free(manifest->items[i]); + array_list_delete(manifest); + status = receive_status(file_descriptor); + } if (status != STATUS_FINISHED) { log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status"); send_status(file_descriptor, STATUS_ERROR); diff --git a/src/shared/config.c b/src/shared/config.c index 43d40af..e5d8637 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -23,11 +23,16 @@ Config *config_create(char *version, char *send_directory, config->use_compression = use_compression; config->use_metadata = use_metadata; config->show_progress = false; + config->dry_run = false; + config->use_delete = false; config->compression_level = compression_level; config->use_sendfile = use_sendfile; config->chunk_size = chunk_size > 0 ? chunk_size : DEFAULT_CHUNK_SIZE; + config->ssh_port = 22; config->transport = TRANSPORT_TCP; config->ssh_destination = NULL; + config->exclude_patterns = NULL; + config->exclude_count = 0; return config; } @@ -57,6 +62,9 @@ void config_delete(Config *config) { free(config->send_directory); free(config->receive_root_directory); free(config->ssh_destination); + for (int i = 0; i < config->exclude_count; i++) + free(config->exclude_patterns[i]); + free(config->exclude_patterns); free(config); } @@ -72,6 +80,7 @@ void config_send(int file_descriptor, Config *config) { send_int(file_descriptor, config->compression_level); send_int(file_descriptor, (int)config->chunk_size); send_int(file_descriptor, config->use_sendfile); + send_int(file_descriptor, config->use_delete); if (receive_status(file_descriptor) != STATUS_OK) { perror("Error transmitting config!"); exit(EXIT_FAILURE); @@ -91,8 +100,14 @@ Config *config_receive(int file_descriptor) { config->compression_level = receive_int(file_descriptor); config->chunk_size = (unsigned long long)receive_int(file_descriptor); config->use_sendfile = receive_int(file_descriptor); + config->use_delete = receive_int(file_descriptor); + config->show_progress = false; + config->dry_run = false; + config->ssh_port = 22; config->transport = TRANSPORT_TCP; config->ssh_destination = NULL; + config->exclude_patterns = NULL; + config->exclude_count = 0; send_status(file_descriptor, STATUS_OK); return config; } diff --git a/src/shared/config.h b/src/shared/config.h index b2a636a..b1c2437 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -19,10 +19,15 @@ typedef struct Config { bool use_sendfile; bool use_metadata; bool show_progress; + bool dry_run; + bool use_delete; int compression_level; unsigned long long chunk_size; + int ssh_port; TransportType transport; char *ssh_destination; + char **exclude_patterns; + int exclude_count; } Config; #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) diff --git a/src/shared/multiprocessing.c b/src/shared/multiprocessing.c index cf2b0a5..5f26d56 100644 --- a/src/shared/multiprocessing.c +++ b/src/shared/multiprocessing.c @@ -1,4 +1,5 @@ #include "multiprocessing.h" +#include "array_list.h" #include "chunk.h" #include "compression.h" #include "config.h" @@ -36,6 +37,11 @@ PipelineContextSender *pipeline_context_sender_create(Config *config, } void pipeline_context_sender_destroy(PipelineContextSender *context) { + if (context->manifest) { + for (int i = 0; i < context->manifest->size; i++) + free(context->manifest->items[i]); + array_list_delete(context->manifest); + } config_delete(context->config); queue_destroy(context->queue_scanner); queue_destroy(context->queue_loader); @@ -119,6 +125,17 @@ int receive_thread(void *pipeline_context) { } status = receive_status(file_descriptor); } + if (status == STATUS_MANIFEST) { + int count = receive_int(file_descriptor); + ArrayList *manifest = array_list_create(free); + for (int i = 0; i < count; i++) + array_list_add(manifest, receive_str(file_descriptor)); + delete_extras(context->config->receive_root_directory, manifest); + for (int i = 0; i < manifest->size; i++) + free(manifest->items[i]); + array_list_delete(manifest); + status = receive_status(file_descriptor); + } mtx_lock(&context->mutex); context->receiver_done = true; cnd_signal(&context->condition_not_empty); diff --git a/src/shared/multiprocessing.h b/src/shared/multiprocessing.h index 81ab589..605c42a 100644 --- a/src/shared/multiprocessing.h +++ b/src/shared/multiprocessing.h @@ -3,6 +3,7 @@ #include +#include "array_list.h" #include "config.h" #include "file.h" #include "queue.h" @@ -19,6 +20,7 @@ typedef struct { cnd_t condition_not_full_loader; cnd_t condition_not_empty_loader; bool loader_done; + ArrayList *manifest; } PipelineContextSender; typedef struct PipelineContextReceiver { diff --git a/src/shared/protocol.h b/src/shared/protocol.h index e2961e6..d981097 100644 --- a/src/shared/protocol.h +++ b/src/shared/protocol.h @@ -5,7 +5,7 @@ #include typedef int Status; -enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK }; +enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST }; void io_set_fds(int read_fd, int write_fd); void send_n_data(int file_descriptor, void *data, size_t data_size); diff --git a/src/shared/transport_ssh.c b/src/shared/transport_ssh.c index 6b2b5d1..4fc54cb 100644 --- a/src/shared/transport_ssh.c +++ b/src/shared/transport_ssh.c @@ -42,7 +42,7 @@ static int parse_remote_dest(const char *dest, RemoteDest *r) { return 0; } -Client *client_connect_ssh(char *destination) { +Client *client_connect_ssh(char *destination, int port) { RemoteDest r; if (parse_remote_dest(destination, &r) != 0) { fprintf(stderr, "Invalid remote destination: %s\n", destination); @@ -90,10 +90,26 @@ Client *client_connect_ssh(char *destination) { else snprintf(ssh_user, sizeof(ssh_user), "%s", r.host); - execlp("ssh", "ssh", "-o", "Compression=no", "-o", - "ControlMaster=auto", "-o", - "ControlPath=~/.cache/fastsync-%r@%h:%p", ssh_user, - "fastsync-server", "--stdio", (char *)NULL); + char *ssh_argv[16]; + int ac = 0; + char port_str[16]; + ssh_argv[ac++] = "ssh"; + ssh_argv[ac++] = "-o"; + ssh_argv[ac++] = "Compression=no"; + ssh_argv[ac++] = "-o"; + ssh_argv[ac++] = "ControlMaster=auto"; + ssh_argv[ac++] = "-o"; + ssh_argv[ac++] = "ControlPath=~/.cache/fastsync-%r@%h:%p"; + if (port > 0 && port != 22) { + ssh_argv[ac++] = "-p"; + snprintf(port_str, sizeof(port_str), "%d", port); + ssh_argv[ac++] = port_str; + } + ssh_argv[ac++] = ssh_user; + ssh_argv[ac++] = "fastsync-server"; + ssh_argv[ac++] = "--stdio"; + ssh_argv[ac] = NULL; + execvp("ssh", ssh_argv); perror("exec of ssh failed"); ssize_t wret = write(exec_pipe[1], "x", 1); (void)wret; diff --git a/src/shared/transport_ssh.h b/src/shared/transport_ssh.h index 3bad43c..f542870 100644 --- a/src/shared/transport_ssh.h +++ b/src/shared/transport_ssh.h @@ -3,6 +3,6 @@ #include "transport_tcp.h" -Client *client_connect_ssh(char *destination); +Client *client_connect_ssh(char *destination, int port); #endif diff --git a/src/shared/utils.c b/src/shared/utils.c index 7054d44..b7c2c7a 100644 --- a/src/shared/utils.c +++ b/src/shared/utils.c @@ -1,9 +1,12 @@ #include "utils.h" +#include "array_list.h" #include "libgen.h" +#include #include #include #include #include +#include void mkdir_r(char *path) { char *path_duplicate = malloc(strlen(path) + 1); @@ -44,6 +47,75 @@ char *str_dup(const char *string) { return new_string; } +bool glob_match(const char *pattern, const char *str) { + while (*pattern) { + if (*pattern == '*') { + pattern++; + while (*str && *str != '/') { + if (glob_match(pattern, str)) + return true; + str++; + } + return glob_match(pattern, str); + } else if (*pattern == '?') { + if (!*str || *str == '/') + return false; + pattern++; + str++; + } else { + if (*pattern != *str) + return false; + pattern++; + str++; + } + } + return *str == '\0'; +} + +static void delete_extras_walk(const char *abs_path, const char *rel_path, + ArrayList *manifest) { + DIR *dir = opendir(abs_path); + if (!dir) + return; + struct dirent *entry; + while ((entry = readdir(dir)) != NULL) { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) + continue; + char *child_abs = path_cat((char *)abs_path, entry->d_name); + char *child_rel = path_cat((char *)rel_path, entry->d_name); + struct stat st; + if (stat(child_abs, &st) != 0) { + free(child_abs); + free(child_rel); + continue; + } + if (S_ISDIR(st.st_mode)) { + delete_extras_walk(child_abs, child_rel, manifest); + } else { + // Check if relative path is in manifest + bool found = false; + for (int i = 0; i < manifest->size; i++) { + if (strcmp((char *)manifest->items[i], child_rel) == 0) { + found = true; + break; + } + } + if (!found) { + unlink(child_abs); + fprintf(stderr, " Deleted: %s\n", child_rel); + } + } + free(child_abs); + free(child_rel); + } + closedir(dir); + rmdir(abs_path); +} + +void delete_extras(const char *dest_root, ArrayList *manifest) { + delete_extras_walk(dest_root, "", manifest); +} + char *path_cat(char *path1, char *path2) { if (path1 == NULL || *path1 == '\0') return str_dup(path2); diff --git a/src/shared/utils.h b/src/shared/utils.h index f54c98d..67030ad 100644 --- a/src/shared/utils.h +++ b/src/shared/utils.h @@ -1,8 +1,13 @@ #ifndef UTILS_H #define UTILS_H +#include "array_list.h" +#include + void mkdir_r(char *path); char *str_dup(const char *string); char *path_cat(char *path1, char *path2); +bool glob_match(const char *pattern, const char *str); +void delete_extras(const char *dest_root, ArrayList *manifest); #endif diff --git a/tests/test_scanner.c b/tests/test_scanner.c index e29ab67..a733491 100644 --- a/tests/test_scanner.c +++ b/tests/test_scanner.c @@ -18,7 +18,7 @@ static void test_scanner_single_file() { mkdir(dir, 0755); create_test_file(file1, content1); - DirectoryScanner *scanner = directory_scanner_create((char *)dir, false, 0); + DirectoryScanner *scanner = directory_scanner_create((char *)dir, false, 0, NULL, 0); EXPECT_NOT_NULL(scanner); Chunk *chunk = directory_scanner_next(scanner); @@ -46,7 +46,7 @@ static void test_scanner_multiple_files() { create_test_file(file1, content1); create_test_file(file2, content2); - DirectoryScanner *scanner = directory_scanner_create((char *)dir, false, 0); + DirectoryScanner *scanner = directory_scanner_create((char *)dir, false, 0, NULL, 0); EXPECT_NOT_NULL(scanner); Chunk *chunk = directory_scanner_next(scanner); @@ -83,7 +83,7 @@ static void test_scanner_subdirectory() { create_test_file(root_file, content); create_test_file(sub_file, content); - DirectoryScanner *scanner = directory_scanner_create((char *)root, false, 0); + DirectoryScanner *scanner = directory_scanner_create((char *)root, false, 0, NULL, 0); EXPECT_NOT_NULL(scanner); int total_files = 0; @@ -106,7 +106,7 @@ static void test_scanner_empty_directory() { mkdir(dir, 0755); - DirectoryScanner *scanner = directory_scanner_create((char *)dir, false, 0); + DirectoryScanner *scanner = directory_scanner_create((char *)dir, false, 0, NULL, 0); EXPECT_NOT_NULL(scanner); Chunk *chunk = directory_scanner_next(scanner);