diff --git a/src/client/client.c b/src/client/client.c index 68c3d56..f2e3875 100644 --- a/src/client/client.c +++ b/src/client/client.c @@ -21,18 +21,18 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) { send_status(client->file_descriptor, STATUS_CHUNK); Data *data; if (config->use_compression) { - data = chunk_compress(chunk, config->compression_level); + data = chunk_compress(chunk, config->compression_level, config->use_metadata); } else { for (int i = 0; i < chunk->element_count; i++) file_load_data(chunk->items[i]); - data = chunk_serialize(chunk); + data = chunk_serialize(chunk, config->use_metadata); } send_data(client->file_descriptor, data->data, data->size); data_destroy(data); } else if (config->use_sendfile && !config->use_compression) { for (int i = 0; i < chunk->element_count; i++) { send_status(client->file_descriptor, STATUS_NEXT); - file_send_sendfile(chunk->items[i], client->file_descriptor); + file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata); } } else { for (int i = 0; i < chunk->element_count; i++) { @@ -44,7 +44,7 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) { data_destroy(file->data); file->data = compressed_data; } - file_send_single_calls(file, client->file_descriptor); + file_send_single_calls(file, client->file_descriptor, config->use_metadata); } } return 0; @@ -54,7 +54,7 @@ int scan_directory_multithreaded(void *pipeline_context) { PipelineContextSender *context = (PipelineContextSender *)pipeline_context; mtx_lock(&context->mutex_scanner); DirectoryScanner *scanner = - directory_scanner_create(context->config->send_directory); + directory_scanner_create(context->config->send_directory, context->config->use_metadata); mtx_unlock(&context->mutex_scanner); Chunk *current_chunk; @@ -134,7 +134,7 @@ int send_files(Config *config) { int port = env_port ? atoi(env_port) : 8080; client_connect(client, (char *)ip, port); config_send(client->file_descriptor, config); - DirectoryScanner *scanner = directory_scanner_create(config->send_directory); + DirectoryScanner *scanner = directory_scanner_create(config->send_directory, config->use_metadata); Chunk *current_chunk; while ((current_chunk = directory_scanner_next(scanner)) != NULL) { if (!config->use_sendfile) { @@ -201,7 +201,7 @@ int main(int argc, char *argv[]) { } Config *config = config_create(str_dup("1.0.0"), source_dir, dest_dir, - save_to_disk, false, false, false, 5, 20, false); + save_to_disk, false, false, false, false, 5, 20, false); for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "-c") == 0) { config->use_compression = true; @@ -224,6 +224,9 @@ int main(int argc, char *argv[]) { config->receive_root_directory = str_dup(argv[++i]); } else if (strcmp(argv[i], "--save-to-disk") == 0) { config->save_to_disk = true; + } else if (strcmp(argv[i], "-M") == 0 || strcmp(argv[i], "--preserve") == 0) { + config->use_metadata = true; + log_message(LOG_LEVEL_INFO, "Enabled metadata preservation"); } else if (strcmp(argv[i], "-f") == 0 || strcmp(argv[i], "--sendfile") == 0) { config->use_sendfile = true; log_message(LOG_LEVEL_INFO, "Enabled sendfile"); diff --git a/src/client/scanner.c b/src/client/scanner.c index bcb3464..386849c 100644 --- a/src/client/scanner.c +++ b/src/client/scanner.c @@ -1,6 +1,7 @@ #include "scanner.h" #include "array_list.h" #include "chunk.h" +#include "file.h" #include "queue.h" #include "utils.h" #include @@ -10,11 +11,12 @@ #include #include -DirectoryScanner *directory_scanner_create(char *root_directory) { +DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata) { DirectoryScanner *scanner = malloc(sizeof(DirectoryScanner)); scanner->directories = queue_create(100, free); scanner->current_dir = NULL; scanner->current_path = NULL; + scanner->use_metadata = use_metadata; queue_enqueue(scanner->directories, str_dup(root_directory)); return scanner; } @@ -91,9 +93,12 @@ Chunk *directory_scanner_next(DirectoryScanner *scanner) { if (!S_ISREG(stats.st_mode)) { queue_enqueue(scanner->directories, (void *)cur_path); } else { - File *file = file_create(cur_path, &stats); + File *file = file_create(cur_path); + file->data->size = stats.st_size; + if (scanner->use_metadata) + file->metadata = file_metadata_create(&stats); array_list_add(chunk_data, file); - chunk_data_size += file->stats.st_size; + chunk_data_size += file->data->size; if (chunk_data_size > DESIRED_CHUNK_SIZE) return chunk_data_to_chunk(chunk_data); free(cur_path); diff --git a/src/client/scanner.h b/src/client/scanner.h index 43596ad..08d1004 100644 --- a/src/client/scanner.h +++ b/src/client/scanner.h @@ -4,14 +4,16 @@ #include "chunk.h" #include "queue.h" #include +#include typedef struct { Queue *directories; DIR *current_dir; char *current_path; + bool use_metadata; } DirectoryScanner; -DirectoryScanner *directory_scanner_create(char *root_directory); +DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata); Chunk *directory_scanner_next(DirectoryScanner *scanner); void directory_scanner_destroy(DirectoryScanner *scanner); diff --git a/src/server/server.c b/src/server/server.c index 3debf97..0a5fb3b 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -12,34 +12,38 @@ #include #include -FileReceive *receive_file_receive(Config *config, int file_descriptor) { +File *file_receive(Config *config, int file_descriptor) { char *path = (char *)receive_str(file_descriptor); + File *file = file_create(path); + free(path); + if (config->use_metadata) + file->metadata = file_receive_metadata(file_descriptor); Data *file_data = receive_data(file_descriptor); if (config->use_compression) { Data *file_data_uncompressed = data_decompress(file_data); data_destroy(file_data); file_data = file_data_uncompressed; } - FileReceive *file = file_receive_create(path, file_data); + data_destroy(file->data); + file->data = file_data; return file; } static void receive_chunk_enqueue(int file_descriptor, Config *config, PipelineContextReceiver *context) { + (void)config; Data *chunk_data = receive_data(file_descriptor); Data *data_to_process = chunk_data; if (config->use_compression) { data_to_process = data_decompress(chunk_data); data_destroy(chunk_data); } - Chunk *chunk = chunk_deserialize(data_to_process); + Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata); data_destroy(data_to_process); for (int i = 0; i < chunk->element_count; i++) { - FileReceive *file = file_receive_create(chunk->items[i]->path, - chunk->items[i]->data); - chunk->items[i]->path = NULL; - chunk->items[i]->data = NULL; + File *file = chunk->items[i]; + chunk->items[i] = NULL; queue_enqueue_multithreaded(context->queue, file, &context->mutex, &context->condition_not_empty, &context->condition_not_full); @@ -60,7 +64,7 @@ int receive_thread(void *pipeline_context) { if (status == STATUS_CHUNK) { receive_chunk_enqueue(file_descriptor, config, context); } else { - FileReceive *file = receive_file_receive(config, file_descriptor); + File *file = file_receive(config, file_descriptor); queue_enqueue_multithreaded(context->queue, file, &context->mutex, &context->condition_not_empty, &context->condition_not_full); @@ -83,16 +87,20 @@ int write_thread(void *pipeline_context) { mtx_unlock(&context->mutex); while (true) { - FileReceive *file = queue_dequeue_multithreaded( + File *file = queue_dequeue_multithreaded( context->queue, &context->mutex, &context->condition_not_empty, &context->condition_not_full, &context->receiver_done); if (file == NULL) { free(root_directory); return thrd_success; } - if (save_to_disk) - to_disk(path_cat(root_directory, file->path), file->data->data, - file->data->size); + if (save_to_disk) { + char *disk_path = path_cat(root_directory, file->path); + to_disk(disk_path, file->data->data, file->data->size); + file_restore_metadata(disk_path, file->metadata); + free(disk_path); + } + file_destroy(file); } } @@ -106,21 +114,27 @@ int receive_files(Config *config, int file_descriptor) { data_to_process = data_decompress(chunk_data); data_destroy(chunk_data); } - Chunk *chunk = chunk_deserialize(data_to_process); + Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata); data_destroy(data_to_process); for (int i = 0; i < chunk->element_count; i++) { - if (config->save_to_disk) - to_disk(path_cat(config->receive_root_directory, chunk->items[i]->path), - chunk->items[i]->data->data, chunk->items[i]->data->size); + if (config->save_to_disk) { + char *disk_path = path_cat(config->receive_root_directory, chunk->items[i]->path); + to_disk(disk_path, chunk->items[i]->data->data, chunk->items[i]->data->size); + file_restore_metadata(disk_path, chunk->items[i]->metadata); + free(disk_path); + } } chunk_destroy(chunk); } else { - FileReceive *file = receive_file_receive(config, file_descriptor); - if (config->save_to_disk) - to_disk(path_cat(config->receive_root_directory, file->path), - file->data->data, file->data->size); - file_receive_destroy(file); + File *file = file_receive(config, file_descriptor); + if (config->save_to_disk) { + char *disk_path = path_cat(config->receive_root_directory, file->path); + to_disk(disk_path, file->data->data, file->data->size); + file_restore_metadata(disk_path, file->metadata); + free(disk_path); + } + file_destroy(file); } status = receive_status(file_descriptor); } @@ -137,7 +151,7 @@ void handler(int file_descriptor) { Config *config = config_receive(file_descriptor); if (config->use_multithreading) { PipelineContextReceiver *context = pipeline_context_receiver_create( - config, queue_create(100, file_receive_destroy), file_descriptor); + config, queue_create(100, file_destroy), file_descriptor); thrd_t receiver, writer; if (thrd_create(&receiver, receive_thread, context) != thrd_success || thrd_create(&writer, write_thread, context) != thrd_success) { diff --git a/src/shared/chunk.c b/src/shared/chunk.c index 27433a8..6157ece 100644 --- a/src/shared/chunk.c +++ b/src/shared/chunk.c @@ -11,6 +11,8 @@ #include "file.h" #include "log.h" +#define FILE_METADATA_WIRE_SIZE (sizeof(mode_t) + sizeof(uid_t) + sizeof(gid_t) + sizeof(time_t) + sizeof(long)) + Chunk *chunk_create(File **items, int element_count) { Chunk *chunk = (Chunk *)malloc(sizeof(Chunk)); if (chunk == NULL) { @@ -56,13 +58,44 @@ void chunk_print(void *item) { file_print(chunk->items[i]); } +static void metadata_to_buf(char **buf, FileMetadata *m) { + int present = (m != NULL) ? 1 : 0; + memcpy(*buf, &present, sizeof(int)); + *buf += sizeof(int); + if (m == NULL) + return; + memcpy(*buf, &m->mode, sizeof(mode_t)); *buf += sizeof(mode_t); + memcpy(*buf, &m->uid, sizeof(uid_t)); *buf += sizeof(uid_t); + memcpy(*buf, &m->gid, sizeof(gid_t)); *buf += sizeof(gid_t); + memcpy(*buf, &m->mtime_sec, sizeof(time_t)); *buf += sizeof(time_t); + memcpy(*buf, &m->mtime_nsec, sizeof(long)); *buf += sizeof(long); +} + +static FileMetadata *metadata_from_buf(char **buf) { + int present; + memcpy(&present, *buf, sizeof(int)); + *buf += sizeof(int); + if (!present) + return NULL; + FileMetadata *m = malloc(sizeof(FileMetadata)); + memcpy(&m->mode, *buf, sizeof(mode_t)); *buf += sizeof(mode_t); + memcpy(&m->uid, *buf, sizeof(uid_t)); *buf += sizeof(uid_t); + memcpy(&m->gid, *buf, sizeof(gid_t)); *buf += sizeof(gid_t); + memcpy(&m->mtime_sec, *buf, sizeof(time_t)); *buf += sizeof(time_t); + memcpy(&m->mtime_nsec, *buf, sizeof(long)); *buf += sizeof(long); + return m; +} + +static unsigned long long per_file_chunk_format_size(File *file) { + return sizeof(int) + strlen(file->path) + sizeof(int) + + (file->metadata ? FILE_METADATA_WIRE_SIZE : 0) + + sizeof(unsigned long long) + file->data->size; +} + Data *chunk_format(Chunk *chunk) { unsigned long long buffer_size = 0; for (int i = 0; i < chunk->element_count; ++i) { - buffer_size += sizeof(int); - buffer_size += strlen(chunk->items[i]->path); - buffer_size += sizeof(unsigned long long); - buffer_size += chunk->items[i]->stats.st_size; + buffer_size += per_file_chunk_format_size(chunk->items[i]); } char *data = malloc(buffer_size); @@ -80,12 +113,14 @@ Data *chunk_format(Chunk *chunk) { // add path memcpy(current_data_pointer, file->path, path_length); current_data_pointer += path_length; + // add metadata + metadata_to_buf(¤t_data_pointer, file->metadata); // add file data len - unsigned long long file_length = file->stats.st_size; + unsigned long long file_length = file->data->size; memcpy(current_data_pointer, &file_length, sizeof(unsigned long long)); current_data_pointer += sizeof(unsigned long long); // add file data - if (file->data == NULL) { + if (file->data->data == NULL) { file_load_data(file); } memcpy(current_data_pointer, file->data->data, file_length); @@ -98,13 +133,16 @@ Data *chunk_format(Chunk *chunk) { return chunk_data_create(data, buffer_size); } -Data *chunk_serialize(Chunk *chunk) { +static unsigned long long per_file_serialize_size(File *file, bool use_metadata) { + return sizeof(size_t) + strlen(file->path) + + (use_metadata ? sizeof(int) + (file->metadata ? FILE_METADATA_WIRE_SIZE : 0) : 0) + + sizeof(size_t) + file->data->size; +} + +Data *chunk_serialize(Chunk *chunk, bool use_metadata) { unsigned long long data_size = 0; for (int i = 0; i < chunk->element_count; i++) { - data_size += sizeof(size_t); - data_size += strlen(chunk->items[i]->path); - data_size += sizeof(size_t); - data_size += chunk->items[i]->stats.st_size; + data_size += per_file_serialize_size(chunk->items[i], use_metadata); } Data *data = data_create_empty(data_size); if (data == NULL) { @@ -114,22 +152,26 @@ Data *chunk_serialize(Chunk *chunk) { } char *data_pointer = data->data; for (int i = 0; i < chunk->element_count; i++) { - size_t path_len = strlen(chunk->items[i]->path); + File *file = chunk->items[i]; + size_t path_len = strlen(file->path); memcpy(data_pointer, &path_len, sizeof(size_t)); data_pointer += sizeof(size_t); - memcpy(data_pointer, chunk->items[i]->path, path_len); + memcpy(data_pointer, file->path, path_len); data_pointer += path_len; - size_t file_data_size = chunk->items[i]->stats.st_size; + if (use_metadata) + metadata_to_buf(&data_pointer, file->metadata); + + size_t file_data_size = file->data->size; memcpy(data_pointer, &file_data_size, sizeof(size_t)); data_pointer += sizeof(size_t); - memcpy(data_pointer, chunk->items[i]->data->data, file_data_size); + memcpy(data_pointer, file->data->data, file_data_size); data_pointer += file_data_size; } return data; } -Chunk *chunk_deserialize(Data *data) { +Chunk *chunk_deserialize(Data *data, bool use_metadata) { ArrayList *files = array_list_create(file_destroy); char *data_pointer = data->data; size_t remaining_size = data->size; @@ -162,9 +204,18 @@ Chunk *chunk_deserialize(Data *data) { data_pointer += path_len; remaining_size -= path_len; + File *file = file_create(path); + free(path); + + if (use_metadata) { + file->metadata = metadata_from_buf(&data_pointer); + remaining_size -= sizeof(int); + if (file->metadata) + remaining_size -= FILE_METADATA_WIRE_SIZE; + } + if (remaining_size < sizeof(size_t)) { log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for data size"); - free(path); array_list_delete(files); return NULL; } @@ -175,16 +226,6 @@ Chunk *chunk_deserialize(Data *data) { if (remaining_size < file_data_size) { log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for file content"); - free(path); - array_list_delete(files); - return NULL; - } - - struct stat st = {0}; - st.st_size = file_data_size; - File *file = file_create(path, &st); - if (file == NULL) { - free(path); array_list_delete(files); return NULL; } @@ -192,17 +233,16 @@ Chunk *chunk_deserialize(Data *data) { void *file_data = malloc(file_data_size); if (file_data == NULL) { perror("Could not allocate memory for file data"); - free(path); array_list_delete(files); return NULL; } memcpy(file_data, data_pointer, file_data_size); + data_destroy(file->data); file->data = data_create(file_data, file_data_size); data_pointer += file_data_size; remaining_size -= file_data_size; array_list_add(files, file); - free(path); } File **file_array = (File **)array_list_to_array(files); @@ -215,16 +255,16 @@ Chunk *chunk_deserialize(Data *data) { return chunk; } -Data *chunk_compress(Chunk *chunk, int compression_level) { +Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata) { log_message(LOG_LEVEL_DEBUG, "Starting to compress chunk"); - Data *serialized = chunk_serialize(chunk); + Data *serialized = chunk_serialize(chunk, use_metadata); Data *compressed = data_compress(serialized, compression_level); data_destroy(serialized); log_message(LOG_LEVEL_DEBUG, "Chunk successfully compressed"); return compressed; } -Chunk *chunk_decompress(Data *compressed_data) { +Chunk *chunk_decompress(Data *compressed_data, bool use_metadata) { log_message(LOG_LEVEL_DEBUG, "Starting to decompress chunk"); Data *uncompressed_data = data_decompress(compressed_data); if (uncompressed_data == NULL) { @@ -232,7 +272,7 @@ Chunk *chunk_decompress(Data *compressed_data) { return NULL; } - Chunk *chunk = chunk_deserialize(uncompressed_data); + Chunk *chunk = chunk_deserialize(uncompressed_data, use_metadata); if (chunk == NULL) { log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk data"); data_destroy(uncompressed_data); @@ -260,36 +300,3 @@ void chunk_data_delete(void *chunk) { free(chunk_data->data); free(chunk_data); } - -// void chunk_data_to_disk(ChunkData *chunk_formated, char *root_directory) { -// char *current_data_pointer = chunk_formated->data; -// while (current_data_pointer - (char *)chunk_formated->data < -// chunk_formated->data_size) { -// // get path length -// int path_length = 0; -// memcpy(&path_length, (int *)current_data_pointer, sizeof(int)); -// current_data_pointer += sizeof(int); -// // get path -// int path_dir_size = -// (strlen(root_directory) + path_length + 1) * sizeof(char); -// char *path = (char *)malloc(path_dir_size); -// if (path == NULL) { -// perror("Could not allocate memory for path!"); -// exit(EXIT_FAILURE); -// } -// snprintf(path, path_dir_size, "%s%.*s", root_directory, path_length, -// current_data_pointer); -// current_data_pointer += sizeof(char) * path_length; -// // get data length -// unsigned long long data_size = 0; -// memcpy(&data_size, (unsigned long long *)current_data_pointer, -// sizeof(unsigned long long)); -// current_data_pointer += sizeof(unsigned long long); -// // create File Receive -// FileReceive *file = -// file_receive_create(path, data_size, current_data_pointer); -// file_receive_print(file); -// file_receive_to_disk(file); -// current_data_pointer += sizeof(char) * data_size; -// } -// } diff --git a/src/shared/chunk.h b/src/shared/chunk.h index 41b62b8..b89896d 100644 --- a/src/shared/chunk.h +++ b/src/shared/chunk.h @@ -3,6 +3,7 @@ #include "data.h" #include "file.h" +#include #include #define DESIRED_CHUNK_SIZE 10 * 1024 * 1024 @@ -16,10 +17,10 @@ Chunk *chunk_create(File **items, int element_count); void chunk_destroy(void *chunk); void chunk_print(void *chunk); Data *chunk_format(Chunk *chunk); -Data *chunk_serialize(Chunk *chunk); -Chunk *chunk_deserialize(Data *data); -Data *chunk_compress(Chunk *chunk, int compression_level); -Chunk *chunk_decompress(Data *compressed_data); +Data *chunk_serialize(Chunk *chunk, bool use_metadata); +Chunk *chunk_deserialize(Data *data, bool use_metadata); +Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata); +Chunk *chunk_decompress(Data *compressed_data, bool use_metadata); Data *chunk_data_create(void *data, unsigned long long data_size); void chunk_data_delete(void *chunk); diff --git a/src/shared/config.c b/src/shared/config.c index b2ed396..149b448 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -7,8 +7,8 @@ Config *config_create(char *version, char *send_directory, char *receive_directory, bool save_to_disk, bool use_multithreading, bool use_chunk_serialization, - bool use_compression, int compression_level, - int num_connections, bool use_sendfile) { + bool use_compression, bool use_metadata, + int compression_level, int num_connections, bool use_sendfile) { Config *config = malloc(sizeof(Config)); config->version = version; @@ -18,6 +18,7 @@ Config *config_create(char *version, char *send_directory, config->use_multithreading = use_multithreading; config->use_chunk_serialization = use_chunk_serialization; config->use_compression = use_compression; + config->use_metadata = use_metadata; config->compression_level = compression_level; config->num_connections = num_connections; config->use_sendfile = use_sendfile; @@ -39,8 +40,10 @@ void config_send(int file_descriptor, Config *config) { send_int(file_descriptor, config->use_multithreading); send_int(file_descriptor, config->use_chunk_serialization); send_int(file_descriptor, config->use_compression); + send_int(file_descriptor, config->use_metadata); send_int(file_descriptor, config->compression_level); send_int(file_descriptor, config->num_connections); + send_int(file_descriptor, config->use_sendfile); if (receive_status(file_descriptor) != STATUS_OK) { perror("Error transmitting config!"); exit(EXIT_FAILURE); @@ -56,8 +59,10 @@ Config *config_receive(int file_descriptor) { config->use_multithreading = receive_int(file_descriptor); config->use_chunk_serialization = receive_int(file_descriptor); config->use_compression = receive_int(file_descriptor); + config->use_metadata = receive_int(file_descriptor); config->compression_level = receive_int(file_descriptor); config->num_connections = receive_int(file_descriptor); + config->use_sendfile = receive_int(file_descriptor); send_status(file_descriptor, STATUS_OK); return config; } diff --git a/src/shared/config.h b/src/shared/config.h index f991fda..2a645d7 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -13,6 +13,7 @@ typedef struct Config { bool use_compression; bool use_sendfile; bool use_single_send_per_file; + bool use_metadata; int compression_level; int num_connections; } Config; @@ -20,8 +21,8 @@ typedef struct Config { Config *config_create(char *version, char *send_directory, char *receive_directory, bool save_to_disk, bool use_multithreading, bool use_chunk_serialization, - bool use_compression, int compression_level, - int num_connections, bool use_sendfile); + bool use_compression, bool use_metadata, + int compression_level, int num_connections, bool use_sendfile); void config_delete(Config *config); void config_send(int file_descriptor, Config *config); Config *config_receive(int file_descriptor); diff --git a/src/shared/data.c b/src/shared/data.c index 689c4d4..7528230 100644 --- a/src/shared/data.c +++ b/src/shared/data.c @@ -13,6 +13,17 @@ Data *data_create_empty(size_t data_size) { return data_create(data, data_size); } +Data *data_create_reserve(size_t size) { + Data *d = malloc(sizeof(Data)); + if (d == NULL) { + log_message(LOG_LEVEL_ERROR, "Could not allocate memory for data"); + exit(EXIT_FAILURE); + } + d->data = NULL; + d->size = size; + return d; +} + Data *data_create(void *data, size_t data_size) { Data *new_data = malloc(sizeof(Data)); if (new_data == NULL) { diff --git a/src/shared/data.h b/src/shared/data.h index fc26fee..0fa33da 100644 --- a/src/shared/data.h +++ b/src/shared/data.h @@ -9,6 +9,7 @@ typedef struct { } Data; Data *data_create_empty(size_t data_size); +Data *data_create_reserve(size_t size); Data *data_create(void *data, size_t data_size); void data_destroy(Data *data); Data *data_compress(Data *data_to_compress, int compression_level); diff --git a/src/shared/file.c b/src/shared/file.c index 5a2f19b..c6190e6 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -14,15 +14,13 @@ #include "log.h" #include "socket.h" -File *file_create(const char *path, struct stat *stats) { +File *file_create(const char *path) { File *file = (File *)malloc(sizeof(File)); if (file == NULL) { perror("FATAL ERROR: Could not allocate memory for file struct"); exit(EXIT_FAILURE); } - file->stats = *stats; - int path_len = strlen(path); file->path = (char *)malloc(path_len + 1); if (file->path == NULL) { @@ -32,7 +30,8 @@ File *file_create(const char *path, struct stat *stats) { } strcpy(file->path, path); - file->data = NULL; + file->data = data_create_reserve(0); + file->metadata = NULL; return file; } @@ -42,15 +41,45 @@ void file_destroy(void *item) { File *file = (File *)item; data_destroy(file->data); file->data = NULL; + file_metadata_destroy(file->metadata); + file->metadata = NULL; free(file->path); file->path = NULL; free(file); } +FileMetadata *file_metadata_create(struct stat *stats) { + FileMetadata *m = malloc(sizeof(FileMetadata)); + if (m == NULL) { + perror("FATAL ERROR: Could not allocate memory for file metadata"); + exit(EXIT_FAILURE); + } + m->mode = stats->st_mode; + m->uid = stats->st_uid; + m->gid = stats->st_gid; + m->mtime_sec = stats->st_mtime; +#ifdef __linux__ + m->mtime_nsec = stats->st_mtim.tv_nsec; +#else + m->mtime_nsec = 0; +#endif + return m; +} + +void file_metadata_destroy(void *metadata) { + free(metadata); +} + void file_load_data(File *file) { if (file == NULL) return; - file->data = data_create_empty(file->stats.st_size); + if (file->data->data == NULL) { + file->data->data = malloc(file->data->size); + if (file->data->data == NULL) { + perror("Could not allocate memory for file data"); + exit(EXIT_FAILURE); + } + } printf("%ld is file big", file->data->size); size_t bytes_read = file_content_to_buffer(file); if (bytes_read != file->data->size) { @@ -65,14 +94,47 @@ void file_print(void *item) { printf("%s\n", ((File *)item)->path); } -void file_send_single_calls(File *file, int file_descriptor) { +static void metadata_send(int file_descriptor, FileMetadata *m) { + if (m == NULL) { + int zero = 0; + send_n_data(file_descriptor, &zero, sizeof(int)); + return; + } + int present = 1; + send_n_data(file_descriptor, &present, sizeof(int)); + send_n_data(file_descriptor, &m->mode, sizeof(mode_t)); + send_n_data(file_descriptor, &m->uid, sizeof(uid_t)); + send_n_data(file_descriptor, &m->gid, sizeof(gid_t)); + send_n_data(file_descriptor, &m->mtime_sec, sizeof(time_t)); + send_n_data(file_descriptor, &m->mtime_nsec, sizeof(long)); +} + +FileMetadata *file_receive_metadata(int file_descriptor) { + int present; + receive_n_data(file_descriptor, &present, sizeof(int)); + if (!present) + return NULL; + FileMetadata *m = malloc(sizeof(FileMetadata)); + receive_n_data(file_descriptor, &m->mode, sizeof(mode_t)); + receive_n_data(file_descriptor, &m->uid, sizeof(uid_t)); + receive_n_data(file_descriptor, &m->gid, sizeof(gid_t)); + receive_n_data(file_descriptor, &m->mtime_sec, sizeof(time_t)); + receive_n_data(file_descriptor, &m->mtime_nsec, sizeof(long)); + return m; +} + +void file_send_single_calls(File *file, int file_descriptor, bool use_metadata) { send_str(file_descriptor, file->path); + if (use_metadata) + metadata_send(file_descriptor, file->metadata); printf("Sending File: %ld", file->data->size); send_data(file_descriptor, file->data->data, file->data->size); } -void file_send_sendfile(File *file, int file_descriptor) { +void file_send_sendfile(File *file, int file_descriptor, bool use_metadata) { send_str(file_descriptor, file->path); + if (use_metadata) + metadata_send(file_descriptor, file->metadata); int fd = open(file->path, O_RDONLY); if (fd == -1) { @@ -80,7 +142,7 @@ void file_send_sendfile(File *file, int file_descriptor) { exit(EXIT_FAILURE); } - unsigned long long file_size = file->stats.st_size; + unsigned long long file_size = file->data->size; send_n_data(file_descriptor, &file_size, sizeof(unsigned long long)); off_t offset = 0; @@ -103,29 +165,11 @@ size_t file_content_to_buffer(File *file) { return 0; } size_t bytes_read = - fread(file->data->data, 1, file->stats.st_size, file_pointer); - if (bytes_read != (size_t)file->stats.st_size) { + fread(file->data->data, 1, file->data->size, file_pointer); + if (bytes_read != (size_t)file->data->size) { perror("Read to many or to less bytes from File!"); return 0; } fclose(file_pointer); return bytes_read; } - -FileReceive *file_receive_create(char *path, Data *data) { - FileReceive *file = malloc(sizeof(FileReceive)); - file->path = path; - file->data = data; - return file; -} - -void file_receive_destroy(void *file_receive) { - if (file_receive == NULL) - return; - FileReceive *file = (FileReceive *)file_receive; - data_destroy(file->data); - free(file->path); - free(file); -} - - diff --git a/src/shared/file.h b/src/shared/file.h index 169ca66..f6d5cb3 100644 --- a/src/shared/file.h +++ b/src/shared/file.h @@ -2,28 +2,32 @@ #define FILE_H #include "data.h" +#include #include typedef struct { - char *path; - struct stat stats; - Data *data; -} File; + mode_t mode; + uid_t uid; + gid_t gid; + time_t mtime_sec; + long mtime_nsec; +} FileMetadata; typedef struct { char *path; Data *data; -} FileReceive; + FileMetadata *metadata; +} File; -File *file_create(const char *path, struct stat *stats); +File *file_create(const char *path); void file_destroy(void *item); void file_load_data(File *file); void file_print(void *item); -void file_send_single_calls(File *file, int file_descriptor); -void file_send_sendfile(File *file, int file_descriptor); +void file_send_single_calls(File *file, int file_descriptor, bool use_metadata); +void file_send_sendfile(File *file, int file_descriptor, bool use_metadata); size_t file_content_to_buffer(File *file); - -FileReceive *file_receive_create(char *path, Data *data); -void file_receive_destroy(void *file_receive); +FileMetadata *file_metadata_create(struct stat *stats); +void file_metadata_destroy(void *metadata); +FileMetadata *file_receive_metadata(int file_descriptor); #endif diff --git a/src/shared/utils.c b/src/shared/utils.c index fd16a59..e848019 100644 --- a/src/shared/utils.c +++ b/src/shared/utils.c @@ -1,9 +1,12 @@ #include "utils.h" #include "libgen.h" #include "sys/stat.h" +#include #include #include #include +#include +#include void mkdir_r(char *path) { char *path_duplicate = malloc(strlen(path) + 1); @@ -59,6 +62,19 @@ void to_disk(char *path, void *data, unsigned long long data_size) { free(dir_to_free); } +void file_restore_metadata(const char *path, FileMetadata *metadata) { + if (metadata == NULL) + return; + chmod(path, metadata->mode & 07777); + chown(path, metadata->uid, metadata->gid); + struct timespec times[2]; + times[0].tv_sec = 0; + times[0].tv_nsec = UTIME_OMIT; + times[1].tv_sec = metadata->mtime_sec; + times[1].tv_nsec = metadata->mtime_nsec; + utimensat(AT_FDCWD, path, times, 0); +} + char *path_cat(char *path1, char *path2) { if (path1 == NULL || *path1 == '\0') return str_dup(path2); diff --git a/src/shared/utils.h b/src/shared/utils.h index c1449d2..a3d8838 100644 --- a/src/shared/utils.h +++ b/src/shared/utils.h @@ -1,9 +1,12 @@ #ifndef UTILS_H #define UTILS_H +#include "file.h" + void mkdir_r(char *path); char *str_dup(char *string); void to_disk(char *path, void *data, unsigned long long data_size); +void file_restore_metadata(const char *path, FileMetadata *metadata); char *path_cat(char *path1, char *path2); #endif diff --git a/test.py b/test.py index a6cf580..e181133 100755 --- a/test.py +++ b/test.py @@ -200,7 +200,7 @@ def run_profile(profile_name, source_dir, dest_dir): client_cmd = ( client_prefix + base_client_cmd - + ["--source-dir", source_dir, "--dest-dir", dest_dir, "--save-to-disk"] + + ["--source-dir", source_dir, "--dest-dir", dest_dir, "--save-to-disk", "-M"] + flags ) print(f" Running: {' '.join(client_cmd)}") @@ -272,6 +272,88 @@ def run_profile(profile_name, source_dir, dest_dir): server_process.kill() server_process.wait() + # Run one case without metadata transfer to verify -M disabled works + print(f"\n --- Standard (no metadata) ---") + if os.path.exists(dest_dir): + shutil.rmtree(dest_dir) + server_process2 = None + try: + server_process2 = subprocess.Popen( + SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None + ) + time.sleep(0.5) + + env = os.environ.copy() + client_cmd = ( + client_prefix + + base_client_cmd + + ["--source-dir", source_dir, "--dest-dir", dest_dir, "--save-to-disk"] + ) + print(f" Running: {' '.join(client_cmd)}") + + start_time = time.monotonic() + client_result = subprocess.run( + client_cmd, env=env, text=True, capture_output=True + ) + end_time = time.monotonic() + duration = end_time - start_time + + if server_process2: + try: + server_process2.wait(timeout=5) + except subprocess.TimeoutExpired: + server_process2.kill() + server_process2.wait() + server_process2 = None + + mismatches, missing = [], [] + if client_result.returncode == 0: + received = os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep)) + mismatches, missing = verify_transfer(source_dir, received) + + entry = { + "name": "Standard (no metadata)", + "suite": profile_name, + "time": f"{duration:.4f}s" if client_result.returncode == 0 else "N/A", + } + + if client_result.returncode == 0 and not mismatches and not missing: + entry["status"] = "Success" + entry["error"] = "" + else: + entry["status"] = "Failed" + errors = [] + if client_result.returncode != 0: + err = ( + client_result.stderr.strip().split("\n")[0] + if client_result.stderr + else ( + client_result.stdout.strip().split("\n")[0] + if client_result.stdout + else "No output" + ) + ) + errors.append(f"Exit code {client_result.returncode}: {err[:80]}") + if missing: + errors.append(f"Missing ({len(missing)}): {', '.join(missing[:5])}") + if mismatches: + errors.append(f"Mismatch ({len(mismatches)}): {', '.join(mismatches[:3])}") + entry["error"] = " | ".join(errors) + + results.append(entry) + + except Exception as e: + results.append( + {"name": "Standard (no metadata)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)} + ) + finally: + if server_process2: + try: + server_process2.wait(timeout=5) + except subprocess.TimeoutExpired: + server_process2.kill() + server_process2.wait() + # Rsync tests (over network via daemon, so tc netem applies) rsync_port = find_free_port() rsyncd_conf = os.path.join(tempfile.gettempdir(), f"rsyncd-{rsync_port}.conf") diff --git a/tests/test_chunk.c b/tests/test_chunk.c index 7e0b350..2d7ab8d 100644 --- a/tests/test_chunk.c +++ b/tests/test_chunk.c @@ -13,43 +13,27 @@ static void test_file_operations() { to_disk(test_path, test_content, test_len); - struct stat st; - int stat_res = stat(test_path, &st); - EXPECT_EQ_INT(stat_res, 0); - EXPECT_EQ_INT((int)st.st_size, (int)test_len); - - File *f = file_create(test_path, &st); + File *f = file_create(test_path); EXPECT_NOT_NULL(f); EXPECT_EQ_STR(f->path, test_path); - EXPECT_NULL(f->data); + EXPECT_NOT_NULL(f->data); + EXPECT_NULL(f->data->data); + EXPECT_EQ_INT((int)f->data->size, 0); + + struct stat st; + stat(test_path, &st); + f->data->size = st.st_size; file_load_data(f); EXPECT_NOT_NULL(f->data); + EXPECT_NOT_NULL(f->data->data); + EXPECT_EQ_INT((int)f->data->size, (int)test_len); EXPECT_EQ_INT(memcmp(f->data->data, test_content, test_len), 0); file_destroy(f); unlink(test_path); } -static void test_file_receive_operations() { - char *path = str_dup("temp_receive.txt"); - char *data = str_dup("receive data content"); - unsigned long long size = strlen(data); - - Data *df = data_create(data, size); - EXPECT_NOT_NULL(df); - EXPECT_EQ_INT((int)df->size, (int)size); - EXPECT_EQ_STR(df->data, "receive data content"); - - FileReceive *fr = file_receive_create(path, df); - EXPECT_NOT_NULL(fr); - EXPECT_EQ_STR(fr->path, "temp_receive.txt"); - EXPECT_NOT_NULL(fr->data); - EXPECT_EQ_STR(fr->data->data, "receive data content"); - - file_receive_destroy(fr); -} - static void test_chunk_operations() { char *path1 = "temp_chunk_1.txt"; char *content1 = "chunk item 1"; @@ -66,8 +50,10 @@ static void test_chunk_operations() { stat(path1, &st1); stat(path2, &st2); - File *f1 = file_create(path1, &st1); - File *f2 = file_create(path2, &st2); + File *f1 = file_create(path1); + f1->data->size = st1.st_size; + File *f2 = file_create(path2); + f2->data->size = st2.st_size; File *files[2] = {f1, f2}; Chunk *chunk = chunk_create(files, 2); @@ -76,12 +62,34 @@ static void test_chunk_operations() { EXPECT_NOT_NULL(chunk->items[0]); EXPECT_NOT_NULL(chunk->items[1]); + // load data before serializing + file_load_data(f1); + file_load_data(f2); + + // Test chunk_serialize / chunk_deserialize round-trip + Data *serialized = chunk_serialize(chunk, false); + EXPECT_NOT_NULL(serialized); + + Chunk *deserialized = chunk_deserialize(serialized, false); + EXPECT_NOT_NULL(deserialized); + EXPECT_EQ_INT(deserialized->element_count, 2); + EXPECT_EQ_STR(deserialized->items[0]->path, path1); + EXPECT_EQ_STR(deserialized->items[1]->path, path2); + EXPECT_EQ_INT((int)deserialized->items[0]->data->size, (int)len1); + EXPECT_EQ_INT((int)deserialized->items[1]->data->size, (int)len2); + EXPECT_EQ_INT(memcmp(deserialized->items[0]->data->data, content1, len1), 0); + EXPECT_EQ_INT(memcmp(deserialized->items[1]->data->data, content2, len2), 0); + + chunk_data_delete(serialized); + chunk_destroy(deserialized); + + // Test chunk_format layout (old format) Data *formatted = chunk_format(chunk); EXPECT_NOT_NULL(formatted); unsigned long long expected_size = - (sizeof(int) + strlen(path1) + sizeof(unsigned long long) + len1) + - (sizeof(int) + strlen(path2) + sizeof(unsigned long long) + len2); + (sizeof(int) + strlen(path1) + sizeof(int) + sizeof(unsigned long long) + len1) + + (sizeof(int) + strlen(path2) + sizeof(int) + sizeof(unsigned long long) + len2); EXPECT_EQ_INT((int)formatted->size, (int)expected_size); char *ptr = (char *)formatted->data; @@ -98,6 +106,11 @@ static void test_chunk_operations() { ptr += p_len1; EXPECT_EQ_STR(read_path1, path1); + int meta_present1; + memcpy(&meta_present1, ptr, sizeof(int)); + ptr += sizeof(int); + EXPECT_EQ_INT(meta_present1, 0); + unsigned long long d_len1; memcpy(&d_len1, ptr, sizeof(unsigned long long)); ptr += sizeof(unsigned long long); @@ -121,6 +134,11 @@ static void test_chunk_operations() { ptr += p_len2; EXPECT_EQ_STR(read_path2, path2); + int meta_present2; + memcpy(&meta_present2, ptr, sizeof(int)); + ptr += sizeof(int); + EXPECT_EQ_INT(meta_present2, 0); + unsigned long long d_len2; memcpy(&d_len2, ptr, sizeof(unsigned long long)); ptr += sizeof(unsigned long long); @@ -141,6 +159,5 @@ static void test_chunk_operations() { void test_chunk() { test_file_operations(); - test_file_receive_operations(); test_chunk_operations(); } diff --git a/tests/test_compression.c b/tests/test_compression.c index 4542b79..869b163 100644 --- a/tests/test_compression.c +++ b/tests/test_compression.c @@ -68,8 +68,10 @@ static void test_chunk_compress_decompress_roundtrip() { EXPECT_EQ_INT(stat(path1, &st1), 0); EXPECT_EQ_INT(stat(path2, &st2), 0); - File *f1 = file_create(path1, &st1); - File *f2 = file_create(path2, &st2); + File *f1 = file_create(path1); + f1->data->size = st1.st_size; + File *f2 = file_create(path2); + f2->data->size = st2.st_size; EXPECT_NOT_NULL(f1); EXPECT_NOT_NULL(f2); @@ -80,10 +82,10 @@ static void test_chunk_compress_decompress_roundtrip() { Chunk *chunk = chunk_create(files, 2); EXPECT_NOT_NULL(chunk); - Data *compressed = chunk_compress(chunk, 3); + Data *compressed = chunk_compress(chunk, 3, false); EXPECT_NOT_NULL(compressed); - Chunk *decompressed_chunk = chunk_decompress(compressed); + Chunk *decompressed_chunk = chunk_decompress(compressed, false); EXPECT_NOT_NULL(decompressed_chunk); EXPECT_EQ_INT(decompressed_chunk->element_count, 2); diff --git a/tests/test_config.c b/tests/test_config.c index 7c587f8..664ec70 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -8,7 +8,7 @@ static void test_config_lifecycle() { Config *cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("/dst"), - true, true, false, false, 1, 4, false); + true, true, false, false, false, 1, 4, false); EXPECT_NOT_NULL(cfg); EXPECT_EQ_STR(cfg->version, "1.0"); EXPECT_EQ_STR(cfg->send_directory, "/src"); @@ -23,7 +23,7 @@ static void test_config_lifecycle() { static void test_pipeline_sender_lifecycle() { Config *cfg = config_create(str_dup("2.0"), str_dup("/src2"), - str_dup("/dst2"), false, false, true, true, 1, 8, false); + str_dup("/dst2"), false, false, true, true, false, 1, 8, false); Queue *q1 = queue_create(5, NULL); Queue *q2 = queue_create(15, NULL); @@ -40,7 +40,7 @@ static void test_pipeline_sender_lifecycle() { static void test_pipeline_receiver_lifecycle() { Config *cfg = config_create(str_dup("3.0"), str_dup("/src3"), - str_dup("/dst3"), true, true, true, true, 1, 2, false); + str_dup("/dst3"), true, true, true, true, false, 1, 2, false); Queue *q = queue_create(20, NULL); PipelineContextReceiver *pcr = pipeline_context_receiver_create(cfg, q, 42); diff --git a/tests/test_scanner.c b/tests/test_scanner.c index 4446333..6187cbe 100644 --- a/tests/test_scanner.c +++ b/tests/test_scanner.c @@ -18,7 +18,7 @@ static void test_scanner_single_file() { mkdir(dir, 0755); create_test_file(file1, content1); - DirectoryScanner *scanner = directory_scanner_create((char *)dir); + DirectoryScanner *scanner = directory_scanner_create((char *)dir, false); EXPECT_NOT_NULL(scanner); Chunk *chunk = directory_scanner_next(scanner); @@ -46,7 +46,7 @@ static void test_scanner_multiple_files() { create_test_file(file1, content1); create_test_file(file2, content2); - DirectoryScanner *scanner = directory_scanner_create((char *)dir); + DirectoryScanner *scanner = directory_scanner_create((char *)dir, false); EXPECT_NOT_NULL(scanner); Chunk *chunk = directory_scanner_next(scanner); @@ -83,7 +83,7 @@ static void test_scanner_subdirectory() { create_test_file(root_file, content); create_test_file(sub_file, content); - DirectoryScanner *scanner = directory_scanner_create((char *)root); + DirectoryScanner *scanner = directory_scanner_create((char *)root, false); EXPECT_NOT_NULL(scanner); int total_files = 0; @@ -106,7 +106,7 @@ static void test_scanner_empty_directory() { mkdir(dir, 0755); - DirectoryScanner *scanner = directory_scanner_create((char *)dir); + DirectoryScanner *scanner = directory_scanner_create((char *)dir, false); EXPECT_NOT_NULL(scanner); Chunk *chunk = directory_scanner_next(scanner);