fix: address PR review comments
- Rename receive_file_receive -> file_receive (comment 62) - Guard metadata receive with config->use_metadata in server.c (comment 63) - Add NULL check after malloc in file_load_data (comment 66) - Make chunk_serialize/chunk_deserialize metadata conditional on use_metadata param, thread through chunk_compress/chunk_decompress (comment 65) - Add missing stdbool.h include to chunk.h
This commit is contained in:
+2
-2
@@ -21,11 +21,11 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) {
|
|||||||
send_status(client->file_descriptor, STATUS_CHUNK);
|
send_status(client->file_descriptor, STATUS_CHUNK);
|
||||||
Data *data;
|
Data *data;
|
||||||
if (config->use_compression) {
|
if (config->use_compression) {
|
||||||
data = chunk_compress(chunk, config->compression_level);
|
data = chunk_compress(chunk, config->compression_level, config->use_metadata);
|
||||||
} else {
|
} else {
|
||||||
for (int i = 0; i < chunk->element_count; i++)
|
for (int i = 0; i < chunk->element_count; i++)
|
||||||
file_load_data(chunk->items[i]);
|
file_load_data(chunk->items[i]);
|
||||||
data = chunk_serialize(chunk);
|
data = chunk_serialize(chunk, config->use_metadata);
|
||||||
}
|
}
|
||||||
send_data(client->file_descriptor, data->data, data->size);
|
send_data(client->file_descriptor, data->data, data->size);
|
||||||
data_destroy(data);
|
data_destroy(data);
|
||||||
|
|||||||
+6
-5
@@ -12,10 +12,11 @@
|
|||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <threads.h>
|
#include <threads.h>
|
||||||
|
|
||||||
File *receive_file_receive(Config *config, int file_descriptor) {
|
File *file_receive(Config *config, int file_descriptor) {
|
||||||
char *path = (char *)receive_str(file_descriptor);
|
char *path = (char *)receive_str(file_descriptor);
|
||||||
File *file = file_create(path);
|
File *file = file_create(path);
|
||||||
free(path);
|
free(path);
|
||||||
|
if (config->use_metadata)
|
||||||
file->metadata = file_receive_metadata(file_descriptor);
|
file->metadata = file_receive_metadata(file_descriptor);
|
||||||
Data *file_data = receive_data(file_descriptor);
|
Data *file_data = receive_data(file_descriptor);
|
||||||
if (config->use_compression) {
|
if (config->use_compression) {
|
||||||
@@ -37,7 +38,7 @@ static void receive_chunk_enqueue(int file_descriptor, Config *config,
|
|||||||
data_to_process = data_decompress(chunk_data);
|
data_to_process = data_decompress(chunk_data);
|
||||||
data_destroy(chunk_data);
|
data_destroy(chunk_data);
|
||||||
}
|
}
|
||||||
Chunk *chunk = chunk_deserialize(data_to_process);
|
Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata);
|
||||||
data_destroy(data_to_process);
|
data_destroy(data_to_process);
|
||||||
|
|
||||||
for (int i = 0; i < chunk->element_count; i++) {
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
@@ -63,7 +64,7 @@ int receive_thread(void *pipeline_context) {
|
|||||||
if (status == STATUS_CHUNK) {
|
if (status == STATUS_CHUNK) {
|
||||||
receive_chunk_enqueue(file_descriptor, config, context);
|
receive_chunk_enqueue(file_descriptor, config, context);
|
||||||
} else {
|
} else {
|
||||||
File *file = receive_file_receive(config, file_descriptor);
|
File *file = file_receive(config, file_descriptor);
|
||||||
queue_enqueue_multithreaded(context->queue, file, &context->mutex,
|
queue_enqueue_multithreaded(context->queue, file, &context->mutex,
|
||||||
&context->condition_not_empty,
|
&context->condition_not_empty,
|
||||||
&context->condition_not_full);
|
&context->condition_not_full);
|
||||||
@@ -113,7 +114,7 @@ int receive_files(Config *config, int file_descriptor) {
|
|||||||
data_to_process = data_decompress(chunk_data);
|
data_to_process = data_decompress(chunk_data);
|
||||||
data_destroy(chunk_data);
|
data_destroy(chunk_data);
|
||||||
}
|
}
|
||||||
Chunk *chunk = chunk_deserialize(data_to_process);
|
Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata);
|
||||||
data_destroy(data_to_process);
|
data_destroy(data_to_process);
|
||||||
|
|
||||||
for (int i = 0; i < chunk->element_count; i++) {
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
@@ -126,7 +127,7 @@ int receive_files(Config *config, int file_descriptor) {
|
|||||||
}
|
}
|
||||||
chunk_destroy(chunk);
|
chunk_destroy(chunk);
|
||||||
} else {
|
} else {
|
||||||
File *file = receive_file_receive(config, file_descriptor);
|
File *file = file_receive(config, file_descriptor);
|
||||||
if (config->save_to_disk) {
|
if (config->save_to_disk) {
|
||||||
char *disk_path = path_cat(config->receive_root_directory, file->path);
|
char *disk_path = path_cat(config->receive_root_directory, file->path);
|
||||||
to_disk(disk_path, file->data->data, file->data->size);
|
to_disk(disk_path, file->data->data, file->data->size);
|
||||||
|
|||||||
+13
-10
@@ -133,16 +133,16 @@ Data *chunk_format(Chunk *chunk) {
|
|||||||
return chunk_data_create(data, buffer_size);
|
return chunk_data_create(data, buffer_size);
|
||||||
}
|
}
|
||||||
|
|
||||||
static unsigned long long per_file_serialize_size(File *file) {
|
static unsigned long long per_file_serialize_size(File *file, bool use_metadata) {
|
||||||
return sizeof(size_t) + strlen(file->path) + sizeof(int) +
|
return sizeof(size_t) + strlen(file->path) +
|
||||||
(file->metadata ? FILE_METADATA_WIRE_SIZE : 0) +
|
(use_metadata ? sizeof(int) + (file->metadata ? FILE_METADATA_WIRE_SIZE : 0) : 0) +
|
||||||
sizeof(size_t) + file->data->size;
|
sizeof(size_t) + file->data->size;
|
||||||
}
|
}
|
||||||
|
|
||||||
Data *chunk_serialize(Chunk *chunk) {
|
Data *chunk_serialize(Chunk *chunk, bool use_metadata) {
|
||||||
unsigned long long data_size = 0;
|
unsigned long long data_size = 0;
|
||||||
for (int i = 0; i < chunk->element_count; i++) {
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
data_size += per_file_serialize_size(chunk->items[i]);
|
data_size += per_file_serialize_size(chunk->items[i], use_metadata);
|
||||||
}
|
}
|
||||||
Data *data = data_create_empty(data_size);
|
Data *data = data_create_empty(data_size);
|
||||||
if (data == NULL) {
|
if (data == NULL) {
|
||||||
@@ -159,6 +159,7 @@ Data *chunk_serialize(Chunk *chunk) {
|
|||||||
memcpy(data_pointer, file->path, path_len);
|
memcpy(data_pointer, file->path, path_len);
|
||||||
data_pointer += path_len;
|
data_pointer += path_len;
|
||||||
|
|
||||||
|
if (use_metadata)
|
||||||
metadata_to_buf(&data_pointer, file->metadata);
|
metadata_to_buf(&data_pointer, file->metadata);
|
||||||
|
|
||||||
size_t file_data_size = file->data->size;
|
size_t file_data_size = file->data->size;
|
||||||
@@ -170,7 +171,7 @@ Data *chunk_serialize(Chunk *chunk) {
|
|||||||
return data;
|
return data;
|
||||||
}
|
}
|
||||||
|
|
||||||
Chunk *chunk_deserialize(Data *data) {
|
Chunk *chunk_deserialize(Data *data, bool use_metadata) {
|
||||||
ArrayList *files = array_list_create(file_destroy);
|
ArrayList *files = array_list_create(file_destroy);
|
||||||
char *data_pointer = data->data;
|
char *data_pointer = data->data;
|
||||||
size_t remaining_size = data->size;
|
size_t remaining_size = data->size;
|
||||||
@@ -206,10 +207,12 @@ Chunk *chunk_deserialize(Data *data) {
|
|||||||
File *file = file_create(path);
|
File *file = file_create(path);
|
||||||
free(path);
|
free(path);
|
||||||
|
|
||||||
|
if (use_metadata) {
|
||||||
file->metadata = metadata_from_buf(&data_pointer);
|
file->metadata = metadata_from_buf(&data_pointer);
|
||||||
remaining_size -= sizeof(int);
|
remaining_size -= sizeof(int);
|
||||||
if (file->metadata)
|
if (file->metadata)
|
||||||
remaining_size -= FILE_METADATA_WIRE_SIZE;
|
remaining_size -= FILE_METADATA_WIRE_SIZE;
|
||||||
|
}
|
||||||
|
|
||||||
if (remaining_size < sizeof(size_t)) {
|
if (remaining_size < sizeof(size_t)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for data size");
|
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for data size");
|
||||||
@@ -252,16 +255,16 @@ Chunk *chunk_deserialize(Data *data) {
|
|||||||
return chunk;
|
return chunk;
|
||||||
}
|
}
|
||||||
|
|
||||||
Data *chunk_compress(Chunk *chunk, int compression_level) {
|
Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata) {
|
||||||
log_message(LOG_LEVEL_DEBUG, "Starting to compress chunk");
|
log_message(LOG_LEVEL_DEBUG, "Starting to compress chunk");
|
||||||
Data *serialized = chunk_serialize(chunk);
|
Data *serialized = chunk_serialize(chunk, use_metadata);
|
||||||
Data *compressed = data_compress(serialized, compression_level);
|
Data *compressed = data_compress(serialized, compression_level);
|
||||||
data_destroy(serialized);
|
data_destroy(serialized);
|
||||||
log_message(LOG_LEVEL_DEBUG, "Chunk successfully compressed");
|
log_message(LOG_LEVEL_DEBUG, "Chunk successfully compressed");
|
||||||
return compressed;
|
return compressed;
|
||||||
}
|
}
|
||||||
|
|
||||||
Chunk *chunk_decompress(Data *compressed_data) {
|
Chunk *chunk_decompress(Data *compressed_data, bool use_metadata) {
|
||||||
log_message(LOG_LEVEL_DEBUG, "Starting to decompress chunk");
|
log_message(LOG_LEVEL_DEBUG, "Starting to decompress chunk");
|
||||||
Data *uncompressed_data = data_decompress(compressed_data);
|
Data *uncompressed_data = data_decompress(compressed_data);
|
||||||
if (uncompressed_data == NULL) {
|
if (uncompressed_data == NULL) {
|
||||||
@@ -269,7 +272,7 @@ Chunk *chunk_decompress(Data *compressed_data) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
Chunk *chunk = chunk_deserialize(uncompressed_data);
|
Chunk *chunk = chunk_deserialize(uncompressed_data, use_metadata);
|
||||||
if (chunk == NULL) {
|
if (chunk == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk data");
|
log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk data");
|
||||||
data_destroy(uncompressed_data);
|
data_destroy(uncompressed_data);
|
||||||
|
|||||||
+5
-4
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
#include "data.h"
|
#include "data.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
|
#include <stdbool.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
|
|
||||||
#define DESIRED_CHUNK_SIZE 10 * 1024 * 1024
|
#define DESIRED_CHUNK_SIZE 10 * 1024 * 1024
|
||||||
@@ -16,10 +17,10 @@ Chunk *chunk_create(File **items, int element_count);
|
|||||||
void chunk_destroy(void *chunk);
|
void chunk_destroy(void *chunk);
|
||||||
void chunk_print(void *chunk);
|
void chunk_print(void *chunk);
|
||||||
Data *chunk_format(Chunk *chunk);
|
Data *chunk_format(Chunk *chunk);
|
||||||
Data *chunk_serialize(Chunk *chunk);
|
Data *chunk_serialize(Chunk *chunk, bool use_metadata);
|
||||||
Chunk *chunk_deserialize(Data *data);
|
Chunk *chunk_deserialize(Data *data, bool use_metadata);
|
||||||
Data *chunk_compress(Chunk *chunk, int compression_level);
|
Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata);
|
||||||
Chunk *chunk_decompress(Data *compressed_data);
|
Chunk *chunk_decompress(Data *compressed_data, bool use_metadata);
|
||||||
|
|
||||||
Data *chunk_data_create(void *data, unsigned long long data_size);
|
Data *chunk_data_create(void *data, unsigned long long data_size);
|
||||||
void chunk_data_delete(void *chunk);
|
void chunk_data_delete(void *chunk);
|
||||||
|
|||||||
+6
-1
@@ -73,8 +73,13 @@ void file_metadata_destroy(void *metadata) {
|
|||||||
void file_load_data(File *file) {
|
void file_load_data(File *file) {
|
||||||
if (file == NULL)
|
if (file == NULL)
|
||||||
return;
|
return;
|
||||||
if (file->data->data == NULL)
|
if (file->data->data == NULL) {
|
||||||
file->data->data = malloc(file->data->size);
|
file->data->data = malloc(file->data->size);
|
||||||
|
if (file->data->data == NULL) {
|
||||||
|
perror("Could not allocate memory for file data");
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
}
|
||||||
|
}
|
||||||
printf("%ld is file big", file->data->size);
|
printf("%ld is file big", file->data->size);
|
||||||
size_t bytes_read = file_content_to_buffer(file);
|
size_t bytes_read = file_content_to_buffer(file);
|
||||||
if (bytes_read != file->data->size) {
|
if (bytes_read != file->data->size) {
|
||||||
|
|||||||
+2
-2
@@ -67,10 +67,10 @@ static void test_chunk_operations() {
|
|||||||
file_load_data(f2);
|
file_load_data(f2);
|
||||||
|
|
||||||
// Test chunk_serialize / chunk_deserialize round-trip
|
// Test chunk_serialize / chunk_deserialize round-trip
|
||||||
Data *serialized = chunk_serialize(chunk);
|
Data *serialized = chunk_serialize(chunk, false);
|
||||||
EXPECT_NOT_NULL(serialized);
|
EXPECT_NOT_NULL(serialized);
|
||||||
|
|
||||||
Chunk *deserialized = chunk_deserialize(serialized);
|
Chunk *deserialized = chunk_deserialize(serialized, false);
|
||||||
EXPECT_NOT_NULL(deserialized);
|
EXPECT_NOT_NULL(deserialized);
|
||||||
EXPECT_EQ_INT(deserialized->element_count, 2);
|
EXPECT_EQ_INT(deserialized->element_count, 2);
|
||||||
EXPECT_EQ_STR(deserialized->items[0]->path, path1);
|
EXPECT_EQ_STR(deserialized->items[0]->path, path1);
|
||||||
|
|||||||
@@ -82,10 +82,10 @@ static void test_chunk_compress_decompress_roundtrip() {
|
|||||||
Chunk *chunk = chunk_create(files, 2);
|
Chunk *chunk = chunk_create(files, 2);
|
||||||
EXPECT_NOT_NULL(chunk);
|
EXPECT_NOT_NULL(chunk);
|
||||||
|
|
||||||
Data *compressed = chunk_compress(chunk, 3);
|
Data *compressed = chunk_compress(chunk, 3, false);
|
||||||
EXPECT_NOT_NULL(compressed);
|
EXPECT_NOT_NULL(compressed);
|
||||||
|
|
||||||
Chunk *decompressed_chunk = chunk_decompress(compressed);
|
Chunk *decompressed_chunk = chunk_decompress(compressed, false);
|
||||||
EXPECT_NOT_NULL(decompressed_chunk);
|
EXPECT_NOT_NULL(decompressed_chunk);
|
||||||
EXPECT_EQ_INT(decompressed_chunk->element_count, 2);
|
EXPECT_EQ_INT(decompressed_chunk->element_count, 2);
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user