perf: downsized test data, increased SSH socket buffer, and made chunk size configurable
- Reduced test data from 50MB to 25MB (test.py) - Increased SSH socketpair buffer to 1MB via setsockopt (transport_ssh.c) - Made chunk size configurable: new Config field, --chunk-size CLI flag, sent over wire, and used in scanner instead of DESIRED_CHUNK_SIZE - Fixed DESIRED_CHUNK_SIZE macro parentheses (chunk.h)
This commit is contained in:
@@ -26,6 +26,7 @@ static void print_usage(void) {
|
||||
printf(" -f Enable sendfile (TCP only, not with -c or -s)\n");
|
||||
printf(" -v, --verbose Enable debug logging\n");
|
||||
printf(" -M, --preserve Preserve file metadata\n");
|
||||
printf(" --chunk-size <n> Chunk size in bytes (default: %d)\n", DEFAULT_CHUNK_SIZE);
|
||||
printf(" --source-dir <path> Source directory\n");
|
||||
printf(" --dest-dir <path> Destination directory\n");
|
||||
printf(" --save-to-disk Write received files to disk\n");
|
||||
@@ -46,7 +47,7 @@ int main(int argc, char *argv[]) {
|
||||
}
|
||||
|
||||
Config *config = config_create(str_dup("1.0.0"), NULL, NULL,
|
||||
save_to_disk, false, false, false, false, 5, false);
|
||||
save_to_disk, false, false, false, false, 5, false, 0);
|
||||
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
@@ -93,6 +94,10 @@ int main(int argc, char *argv[]) {
|
||||
server_host = str_dup(argv[++i]);
|
||||
} else if (strcmp(argv[i], "--server-port") == 0 && i + 1 < argc) {
|
||||
server_port = atoi(argv[++i]);
|
||||
} else if (strcmp(argv[i], "--chunk-size") == 0 && i + 1 < argc) {
|
||||
unsigned long long val = strtoull(argv[++i], NULL, 10);
|
||||
if (val > 0)
|
||||
config->chunk_size = val;
|
||||
} else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
|
||||
set_log_level(LOG_LEVEL_DEBUG);
|
||||
} else if (argv[i][0] == '-') {
|
||||
|
||||
@@ -82,7 +82,7 @@ static int scan_directory_multithreaded(void *pipeline_context) {
|
||||
PipelineContextSender *context = (PipelineContextSender *)pipeline_context;
|
||||
mtx_lock(&context->mutex_scanner);
|
||||
DirectoryScanner *scanner =
|
||||
directory_scanner_create(context->config->send_directory, context->config->use_metadata);
|
||||
directory_scanner_create(context->config->send_directory, context->config->use_metadata, context->config->chunk_size);
|
||||
mtx_unlock(&context->mutex_scanner);
|
||||
|
||||
Chunk *current_chunk;
|
||||
@@ -138,7 +138,7 @@ int send_files(Config *config) {
|
||||
client_connect(client, server_host, server_port);
|
||||
}
|
||||
config_send(client->file_descriptor, config);
|
||||
DirectoryScanner *scanner = directory_scanner_create(config->send_directory, config->use_metadata);
|
||||
DirectoryScanner *scanner = directory_scanner_create(config->send_directory, config->use_metadata, config->chunk_size);
|
||||
Chunk *current_chunk;
|
||||
while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
|
||||
if (!config->use_sendfile) {
|
||||
|
||||
@@ -11,12 +11,13 @@
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata) {
|
||||
DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size) {
|
||||
DirectoryScanner *scanner = malloc(sizeof(DirectoryScanner));
|
||||
scanner->directories = queue_create(100, free);
|
||||
scanner->current_dir = NULL;
|
||||
scanner->current_path = NULL;
|
||||
scanner->use_metadata = use_metadata;
|
||||
scanner->chunk_size = chunk_size > 0 ? chunk_size : DESIRED_CHUNK_SIZE;
|
||||
queue_enqueue(scanner->directories, str_dup(root_directory));
|
||||
return scanner;
|
||||
}
|
||||
@@ -99,7 +100,7 @@ Chunk *directory_scanner_next(DirectoryScanner *scanner) {
|
||||
file->metadata = file_metadata_create(&stats);
|
||||
array_list_add(chunk_data, file);
|
||||
chunk_data_size += file->data->size;
|
||||
if (chunk_data_size > DESIRED_CHUNK_SIZE) {
|
||||
if (chunk_data_size > scanner->chunk_size) {
|
||||
free(cur_path);
|
||||
return chunk_data_to_chunk(chunk_data);
|
||||
}
|
||||
|
||||
@@ -11,9 +11,10 @@ typedef struct {
|
||||
DIR *current_dir;
|
||||
char *current_path;
|
||||
bool use_metadata;
|
||||
unsigned long long chunk_size;
|
||||
} DirectoryScanner;
|
||||
|
||||
DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata);
|
||||
DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size);
|
||||
Chunk *directory_scanner_next(DirectoryScanner *scanner);
|
||||
void directory_scanner_destroy(DirectoryScanner *scanner);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user