Apply PR #143 content on top of latest main

src/client/client_send.c

@@ -21,6 +21,9 @@
 #include <string.h>
 #include <threads.h>
 #include <time.h>
+#include <unistd.h>
+
+#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
 
 static int incremental_check(Client* client, File* file, DeltaSignature** out_sig) {
   *out_sig = NULL;
@@ -208,10 +211,13 @@ int send_chunk(Client* client, Chunk* chunk, Config* config) {
     return 0;
   }
 
-  bool use_sendfile = config->use_sendfile && !config->use_compression;
   for (int i = 0; i < chunk->element_count; i++) {
-    int rc =
-        send_single_file(client, chunk->items[i], config, config->use_incremental, use_sendfile);
+    File* f = chunk->items[i];
+    if (f == NULL)
+      continue;
+    bool stream = f->data->data == NULL && f->data->size > 0;
+    bool use_sendfile = (config->use_sendfile && !config->use_compression) || stream;
+    int rc = send_single_file(client, f, config, config->use_incremental, use_sendfile);
     if (rc == 1)
       continue;
     if (rc < 0)
@@ -296,16 +302,14 @@ static int send_chunks_multithreaded(void* pipeline_context) {
 
 static int scan_directory_multithreaded(void* pipeline_context) {
   PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
-  mtx_lock(&context->mutex_scanner);
-  DirectoryScanner* scanner = directory_scanner_create(
+  ParallelScanner* scanner = parallel_scanner_create(
       context->config->send_directory, context->config->use_metadata, context->config->chunk_size,
       context->config->exclude_patterns, context->config->exclude_count,
       context->config->include_patterns, context->config->include_count, context->config->max_size,
-      context->config->min_size);
-  mtx_unlock(&context->mutex_scanner);
+      context->config->min_size, context->config->max_depth, 4);
 
   Chunk* current_chunk;
-  while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
+  while ((current_chunk = parallel_scanner_next(scanner)) != NULL) {
     if (context->config->use_delete) {
       mtx_lock(&context->mutex_scanner);
       for (int i = 0; i < current_chunk->element_count; i++) {
@@ -325,7 +329,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
   cnd_signal(&context->condition_not_empty_scanner);
   mtx_unlock(&context->mutex_scanner);
 
-  directory_scanner_destroy(scanner);
+  parallel_scanner_destroy(scanner);
   return thrd_success;
 }
 
@@ -344,9 +348,12 @@ static int load_files_multithreaded(void* pipeline_context) {
     }
     if (!context->config->use_sendfile) {
       for (int i = 0; i < chunk->element_count; i++) {
-        if (!file_load_data(chunk->items[i])) {
+        File* f = chunk->items[i];
+        if (f->data->size > STREAM_THRESHOLD)
+          continue;
+        if (!file_load_data(f)) {
           log_message(LOG_LEVEL_ERROR, "Failed to load file data, skipping");
-          file_destroy(chunk->items[i]);
+          file_destroy(f);
           chunk->items[i] = NULL;
         }
       }
@@ -362,7 +369,7 @@ int send_files(Config* config) {
     DirectoryScanner* scanner = directory_scanner_create(
         config->send_directory, config->use_metadata, config->chunk_size, config->exclude_patterns,
         config->exclude_count, config->include_patterns, config->include_count, config->max_size,
-        config->min_size);
+        config->min_size, config->max_depth);
     Chunk* chunk;
     int file_count = 0;
     unsigned long long total_bytes = 0;
@@ -416,7 +423,7 @@ int send_files(Config* config) {
   DirectoryScanner* scanner = directory_scanner_create(
       config->send_directory, config->use_metadata, config->chunk_size, config->exclude_patterns,
       config->exclude_count, config->include_patterns, config->include_count, config->max_size,
-      config->min_size);
+      config->min_size, config->max_depth);
   Chunk* current_chunk;
   unsigned long long total_bytes = 0;
   time_t last_progress = 0;
@@ -435,7 +442,10 @@ int send_files(Config* config) {
     }
     if (!config->use_sendfile) {
       for (int i = 0; i < current_chunk->element_count; i++) {
-        if (!file_load_data(current_chunk->items[i])) {
+        File* f = current_chunk->items[i];
+        if (f->data->size > STREAM_THRESHOLD)
+          continue;
+        if (!file_load_data(f)) {
           log_message(LOG_LEVEL_ERROR, "Failed to load file data");
           continue;
         }
@@ -502,7 +512,7 @@ int send_files_multithreaded(Config* config) {
     DirectoryScanner* scanner = directory_scanner_create(
         config->send_directory, config->use_metadata, config->chunk_size, config->exclude_patterns,
         config->exclude_count, config->include_patterns, config->include_count, config->max_size,
-        config->min_size);
+        config->min_size, config->max_depth);
     Chunk* chunk;
     int file_count = 0;
     unsigned long long total_bytes = 0;
@@ -520,8 +530,20 @@ int send_files_multithreaded(Config* config) {
     return 0;
   }
 
-  Queue* q1 = queue_create(100, chunk_destroy);
-  Queue* q2 = queue_create(100, chunk_destroy);
+  long pages = sysconf(_SC_AVPHYS_PAGES);
+  long page_size = sysconf(_SC_PAGE_SIZE);
+  unsigned long long available_memory =
+      pages > 0 && page_size > 0 ? (unsigned long long)pages * (unsigned long long)page_size
+                                 : 512ULL * 1024 * 1024;
+  unsigned long long avg_file_size = 1024 * 1024;
+  int qsize = (int)(available_memory / avg_file_size);
+  if (qsize < 10)
+    qsize = 10;
+  if (qsize > 1000)
+    qsize = 1000;
+
+  Queue* q1 = queue_create(qsize, chunk_destroy);
+  Queue* q2 = queue_create(qsize, chunk_destroy);
   if (!q1 || !q2) {
     if (q1)
       queue_destroy(q1);