Merge fix/security-hardening into dev

fixes #254 (receiver queue byte-budget) #258 (inplace setuid/truncate)

# Conflicts:
#	src/shared/multiprocessing.c

src/server/server.c

@@ -4,6 +4,7 @@
 #include "log.h"
 #include "metadata.h"
 #include "multiprocessing.h"
+#include "protocol.h"
 #include "queue.h"
 #include "receiver.h"
 #include "transport_tcp.h"
@@ -26,6 +27,13 @@ static bool allow_delete;
 static bool allow_unauthenticated;
 static const char* required_client_cn;
 
+/* Aggregate payload bytes the multithreaded receiver may buffer ahead of the
+   slow disk writer.  Receiving one more chunk adds up to ~2 * MAX_CHUNK_SIZE
+   of transient wire/decompression buffers on top of the queued payloads, so
+   this ceiling keeps total per-connection receive memory (decompressed and
+   per-file copied chunk buffers included) within MAX_CONNECTION_MEMORY. */
+#define RECEIVER_QUEUE_MAX_BYTES (MAX_CONNECTION_MEMORY - 2 * MAX_CHUNK_SIZE)
+
 static bool tls_client_identity_allowed(SSL* ssl) {
   if (!ssl || !required_client_cn)
     return false;
@@ -314,6 +322,7 @@ void handler(int file_descriptor) {
     protocol_session_set_max_alloc(&context->session, config->max_alloc);
     atomic_store(&context->session.total_allocated_bytes,
                  atomic_load(&session.total_allocated_bytes));
+    pipeline_context_receiver_set_queue_byte_limit(context, RECEIVER_QUEUE_MAX_BYTES);
     thrd_t receiver, writer;
     bool receiver_created = thrd_create(&receiver, receive_thread, context) == thrd_success;
     bool writer_created = false;

src/shared/file.c

@@ -348,10 +348,25 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
       if (newer) {
         ok = true;
       } else {
-        if (!sparse || data_size == 0 || ftruncate(fd, (off_t)data_size) == 0)
+        /* In-place overwrites: pre-size sparse targets and always trim the
+           file to the new payload length afterwards so shorter payloads can
+           never leave stale trailing bytes from a previous version. */
+        if (sparse && data_size > 0)
+          ok = ftruncate(fd, (off_t)data_size) == 0;
+        if (ok || !sparse || data_size == 0)
           ok = write_all(fd, data, data_size);
-        if (ok && metadata)
-          ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
+        if (ok)
+          ok = ftruncate(fd, (off_t)data_size) == 0;
+        /* Normalize the mode: apply the metadata-derived safe mode when the
+           sender supplied metadata (setuid/setgid/sticky are never honored);
+           otherwise fall back to a safe default so dangerous bits on an
+           existing destination cannot survive an overwrite. */
+        if (ok) {
+          if (metadata)
+            ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
+          else if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) != 0)
+            ok = false;
+        }
         if (ok && use_fsync)
           ok = fsync(fd) == 0;
       }

src/shared/multiprocessing.c

@@ -111,6 +111,8 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
   protocol_session_init(&context->session, file_descriptor, file_descriptor);
   protocol_session_set_ssl(&context->session, ssl);
   context->receiver_done = false;
+  context->queued_bytes = 0;
+  context->max_queue_bytes = 0;
   atomic_init(&context->cancelled, false);
   int init = 0;
   if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
@@ -147,14 +149,73 @@ void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
   free(context);
 }
 
+void pipeline_context_receiver_set_queue_byte_limit(PipelineContextReceiver* context,
+                                                    size_t max_bytes) {
+  if (context == NULL)
+    return;
+  mtx_lock(&context->mutex);
+  context->max_queue_bytes = max_bytes;
+  context->queued_bytes = 0;
+  cnd_broadcast(&context->condition_not_full);
+  mtx_unlock(&context->mutex);
+}
+
+void pipeline_context_receiver_note_bytes_released(PipelineContextReceiver* context,
+                                                   size_t released_bytes) {
+  if (context == NULL || context->max_queue_bytes == 0 || released_bytes == 0)
+    return;
+  mtx_lock(&context->mutex);
+  if (released_bytes >= context->queued_bytes)
+    context->queued_bytes = 0;
+  else
+    context->queued_bytes -= released_bytes;
+  cnd_signal(&context->condition_not_full);
+  mtx_unlock(&context->mutex);
+}
+
+bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, File* file) {
+  if (context == NULL || file == NULL)
+    return false;
+  size_t file_bytes = file->data ? file->data->size : 0;
+  mtx_lock(&context->mutex);
+  while (!atomic_load(&context->cancelled)) {
+    bool blocked_by_count = queue_is_full(context->queue);
+    bool blocked_by_budget = false;
+    if (context->max_queue_bytes > 0) {
+      size_t budget = context->max_queue_bytes;
+      size_t used = context->queued_bytes;
+      if (used >= budget) {
+        blocked_by_budget = true;
+      } else if (file_bytes > budget - used) {
+        /* A single payload larger than the whole budget (not possible with
+           the per-file receive cap) is only admitted to an empty pipeline so
+           the wait can never deadlock. */
+        blocked_by_budget = used != 0;
+      }
+    }
+    if (!blocked_by_count && !blocked_by_budget)
+      break;
+    cnd_wait(&context->condition_not_full, &context->mutex);
+  }
+  if (atomic_load(&context->cancelled)) {
+    mtx_unlock(&context->mutex);
+    file_destroy(file);
+    return false;
+  }
+  if (!queue_enqueue(context->queue, file)) {
+    mtx_unlock(&context->mutex);
+    file_destroy(file);
+    return false;
+  }
+  context->queued_bytes += file_bytes;
+  cnd_signal(&context->condition_not_empty);
+  mtx_unlock(&context->mutex);
+  return true;
+}
+
 static bool receiver_enqueue_file(File* file, void* context_pointer) {
-  PipelineContextReceiver* context = context_pointer;
-  if (queue_enqueue_multithreaded_cancel(context->queue, file, &context->mutex,
-                                         &context->condition_not_empty,
-                                         &context->condition_not_full, &context->cancelled))
-    return true;
-  file_destroy(file);
-  return false;
+  PipelineContextReceiver* context = (PipelineContextReceiver*)context_pointer;
+  return pipeline_context_receiver_enqueue_file(context, file);
 }
 
 static void receiver_thread_fail(PipelineContextReceiver* context) {
@@ -215,11 +276,13 @@ int write_thread(void* pipeline_context) {
       protocol_session_unbind();
       return thrd_success;
     }
+    size_t file_bytes = file->data ? file->data->size : 0;
     FileSaveResult result = FILE_SAVE_SKIPPED;
     if (save_to_disk) {
       result = file_save_to_disk_full(root_directory, file, context->config);
       if (result == FILE_SAVE_ERROR) {
         file_destroy(file);
+        pipeline_context_receiver_note_bytes_released(context, file_bytes);
         mtx_lock(&context->mutex);
         atomic_store(&context->cancelled, true);
         context->receiver_done = true;
@@ -236,6 +299,7 @@ int write_thread(void* pipeline_context) {
     if (context->config->remove_source_files &&
         !receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
       file_destroy(file);
+      pipeline_context_receiver_note_bytes_released(context, file_bytes);
       mtx_lock(&context->mutex);
       atomic_store(&context->cancelled, true);
       context->receiver_done = true;
@@ -247,5 +311,6 @@ int write_thread(void* pipeline_context) {
       return thrd_error;
     }
     file_destroy(file);
+    pipeline_context_receiver_note_bytes_released(context, file_bytes);
   }
 }

src/shared/multiprocessing.h

@@ -47,6 +47,14 @@ typedef struct PipelineContextReceiver {
   cnd_t condition_not_empty;
   bool receiver_done;
   atomic_bool cancelled;
+  /* Aggregate payload bytes that have been received but not yet released by
+     the disk writer (queued or in the writer's hand).  Guarded by `mutex`.
+     When `max_queue_bytes` is non-zero the receiver blocks before enqueuing
+     once this total would exceed it, so decompressed/copied file payloads
+     buffered ahead of a slow disk writer respect the per-connection memory
+     budget instead of growing without bound. */
+  size_t queued_bytes;
+  size_t max_queue_bytes;
 } PipelineContextReceiver;
 
 PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* queue_scanner,
@@ -55,6 +63,18 @@ void pipeline_context_sender_destroy(PipelineContextSender* context);
 PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue_receiver,
                                                           int file_descriptor, SSL* ssl);
 void pipeline_context_receiver_destroy(PipelineContextReceiver* context);
+/* Bound the bytes buffered ahead of the disk writer (see max_queue_bytes). */
+void pipeline_context_receiver_set_queue_byte_limit(PipelineContextReceiver* context,
+                                                    size_t max_bytes);
+/* Blocking enqueue used by the receive pipeline sink.  Blocks while the queue
+   is full by element count or when adding `file` would push queued_bytes over
+   the configured byte limit; waits until the disk writer releases bytes.
+   Takes ownership of `file` on success and destroys it on failure/cancel. */
+bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, File* file);
+/* Account for `released_bytes` of payload memory that has been freed by the
+   disk writer, unblocking a receiver that is waiting on the byte limit. */
+void pipeline_context_receiver_note_bytes_released(PipelineContextReceiver* context,
+                                                   size_t released_bytes);
 int receive_thread(void* pipeline_context);
 int write_thread(void* pipeline_context);
 #endif

src/shared/protocol.c

@@ -14,7 +14,6 @@
 
 #define RECEIVE_TIMEOUT_SEC 60 /* 60 second per-message timeout */
 #define SEND_TIMEOUT_SEC 60
-#define MAX_CONNECTION_MEMORY (256ULL * 1024 * 1024) /* bounded cumulative receive budget */
 
 static __thread int io_read_fd = -1;
 static __thread int io_write_fd = -1;

src/shared/protocol.h

@@ -28,6 +28,10 @@
 #define DEFAULT_MAX_ALLOC (1ULL * 1024 * 1024 * 1024)
 /* Server policy ceiling for a client-provided allocation limit. */
 #define MAX_SERVER_ALLOC (256ULL * 1024 * 1024)
+/* Bounded cumulative per-connection receive budget.  In-flight wire buffers,
+   decompression buffers and queued (not yet written) file payloads for a
+   connection must stay within this ceiling. */
+#define MAX_CONNECTION_MEMORY (256ULL * 1024 * 1024)
 
 typedef struct ssl_st SSL;