build: explicit CMake targets; move receiver pipeline out of shared

src/server/receiver_pipeline.c

@@ -0,0 +1,254 @@
+#include "receiver_pipeline.h"
+
+#include "log.h"
+#include "protocol.h"
+#include "queue.h"
+#include "utils.h"
+#include <stdlib.h>
+#include <string.h>
+#include <threads.h>
+
+PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue,
+                                                          int file_descriptor, SSL* ssl) {
+  PipelineContextReceiver* context = malloc(sizeof(PipelineContextReceiver));
+  if (context == NULL)
+    return NULL;
+  context->config = config;
+  context->queue = queue;
+  context->file_descriptor = file_descriptor;
+  context->ssl = ssl;
+  context->outcomes.entries = NULL;
+  context->outcomes.count = 0;
+  context->outcomes.capacity = 0;
+  dir_time_list_init(&context->dir_times);
+  protocol_session_init(&context->session, file_descriptor, file_descriptor);
+  protocol_session_set_ssl(&context->session, ssl);
+  context->receiver_done = false;
+  context->queued_bytes = 0;
+  context->max_queue_bytes = 0;
+  context->deferred_manifest = NULL;
+  atomic_init(&context->cancelled, false);
+  int init = 0;
+  if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
+    goto fail;
+  init++;
+  if (cnd_init(&context->condition_not_full) != thrd_success)
+    goto fail;
+  init++;
+  if (cnd_init(&context->condition_not_empty) != thrd_success)
+    goto fail;
+  // cppcheck-suppress unreadVariable
+  init++;
+  return context;
+
+fail:
+  log_perror("Error initializing synchronization objects");
+  if (init >= 3)
+    cnd_destroy(&context->condition_not_empty);
+  if (init >= 2)
+    cnd_destroy(&context->condition_not_full);
+  if (init >= 1)
+    mtx_destroy(&context->mutex);
+  free(context);
+  return NULL;
+}
+
+void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
+  config_delete(context->config);
+  if (context->deferred_manifest)
+    delete_manifest_free(context->deferred_manifest);
+  queue_destroy(context->queue);
+  receiver_outcomes_destroy(&context->outcomes);
+  dir_time_list_free(&context->dir_times);
+  mtx_destroy(&context->mutex);
+  cnd_destroy(&context->condition_not_full);
+  cnd_destroy(&context->condition_not_empty);
+  free(context);
+}
+
+void pipeline_context_receiver_set_queue_byte_limit(PipelineContextReceiver* context,
+                                                    size_t max_bytes) {
+  if (context == NULL)
+    return;
+  mtx_lock(&context->mutex);
+  context->max_queue_bytes = max_bytes;
+  context->queued_bytes = 0;
+  cnd_broadcast(&context->condition_not_full);
+  mtx_unlock(&context->mutex);
+}
+
+void pipeline_context_receiver_note_bytes_released(PipelineContextReceiver* context,
+                                                   size_t released_bytes) {
+  if (context == NULL || context->max_queue_bytes == 0 || released_bytes == 0)
+    return;
+  mtx_lock(&context->mutex);
+  if (released_bytes >= context->queued_bytes)
+    context->queued_bytes = 0;
+  else
+    context->queued_bytes -= released_bytes;
+  cnd_signal(&context->condition_not_full);
+  mtx_unlock(&context->mutex);
+}
+
+bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, File* file) {
+  if (context == NULL || file == NULL)
+    return false;
+  size_t file_bytes = file->data ? file->data->size : 0;
+  mtx_lock(&context->mutex);
+  while (!atomic_load(&context->cancelled)) {
+    bool blocked_by_count = queue_is_full(context->queue);
+    bool blocked_by_budget = false;
+    if (context->max_queue_bytes > 0) {
+      size_t budget = context->max_queue_bytes;
+      size_t used = context->queued_bytes;
+      if (used >= budget) {
+        blocked_by_budget = true;
+      } else if (file_bytes > budget - used) {
+        /* A single payload larger than the whole budget (not possible with
+           the per-file receive cap) is only admitted to an empty pipeline so
+           the wait can never deadlock. */
+        blocked_by_budget = used != 0;
+      }
+    }
+    if (!blocked_by_count && !blocked_by_budget)
+      break;
+    cnd_wait(&context->condition_not_full, &context->mutex);
+  }
+  if (atomic_load(&context->cancelled)) {
+    mtx_unlock(&context->mutex);
+    file_destroy(file);
+    return false;
+  }
+  if (!queue_enqueue(context->queue, file)) {
+    mtx_unlock(&context->mutex);
+    file_destroy(file);
+    return false;
+  }
+  context->queued_bytes += file_bytes;
+  cnd_signal(&context->condition_not_empty);
+  mtx_unlock(&context->mutex);
+  return true;
+}
+
+static bool receiver_enqueue_file(File* file, void* context_pointer) {
+  PipelineContextReceiver* context = (PipelineContextReceiver*)context_pointer;
+  return pipeline_context_receiver_enqueue_file(context, file);
+}
+
+static void receiver_thread_fail(PipelineContextReceiver* context) {
+  mtx_lock(&context->mutex);
+  atomic_store(&context->cancelled, true);
+  context->receiver_done = true;
+  cnd_broadcast(&context->condition_not_empty);
+  cnd_broadcast(&context->condition_not_full);
+  mtx_unlock(&context->mutex);
+}
+
+int receive_thread(void* pipeline_context) {
+  PipelineContextReceiver* context = (PipelineContextReceiver*)pipeline_context;
+  protocol_session_bind(&context->session);
+  mtx_lock(&context->mutex);
+  int file_descriptor = context->file_descriptor;
+  const Config* config = context->config;
+  mtx_unlock(&context->mutex);
+
+  ReceiverSink sink = {receiver_enqueue_file, context, false, false, NULL};
+  if (receiver_process_pending((Config*)config, file_descriptor, &sink,
+                               &context->deferred_manifest) != 0) {
+    receiver_thread_fail(context);
+    protocol_session_unbind();
+    return thrd_error;
+  }
+  mtx_lock(&context->mutex);
+  context->receiver_done = true;
+  cnd_signal(&context->condition_not_empty);
+  mtx_unlock(&context->mutex);
+  protocol_session_unbind();
+  return thrd_success;
+}
+
+int write_thread(void* pipeline_context) {
+  PipelineContextReceiver* context = (PipelineContextReceiver*)pipeline_context;
+  protocol_session_bind(&context->session);
+  mtx_lock(&context->mutex);
+  bool save_to_disk = context->config->save_to_disk;
+  char* root_directory = str_dup(context->config->receive_root_directory);
+  mtx_unlock(&context->mutex);
+  if (save_to_disk && !root_directory) {
+    mtx_lock(&context->mutex);
+    atomic_store(&context->cancelled, true);
+    context->receiver_done = true;
+    cnd_broadcast(&context->condition_not_full);
+    cnd_broadcast(&context->condition_not_empty);
+    mtx_unlock(&context->mutex);
+    protocol_session_unbind();
+    return thrd_error;
+  }
+
+  while (true) {
+    File* file =
+        queue_dequeue_multithreaded(context->queue, &context->mutex, &context->condition_not_empty,
+                                    &context->condition_not_full, &context->receiver_done);
+    if (file == NULL) {
+      free(root_directory);
+      protocol_session_unbind();
+      return thrd_success;
+    }
+    size_t file_bytes = file->data ? file->data->size : 0;
+    FileSaveResult result = FILE_SAVE_SKIPPED;
+    if (save_to_disk) {
+      result = file_save_to_disk_full(root_directory, file, context->config);
+      if (result == FILE_SAVE_ERROR) {
+        file_destroy(file);
+        pipeline_context_receiver_note_bytes_released(context, file_bytes);
+        mtx_lock(&context->mutex);
+        atomic_store(&context->cancelled, true);
+        context->receiver_done = true;
+        cnd_broadcast(&context->condition_not_full);
+        cnd_broadcast(&context->condition_not_empty);
+        mtx_unlock(&context->mutex);
+        free(root_directory);
+        protocol_session_unbind();
+        return thrd_error;
+      }
+    }
+    /* P7 Wave D: a directory's times are never applied inline (a later child
+       write would clobber them); accumulate the metadata here and let the
+       caller apply it once every writer has drained. */
+    if (result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
+        dir_times_should_capture(context->config) &&
+        !dir_time_list_add(&context->dir_times, file->path, file->metadata)) {
+      file_destroy(file);
+      pipeline_context_receiver_note_bytes_released(context, file_bytes);
+      mtx_lock(&context->mutex);
+      atomic_store(&context->cancelled, true);
+      context->receiver_done = true;
+      cnd_broadcast(&context->condition_not_full);
+      cnd_broadcast(&context->condition_not_empty);
+      mtx_unlock(&context->mutex);
+      free(root_directory);
+      protocol_session_unbind();
+      return thrd_error;
+    }
+    /* Record the per-file outcome so a --remove-source-files sender learns
+       which sources were actually written versus skipped on the receiver.
+       Explicit directory entries and recreated device/special nodes have no
+       source and are never acknowledged (mirrors receiver.c). */
+    if (context->config->remove_source_files && !file->is_dir && !file->is_special && !file->skip &&
+        !receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
+      file_destroy(file);
+      pipeline_context_receiver_note_bytes_released(context, file_bytes);
+      mtx_lock(&context->mutex);
+      atomic_store(&context->cancelled, true);
+      context->receiver_done = true;
+      cnd_broadcast(&context->condition_not_full);
+      cnd_broadcast(&context->condition_not_empty);
+      mtx_unlock(&context->mutex);
+      free(root_directory);
+      protocol_session_unbind();
+      return thrd_error;
+    }
+    file_destroy(file);
+    pipeline_context_receiver_note_bytes_released(context, file_bytes);
+  }
+}

src/server/receiver_pipeline.h

@@ -0,0 +1,68 @@
+#ifndef RECEIVER_PIPELINE_H
+#define RECEIVER_PIPELINE_H
+
+#include <stdatomic.h>
+#include <stdbool.h>
+#include <threads.h>
+
+#include "config.h"
+#include "file.h"
+#include "file_receive.h"
+#include "protocol.h"
+#include "queue.h"
+#include "receiver.h"
+#include <openssl/ssl.h>
+
+typedef struct PipelineContextReceiver {
+  Queue* queue;
+  Config* config;
+  int file_descriptor;
+  SSL* ssl;
+  ProtocolSession session;
+  ReceiverOutcomes outcomes;
+  mtx_t mutex;
+  cnd_t condition_not_full;
+  cnd_t condition_not_empty;
+  bool receiver_done;
+  atomic_bool cancelled;
+  /* Aggregate payload bytes that have been received but not yet released by
+     the disk writer (queued or in the writer's hand).  Guarded by `mutex`.
+     When `max_queue_bytes` is non-zero the receiver blocks before enqueuing
+     once this total would exceed it, so decompressed/copied file payloads
+     buffered ahead of a slow disk writer respect the per-connection memory
+     budget instead of growing without bound. */
+  size_t queued_bytes;
+  size_t max_queue_bytes;
+  /* Keep-set manifest for the commit-style (late) deletion
+     (--delete/--delete-after/--delete-delay).  receive_thread parses the whole
+     protocol stream but hands the manifest here instead of deleting while the
+     disk writer may still be draining; the caller (server.c) commits the
+     deletion after both threads have joined, so no extra is removed unless the
+     transfer truly succeeded.  NULL in the early delete modes (which delete at
+     the manifest). */
+  DeleteManifest* deferred_manifest;
+  /* P7 Wave D: directory metadata collected by write_thread from received
+     directory entries.  Only write_thread mutates it (before it joins); the
+     caller (server.c) applies it after the delete/delay-updates phase. */
+  DirTimeList dir_times;
+} PipelineContextReceiver;
+
+PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue_receiver,
+                                                          int file_descriptor, SSL* ssl);
+void pipeline_context_receiver_destroy(PipelineContextReceiver* context);
+/* Bound the bytes buffered ahead of the disk writer (see max_queue_bytes). */
+void pipeline_context_receiver_set_queue_byte_limit(PipelineContextReceiver* context,
+                                                    size_t max_bytes);
+/* Blocking enqueue used by the receive pipeline sink.  Blocks while the queue
+   is full by element count or when adding `file` would push queued_bytes over
+   the configured byte limit; waits until the disk writer releases bytes.
+   Takes ownership of `file` on success and destroys it on failure/cancel. */
+bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, File* file);
+/* Account for `released_bytes` of payload memory that has been freed by the
+   disk writer, unblocking a receiver that is waiting on the byte limit. */
+void pipeline_context_receiver_note_bytes_released(PipelineContextReceiver* context,
+                                                   size_t released_bytes);
+int receive_thread(void* pipeline_context);
+int write_thread(void* pipeline_context);
+
+#endif

src/server/server.c

@@ -7,10 +7,10 @@
 #include "identity.h"
 #include "log.h"
 #include "motd.h"
-#include "multiprocessing.h"
 #include "protocol.h"
 #include "queue.h"
 #include "receiver.h"
+#include "receiver_pipeline.h"
 #include "server_cli.h"
 #include "transport_tcp.h"
 #include "transport_tls.h"