Merge remote-tracking branch 'origin/feat/compress-threads' into dev

src/client/client_cli.c

@@ -1,6 +1,7 @@
 #include "client_send.h"
 #include "client_validation.h"
 #include "chmod.h"
+#include "compression.h"
 #include "config.h"
 #include "delta.h"
 #include "log.h"
@@ -90,6 +91,17 @@ static int set_compression_choice(Config* config, const char* value) {
   return 0;
 }
 
+static int set_compression_threads_option(int* dest, const char* value) {
+  if (set_positive_int_option(dest, value, "--compress-threads") != 0)
+    return -1;
+  if (*dest > COMPRESSION_MAX_THREADS) {
+    log_message(LOG_LEVEL_ERROR, "--compress-threads must be between 1 and %d",
+                COMPRESSION_MAX_THREADS);
+    return -1;
+  }
+  return 0;
+}
+
 /* Parse a string as a non-negative integer into *dest. Returns true on success, false on error. */
 static int set_nonneg_int_option(int* dest, const char* value, const char* option_name) {
   if (!parse_nonneg_int(value, dest)) {
@@ -369,6 +381,13 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
         return -1;
       continue;
     }
+    const char* threads_prefix = "--compress-threads=";
+    if (strncmp(argv[i], threads_prefix, strlen(threads_prefix)) == 0) {
+      if (set_compression_threads_option(&config->compression_threads,
+                                         argv[i] + strlen(threads_prefix)) != 0)
+        return -1;
+      continue;
+    }
     const OptionEntry* entry = find_table_option(argv[i]);
     const char* inline_value = NULL;
     if (!entry)
@@ -586,6 +605,9 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
     } else if (opt_is(argv[i], "--skip-compress", NULL) && i + 1 < argc) {
       if (parse_skip_compress(config, argv[++i]) != 0)
         return -1;
+    } else if (opt_is(argv[i], "--compress-threads", NULL) && i + 1 < argc) {
+      if (set_compression_threads_option(&config->compression_threads, argv[++i]) != 0)
+        return -1;
     } else if (argv[i][0] == '-') {
       char* escaped = output_escape(argv[i], false);
       fprintf(stderr, "Unknown option: %s\n", escaped ? escaped : "<allocation failed>");

src/client/client_send.c

@@ -354,7 +354,8 @@ static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* c
   int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
   if (config->use_compression && !compression_should_skip_with_suffixes(
                                      file->path, config->skip_compress_suffixes, skip_count)) {
-    to_send = data_compress(delta_data, config->compression_level);
+    to_send = data_compress_with_threads(delta_data, config->compression_level,
+                                         config->compression_threads);
     data_destroy(delta_data);
     if (!to_send)
       return send_status(client->file_descriptor, STATUS_NEXT) ? 1 : -1;
@@ -377,7 +378,8 @@ static bool send_file_direct(File* file, int fd, bool use_metadata, int compress
     return false;
   int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
   return file_send_single_calls_with_skip(file, fd, use_metadata, compression_level, true,
-                                          config->skip_compress_suffixes, skip_count);
+                                          config->skip_compress_suffixes, skip_count,
+                                          config->compression_threads);
 }
 
 // Send a single file directly via sendfile (non-incremental path).
@@ -386,7 +388,8 @@ static bool send_file_direct_sendfile(File* file, int fd, bool use_metadata, con
     return false;
   int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
   return file_send_sendfile_with_skip(file, fd, use_metadata, 0, true,
-                                      config->skip_compress_suffixes, skip_count);
+                                      config->skip_compress_suffixes, skip_count,
+                                      config->compression_threads);
 }
 
 // Process one file in a chunk: either via incremental check or direct send.
@@ -430,7 +433,8 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
     // Fall through: send full file via sendfile (pass 0 for compression_level)
     int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
     if (!file_send_sendfile_with_skip(file, client->file_descriptor, config->use_metadata, 0, false,
-                                      config->skip_compress_suffixes, skip_count))
+                                      config->skip_compress_suffixes, skip_count,
+                                      config->compression_threads))
       return -1;
     return 0;
   }
@@ -466,7 +470,7 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
   int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
   if (!file_send_single_calls_with_skip(file, client->file_descriptor, config->use_metadata,
                                         compression_level, false, config->skip_compress_suffixes,
-                                        skip_count))
+                                        skip_count, config->compression_threads))
     return -1;
   return 0;
 }
@@ -484,7 +488,8 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
       return -1;
     Data* data;
     if (config->use_compression) {
-      data = chunk_compress(chunk, config->compression_level, config->use_metadata);
+      data = chunk_compress_with_threads(chunk, config->compression_level, config->use_metadata,
+                                         config->compression_threads);
     } else {
       data = chunk_serialize(chunk, config->use_metadata);
     }

src/client/client_validation.c

@@ -15,6 +15,10 @@ bool validate_config(const Config* config) {
                                  "(chunk serialization)");
     return false;
   }
+  if (config->compression_threads > 0 && !config->use_compression) {
+    log_message(LOG_LEVEL_ERROR, "--compress-threads requires compression (-c or -z)");
+    return false;
+  }
   if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
     log_message(LOG_LEVEL_ERROR, "-f/--sendfile is not supported with SSH transport");
     return false;

src/client/usage.c

@@ -90,6 +90,7 @@ void print_usage(void) {
   printf("  --compress-level <n>    Compression level (default: 5)\n");
   printf("  --zl <n>             Alias for --compress-level\n");
   printf("  --skip-compress=LIST    Skip compression for comma-separated suffixes\n");
+  printf("  --compress-threads <n> Compression worker threads (requires zstd threaded support)\n");
   printf("  --help              Show this help\n");
   printf("  -V, --version       Show version\n");
 }

src/shared/chunk.c

@@ -290,11 +290,16 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
 }
 
 Data* chunk_compress(Chunk* chunk, int compression_level, bool use_metadata) {
-  log_debug_message(LOG_DEBUG_PACK, "Starting to compress chunk");
+  return chunk_compress_with_threads(chunk, compression_level, use_metadata, 0);
+}
+
+Data* chunk_compress_with_threads(Chunk* chunk, int compression_level, bool use_metadata,
+                                  int compression_threads) {
+  log_message(LOG_LEVEL_DEBUG, "Starting to compress chunk");
   Data* serialized = chunk_serialize(chunk, use_metadata);
   if (serialized == NULL)
     return NULL;
-  Data* compressed = data_compress(serialized, compression_level);
+  Data* compressed = data_compress_with_threads(serialized, compression_level, compression_threads);
   data_destroy(serialized);
   if (compressed == NULL)
     return NULL;

src/shared/chunk.h

@@ -19,6 +19,8 @@ void chunk_destroy(void* chunk);
 Data* chunk_serialize(Chunk* chunk, bool use_metadata);
 Chunk* chunk_deserialize(Data* data, bool use_metadata);
 Data* chunk_compress(Chunk* chunk, int compression_level, bool use_metadata);
+Data* chunk_compress_with_threads(Chunk* chunk, int compression_level, bool use_metadata,
+                                  int compression_threads);
 Chunk* receive_chunk_data(int fd, const Config* config);
 
 #endif

src/shared/compression.c

@@ -6,6 +6,7 @@
 #include <stdint.h>
 #include <string.h>
 #include <strings.h>
+#include <unistd.h>
 #include <zstd.h>
 
 #define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
@@ -38,7 +39,15 @@ bool compression_should_skip_with_suffixes(const char* path, char* const* suffix
 }
 
 Data* data_compress(Data* data_to_compress, int compression_level) {
-  log_debug_message(LOG_DEBUG_UTIL, "Starting to compress data");
+  return data_compress_with_threads(data_to_compress, compression_level, 0);
+}
+
+Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
+                                 int compression_threads) {
+  if (!data_to_compress || (!data_to_compress->data && data_to_compress->size != 0) ||
+      compression_threads < 0 || compression_threads > COMPRESSION_MAX_THREADS)
+    return NULL;
+  log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
   size_t dst_size = ZSTD_compressBound(data_to_compress->size);
   Data* compressed_data = data_create_empty(dst_size);
   if (compressed_data == NULL)
@@ -59,6 +68,30 @@ Data* data_compress(Data* data_to_compress, int compression_level) {
     return NULL;
   }
 
+  if (compression_threads > 0) {
+    long online_cpus = sysconf(_SC_NPROCESSORS_ONLN);
+    int available_threads = online_cpus > 0 && online_cpus < compression_threads
+                                ? (int)online_cpus
+                                : compression_threads;
+    zret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, available_threads);
+    if (ZSTD_isError(zret)) {
+      log_message(LOG_LEVEL_ERROR, "Failed to set compression threads: %s",
+                  ZSTD_getErrorName(zret));
+      ZSTD_freeCCtx(cctx);
+      data_destroy(compressed_data);
+      return NULL;
+    }
+    /* Streaming compression needs the source size before threaded mode can end a frame. */
+    zret = ZSTD_CCtx_setPledgedSrcSize(cctx, data_to_compress->size);
+    if (ZSTD_isError(zret)) {
+      log_message(LOG_LEVEL_ERROR, "Failed to set compression source size: %s",
+                  ZSTD_getErrorName(zret));
+      ZSTD_freeCCtx(cctx);
+      data_destroy(compressed_data);
+      return NULL;
+    }
+  }
+
   ZSTD_inBuffer input = {data_to_compress->data, data_to_compress->size, 0};
   ZSTD_outBuffer output = {compressed_data->data, dst_size, 0};
 

src/shared/compression.h

@@ -4,7 +4,11 @@
 #include "data.h"
 #include <stdbool.h>
 
+#define COMPRESSION_MAX_THREADS 64
+
 Data* data_compress(Data* data_to_compress, int compression_level);
+Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
+                                 int compression_threads);
 Data* data_decompress(Data* compressed_data);
 Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
 bool compression_should_skip(const char* path);

src/shared/config.c

@@ -24,6 +24,7 @@ static void config_set_defaults(Config* config) {
   config->remove_source_files = false;
   config->use_delete = false;
   config->compression_level = 5;
+  config->compression_threads = 0;
   config->use_sendfile = false;
   config->chunk_size = DEFAULT_CHUNK_SIZE;
   config->ssh_port = 22;

src/shared/config.h

@@ -24,6 +24,7 @@ typedef struct Config {
   bool remove_source_files;
   bool use_delete;
   int compression_level;
+  int compression_threads;
   unsigned long long chunk_size;
   int ssh_port;
   TransportType transport;

src/shared/file_send.c

@@ -20,19 +20,21 @@
 bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
                             int compression_level, bool send_path) {
   return file_send_single_calls_with_skip(file, file_descriptor, use_metadata, compression_level,
-                                          send_path, NULL, -1);
+                                          send_path, NULL, -1, 0);
 }
 
 bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_metadata,
                                       int compression_level, bool send_path,
-                                      char* const* skip_suffixes, int skip_count) {
+                                      char* const* skip_suffixes, int skip_count,
+                                      int compression_threads) {
   if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data))
     return false;
   const Data* data_to_send = file->data;
   Data* compressed_data = NULL;
   if (compression_level > 0 &&
       !compression_should_skip_with_suffixes(file->path, skip_suffixes, skip_count)) {
-    compressed_data = data_compress(file->data, compression_level);
+    compressed_data = data_compress_with_threads(file->data, compression_level,
+                                                 compression_threads);
     if (compressed_data == NULL) {
       log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
       return false;
@@ -58,17 +60,18 @@ bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_
 bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
                         bool send_path) {
   return file_send_sendfile_with_skip(file, file_descriptor, use_metadata, compression_level,
-                                      send_path, NULL, -1);
+                                      send_path, NULL, -1, 0);
 }
 
 bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_metadata,
                                   int compression_level, bool send_path, char* const* skip_suffixes,
-                                  int skip_count) {
+                                  int skip_count, int compression_threads) {
   if (!file || !file->path || !file->data)
     return false;
   if (compression_level > 0)
     return file_send_single_calls_with_skip(file, file_descriptor, use_metadata, compression_level,
-                                            send_path, skip_suffixes, skip_count);
+                                            send_path, skip_suffixes, skip_count,
+                                            compression_threads);
 
   if (send_path && !send_str(file_descriptor, file->path))
     return false;

src/shared/file_send.h

@@ -10,11 +10,12 @@ bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
                             int compression_level, bool send_path);
 bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_metadata,
                                       int compression_level, bool send_path,
-                                      char* const* skip_suffixes, int skip_count);
+                                      char* const* skip_suffixes, int skip_count,
+                                      int compression_threads);
 bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
                         bool send_path);
 bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_metadata,
                                   int compression_level, bool send_path, char* const* skip_suffixes,
-                                  int skip_count);
+                                  int skip_count, int compression_threads);
 
 #endif