feat(codec): negotiate checksum/compression algorithms (protocol 2.26.0)

Accept the full rsync 3.4.1 --compress-choice set (zstd/lz4/zlib/zlibx/ none/auto) and the two-name --checksum-choice TRANSFER,PRE-TRANSFER form, including rsync's 'none' rules (rejected with --checksum at exit 4, and forcing --whole-file as the transfer half) and unknown names at exit 4. The checksum default becomes the auto-negotiated xxh128. Negotiation is deterministic and symmetric: both peers run the same preference resolver (rsync's --version order). The resolved compression_algo crosses the wire as a new trailing config-frame int so the receiver validates and installs the exact codec; an unsupported choice is refused before STATUS_OK like rsync's failed negotiation. Bump PROTOCOL_VERSION to 2.26.0.
2026-09-16 23:27:40 +02:00
parent 0e33f84f38
commit 24b81c7e5a
5 changed files with 240 additions and 70 deletions
@@ -14,6 +14,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <strings.h>
 #include <limits.h>
 #include <errno.h>

@@ -67,6 +68,8 @@ static void config_set_defaults(Config* config) {
  config->human_readable = false;
  config->ignore_errors = false;
  config->ignore_missing_args = false;
+  config->checksum_transfer_algo = CHECKSUM_ALGO_DEFAULT;
+  config->cli_exit_code = 0;
  config->filters = NULL;
  config->files_from = NULL;
  config->files_from_set = NULL;
@@ -196,12 +199,13 @@ static bool validate_received_config(const Config* config) {
         valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
         valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
         valid_wire_bool(config->dry_run) && checksum_algo_valid(config->checksum_algo) &&
-         identity_wire_valid(config) && valid_wire_bool(config->preserve_atimes) &&
-         valid_wire_bool(config->preserve_crtimes) && valid_wire_bool(config->omit_dir_times) &&
-         valid_wire_bool(config->omit_link_times) && valid_wire_bool(config->preserve_perms) &&
-         valid_wire_bool(config->preserve_times) && valid_wire_bool(config->preserve_owner) &&
-         valid_wire_bool(config->preserve_group) && valid_wire_bool(config->munge_links) &&
-         valid_wire_bool(config->keep_dirlinks) && valid_wire_bool(config->fake_super) &&
+         compression_algo_valid(config->compression_algo) && identity_wire_valid(config) &&
+         valid_wire_bool(config->preserve_atimes) && valid_wire_bool(config->preserve_crtimes) &&
+         valid_wire_bool(config->omit_dir_times) && valid_wire_bool(config->omit_link_times) &&
+         valid_wire_bool(config->preserve_perms) && valid_wire_bool(config->preserve_times) &&
+         valid_wire_bool(config->preserve_owner) && valid_wire_bool(config->preserve_group) &&
+         valid_wire_bool(config->munge_links) && valid_wire_bool(config->keep_dirlinks) &&
+         valid_wire_bool(config->fake_super) &&
         (!config->copy_as_set || (config->copy_as_uid >= 0 && config->copy_as_gid >= 0)) &&
         (!config->use_compression ||
          (config->compression_level >= 1 && config->compression_level <= 22)) &&
@@ -880,6 +884,14 @@ static bool config_receive_checksum_algo(int fd, int* value) {
  return true;
 }

+static bool config_receive_compression_algo(int fd, int* value) {
+  int algo;
+  if (!receive_int(fd, &algo) || !compression_algo_valid(algo))
+    return false;
+  *value = algo;
+  return true;
+}
+
 static bool config_receive_super_mode(int fd, SuperMode* value) {
  int mode;
  if (!receive_int(fd, &mode) || mode < SUPER_MODE_AUTO || mode > SUPER_MODE_OFF)
@@ -1081,6 +1093,9 @@ fail:
 #define CONFIG_SEND_INT_CHECKSUM_ALGO(name) send_int(fd, c->name)
 #define CONFIG_RECV_INT_CHECKSUM_ALGO(name) config_receive_checksum_algo(fd, &c->name)

+#define CONFIG_SEND_INT_COMPRESSION_ALGO(name) send_int(fd, c->name)
+#define CONFIG_RECV_INT_COMPRESSION_ALGO(name) config_receive_compression_algo(fd, &c->name)
+
 #define CONFIG_SEND_SUPERMODE(name) send_int(fd, (int)c->name)
 #define CONFIG_RECV_SUPERMODE(name) config_receive_super_mode(fd, &c->name)

@@ -1156,6 +1171,7 @@ CONFIG_DEFINE_SEND(send_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
 CONFIG_DEFINE_SEND(send_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
 CONFIG_DEFINE_SEND(send_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
 CONFIG_DEFINE_SEND(send_output_options, CONFIG_WIRE_OUTPUT_FIELDS)
+CONFIG_DEFINE_SEND(send_codec_options, CONFIG_WIRE_CODEC_FIELDS)

 CONFIG_DEFINE_RECV(receive_core_fields, CONFIG_WIRE_CORE_FIELDS)
 CONFIG_DEFINE_RECV(receive_delta_fields, CONFIG_WIRE_DELTA_FIELDS)
@@ -1175,6 +1191,7 @@ CONFIG_DEFINE_RECV(receive_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
 CONFIG_DEFINE_RECV(receive_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
 CONFIG_DEFINE_RECV(receive_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
 CONFIG_DEFINE_RECV(receive_output_options, CONFIG_WIRE_OUTPUT_FIELDS)
+CONFIG_DEFINE_RECV(receive_codec_options, CONFIG_WIRE_CODEC_FIELDS)

 #undef XSEND
 #undef XRECV
@@ -1292,7 +1309,8 @@ bool config_send_wire_block(int file_descriptor, const Config* config) {
         send_iconv_spec(file_descriptor, config) &&
         send_privilege_options(file_descriptor, config) &&
         send_copy_as_options(file_descriptor, config) &&
-         send_output_options(file_descriptor, config);
+         send_output_options(file_descriptor, config) &&
+         send_codec_options(file_descriptor, config);
 }

 bool config_send(int file_descriptor, const Config* config) {
@@ -1363,29 +1381,59 @@ Config* config_receive_with_validate(int file_descriptor, ConfigValidateFunc val
      !receive_iconv_spec(file_descriptor, config, &budget) ||
      !receive_privilege_options(file_descriptor, config, &budget) ||
      !receive_copy_as_options(file_descriptor, config, &budget) ||
-      !receive_output_options(file_descriptor, config, &budget))
+      !receive_output_options(file_descriptor, config, &budget) ||
+      !receive_codec_options(file_descriptor, config, &budget))
    goto error;
-  if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
-      strcmp(config->compress_choice, "none") != 0 &&
-      strcmp(config->compress_choice, "auto") != 0) {
-    char* escaped_choice = output_escape(config->compress_choice, config->eight_bit_output);
-    log_message(LOG_LEVEL_ERROR, "Unsupported compression choice: %s",
-                escaped_choice ? escaped_choice : "<allocation failed>");
-    char detail[128];
-    snprintf(detail, sizeof(detail), "unsupported compression choice: %s",
-             escaped_choice ? escaped_choice : "<allocation failed>");
-    send_error_detail(file_descriptor, detail);
-    free(escaped_choice);
+  /* Validate/normalize the negotiated codec.  compress_choice is the human
+   * spelling (NULL or "" when -z was not given); compression_algo is the
+   * concrete codec id the sender used.  They must agree, and "auto" is
+   * canonicalized to FastSync's negotiated default so the stored spelling is
+   * always concrete (a hostile/older client may still send "auto"). */
+  if (config->compress_choice && config->compress_choice[0] != '\0') {
+    int choice_algo = compression_algo_from_name(config->compress_choice);
+    if (choice_algo < 0 && strcasecmp(config->compress_choice, "auto") != 0) {
+      char* escaped_choice = output_escape(config->compress_choice, config->eight_bit_output);
+      log_message(LOG_LEVEL_ERROR, "Unsupported compression choice: %s",
+                  escaped_choice ? escaped_choice : "<allocation failed>");
+      char detail[160];
+      snprintf(detail, sizeof(detail), "unsupported compression choice: %s",
+               escaped_choice ? escaped_choice : "<allocation failed>");
+      send_error_detail(file_descriptor, detail);
+      free(escaped_choice);
+      goto error;
+    }
+    if (choice_algo < 0)
+      choice_algo = (int)compression_negotiate_default();
+    if (strcasecmp(config->compress_choice, "auto") == 0 ||
+        choice_algo == (int)COMPRESSION_ALGO_NONE) {
+      const char* canonical = compression_algo_name((CompressionAlgo)choice_algo);
+      char* dup = str_dup(canonical);
+      if (!dup)
+        goto error;
+      free(config->compress_choice);
+      config->compress_choice = dup;
+    }
+    if (config->compression_algo != choice_algo) {
+      log_message(LOG_LEVEL_ERROR, "Compression choice '%s' does not match codec id %d",
+                  config->compress_choice, config->compression_algo);
+      send_error_detail(file_descriptor, "compression choice/codec mismatch");
+      goto error;
+    }
+  }
+  /* The concrete codec must exist only when compression is on.  A client that
+   * left -z off has no codec in effect, but the field keeps whatever id it
+   * carried (the receiver never dispatches on it without use_compression), so
+   * the wire value round-trips untouched. */
+  if (config->use_compression && config->compression_algo == (int)COMPRESSION_ALGO_NONE) {
+    log_message(LOG_LEVEL_ERROR, "Compression requested with the 'none' codec");
+    send_error_detail(file_descriptor, "compression requested with the none codec");
    goto error;
  }
-  /* Defensive: an older/hostile client may still send "auto"; canonicalize it
-     to zstd (its effective choice) so the stored value is always concrete. */
-  if (strcmp(config->compress_choice, "auto") == 0) {
-    char* canonical = str_dup("zstd");
-    if (!canonical)
-      goto error;
-    free(config->compress_choice);
-    config->compress_choice = canonical;
+  /* rsync: "none" as the pre-transfer checksum is invalid with --checksum. */
+  if (config->checksum && config->checksum_algo == (int)CHECKSUM_ALGO_NONE) {
+    log_message(LOG_LEVEL_ERROR, "Invalid checksum-choice for --checksum: none");
+    send_error_detail(file_descriptor, "checksum-choice 'none' cannot be used with --checksum");
+    goto error;
  }
  if (!validate_received_config(config)) {
    log_message(LOG_LEVEL_ERROR, "Invalid configuration received from client");
@@ -3,6 +3,7 @@

 #include "array_list.h"
 #include "checksum.h"
+#include "compression.h"
 #include <stdbool.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -81,7 +82,7 @@ typedef struct {
 typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode;

 /* ===========================================================================
- * Config wire-field table (single source of truth for protocol 2.23.0).
+ * Config wire-field table (single source of truth for protocol 2.26.0).
 *
 * Every field below crosses the wire.  The table is the ONLY place a
 * serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare
@@ -203,7 +204,7 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
 #define CONFIG_WIRE_FUZZY_FIELDS(X) X(fuzzy, bool, false, BOOL)

 #define CONFIG_WIRE_CHECKSUM_FIELDS(X)                                                             \
-  X(checksum_algo, int, CHECKSUM_ALGO_XXH64, INT_CHECKSUM_ALGO)                                    \
+  X(checksum_algo, int, CHECKSUM_ALGO_DEFAULT, INT_CHECKSUM_ALGO)                                  \
  X(checksum_seed, uint64_t, 0, RAW)

 #define CONFIG_WIRE_IDENTITY_FIELDS(X)                                                             \
@@ -253,6 +254,27 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
 * output; the transfer decision itself is unchanged. */
 #define CONFIG_WIRE_OUTPUT_FIELDS(X) X(report_dest_info, bool, false, BOOL)

+/* Codec-negotiation wave (protocol 2.26.0).  compression_algo is the concrete
+ * codec the client selected for this transfer (a CompressionAlgo id) and is the
+ * value the receiver validates and installs.  It is the resolved result of
+ * --compress-choice / the "auto" negotiation so both peers agree exactly.
+ *
+ * Negotiation model: FastSync enforces a strict same-version handshake, so both
+ * peers carry the identical compiled-in codec set.  The client resolves the
+ * effective algorithm deterministically and serializes it here; "auto" picks
+ * the first entry of the rsync 3.4.1 preference order
+ * (compression: zstd lz4 zlibx zlib none; checksum: xxh128 xxh3 xxh64 md5 md4
+ * sha1 none), and an explicit request wins.  The receiver rejects (before
+ * STATUS_OK) any algorithm outside its own supported set, which is rsync's
+ * "no common choice is an error" behavior.  The same resolver runs on both
+ * sides (compression_negotiate_default / checksum_negotiate_default), so the
+ * fallback is consistent.
+ *
+ * The field is appended after the output block so every pre-2.26 field keeps
+ * its wire position. */
+#define CONFIG_WIRE_CODEC_FIELDS(X)                                                                \
+  X(compression_algo, int, COMPRESSION_ALGO_ZSTD, INT_COMPRESSION_ALGO)
+
 /* All serialized fields, in exact wire order.  Concatenating the per-segment
 * lists here is what keeps the declaration order = the wire order. */
 #define CONFIG_WIRE_FIELDS(X)                                                                      \
@@ -274,7 +296,8 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
  CONFIG_WIRE_ICONV_FIELDS(X)                                                                      \
  CONFIG_WIRE_PRIVILEGE_FIELDS(X)                                                                  \
  CONFIG_WIRE_COPY_AS_FIELDS(X)                                                                    \
-  CONFIG_WIRE_OUTPUT_FIELDS(X)
+  CONFIG_WIRE_OUTPUT_FIELDS(X)                                                                     \
+  CONFIG_WIRE_CODEC_FIELDS(X)

 typedef struct Config {
  /* -j/--threads=N: number of parallel scanner worker threads for the -m
@@ -369,6 +392,16 @@ typedef struct Config {
   * enters the keep-set.  Implied by --delete-missing-args. */
  bool ignore_missing_args;

+  /* Codec-negotiation CLI state (all client-only, never serialized).  The
+   * effective pre-transfer checksum is Config->checksum_algo (serialized);
+   * checksum_transfer_algo is the rsync "transfer" half of a two-name
+   * --checksum-choice form (validated and used only to mirror rsync's
+   * whole-file forcing, since FastSync's per-block strong hash is fixed).
+   * cli_exit_code carries a parser-requested process exit status (rsync uses 4
+   * for an unsupported checksum/compress algorithm) so main() can mirror it. */
+  int checksum_transfer_algo;
+  int cli_exit_code;
+
  // Issue #129: Advanced file selection. These fields are CLIENT-ONLY: they are
  // never serialized to the wire (the receiver must not learn them).
  ArrayList* filters;   /* --filter=RULE rule strings, in order */
@@ -911,8 +944,23 @@ typedef struct Config {
 * snapshot of the old entry) before its ordinary verdict when the config frame
 * carries the new report_dest_info bool appended after the --copy-as block.
 * This is both a config-frame layout change (one trailing bool) and a frame
- * sequence change (the new status). */
-#define PROTOCOL_VERSION "2.23.0"
+ * sequence change (the new status).
+ *
+ * Codec-Breadth + Negotiation Wave: 2.23.0 -> 2.26.0.
+ *
+ * WHY the bump, grounded in the wire: the config frame gains one trailing int,
+ * compression_algo (a CompressionAlgo id), appended after the output block.
+ * It is the negotiated/effective compression codec and is what the receiver's
+ * self-describing decompressor validates against its own supported set.  The
+ * checksum_algo wire value now also accepts md4/sha1/none, and its default
+ * changes to the rsync 3.4.1 auto-negotiated xxh128.  Any config-frame layout
+ * change must bump the protocol version: a peer that does not parse the new
+ * trailing int would desynchronize on the frame boundary, and the strict
+ * same-version handshake (config_receive rejects a mismatched version before
+ * parsing anything else) is what keeps a 2.26 client and an older server from
+ * ever reaching that state.  (2.24/2.25 are reserved for the other waves
+ * landing alongside this one; this busy-work bump keeps 2.26.0 for codecs.) */
+#define PROTOCOL_VERSION "2.26.0"
 #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
 /* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
 #define MAX_BASIS_DIRS 64