test(p6-iconv): iconv unit, config wire-roundtrip, integration tests

tests/integration/test_iconv.py

@@ -0,0 +1,139 @@
+"""--iconv=CONVERT_SPEC file-NAME charset conversion integration tests.
+
+The client converts every source file name from LOCAL to REMOTE before it goes
+on the wire, and the receiver converts it back from REMOTE to LOCAL, so a
+source tree using one charset can be written into a destination tree using
+another (rsync compatibility; content bytes are never touched).
+"""
+import os
+import shutil
+
+import pytest
+
+from common import TEST_DATA_DIR, run_client, clean_dir, ServerManager
+
+LATIN1_NAME = b"caf\xe9.txt"
+UTF8_NAME = "caf\u00e9.txt".encode("utf-8")
+
+
+def _make(tag):
+    source = os.path.join(TEST_DATA_DIR, f"iconv_{tag}_src")
+    dest = os.path.join(TEST_DATA_DIR, f"iconv_{tag}_dst")
+    clean_dir(source)
+    shutil.rmtree(dest, ignore_errors=True)
+    # The destination ROOT must pre-exist on the receiver (the --mkpath contract:
+    # without --mkpath the server requires the root directory to exist).
+    os.makedirs(dest, exist_ok=True)
+    return source, dest
+
+
+def _place_bytes(root, name_bytes, data=b"latin1 payload\n"):
+    full = os.path.join(os.fsencode(root), name_bytes)
+    os.makedirs(os.path.dirname(full), exist_ok=True)
+    with open(full, "wb") as fh:
+        fh.write(data)
+    return full
+
+
+def _dest_file(source, dest, name):
+    base = os.path.join(dest, os.path.abspath(source).lstrip(os.sep))
+    return os.path.join(os.fsencode(base), name)
+
+
+@pytest.mark.ci
+def test_iconv_latin1_roundtrip(shared_server):
+    """A source file whose name is ISO-8859-1 bytes is transferred with
+    --iconv=iso-8859-1,utf-8 and lands on the destination with the ORIGINAL
+    latin1 name (the wire carried it as UTF-8)."""
+    source, dest = _make("latin1")
+    _place_bytes(source, LATIN1_NAME)
+
+    result, _ = run_client(
+        source, dest, flags=["--iconv=iso-8859-1,utf-8"], port=shared_server.port
+    )
+    assert result.returncode == 0, (result.stderr or result.stdout)[:400]
+
+    dst = _dest_file(source, dest, LATIN1_NAME)
+    assert os.path.exists(dst), f"dest latin1-named file not found under {dest}"
+
+
+@pytest.mark.ci
+def test_iconv_to_utf8_on_wire(shared_server):
+    """--iconv=utf-8 (single, identity both ways) on an ascii filename transfers
+    cleanly with no error."""
+    source, dest = _make("utf8")
+    src_path = os.path.join(source, "plain.txt")
+    with open(src_path, "wb") as fh:
+        fh.write(b"identity\n")
+
+    result, _ = run_client(source, dest, flags=["--iconv=utf-8"], port=shared_server.port)
+    assert result.returncode == 0, (result.stderr or result.stdout)[:400]
+
+    dst = _dest_file(source, dest, os.fsencode("plain.txt"))
+    assert os.path.exists(dst)
+
+
+@pytest.mark.ci
+def test_iconv_passthrough_identity(shared_server):
+    """No --iconv flag: the transfer is unchanged (regression guard -- the common
+    path must not go through iconv at all)."""
+    source, dest = _make("identity")
+    for name, data in (("a.txt", b"aaa\n"), ("sub/b.txt", b"bbb\n")):
+        p = os.path.join(source, name)
+        os.makedirs(os.path.dirname(p), exist_ok=True)
+        with open(p, "wb") as fh:
+            fh.write(data)
+
+    result, _ = run_client(source, dest, port=shared_server.port)
+    assert result.returncode == 0, (result.stderr or result.stdout)[:400]
+
+    for name in ("a.txt", "sub/b.txt"):
+        assert os.path.exists(_dest_file(source, dest, os.fsencode(name)))
+
+
+@pytest.mark.ci
+def test_iconv_receiver_own_charset(shared_server):
+    """A dedicated server started with its OWN --iconv converts received names
+    to ITS charset: the source holds a latin1-named file, the wire carries it
+    as UTF-8 (from the client's spec), and the receiver re-decodes it to UTF-8
+    on disk.  This discriminates a real wire conversion from a no-op passthrough
+    (a latin1 byte sequence is not valid UTF-8, so the receiver decoding it as
+    UTF-8 would fail the transfer)."""
+    with ServerManager() as server:
+        server.start(extra_args=["--iconv=utf-8"])
+        source, dest = _make("recv_charset")
+        _place_bytes(source, LATIN1_NAME)
+
+        result, _ = run_client(
+            source, dest, flags=["--iconv=iso-8859-1,utf-8"], port=server.port
+        )
+        assert result.returncode == 0, (result.stderr or result.stdout)[:400]
+
+        dst = _dest_file(source, dest, UTF8_NAME)
+        assert os.path.exists(dst), f"dest UTF-8-named file not found under {dest}"
+
+
+@pytest.mark.ci
+def test_iconv_invalid_charset_rejected(shared_server):
+    """An unsupported charset name is rejected at startup with a nonzero exit."""
+    source, dest = _make("badcharset")
+    src_path = os.path.join(source, "f.txt")
+    with open(src_path, "wb") as fh:
+        fh.write(b"x")
+
+    result, _ = run_client(
+        source, dest, flags=["--iconv=no-such-charset,utf-8"], port=shared_server.port
+    )
+    assert result.returncode != 0
+
+
+@pytest.mark.ci
+def test_iconv_garbage_spec_rejected(shared_server):
+    """A malformed CONVERT_SPEC is rejected at startup with a nonzero exit."""
+    source, dest = _make("garbage")
+    src_path = os.path.join(source, "f.txt")
+    with open(src_path, "wb") as fh:
+        fh.write(b"x")
+
+    result, _ = run_client(source, dest, flags=["--iconv=,,,"], port=shared_server.port)
+    assert result.returncode != 0

tests/runner.c

@@ -14,6 +14,7 @@
 #include "test_file_sendfile.h"
 #include "test_fuzz_smoke.h"
 #include "test_glob.h"
+#include "test_iconv.h"
 #include "test_log.h"
 #include "test_metadata.h"
 #include "test_motd.h"
@@ -59,6 +60,7 @@ int main() {
   RUN_TEST(test_protocol);
   RUN_TEST(test_metadata);
   RUN_TEST(test_glob);
+  RUN_TEST(test_iconv);
   RUN_TEST(test_file);
   RUN_TEST(test_trust_sender);
   RUN_TEST(test_delay_updates);

tests/test_config.c

@@ -1565,6 +1565,110 @@ static void test_config_local_only_fields_not_serialized() {
   EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
 }
 
+static void test_config_iconv_spec_wire_roundtrip() {
+  Config* send_cfg = config_create();
+  EXPECT_NOT_NULL(send_cfg);
+  send_cfg->send_directory = str_dup("/src");
+  send_cfg->receive_root_directory = str_dup("rel/path");
+  send_cfg->iconv_spec = str_dup("utf-8,iso-8859-1");
+
+  int p[2];
+  EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
+  io_set_fds(p[0], p[1]);
+  io_set_bwlimit(0);
+
+  pid_t pid = fork();
+  if (pid == 0) {
+    close(p[1]);
+    io_set_fds(p[0], p[0]);
+    Config* recv_cfg = config_receive(p[0]);
+    bool ok = recv_cfg != NULL && recv_cfg->iconv_spec != NULL &&
+              strcmp(recv_cfg->iconv_spec, "utf-8,iso-8859-1") == 0;
+    config_delete(recv_cfg);
+    close(p[0]);
+    _exit(ok ? 0 : 1);
+  } else {
+    close(p[0]);
+    io_set_fds(p[1], p[1]);
+    bool sent = config_send(p[1], send_cfg);
+    int status;
+    waitpid(pid, &status, 0);
+    close(p[1]);
+    config_delete(send_cfg);
+    EXPECT_TRUE(sent);
+    EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
+  }
+}
+
+static void test_config_iconv_spec_empty_canonicalizes_to_null() {
+  Config* send_cfg = config_create();
+  EXPECT_NOT_NULL(send_cfg);
+  send_cfg->send_directory = str_dup("/src");
+  send_cfg->receive_root_directory = str_dup("/dst");
+  /* iconv_spec left NULL -> serialized as "" -> received back as NULL. */
+
+  int p[2];
+  EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
+  io_set_fds(p[0], p[1]);
+  io_set_bwlimit(0);
+
+  pid_t pid = fork();
+  if (pid == 0) {
+    close(p[1]);
+    io_set_fds(p[0], p[0]);
+    Config* recv_cfg = config_receive(p[0]);
+    bool ok = recv_cfg != NULL && recv_cfg->iconv_spec == NULL;
+    config_delete(recv_cfg);
+    close(p[0]);
+    _exit(ok ? 0 : 1);
+  } else {
+    close(p[0]);
+    io_set_fds(p[1], p[1]);
+    bool sent = config_send(p[1], send_cfg);
+    int status;
+    waitpid(pid, &status, 0);
+    close(p[1]);
+    config_delete(send_cfg);
+    EXPECT_TRUE(sent);
+    EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
+  }
+}
+
+static void test_config_receive_rejects_invalid_iconv_spec() {
+  Config* send_cfg = config_create();
+  EXPECT_NOT_NULL(send_cfg);
+  send_cfg->send_directory = str_dup("/src");
+  send_cfg->receive_root_directory = str_dup("/dst");
+  send_cfg->iconv_spec = str_dup("no-such-charset,utf-8");
+
+  int p[2];
+  EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
+  io_set_fds(p[0], p[1]);
+  io_set_bwlimit(0);
+
+  pid_t pid = fork();
+  if (pid == 0) {
+    close(p[1]);
+    io_set_fds(p[0], p[0]);
+    /* A malformed/unsupported spec must be refused at the config handshake
+       (STATUS_ERROR makes config_send fail on the parent). */
+    Config* recv_cfg = config_receive(p[0]);
+    config_delete(recv_cfg);
+    close(p[0]);
+    _exit(recv_cfg ? 1 : 0);
+  } else {
+    close(p[0]);
+    io_set_fds(p[1], p[1]);
+    bool sent = config_send(p[1], send_cfg);
+    int status;
+    waitpid(pid, &status, 0);
+    close(p[1]);
+    config_delete(send_cfg);
+    EXPECT_FALSE(sent);
+    EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
+  }
+}
+
 void test_config() {
   test_config_lifecycle();
   test_config_ssh_dest();
@@ -1608,6 +1712,9 @@ void test_config() {
     test_config_module_wire_empty_canonicalizes_to_null();
     test_config_daemon_auth_wire_roundtrip();
     test_config_daemon_auth_wire_rejects_malformed();
+    test_config_iconv_spec_wire_roundtrip();
+    test_config_iconv_spec_empty_canonicalizes_to_null();
+    test_config_receive_rejects_invalid_iconv_spec();
     test_config_receive_with_validate_rejects();
   }
   test_config_delete_timing_early_helper();

tests/test_iconv.c

@@ -0,0 +1,171 @@
+#include "test_iconv.h"
+#include "charset.h"
+#include "protocol.h"
+#include "test_utils.h"
+#include "utils.h"
+#include <errno.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+/* --- CONVERT_SPEC parsing ------------------------------------------------ */
+
+static void test_iconv_spec_parse_split() {
+  char* local = NULL;
+  char* remote = NULL;
+  EXPECT_EQ_INT(charset_spec_parse("utf-8,iso-8859-1", &local, &remote), 0);
+  EXPECT_EQ_STR(local, "utf-8");
+  EXPECT_EQ_STR(remote, "iso-8859-1");
+  free(local);
+  free(remote);
+}
+
+static void test_iconv_spec_parse_single_defaults_to_local() {
+  char* local = NULL;
+  char* remote = NULL;
+  EXPECT_EQ_INT(charset_spec_parse("utf-8", &local, &remote), 0);
+  EXPECT_EQ_STR(local, "utf-8");
+  EXPECT_EQ_STR(remote, "utf-8");
+  free(local);
+  free(remote);
+}
+
+static void test_iconv_spec_parse_garbage() {
+  char* local = NULL;
+  char* remote = NULL;
+  EXPECT_EQ_INT(charset_spec_parse(NULL, &local, &remote), -1);
+  EXPECT_EQ_INT(charset_spec_parse("", &local, &remote), -1);
+  EXPECT_EQ_INT(charset_spec_parse(",", &local, &remote), -1);
+  EXPECT_EQ_INT(charset_spec_parse("utf-8,", &local, &remote), -1);
+  EXPECT_EQ_INT(charset_spec_parse(",utf-8", &local, &remote), -1);
+}
+
+static void test_iconv_spec_valid() {
+  EXPECT_TRUE(charset_spec_valid(NULL));
+  EXPECT_TRUE(charset_spec_valid("utf-8"));
+  EXPECT_TRUE(charset_spec_valid("utf-8,iso-8859-1"));
+  EXPECT_TRUE(charset_spec_valid("iso-8859-1,ascii"));
+  EXPECT_FALSE(charset_spec_valid("no-such-charset,utf-8"));
+  EXPECT_FALSE(charset_spec_valid("utf-8,no-such-charset"));
+  EXPECT_FALSE(charset_spec_valid(",,,"));
+  EXPECT_FALSE(charset_spec_valid("utf-8,"));
+}
+
+/* --- one-shot conversion ------------------------------------------------ */
+
+static void test_iconv_utf8_to_latin1() {
+  void* conv = charset_conversion_open("utf-8", "iso-8859-1");
+  EXPECT_NOT_NULL(conv);
+  char* out = charset_convert(conv, "caf\xc3\xa9", NULL);
+  EXPECT_NOT_NULL(out);
+  EXPECT_EQ_INT(strcmp(out, "caf\xe9"), 0);
+  free(out);
+  charset_conversion_close(conv);
+}
+
+static void test_iconv_latin1_to_utf8() {
+  void* conv = charset_conversion_open("iso-8859-1", "utf-8");
+  EXPECT_NOT_NULL(conv);
+  char* out = charset_convert(conv, "caf\xe9", NULL);
+  EXPECT_NOT_NULL(out);
+  EXPECT_EQ_INT(strcmp(out, "caf\xc3\xa9"), 0);
+  free(out);
+  charset_conversion_close(conv);
+}
+
+static void test_iconv_invalid_sequence_fails() {
+  int err = 0;
+  /* 0xff is not a valid UTF-8 sequence. */
+  void* conv = charset_conversion_open("utf-8", "ascii");
+  EXPECT_NOT_NULL(conv);
+  EXPECT_TRUE(charset_convert(conv, "bad\xff", &err) == NULL);
+  EXPECT_TRUE(err == EILSEQ || err == EINVAL);
+  charset_conversion_close(conv);
+}
+
+static void test_iconv_unrepresentable_fails() {
+  /* "caf\xc3\xa9" (UTF-8 for cafe) has no ASCII representation. */
+  void* conv = charset_conversion_open("utf-8", "ascii");
+  EXPECT_NOT_NULL(conv);
+  EXPECT_TRUE(charset_convert(conv, "caf\xc3\xa9", NULL) == NULL);
+  charset_conversion_close(conv);
+}
+
+/* --- process-wide wire conversion ---------------------------------------- */
+
+static void test_iconv_wire_sender_converts_local_to_remote() {
+  EXPECT_TRUE(charset_wire_init_sender("utf-8,iso-8859-1"));
+  char* wire = charset_wire_apply("caf\xc3\xa9");
+  EXPECT_NOT_NULL(wire);
+  EXPECT_EQ_INT(strcmp(wire, "caf\xe9"), 0);
+  free(wire);
+  charset_wire_free();
+}
+
+static void test_iconv_wire_receiver_converts_remote_to_local() {
+  EXPECT_TRUE(charset_wire_init_receiver("utf-8,iso-8859-1", NULL));
+  char* local = charset_wire_apply("caf\xe9");
+  EXPECT_NOT_NULL(local);
+  EXPECT_EQ_INT(strcmp(local, "caf\xc3\xa9"), 0);
+  free(local);
+  charset_wire_free();
+}
+
+static void test_iconv_wire_disabled_passthrough() {
+  charset_wire_init_sender(NULL);
+  EXPECT_FALSE(charset_wire_active());
+  char* out = charset_wire_apply("plain/name\xff");
+  EXPECT_NOT_NULL(out);
+  EXPECT_EQ_INT(strcmp(out, "plain/name\xff"), 0);
+  free(out);
+  charset_wire_free();
+}
+
+static void test_iconv_wire_str_roundtrip() {
+  EXPECT_TRUE(charset_wire_init_sender("utf-8,iso-8859-1"));
+  int p[2];
+  EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
+  io_set_fds(p[0], p[1]);
+  io_set_bwlimit(0);
+
+  pid_t pid = fork();
+  if (pid == 0) {
+    close(p[1]);
+    io_set_fds(p[0], p[0]);
+    charset_wire_free();
+    charset_wire_init_receiver("utf-8,iso-8859-1", NULL);
+    char* got = receive_wire_str(p[0]);
+    bool ok = got != NULL && strcmp(got, "caf\xc3\xa9") == 0;
+    free(got);
+    charset_wire_free();
+    close(p[0]);
+    _exit(ok ? 0 : 1);
+  } else {
+    close(p[0]);
+    io_set_fds(p[1], p[1]);
+    bool sent = send_wire_str(p[1], "caf\xc3\xa9");
+    int status;
+    waitpid(pid, &status, 0);
+    close(p[1]);
+    charset_wire_free();
+    EXPECT_TRUE(sent);
+    EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
+  }
+}
+
+void test_iconv() {
+  test_iconv_spec_parse_split();
+  test_iconv_spec_parse_single_defaults_to_local();
+  test_iconv_spec_parse_garbage();
+  test_iconv_spec_valid();
+  test_iconv_utf8_to_latin1();
+  test_iconv_latin1_to_utf8();
+  test_iconv_invalid_sequence_fails();
+  test_iconv_unrepresentable_fails();
+  test_iconv_wire_sender_converts_local_to_remote();
+  test_iconv_wire_receiver_converts_remote_to_local();
+  test_iconv_wire_disabled_passthrough();
+  test_iconv_wire_str_roundtrip();
+}

tests/test_iconv.h

@@ -0,0 +1,6 @@
+#ifndef TEST_ICONV_H
+#define TEST_ICONV_H
+
+void test_iconv(void);
+
+#endif