diff --git a/tests/integration/test_iconv.py b/tests/integration/test_iconv.py new file mode 100644 index 0000000..c8283b0 --- /dev/null +++ b/tests/integration/test_iconv.py @@ -0,0 +1,139 @@ +"""--iconv=CONVERT_SPEC file-NAME charset conversion integration tests. + +The client converts every source file name from LOCAL to REMOTE before it goes +on the wire, and the receiver converts it back from REMOTE to LOCAL, so a +source tree using one charset can be written into a destination tree using +another (rsync compatibility; content bytes are never touched). +""" +import os +import shutil + +import pytest + +from common import TEST_DATA_DIR, run_client, clean_dir, ServerManager + +LATIN1_NAME = b"caf\xe9.txt" +UTF8_NAME = "caf\u00e9.txt".encode("utf-8") + + +def _make(tag): + source = os.path.join(TEST_DATA_DIR, f"iconv_{tag}_src") + dest = os.path.join(TEST_DATA_DIR, f"iconv_{tag}_dst") + clean_dir(source) + shutil.rmtree(dest, ignore_errors=True) + # The destination ROOT must pre-exist on the receiver (the --mkpath contract: + # without --mkpath the server requires the root directory to exist). + os.makedirs(dest, exist_ok=True) + return source, dest + + +def _place_bytes(root, name_bytes, data=b"latin1 payload\n"): + full = os.path.join(os.fsencode(root), name_bytes) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "wb") as fh: + fh.write(data) + return full + + +def _dest_file(source, dest, name): + base = os.path.join(dest, os.path.abspath(source).lstrip(os.sep)) + return os.path.join(os.fsencode(base), name) + + +@pytest.mark.ci +def test_iconv_latin1_roundtrip(shared_server): + """A source file whose name is ISO-8859-1 bytes is transferred with + --iconv=iso-8859-1,utf-8 and lands on the destination with the ORIGINAL + latin1 name (the wire carried it as UTF-8).""" + source, dest = _make("latin1") + _place_bytes(source, LATIN1_NAME) + + result, _ = run_client( + source, dest, flags=["--iconv=iso-8859-1,utf-8"], port=shared_server.port + ) + assert result.returncode == 0, (result.stderr or result.stdout)[:400] + + dst = _dest_file(source, dest, LATIN1_NAME) + assert os.path.exists(dst), f"dest latin1-named file not found under {dest}" + + +@pytest.mark.ci +def test_iconv_to_utf8_on_wire(shared_server): + """--iconv=utf-8 (single, identity both ways) on an ascii filename transfers + cleanly with no error.""" + source, dest = _make("utf8") + src_path = os.path.join(source, "plain.txt") + with open(src_path, "wb") as fh: + fh.write(b"identity\n") + + result, _ = run_client(source, dest, flags=["--iconv=utf-8"], port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:400] + + dst = _dest_file(source, dest, os.fsencode("plain.txt")) + assert os.path.exists(dst) + + +@pytest.mark.ci +def test_iconv_passthrough_identity(shared_server): + """No --iconv flag: the transfer is unchanged (regression guard -- the common + path must not go through iconv at all).""" + source, dest = _make("identity") + for name, data in (("a.txt", b"aaa\n"), ("sub/b.txt", b"bbb\n")): + p = os.path.join(source, name) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "wb") as fh: + fh.write(data) + + result, _ = run_client(source, dest, port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:400] + + for name in ("a.txt", "sub/b.txt"): + assert os.path.exists(_dest_file(source, dest, os.fsencode(name))) + + +@pytest.mark.ci +def test_iconv_receiver_own_charset(shared_server): + """A dedicated server started with its OWN --iconv converts received names + to ITS charset: the source holds a latin1-named file, the wire carries it + as UTF-8 (from the client's spec), and the receiver re-decodes it to UTF-8 + on disk. This discriminates a real wire conversion from a no-op passthrough + (a latin1 byte sequence is not valid UTF-8, so the receiver decoding it as + UTF-8 would fail the transfer).""" + with ServerManager() as server: + server.start(extra_args=["--iconv=utf-8"]) + source, dest = _make("recv_charset") + _place_bytes(source, LATIN1_NAME) + + result, _ = run_client( + source, dest, flags=["--iconv=iso-8859-1,utf-8"], port=server.port + ) + assert result.returncode == 0, (result.stderr or result.stdout)[:400] + + dst = _dest_file(source, dest, UTF8_NAME) + assert os.path.exists(dst), f"dest UTF-8-named file not found under {dest}" + + +@pytest.mark.ci +def test_iconv_invalid_charset_rejected(shared_server): + """An unsupported charset name is rejected at startup with a nonzero exit.""" + source, dest = _make("badcharset") + src_path = os.path.join(source, "f.txt") + with open(src_path, "wb") as fh: + fh.write(b"x") + + result, _ = run_client( + source, dest, flags=["--iconv=no-such-charset,utf-8"], port=shared_server.port + ) + assert result.returncode != 0 + + +@pytest.mark.ci +def test_iconv_garbage_spec_rejected(shared_server): + """A malformed CONVERT_SPEC is rejected at startup with a nonzero exit.""" + source, dest = _make("garbage") + src_path = os.path.join(source, "f.txt") + with open(src_path, "wb") as fh: + fh.write(b"x") + + result, _ = run_client(source, dest, flags=["--iconv=,,,"], port=shared_server.port) + assert result.returncode != 0 \ No newline at end of file diff --git a/tests/runner.c b/tests/runner.c index 5cf2d54..1492be0 100644 --- a/tests/runner.c +++ b/tests/runner.c @@ -14,6 +14,7 @@ #include "test_file_sendfile.h" #include "test_fuzz_smoke.h" #include "test_glob.h" +#include "test_iconv.h" #include "test_log.h" #include "test_metadata.h" #include "test_motd.h" @@ -59,6 +60,7 @@ int main() { RUN_TEST(test_protocol); RUN_TEST(test_metadata); RUN_TEST(test_glob); + RUN_TEST(test_iconv); RUN_TEST(test_file); RUN_TEST(test_trust_sender); RUN_TEST(test_delay_updates); diff --git a/tests/test_config.c b/tests/test_config.c index 49775d4..d5b8c4e 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -1565,6 +1565,110 @@ static void test_config_local_only_fields_not_serialized() { EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0); } +static void test_config_iconv_spec_wire_roundtrip() { + Config* send_cfg = config_create(); + EXPECT_NOT_NULL(send_cfg); + send_cfg->send_directory = str_dup("/src"); + send_cfg->receive_root_directory = str_dup("rel/path"); + send_cfg->iconv_spec = str_dup("utf-8,iso-8859-1"); + + int p[2]; + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + io_set_fds(p[0], p[1]); + io_set_bwlimit(0); + + pid_t pid = fork(); + if (pid == 0) { + close(p[1]); + io_set_fds(p[0], p[0]); + Config* recv_cfg = config_receive(p[0]); + bool ok = recv_cfg != NULL && recv_cfg->iconv_spec != NULL && + strcmp(recv_cfg->iconv_spec, "utf-8,iso-8859-1") == 0; + config_delete(recv_cfg); + close(p[0]); + _exit(ok ? 0 : 1); + } else { + close(p[0]); + io_set_fds(p[1], p[1]); + bool sent = config_send(p[1], send_cfg); + int status; + waitpid(pid, &status, 0); + close(p[1]); + config_delete(send_cfg); + EXPECT_TRUE(sent); + EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0); + } +} + +static void test_config_iconv_spec_empty_canonicalizes_to_null() { + Config* send_cfg = config_create(); + EXPECT_NOT_NULL(send_cfg); + send_cfg->send_directory = str_dup("/src"); + send_cfg->receive_root_directory = str_dup("/dst"); + /* iconv_spec left NULL -> serialized as "" -> received back as NULL. */ + + int p[2]; + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + io_set_fds(p[0], p[1]); + io_set_bwlimit(0); + + pid_t pid = fork(); + if (pid == 0) { + close(p[1]); + io_set_fds(p[0], p[0]); + Config* recv_cfg = config_receive(p[0]); + bool ok = recv_cfg != NULL && recv_cfg->iconv_spec == NULL; + config_delete(recv_cfg); + close(p[0]); + _exit(ok ? 0 : 1); + } else { + close(p[0]); + io_set_fds(p[1], p[1]); + bool sent = config_send(p[1], send_cfg); + int status; + waitpid(pid, &status, 0); + close(p[1]); + config_delete(send_cfg); + EXPECT_TRUE(sent); + EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0); + } +} + +static void test_config_receive_rejects_invalid_iconv_spec() { + Config* send_cfg = config_create(); + EXPECT_NOT_NULL(send_cfg); + send_cfg->send_directory = str_dup("/src"); + send_cfg->receive_root_directory = str_dup("/dst"); + send_cfg->iconv_spec = str_dup("no-such-charset,utf-8"); + + int p[2]; + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + io_set_fds(p[0], p[1]); + io_set_bwlimit(0); + + pid_t pid = fork(); + if (pid == 0) { + close(p[1]); + io_set_fds(p[0], p[0]); + /* A malformed/unsupported spec must be refused at the config handshake + (STATUS_ERROR makes config_send fail on the parent). */ + Config* recv_cfg = config_receive(p[0]); + config_delete(recv_cfg); + close(p[0]); + _exit(recv_cfg ? 1 : 0); + } else { + close(p[0]); + io_set_fds(p[1], p[1]); + bool sent = config_send(p[1], send_cfg); + int status; + waitpid(pid, &status, 0); + close(p[1]); + config_delete(send_cfg); + EXPECT_FALSE(sent); + EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0); + } +} + void test_config() { test_config_lifecycle(); test_config_ssh_dest(); @@ -1608,6 +1712,9 @@ void test_config() { test_config_module_wire_empty_canonicalizes_to_null(); test_config_daemon_auth_wire_roundtrip(); test_config_daemon_auth_wire_rejects_malformed(); + test_config_iconv_spec_wire_roundtrip(); + test_config_iconv_spec_empty_canonicalizes_to_null(); + test_config_receive_rejects_invalid_iconv_spec(); test_config_receive_with_validate_rejects(); } test_config_delete_timing_early_helper(); diff --git a/tests/test_iconv.c b/tests/test_iconv.c new file mode 100644 index 0000000..27b1840 --- /dev/null +++ b/tests/test_iconv.c @@ -0,0 +1,171 @@ +#include "test_iconv.h" +#include "charset.h" +#include "protocol.h" +#include "test_utils.h" +#include "utils.h" +#include +#include +#include +#include +#include +#include + +/* --- CONVERT_SPEC parsing ------------------------------------------------ */ + +static void test_iconv_spec_parse_split() { + char* local = NULL; + char* remote = NULL; + EXPECT_EQ_INT(charset_spec_parse("utf-8,iso-8859-1", &local, &remote), 0); + EXPECT_EQ_STR(local, "utf-8"); + EXPECT_EQ_STR(remote, "iso-8859-1"); + free(local); + free(remote); +} + +static void test_iconv_spec_parse_single_defaults_to_local() { + char* local = NULL; + char* remote = NULL; + EXPECT_EQ_INT(charset_spec_parse("utf-8", &local, &remote), 0); + EXPECT_EQ_STR(local, "utf-8"); + EXPECT_EQ_STR(remote, "utf-8"); + free(local); + free(remote); +} + +static void test_iconv_spec_parse_garbage() { + char* local = NULL; + char* remote = NULL; + EXPECT_EQ_INT(charset_spec_parse(NULL, &local, &remote), -1); + EXPECT_EQ_INT(charset_spec_parse("", &local, &remote), -1); + EXPECT_EQ_INT(charset_spec_parse(",", &local, &remote), -1); + EXPECT_EQ_INT(charset_spec_parse("utf-8,", &local, &remote), -1); + EXPECT_EQ_INT(charset_spec_parse(",utf-8", &local, &remote), -1); +} + +static void test_iconv_spec_valid() { + EXPECT_TRUE(charset_spec_valid(NULL)); + EXPECT_TRUE(charset_spec_valid("utf-8")); + EXPECT_TRUE(charset_spec_valid("utf-8,iso-8859-1")); + EXPECT_TRUE(charset_spec_valid("iso-8859-1,ascii")); + EXPECT_FALSE(charset_spec_valid("no-such-charset,utf-8")); + EXPECT_FALSE(charset_spec_valid("utf-8,no-such-charset")); + EXPECT_FALSE(charset_spec_valid(",,,")); + EXPECT_FALSE(charset_spec_valid("utf-8,")); +} + +/* --- one-shot conversion ------------------------------------------------ */ + +static void test_iconv_utf8_to_latin1() { + void* conv = charset_conversion_open("utf-8", "iso-8859-1"); + EXPECT_NOT_NULL(conv); + char* out = charset_convert(conv, "caf\xc3\xa9", NULL); + EXPECT_NOT_NULL(out); + EXPECT_EQ_INT(strcmp(out, "caf\xe9"), 0); + free(out); + charset_conversion_close(conv); +} + +static void test_iconv_latin1_to_utf8() { + void* conv = charset_conversion_open("iso-8859-1", "utf-8"); + EXPECT_NOT_NULL(conv); + char* out = charset_convert(conv, "caf\xe9", NULL); + EXPECT_NOT_NULL(out); + EXPECT_EQ_INT(strcmp(out, "caf\xc3\xa9"), 0); + free(out); + charset_conversion_close(conv); +} + +static void test_iconv_invalid_sequence_fails() { + int err = 0; + /* 0xff is not a valid UTF-8 sequence. */ + void* conv = charset_conversion_open("utf-8", "ascii"); + EXPECT_NOT_NULL(conv); + EXPECT_TRUE(charset_convert(conv, "bad\xff", &err) == NULL); + EXPECT_TRUE(err == EILSEQ || err == EINVAL); + charset_conversion_close(conv); +} + +static void test_iconv_unrepresentable_fails() { + /* "caf\xc3\xa9" (UTF-8 for cafe) has no ASCII representation. */ + void* conv = charset_conversion_open("utf-8", "ascii"); + EXPECT_NOT_NULL(conv); + EXPECT_TRUE(charset_convert(conv, "caf\xc3\xa9", NULL) == NULL); + charset_conversion_close(conv); +} + +/* --- process-wide wire conversion ---------------------------------------- */ + +static void test_iconv_wire_sender_converts_local_to_remote() { + EXPECT_TRUE(charset_wire_init_sender("utf-8,iso-8859-1")); + char* wire = charset_wire_apply("caf\xc3\xa9"); + EXPECT_NOT_NULL(wire); + EXPECT_EQ_INT(strcmp(wire, "caf\xe9"), 0); + free(wire); + charset_wire_free(); +} + +static void test_iconv_wire_receiver_converts_remote_to_local() { + EXPECT_TRUE(charset_wire_init_receiver("utf-8,iso-8859-1", NULL)); + char* local = charset_wire_apply("caf\xe9"); + EXPECT_NOT_NULL(local); + EXPECT_EQ_INT(strcmp(local, "caf\xc3\xa9"), 0); + free(local); + charset_wire_free(); +} + +static void test_iconv_wire_disabled_passthrough() { + charset_wire_init_sender(NULL); + EXPECT_FALSE(charset_wire_active()); + char* out = charset_wire_apply("plain/name\xff"); + EXPECT_NOT_NULL(out); + EXPECT_EQ_INT(strcmp(out, "plain/name\xff"), 0); + free(out); + charset_wire_free(); +} + +static void test_iconv_wire_str_roundtrip() { + EXPECT_TRUE(charset_wire_init_sender("utf-8,iso-8859-1")); + int p[2]; + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + io_set_fds(p[0], p[1]); + io_set_bwlimit(0); + + pid_t pid = fork(); + if (pid == 0) { + close(p[1]); + io_set_fds(p[0], p[0]); + charset_wire_free(); + charset_wire_init_receiver("utf-8,iso-8859-1", NULL); + char* got = receive_wire_str(p[0]); + bool ok = got != NULL && strcmp(got, "caf\xc3\xa9") == 0; + free(got); + charset_wire_free(); + close(p[0]); + _exit(ok ? 0 : 1); + } else { + close(p[0]); + io_set_fds(p[1], p[1]); + bool sent = send_wire_str(p[1], "caf\xc3\xa9"); + int status; + waitpid(pid, &status, 0); + close(p[1]); + charset_wire_free(); + EXPECT_TRUE(sent); + EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0); + } +} + +void test_iconv() { + test_iconv_spec_parse_split(); + test_iconv_spec_parse_single_defaults_to_local(); + test_iconv_spec_parse_garbage(); + test_iconv_spec_valid(); + test_iconv_utf8_to_latin1(); + test_iconv_latin1_to_utf8(); + test_iconv_invalid_sequence_fails(); + test_iconv_unrepresentable_fails(); + test_iconv_wire_sender_converts_local_to_remote(); + test_iconv_wire_receiver_converts_remote_to_local(); + test_iconv_wire_disabled_passthrough(); + test_iconv_wire_str_roundtrip(); +} \ No newline at end of file diff --git a/tests/test_iconv.h b/tests/test_iconv.h new file mode 100644 index 0000000..8d54ec6 --- /dev/null +++ b/tests/test_iconv.h @@ -0,0 +1,6 @@ +#ifndef TEST_ICONV_H +#define TEST_ICONV_H + +void test_iconv(void); + +#endif \ No newline at end of file