diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 4b14d88..7a70bb7 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -1,5 +1,6 @@ #include "client_send.h" #include "client_validation.h" +#include "charset.h" #include "chmod.h" #include "compression.h" #include "config.h" @@ -590,6 +591,11 @@ static const OptionEntry OPTION_TABLE[] = { * path; main() reads it (after the destination form is known) and derives * the wire credentials. Never crosses the wire. */ {"--password-file", NULL, OPT_STRING, offsetof(Config, password_file)}, + /* --iconv (protocol 2.16.0): convert file-NAME charsets at the wire + * boundary. The CONVERT_SPEC (LOCAL[,REMOTE]) is validated for real iconv + * charsets at startup (client_validation.c) and the full spec rides the + * config frame so the receiver derives the wire charset symmetrically. */ + {"--iconv", NULL, OPT_STRING, offsetof(Config, iconv_spec)}, {"--delete-before", NULL, OPT_FLAG, offsetof(Config, delete_before)}, {"--delete-during", "--del", OPT_FLAG, offsetof(Config, delete_during)}, {"--delete-delay", NULL, OPT_FLAG, offsetof(Config, delete_delay)}, @@ -1593,6 +1599,17 @@ int main(int argc, char* argv[]) { goto cleanup; } + /* --iconv: install the sender-side local->wire conversion before any path is + scanned or serialized (the scanner and the chunk/data path read windows are + all driven from this process, so one global initialization covers every + send site). */ + if (!charset_wire_init_sender(config->iconv_spec)) { + log_message(LOG_LEVEL_ERROR, + "--iconv has an invalid CONVERT_SPEC or an unsupported charset name"); + exit_code = 1; + goto cleanup; + } + /* Apply the requested --outbuf style now that the mode is parsed. */ apply_output_buffering(config); @@ -1614,6 +1631,7 @@ int main(int argc, char* argv[]) { } cleanup: + charset_wire_free(); if (config) { config_delete(config); } diff --git a/src/client/client_send.c b/src/client/client_send.c index e909810..bc86762 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -1,6 +1,7 @@ #include "client_send.h" #include "array_list.h" #include "change_list.h" +#include "charset.h" #include "chunk.h" #include "compression.h" #include "config.h" @@ -810,21 +811,21 @@ static int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protecte if (!send_int(fd, keep_count)) return -1; for (int i = 0; i < keep_count; i++) { - if (!send_str(fd, (char*)manifest->items[i])) + if (!send_wire_str(fd, (char*)manifest->items[i])) return -1; } int protected_count = protected_prefixes ? protected_prefixes->size : 0; if (!send_int(fd, protected_count)) return -1; for (int i = 0; i < protected_count; i++) { - if (!send_str(fd, (char*)protected_prefixes->items[i])) + if (!send_wire_str(fd, (char*)protected_prefixes->items[i])) return -1; } int missing_count = missing_args ? missing_args->size : 0; if (!send_int(fd, missing_count)) return -1; for (int i = 0; i < missing_count; i++) { - if (!send_str(fd, (char*)missing_args->items[i])) + if (!send_wire_str(fd, (char*)missing_args->items[i])) return -1; } return 0; @@ -899,7 +900,7 @@ static int incremental_check(Client* client, File* file, const Config* config, *resume_offset = 0; if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1; - if (!send_str(client->file_descriptor, file_wire_path(file))) + if (!send_wire_str(client->file_descriptor, file_wire_path(file))) return -1; unsigned long long fsize = file->data->size; long long mtime = file->metadata ? file->metadata->mtime_sec : 0; @@ -1119,7 +1120,7 @@ static bool send_directory_entry(Client* client, File* file) { return false; if (!send_status(client->file_descriptor, STATUS_MKDIR)) return false; - return send_str(client->file_descriptor, file_wire_path(file)); + return send_wire_str(client->file_descriptor, file_wire_path(file)); } /* Transmit one symlink entry: a STATUS_SYMLINK frame carrying the destination @@ -1130,8 +1131,8 @@ static bool send_symlink_entry(const Client* client, File* file, const Config* c if (!file || !file_wire_path(file) || !file->symlink_target) return false; int fd = client->file_descriptor; - if (!send_status(fd, STATUS_SYMLINK) || !send_str(fd, file_wire_path(file)) || - !send_str(fd, file->symlink_target)) + if (!send_status(fd, STATUS_SYMLINK) || !send_wire_str(fd, file_wire_path(file)) || + !send_wire_str(fd, file->symlink_target)) return false; return !config->use_metadata || metadata_send(fd, file->metadata); } @@ -1311,9 +1312,9 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config, wire path so the receiver links this entry to that installed file. */ if (f->link_group != 0 && !f->link_first && f->hardlink_target != NULL) { if (!send_status(client->file_descriptor, STATUS_HARDLINK) || - !send_str(client->file_descriptor, file_wire_path(f)) || + !send_wire_str(client->file_descriptor, file_wire_path(f)) || !send_int(client->file_descriptor, f->link_group) || - !send_str(client->file_descriptor, f->hardlink_target)) + !send_wire_str(client->file_descriptor, f->hardlink_target)) return -1; change_emit_file_sent(config, f); continue; diff --git a/src/client/client_validation.c b/src/client/client_validation.c index c0bc08a..d42da2e 100644 --- a/src/client/client_validation.c +++ b/src/client/client_validation.c @@ -1,4 +1,5 @@ #include "client_validation.h" +#include "charset.h" #include "delay_updates.h" #include "log.h" #include "usage.h" @@ -123,5 +124,13 @@ bool validate_config(const Config* config) { "timing; at most one may be given and each implies --delete"); return false; } + /* --iconv: reject a malformed CONVERT_SPEC or an unsupported charset name at + startup (a probe iconv_open is attempted), so a typo'd charset never fails + the run mid-transfer with per-file errors. */ + if (!charset_spec_valid(config->iconv_spec)) { + log_message(LOG_LEVEL_ERROR, + "--iconv requires LOCAL[,REMOTE] charset names supported by iconv"); + return false; + } return true; } diff --git a/src/client/usage.c b/src/client/usage.c index 24e9878..f2159c3 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -35,6 +35,12 @@ void print_usage(void) { printf(" --progress Show transfer progress\n"); printf(" -P Partial mode with progress (retention incomplete)\n"); printf(" -8, --8-bit-output Leave high-bit characters unescaped in output\n"); + printf(" --iconv=LOCAL[,REMOTE] Convert file-NAME charsets at the wire boundary:\n"); + printf(" LOCAL is the charset of our file names, REMOTE is the\n"); + printf(" remote side's charset (defaults to LOCAL). Names are\n"); + printf(" converted before transmission and back on receipt; a\n"); + printf(" name that cannot be represented in the target charset\n"); + printf(" fails that transfer cleanly (rsync-compatible)\n"); printf(" --delete Delete files on receiver not in source\n"); printf(" (default timing: delete only after the whole\n"); printf(" transfer has succeeded)\n"); diff --git a/src/server/receiver.c b/src/server/receiver.c index 63da749..df1dc0f 100644 --- a/src/server/receiver.c +++ b/src/server/receiver.c @@ -1,5 +1,6 @@ #include "receiver.h" +#include "charset.h" #include "chunk.h" #include "config.h" #include "delay_updates.h" @@ -79,7 +80,7 @@ static bool receiver_process_batch(Config* config, int file_descriptor) { count > MAX_MANIFEST_ENTRIES) return false; for (int i = 0; i < count; i++) { - char* check_path = receive_str(file_descriptor); + char* check_path = receive_wire_str(file_descriptor); if (!check_path) return false; unsigned long long check_size; diff --git a/src/server/server.c b/src/server/server.c index 17fce6b..2d66777 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -1,4 +1,5 @@ #include "config.h" +#include "charset.h" #include "credentials.h" #include "daemon_conf.h" #include "delay_updates.h" @@ -31,6 +32,10 @@ static bool allow_delete; static bool trust_sender; static bool allow_unauthenticated; static const char* required_client_cn; +/* --iconv CONVERT_SPEC the server was itself started with (borrowed argv + * pointer). Its LOCAL half may override the local charset the client assumed; + * see charset_wire_init_receiver. */ +static const char* server_iconv_spec; /* Non-NULL exactly when the listener runs in --daemon mode. Loaded once in * main before any accept-loop fork, then shared read-only by every forked @@ -170,6 +175,15 @@ static const char* server_module_gate(const Config* config, void* context) { ModuleGateContext* gate_ctx = (ModuleGateContext*)context; if (!config) return "missing config frame"; + /* --iconv (protocol 2.16.0): the receiver's exact conversion direction (the + client spec's wire charset into this server's local charset, including a + server-side --iconv override) must be usable BEFORE the STATUS_OK ack, so + an impossible conversion is refused at the handshake instead of failing + the first file mid-transfer. The client spec itself was already sanity + checked by validate_received_config. */ + if (config->iconv_spec && + !charset_wire_receiver_spec_valid(config->iconv_spec, server_iconv_spec)) + return "client --iconv conversion cannot be honored by this server"; bool is_daemon = g_daemon_conf != NULL; bool has_module = config->module != NULL && config->module[0] != '\0'; @@ -318,6 +332,20 @@ void handler(int file_descriptor) { return; } config->use_delete = config->use_delete && allow_delete; + /* --iconv (protocol 2.16.0): install the receiver-side wire->local conversion + now that the client's full CONVERT_SPEC has been received and validated, + before any received file name is decoded. The server's own --iconv (if + any) may override the local charset; a spec the client is known to have + validated cannot fail here unless the server's override names an + unsupported charset. */ + if (config->iconv_spec && !charset_wire_init_receiver(config->iconv_spec, server_iconv_spec)) { + log_message(LOG_LEVEL_ERROR, + "--iconv: unsupported charset conversion requested (LOCAL[,REMOTE])"); + config_delete(config); + close(file_descriptor); + protocol_session_unbind(); + return; + } /* --delete-missing-args deletes destination mirrors receiver-side, so it is deletion and stays gated by the same --allow-delete server policy. When the server policy is off the flag is inert (the missing entries are still @@ -485,6 +513,7 @@ void handler(int file_descriptor) { } protocol_session_unbind(); identity_clear_active(); + charset_wire_free(); close(file_descriptor); } @@ -535,6 +564,11 @@ static void print_server_usage(void) { printf(" -6, --ipv6 Bind an IPv6 socket\n"); printf(" --allow-delete Permit manifest deletion\n"); printf(" --trust-sender Trust the remote sender's file list\n"); + printf(" --iconv=LOCAL[,REMOTE] Declare this server's LOCAL charset for file-name\n"); + printf(" conversion: received names are translated to this\n"); + printf(" charset (the wire charset still comes from the\n"); + printf(" client's CONVERT_SPEC). A name that cannot be\n"); + printf(" represented fails the run cleanly\n"); printf(" --allow-unauthenticated Allow plaintext/anonymous network clients\n"); printf(" -v, --verbose Enable debug logging\n"); printf(" --help Show this help\n"); @@ -621,6 +655,7 @@ int main(int argc, char* argv[]) { allow_delete = opts.allow_delete; trust_sender = opts.trust_sender; allow_unauthenticated = opts.allow_unauthenticated; + server_iconv_spec = opts.iconv_spec; signal(SIGINT, cleanup); signal(SIGTERM, cleanup); diff --git a/src/server/server_cli.c b/src/server/server_cli.c index 67b434a..e0168c8 100644 --- a/src/server/server_cli.c +++ b/src/server/server_cli.c @@ -1,4 +1,5 @@ #include "server_cli.h" +#include "charset.h" #include "utils.h" #include #include @@ -143,6 +144,12 @@ int server_cli_parse(int argc, char* argv[], ServerCliOptions* opts, char* err, opts->trust_sender = true; } else if (arg_is(argv[i], "--allow-unauthenticated")) { opts->allow_unauthenticated = true; + } else if (arg_is(argv[i], "--iconv")) { + if (i + 1 >= argc) { + set_error(err, err_size, "missing argument for --iconv"); + return -1; + } + opts->iconv_spec = argv[++i]; } else if (arg_is(argv[i], "-p")) { if (i + 1 >= argc) { set_error(err, err_size, "missing argument for -p"); @@ -180,6 +187,15 @@ int server_cli_parse(int argc, char* argv[], ServerCliOptions* opts, char* err, inline_value = argv[++i]; } opts->early_input_file = inline_value; + } else if (arg_has_value(argv[i], "--iconv", &inline_value)) { + if (!inline_value) { + if (i + 1 >= argc) { + set_error(err, err_size, "missing argument for --iconv"); + return -1; + } + inline_value = argv[++i]; + } + opts->iconv_spec = inline_value; } else if (arg_has_value(argv[i], "--dparam", &inline_value)) { if (!inline_value) { if (i + 1 >= argc) { @@ -227,6 +243,12 @@ int server_cli_parse(int argc, char* argv[], ServerCliOptions* opts, char* err, "--daemon"); return -1; } + /* --iconv: reject a malformed CONVERT_SPEC or an unsupported charset name at + startup (a probe iconv_open is attempted). */ + if (opts->iconv_spec != NULL && !charset_spec_valid(opts->iconv_spec)) { + set_error(err, err_size, "--iconv requires LOCAL[,REMOTE] charset names supported by iconv"); + return -1; + } return 0; } diff --git a/src/server/server_cli.h b/src/server/server_cli.h index c268bcc..4bb2351 100644 --- a/src/server/server_cli.h +++ b/src/server/server_cli.h @@ -33,6 +33,12 @@ typedef struct ServerCliOptions { bool allow_delete; /* --allow-delete */ bool trust_sender; /* --trust-sender */ bool allow_unauthenticated; /* --allow-unauthenticated */ + /* --iconv=CONVERT_SPEC: the server's own LOCAL charset declaration. The + * client's full spec rides the wire config frame anyway; when the server is + * started with its own --iconv, its LOCAL half overrides the local charset + * the client assumed so the server converts received names to ITS charset. + * Borrowed pointer into argv (never owns heap). */ + const char* iconv_spec; /* --iconv value, or NULL */ } ServerCliOptions; /* Parse argc/argv into *opts. Zero-initialize *opts before calling (or use diff --git a/src/shared/charset.c b/src/shared/charset.c new file mode 100644 index 0000000..75a2ac5 --- /dev/null +++ b/src/shared/charset.c @@ -0,0 +1,384 @@ +#include "charset.h" +#include "log.h" +#include "protocol.h" +#include "utils.h" +#include +#include +#include +#include + +typedef struct { + iconv_t cd; +} CharsetConversion; + +/* Process-wide wire conversion descriptor (one direction per process: a client + * only sends, a server only receives). CONCURRENCY CONTRACT: iconv_t is not + * guaranteed thread-safe, so every conversion MUST run on a single thread at a + * time. This holds today -- on the client the conversions run on the sender + * thread (in the -m pipeline chunk_serialize/send happen on the sender thread + * only), on the server on the receive-loop thread; the descriptor is + * initialized on one thread before any transfer thread spawns and torn down + * (charset_wire_free) only after all threads have joined. Do not add a + * concurrent conversion path (e.g. parallel chunk serialization) without + * guarding access with a mutex. */ +static CharsetConversion* g_wire_conv; + +/* Grow *buf to double capacity, freeing it on failure. realloc preserves the + * already-written prefix, so the caller only tracks its write offset. */ +static bool grow_charset_buffer(char** buf, size_t* cap) { + size_t new_cap = *cap * 2; + if (new_cap <= *cap) { + free(*buf); + *buf = NULL; + return false; + } + char* grown = realloc(*buf, new_cap); + if (!grown) { + free(*buf); + *buf = NULL; + return false; + } + *buf = grown; + *cap = new_cap; + return true; +} + +/* Throw away any pending shift state so a subsequent conversion starts clean. + * The flush output is discarded; for the stateless single-byte/UTF charsets + * this feature targets it is a no-op. */ +static void charset_conversion_reset(const CharsetConversion* conv) { + char scratch[64]; + char* sp = scratch; + size_t sl = sizeof(scratch); + (void)iconv(conv->cd, NULL, NULL, &sp, &sl); +} + +int charset_spec_parse(const char* spec, char** local_out, char** remote_out) { + if (!local_out || !remote_out) + return -1; + *local_out = NULL; + *remote_out = NULL; + if (!spec || spec[0] == '\0') + return -1; + char* dup = str_dup(spec); + if (!dup) + return -1; + char* comma = strchr(dup, ','); + if (comma) { + if (comma == dup || comma[1] == '\0') { + free(dup); + return -1; + } + *comma = '\0'; + *local_out = str_dup(dup); + *remote_out = str_dup(comma + 1); + free(dup); + } else { + *local_out = str_dup(dup); + *remote_out = str_dup(dup); + free(dup); + } + if (!*local_out || !*remote_out) { + free(*local_out); + free(*remote_out); + *local_out = NULL; + *remote_out = NULL; + return -1; + } + return 0; +} + +void* charset_conversion_open(const char* from_charset, const char* to_charset) { + if (!from_charset || !to_charset) + return NULL; + iconv_t cd = iconv_open(to_charset, from_charset); + if (cd == (iconv_t)-1) + return NULL; + CharsetConversion* conv = malloc(sizeof(CharsetConversion)); + if (!conv) { + iconv_close(cd); + return NULL; + } + conv->cd = cd; + return conv; +} + +void charset_conversion_close(void* conversion) { + if (!conversion) + return; + CharsetConversion* conv = (CharsetConversion*)conversion; + iconv_close(conv->cd); + free(conv); +} + +/* Probe a single conversion direction: the from/to charsets both open AND a + * representative ASCII name converts to a byte string containing no embedded + * NUL (so a target charset like UTF-16 that emits NUL bytes for ordinary ASCII + * names is rejected up front -- such an output would be silently truncated by + * the C-string wire helpers). */ +static bool direction_probe_valid(const char* from, const char* to) { + if (!from || !to) + return false; + void* conv = charset_conversion_open(from, to); + if (!conv) + return false; + bool ok = true; + char input = 'a'; + char* in_ptr = &input; + size_t in_left = 1; + char out_buf[64]; + char* out_ptr = out_buf; + size_t out_left = sizeof(out_buf); + if (iconv(((CharsetConversion*)conv)->cd, &in_ptr, &in_left, &out_ptr, &out_left) == (size_t)-1) + ok = false; + char flush_buf[64]; + char* flush_ptr = flush_buf; + size_t flush_left = sizeof(flush_buf); + if (ok && + iconv(((CharsetConversion*)conv)->cd, NULL, NULL, &flush_ptr, &flush_left) == (size_t)-1) + ok = false; + size_t produced = (size_t)(out_ptr - out_buf); + if (ok && produced > 0 && memchr(out_buf, '\0', produced) != NULL) + ok = false; + charset_conversion_close(conv); + return ok; +} + +bool charset_pair_valid(const char* local, const char* remote) { + /* Both ends convert in opposite directions with the same two charsets, so a + * valid spec must open (and be NUL-free) in BOTH directions: the sender + * opens local->remote, the receiver opens remote->local. */ + return direction_probe_valid(local, remote) && direction_probe_valid(remote, local); +} + +bool charset_spec_valid(const char* spec) { + if (!spec) + return true; + char* local; + char* remote; + if (charset_spec_parse(spec, &local, &remote) != 0) + return false; + bool ok = charset_pair_valid(local, remote); + free(local); + free(remote); + return ok; +} + +bool charset_spec_valid_direction(const char* from_charset, const char* to_charset) { + return direction_probe_valid(from_charset, to_charset); +} + +/* The receiver's real conversion is wire(client REMOTE) -> server-local (the + * server's own --iconv LOCAL half, or the client's LOCAL half when the server + * has no --iconv). A dedicated pre-ack check so an impossible direction is + * rejected before the connection instead of refusing mid-transfer. */ +bool charset_wire_receiver_spec_valid(const char* spec, const char* server_spec) { + if (!spec) + return true; + char* local; + char* remote; + if (charset_spec_parse(spec, &local, &remote) != 0) + return false; + const char* wire = remote; + const char* target_local = local; + char* server_local = NULL; + char* server_remote = NULL; + if (server_spec) { + if (charset_spec_parse(server_spec, &server_local, &server_remote) != 0) { + free(local); + free(remote); + return false; + } + target_local = server_local; + } + bool ok = charset_spec_valid_direction(wire, target_local); + free(server_local); + free(server_remote); + free(local); + free(remote); + return ok; +} + +char* charset_convert(const void* conversion, const char* in, int* err_out) { + if (!conversion || !in) + return NULL; + const CharsetConversion* conv = (const CharsetConversion*)conversion; + size_t in_len = strlen(in); + size_t cap = in_len + 16; + char* out = malloc(cap); + if (!out) + return NULL; + size_t in_left = in_len; + char* in_ptr = (char*)in; + size_t out_used = 0; + + while (in_left > 0) { + char* out_ptr = out + out_used; + size_t out_left = cap - out_used; + if (iconv(conv->cd, &in_ptr, &in_left, &out_ptr, &out_left) == (size_t)-1) { + if (errno != E2BIG) { + if (err_out) + *err_out = errno; + charset_conversion_reset(conv); + free(out); + return NULL; + } + /* Output exhausted but input remains. E2BIG does not roll the output + pointer back: the bytes iconv already emitted before the failure must + be preserved, so advance out_used before growing. */ + out_used = (size_t)(out_ptr - out); + if (!grow_charset_buffer(&out, &cap)) + return NULL; + continue; + } + out_used = (size_t)(out_ptr - out); + } + + /* Flush any pending shift state (a no-op for the stateless single-byte and + UTF charsets this feature targets, but keeps the descriptor clean). */ + for (;;) { + char* out_ptr = out + out_used; + size_t out_left = cap - out_used; + if (iconv(conv->cd, NULL, NULL, &out_ptr, &out_left) == (size_t)-1) { + if (errno != E2BIG) { + if (err_out) + *err_out = errno; + charset_conversion_reset(conv); + free(out); + return NULL; + } + out_used = (size_t)(out_ptr - out); + if (!grow_charset_buffer(&out, &cap)) + return NULL; + continue; + } + out_used = (size_t)(out_ptr - out); + break; + } + + /* A successful iconv call may legitimately consume the whole buffer (output + exactly fills cap), leaving no room for the terminator: guarantee headroom + before the final write. */ + if (out_used >= cap && !grow_charset_buffer(&out, &cap)) + return NULL; + + /* Defense in depth: a target charset that emits embedded NUL bytes would + truncate at the first NUL in the C-string wire helpers; fail cleanly + (validation already rejects such charsets up front). */ + if (memchr(out, '\0', out_used) != NULL) { + if (err_out) + *err_out = EILSEQ; + charset_conversion_reset(conv); + free(out); + return NULL; + } + + out[out_used] = '\0'; + return out; +} + +bool charset_wire_init_sender(const char* spec) { + charset_wire_free(); + if (!spec) + return true; + char* local; + char* remote; + if (charset_spec_parse(spec, &local, &remote) != 0) + return false; + void* conv = charset_conversion_open(local, remote); + free(local); + free(remote); + if (!conv) + return false; + g_wire_conv = (CharsetConversion*)conv; + return true; +} + +bool charset_wire_init_receiver(const char* spec, const char* server_spec) { + charset_wire_free(); + if (!spec) + return true; + char* local; + char* remote; + if (charset_spec_parse(spec, &local, &remote) != 0) + return false; + /* The wire charset is the client spec's REMOTE half; the local charset is + * the client spec's LOCAL half unless the server was itself started with + * --iconv naming a different local charset (the server halves above never + * travel, so the server's own flag is the only way its local charset can + * differ from what the client assumed). */ + const char* wire = remote; + const char* target_local = local; + char* server_local = NULL; + char* server_remote = NULL; + if (server_spec) { + if (charset_spec_parse(server_spec, &server_local, &server_remote) != 0) { + free(local); + free(remote); + return false; + } + target_local = server_local; + } + void* conv = charset_conversion_open(wire, target_local); + free(server_local); + free(server_remote); + free(local); + free(remote); + if (!conv) + return false; + g_wire_conv = (CharsetConversion*)conv; + return true; +} + +void charset_wire_free(void) { + if (g_wire_conv) { + charset_conversion_close(g_wire_conv); + g_wire_conv = NULL; + } +} + +bool charset_wire_active(void) { + return g_wire_conv != NULL; +} + +char* charset_wire_apply(const char* path) { + if (!g_wire_conv) + return str_dup(path); + return charset_convert(g_wire_conv, path, NULL); +} + +static void charset_convert_failure_log(const char* path) { + char* escaped = output_escape(path, false); + log_message(LOG_LEVEL_ERROR, "--iconv: cannot convert file name '%s' to the target charset", + escaped ? escaped : ""); + free(escaped); +} + +bool send_wire_str(int file_descriptor, const char* local_path) { + if (!g_wire_conv) + return send_str(file_descriptor, local_path); + char* wire = charset_wire_apply(local_path); + if (!wire) { + charset_convert_failure_log(local_path); + return false; + } + bool ok = send_str(file_descriptor, wire); + free(wire); + return ok; +} + +char* receive_wire_str(int file_descriptor) { + char* raw = receive_str(file_descriptor); + if (!raw) + return NULL; + if (!g_wire_conv) + return raw; + char* local = charset_convert(g_wire_conv, raw, NULL); + if (!local) { + charset_convert_failure_log(raw); + free(raw); + return NULL; + } + free(raw); + return local; +} \ No newline at end of file diff --git a/src/shared/charset.h b/src/shared/charset.h new file mode 100644 index 0000000..49cd0f3 --- /dev/null +++ b/src/shared/charset.h @@ -0,0 +1,85 @@ +#ifndef CHARSET_H +#define CHARSET_H + +#include +#include + +/* --iconv=CONVERT_SPEC file-name charset conversion (rsync compatibility). + * + * CONVERT_SPEC is "LOCAL[,REMOTE]": LOCAL is the charset of our own file + * names, REMOTE is the charset of the remote side's file names and defaults + * to LOCAL when the comma half is omitted. The sender converts every local + * path from LOCAL to REMOTE before it goes on the wire; the receiver converts + * every received path back from REMOTE to LOCAL. A NULL/disabled spec means + * identity with zero overhead (the common path never consults iconv). + * + * All helpers are friendly to the strict cold path: the wire conversion state + * is process-global (one direction per process -- a client only sends, a + * server only receives) and is initialized once, before any path is + * serialized, so conversion compiles to a single non-NULL check when disabled. + */ + +/* Parse CONVERT_SPEC into malloc'd LOCAL and REMOTE charset names (caller + * frees both). REMOTE is a separate copy of LOCAL when no comma is present. + * Returns 0 on success, -1 on a malformed spec (empty halves / missing value / + * allocation failure); nothing is allocated on the -1 path. Both output + * pointers are REQUIRED (non-NULL). */ +int charset_spec_parse(const char* spec, char** local_out, char** remote_out); + +/* True when a CONVERT_SPEC is well-formed AND its charsets are usable for this + * feature: each pair opens in a probe iconv_open in BOTH directions (a sender + * converts local->remote, the receiver converts remote->local) and converting + * a representative ASCII name emits no embedded NUL byte (a UTF-16-style NUL + * emitter would be silently truncated by the C-string wire helpers). A typo'd + * charset name is therefore rejected at startup, not mid-run. NULL (iconv + * disabled) is always valid. */ +bool charset_spec_valid(const char* spec); + +/* Probe a concrete from->to conversion pair without keeping the descriptor: + * both charsets open AND a representative ASCII name converts with no embedded + * NUL. Used for direction-specific validation (e.g. the receiver's exact + * wire->local direction including a server-side charset override). */ +bool charset_spec_valid_direction(const char* from_charset, const char* to_charset); +bool charset_pair_valid(const char* local, const char* remote); + +/* One-shot conversion of a NUL-terminated input to a malloc'd NUL-terminated + * result, or NULL on failure. On failure *err_out (when non-NULL) receives + * the iconv errno (EILSEQ/EINVAL = the input is not representable in the + * target charset). The caller must free the result. */ +char* charset_convert(const void* conversion, const char* in, int* err_out); + +/* Open a conversion descriptor for direction from_charset -> to_charset. + * Returns NULL (errno = EINVAL) when a charset name is unsupported. Freed + * with charset_conversion_close. */ +void* charset_conversion_open(const char* from_charset, const char* to_charset); +void charset_conversion_close(void* conversion); + +/* Process-wide wire conversion. charset_wire_init_sender (client side) opens + * LOCAL->REMOTE; charset_wire_init_receiver (server side) opens + * wire(REMOTE)->server-local. server_spec is the server's own --iconv, whose + * LOCAL half may override the local charset the client assumed; NULL reuses + * the client spec's LOCAL half. Both return false on an unsupported spec. + * The state is freed with charset_wire_free. */ +bool charset_wire_init_sender(const char* spec); +bool charset_wire_init_receiver(const char* spec, const char* server_spec); +void charset_wire_free(void); +bool charset_wire_active(void); + +/* Pre-ack receiver-direction sanity (see charset_wire_init_receiver): true + * when the exact wire->server-local conversion the receiver will use (client + * spec's REMOTE half into the server's own LOCAL half, or the client's LOCAL + * half when the server has no --iconv) opens and produces NUL-free output. */ +bool charset_wire_receiver_spec_valid(const char* spec, const char* server_spec); + +/* Convert a path across the wire in the process direction. Returns a malloc'd + * string, or NULL when the name cannot be represented in the target charset. */ +char* charset_wire_apply(const char* path); + +/* Convenience wire string I/O: encode+send_str / receive_str+decode. Both + * return false/NULL (logging a clear --iconv error) on conversion failure, so + * an unconvertible path FAILS the transfer cleanly instead of silently sending + * a mangled name. */ +bool send_wire_str(int file_descriptor, const char* local_path); +char* receive_wire_str(int file_descriptor); + +#endif \ No newline at end of file diff --git a/src/shared/chunk.c b/src/shared/chunk.c index 5b0069e..c5baeb4 100644 --- a/src/shared/chunk.c +++ b/src/shared/chunk.c @@ -6,6 +6,7 @@ #include #include "array_list.h" +#include "charset.h" #include "chunk.h" #include "compression.h" #include "data.h" @@ -63,9 +64,22 @@ void chunk_destroy(void* item) { free(chunk); } +/* --iconv: a chunk blob carries wire-charset path/target bytes. Encode the + * sender-side path (a no-op copy when iconv is disabled) so the blob is in the + * same charset as every other wire string. */ +static char* chunk_encode_wire(const char* path) { + if (!charset_wire_active()) + return str_dup(path); + return charset_wire_apply(path); +} + static unsigned long long per_file_serialize_size(File* file, bool use_metadata) { unsigned long long size = sizeof(size_t); - size_t path_len = strlen(file_wire_path(file)); + char* wire_path = chunk_encode_wire(file_wire_path(file)); + if (!wire_path) + return 0; + size_t path_len = strlen(wire_path); + free(wire_path); unsigned long long metadata_size = use_metadata ? sizeof(int) + (file->metadata ? FILE_METADATA_WIRE_SIZE : 0) : 0; if ((unsigned long long)path_len > ULLONG_MAX - size) @@ -94,7 +108,11 @@ static unsigned long long per_file_serialize_size(File* file, bool use_metadata) size += file->data->size; /* Symlink entries append the target string (length-prefixed). */ if (file->is_symlink) { - size_t target_len = file->symlink_target ? strlen(file->symlink_target) : 0; + char* wire_target = chunk_encode_wire(file->symlink_target ? file->symlink_target : ""); + if (!wire_target) + return 0; + size_t target_len = strlen(wire_target); + free(wire_target); if (sizeof(size_t) > ULLONG_MAX - size) return 0; size += sizeof(size_t); @@ -128,12 +146,17 @@ Data* chunk_serialize(Chunk* chunk, bool use_metadata) { char* data_pointer = data->data; for (int i = 0; i < chunk->element_count; i++) { File* file = chunk->items[i]; - const char* wire_path = file_wire_path(file); + char* wire_path = chunk_encode_wire(file_wire_path(file)); + if (wire_path == NULL) { + data_destroy(data); + return NULL; + } size_t path_len = strlen(wire_path); memcpy(data_pointer, &path_len, sizeof(size_t)); data_pointer += sizeof(size_t); memcpy(data_pointer, wire_path, path_len); data_pointer += path_len; + free(wire_path); int entry_type = file->is_dir ? 1 : (file->is_symlink ? 2 : (file->is_special ? 3 : 0)); memcpy(data_pointer, &entry_type, sizeof(int)); @@ -159,12 +182,18 @@ Data* chunk_serialize(Chunk* chunk, bool use_metadata) { data_pointer += file_data_size; if (file->is_symlink) { - size_t target_len = file->symlink_target ? strlen(file->symlink_target) : 0; + char* wire_target = chunk_encode_wire(file->symlink_target ? file->symlink_target : ""); + if (wire_target == NULL) { + data_destroy(data); + return NULL; + } + size_t target_len = strlen(wire_target); memcpy(data_pointer, &target_len, sizeof(size_t)); data_pointer += sizeof(size_t); if (target_len > 0) - memcpy(data_pointer, file->symlink_target, target_len); + memcpy(data_pointer, wire_target, target_len); data_pointer += target_len; + free(wire_target); } } return data; @@ -222,6 +251,22 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) { data_pointer += path_len; remaining_size -= path_len; + /* --iconv: the blob holds the wire charset; translate it to the receiver's + local charset before validation and creation so the destination gets the + local name. A name that cannot be decoded fails the file cleanly. */ + if (charset_wire_active()) { + char* local_path = charset_wire_apply(path); + free(path); + if (local_path == NULL) { + log_message(LOG_LEVEL_ERROR, + "--iconv: received chunk file name cannot be converted to the local charset"); + array_list_delete(files); + return NULL; + } + path = local_path; + path_len = strlen(path); + } + if (path_len == 0 || has_path_traversal(path)) { free(path); array_list_delete(files); @@ -392,6 +437,21 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) { array_list_delete(files); return NULL; } + /* The symlink target also rides the wire charset; decode it to the local + charset like the path (a target is a path). */ + if (charset_wire_active()) { + char* local_target = charset_wire_apply(target); + free(target); + if (local_target == NULL) { + log_message(LOG_LEVEL_ERROR, + "--iconv: received chunk symlink target cannot be converted to the local " + "charset"); + file_destroy(file); + array_list_delete(files); + return NULL; + } + target = local_target; + } file->symlink_target = target; data_pointer += target_len; remaining_size -= target_len; diff --git a/src/shared/config.c b/src/shared/config.c index 0b8c2d3..2c0b98f 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -1,4 +1,5 @@ #include "config.h" +#include "charset.h" #include "chmod.h" #include "credentials.h" #include "daemon_conf.h" @@ -42,6 +43,7 @@ static void config_set_defaults(Config* config) { config->auth_user = NULL; config->auth_password_hash = NULL; config->password_file = NULL; + config->iconv_spec = NULL; config->fastsync_server_path = NULL; config->exclude_patterns = NULL; config->exclude_count = 0; @@ -242,7 +244,13 @@ static bool validate_received_config(const Config* config) { config->max_delete >= -1 && config->skip_compress_count >= 0 && config->skip_compress_count <= 10000 && config->max_alloc > 0 && (!config->chmod_spec || !*config->chmod_spec || - chmod_apply(0, config->chmod_spec, &(mode_t){0})); + chmod_apply(0, config->chmod_spec, &(mode_t){0})) && + /* The received --iconv CONVERT_SPEC is untrusted input that drives + the receiver's path decoding: reject a malformed spec or an + unsupported charset name so the run is refused up front instead of + every received file name failing mid-transfer. A NULL spec (iconv + disabled) is always accepted. */ + (!config->iconv_spec || charset_spec_valid(config->iconv_spec)); } Config* config_create(void) { @@ -619,6 +627,7 @@ void config_delete(Config* config) { free(config->auth_user); free(config->auth_password_hash); free(config->password_file); + free(config->iconv_spec); free(config->fastsync_server_path); for (int i = 0; i < config->exclude_count; i++) free(config->exclude_patterns[i]); @@ -1144,6 +1153,29 @@ static bool receive_daemon_auth(int fd, Config* c) { return true; } +/* --iconv CONVERT_SPEC (protocol 2.16.0). Trailing string on the config frame, + * sent after the Wave A/B daemon-auth block and before the ack, so the + * receiver knows the wire charset before the first file name arrives. The full + * spec travels (LOCAL,REMOTE) and each end derives its own LOCAL and the wire + * (REMOTE) charset symmetrically; an unset spec is serialized as "" and + * canonicalized back to NULL on receive. */ +static bool send_iconv_spec(int fd, const Config* c) { + return send_str(fd, c->iconv_spec ? c->iconv_spec : ""); +} + +static bool receive_iconv_spec(int fd, Config* c) { + char* spec = receive_str(fd); + if (!spec) + return false; + if (*spec == '\0') { + free(spec); + c->iconv_spec = NULL; + return true; + } + c->iconv_spec = spec; + return true; +} + bool config_send(int file_descriptor, const Config* config) { protocol_session_set_max_alloc(NULL, config->max_alloc); if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) || @@ -1156,7 +1188,8 @@ bool config_send(int file_descriptor, const Config* config) { !send_metadata_times_options(file_descriptor, config) || !send_symlink_trust_options(file_descriptor, config) || !send_phase4_xattr_options(file_descriptor, config) || - !send_daemon_module(file_descriptor, config) || !send_daemon_auth(file_descriptor, config)) + !send_daemon_module(file_descriptor, config) || !send_daemon_auth(file_descriptor, config) || + !send_iconv_spec(file_descriptor, config)) return false; Status status; if (!receive_status(file_descriptor, &status)) @@ -1198,7 +1231,7 @@ Config* config_receive_with_validate(int file_descriptor, ConfigValidateFunc val !receive_symlink_trust_options(file_descriptor, config) || !receive_phase4_xattr_options(file_descriptor, config) || !receive_daemon_module(file_descriptor, config) || - !receive_daemon_auth(file_descriptor, config)) + !receive_daemon_auth(file_descriptor, config) || !receive_iconv_spec(file_descriptor, config)) goto error; if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 && strcmp(config->compress_choice, "none") != 0) { diff --git a/src/shared/config.h b/src/shared/config.h index 2be5acb..e7e01a2 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -110,6 +110,17 @@ typedef struct Config { * populate auth_user/auth_password_hash before connecting). */ char* password_file; char* fastsync_server_path; + /* --iconv=CONVERT_SPEC (protocol 2.16.0, rsync compatibility): convert the + * charset of FILE NAMES at the wire boundary. CONVERT_SPEC is + * "LOCAL[,REMOTE]": LOCAL is the charset of our own file names, REMOTE is + * the remote side's charset and defaults to LOCAL. The sender converts + * every path LOCAL->REMOTE before transmitting it; the receiver converts + * every received path back REMOTE->LOCAL before creating/writing it. The + * FULL SPEC crosses the wire as a trailing config-frame string so each end + * derives its own LOCAL and the wire (REMOTE) charset symmetrically. NULL + * (or "") means no conversion: identity with zero overhead. See charset.c + * and the PROTOCOL_VERSION note below. */ + char* iconv_spec; char** exclude_patterns; int exclude_count; char** include_patterns; @@ -493,8 +504,21 @@ typedef struct Config { * reads that frame right after the ack (client_send.c) -- symmetric * server->client in every build, so the strict same-version handshake keeps the * two peers in lockstep and nothing can desynchronize. The --stdio SSH path - * sends/reads no MOTD at all. */ -#define PROTOCOL_VERSION "2.15.0" + * sends/reads no MOTD at all. + * + * --iconv Wave (P6): 2.15.0 -> 2.16.0. + * + * WHY the bump, grounded in the wire: the --iconv feature adds a serialized + * field to the binary config frame. The client sends the full CONVERT_SPEC + * (Config->iconv_spec) as a new trailing string AFTER the Wave A/B daemon-auth + * block (in config_send/config_receive), so the receiver knows the wire charset + * (the REMOTE half) before the first file name arrives. Any config-frame + * layout change must bump the protocol version: a peer that does not parse the + * new trailing bytes would desynchronize on the frame boundary, and the strict + * same-version handshake (config_receive rejects a mismatched version before + * parsing anything else) is what keeps a 2.16 client and a 2.15 server from + * ever reaching that state. */ +#define PROTOCOL_VERSION "2.16.0" #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) /* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */ #define MAX_BASIS_DIRS 64 diff --git a/src/shared/file_receive.c b/src/shared/file_receive.c index 24c8233..95ee6aa 100644 --- a/src/shared/file_receive.c +++ b/src/shared/file_receive.c @@ -10,6 +10,7 @@ #include #include "array_list.h" +#include "charset.h" #include "chmod.h" #include "compression.h" #include "config.h" @@ -1555,7 +1556,7 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) { return NULL; } *skipped = false; - char* check_path = receive_str(fd); + char* check_path = receive_wire_str(fd); if (check_path == NULL) { return NULL; } @@ -2082,7 +2083,7 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) { } File* file_receive(const Config* config, int file_descriptor) { - char* path = receive_str(file_descriptor); + char* path = receive_wire_str(file_descriptor); if (path == NULL) return NULL; if (path[0] == '\0' || (!file_get_trust_sender() && has_path_traversal(path))) { @@ -2142,7 +2143,7 @@ File* file_receive(const Config* config, int file_descriptor) { traversal), and the created File is routed through the regular store_file sink so single-threaded and -m receivers handle directories identically. */ File* file_receive_directory(int file_descriptor) { - char* path = receive_str(file_descriptor); + char* path = receive_wire_str(file_descriptor); if (path == NULL) return NULL; if (path[0] == '\0' || (!file_get_trust_sender() && has_path_traversal(path))) { @@ -2169,7 +2170,7 @@ File* file_receive_directory(int file_descriptor) { member. All paths are validated like every other received path (non-empty, relative, no traversal). */ File* file_receive_hardlink(int file_descriptor) { - char* path = receive_str(file_descriptor); + char* path = receive_wire_str(file_descriptor); if (path == NULL) return NULL; if (path[0] == '\0' || (!file_get_trust_sender() && has_path_traversal(path))) { @@ -2186,7 +2187,7 @@ File* file_receive_hardlink(int file_descriptor) { free(path); return NULL; } - char* target = receive_str(file_descriptor); + char* target = receive_wire_str(file_descriptor); if (!target) { free(path); return NULL; @@ -2219,7 +2220,7 @@ File* file_receive_hardlink(int file_descriptor) { routed through the regular store_file sink, which creates the link beneath the receive root (unmungeing the target first). */ File* file_receive_symlink(int file_descriptor, const Config* config) { - char* path = receive_str(file_descriptor); + char* path = receive_wire_str(file_descriptor); if (path == NULL) return NULL; if (path[0] == '\0' || (!file_get_trust_sender() && has_path_traversal(path))) { @@ -2231,7 +2232,7 @@ File* file_receive_symlink(int file_descriptor, const Config* config) { send_status(file_descriptor, STATUS_ERROR); return NULL; } - char* target = receive_str(file_descriptor); + char* target = receive_wire_str(file_descriptor); if (!target) { free(path); return NULL; @@ -2274,7 +2275,7 @@ File* file_receive_symlink(int file_descriptor, const Config* config) { * confined). rdev is validated here (non-negative, range-checked) so a bogus * value cannot drive a dangerous node on the receiver. */ File* file_receive_special(int file_descriptor) { - char* path = receive_str(file_descriptor); + char* path = receive_wire_str(file_descriptor); if (path == NULL) return NULL; if (path[0] == '\0' || (!file_get_trust_sender() && has_path_traversal(path))) { @@ -2354,7 +2355,7 @@ static bool receive_manifest_section(int fd, ArrayList* list, size_t* manifest_b return false; } for (int i = 0; i < count; i++) { - char* s = receive_str(fd); + char* s = receive_wire_str(fd); size_t entry_size = s ? strlen(s) : 0; if (!s || s[0] == '\0' || s[0] == '/' || has_path_traversal(s) || entry_size > MAX_MANIFEST_BYTES - *manifest_bytes || diff --git a/src/shared/file_send.c b/src/shared/file_send.c index 8da018a..e7bcffb 100644 --- a/src/shared/file_send.c +++ b/src/shared/file_send.c @@ -10,6 +10,7 @@ #include #include +#include "charset.h" #include "compression.h" #include "data.h" #include "file.h" @@ -27,7 +28,7 @@ bool file_send_special(const File* file, int file_descriptor, bool use_metadata) return false; if (!send_status(file_descriptor, STATUS_SPECIAL)) return false; - if (!send_str(file_descriptor, file_wire_path(file))) + if (!send_wire_str(file_descriptor, file_wire_path(file))) return false; if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; @@ -61,7 +62,7 @@ bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_ } data_to_send = compressed_data; } - if (send_path && !send_str(file_descriptor, file_wire_path(file))) { + if (send_path && !send_wire_str(file_descriptor, file_wire_path(file))) { data_destroy(compressed_data); return false; } @@ -97,7 +98,7 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta send_path, skip_suffixes, skip_count, compression_threads, send_xattrs); - if (send_path && !send_str(file_descriptor, file_wire_path(file))) + if (send_path && !send_wire_str(file_descriptor, file_wire_path(file))) return false; if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; diff --git a/tests/integration/test_iconv.py b/tests/integration/test_iconv.py new file mode 100644 index 0000000..4ec1db7 --- /dev/null +++ b/tests/integration/test_iconv.py @@ -0,0 +1,250 @@ +"""--iconv=CONVERT_SPEC file-NAME charset conversion integration tests. + +The client converts every source file name from LOCAL to REMOTE before it goes +on the wire, and the receiver converts it back from REMOTE to LOCAL, so a +source tree using one charset can be written into a destination tree using +another (rsync compatibility; content bytes are never touched). +""" +import os +import shutil + +import pytest + +from common import TEST_DATA_DIR, run_client, clean_dir, ServerManager + +LATIN1_NAME = b"caf\xe9.txt" +UTF8_NAME = "caf\u00e9.txt".encode("utf-8") + + +def _make(tag): + source = os.path.join(TEST_DATA_DIR, f"iconv_{tag}_src") + dest = os.path.join(TEST_DATA_DIR, f"iconv_{tag}_dst") + clean_dir(source) + shutil.rmtree(dest, ignore_errors=True) + # The destination ROOT must pre-exist on the receiver (the --mkpath contract: + # without --mkpath the server requires the root directory to exist). + os.makedirs(dest, exist_ok=True) + return source, dest + + +def _place_bytes(root, name_bytes, data=b"latin1 payload\n"): + full = os.path.join(os.fsencode(root), name_bytes) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "wb") as fh: + fh.write(data) + return full + + +def _dest_file(source, dest, name): + base = os.path.join(dest, os.path.abspath(source).lstrip(os.sep)) + return os.path.join(os.fsencode(base), name) + + +@pytest.mark.ci +def test_iconv_latin1_roundtrip(shared_server): + """A source file whose name is ISO-8859-1 bytes is transferred with + --iconv=iso-8859-1,utf-8 and lands on the destination with the ORIGINAL + latin1 name (the wire carried it as UTF-8).""" + source, dest = _make("latin1") + _place_bytes(source, LATIN1_NAME) + + result, _ = run_client( + source, dest, flags=["--iconv=iso-8859-1,utf-8"], port=shared_server.port + ) + assert result.returncode == 0, (result.stderr or result.stdout)[:400] + + dst = _dest_file(source, dest, LATIN1_NAME) + assert os.path.exists(dst), f"dest latin1-named file not found under {dest}" + + +@pytest.mark.ci +def test_iconv_to_utf8_on_wire(shared_server): + """--iconv=utf-8 (single, identity both ways) on an ascii filename transfers + cleanly with no error.""" + source, dest = _make("utf8") + src_path = os.path.join(source, "plain.txt") + with open(src_path, "wb") as fh: + fh.write(b"identity\n") + + result, _ = run_client(source, dest, flags=["--iconv=utf-8"], port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:400] + + dst = _dest_file(source, dest, os.fsencode("plain.txt")) + assert os.path.exists(dst) + + +@pytest.mark.ci +def test_iconv_passthrough_identity(shared_server): + """No --iconv flag: the transfer is unchanged (regression guard -- the common + path must not go through iconv at all).""" + source, dest = _make("identity") + for name, data in (("a.txt", b"aaa\n"), ("sub/b.txt", b"bbb\n")): + p = os.path.join(source, name) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "wb") as fh: + fh.write(data) + + result, _ = run_client(source, dest, port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:400] + + for name in ("a.txt", "sub/b.txt"): + assert os.path.exists(_dest_file(source, dest, os.fsencode(name))) + + +@pytest.mark.ci +def test_iconv_receiver_own_charset(shared_server): + """A dedicated server started with its OWN --iconv converts received names + to ITS charset: the source holds a latin1-named file, the wire carries it + as UTF-8 (from the client's spec), and the receiver re-decodes it to UTF-8 + on disk. This discriminates a real wire conversion from a no-op passthrough + (a latin1 byte sequence is not valid UTF-8, so the receiver decoding it as + UTF-8 would fail the transfer).""" + with ServerManager() as server: + server.start(extra_args=["--iconv=utf-8"]) + source, dest = _make("recv_charset") + _place_bytes(source, LATIN1_NAME) + + result, _ = run_client( + source, dest, flags=["--iconv=iso-8859-1,utf-8"], port=server.port + ) + assert result.returncode == 0, (result.stderr or result.stdout)[:400] + + dst = _dest_file(source, dest, UTF8_NAME) + assert os.path.exists(dst), f"dest UTF-8-named file not found under {dest}" + + +@pytest.mark.ci +def test_iconv_invalid_charset_rejected(shared_server): + """An unsupported charset name is rejected at startup with a nonzero exit.""" + source, dest = _make("badcharset") + src_path = os.path.join(source, "f.txt") + with open(src_path, "wb") as fh: + fh.write(b"x") + + result, _ = run_client( + source, dest, flags=["--iconv=no-such-charset,utf-8"], port=shared_server.port + ) + assert result.returncode != 0 + + +@pytest.mark.ci +def test_iconv_garbage_spec_rejected(shared_server): + """A malformed CONVERT_SPEC is rejected at startup with a nonzero exit.""" + source, dest = _make("garbage") + src_path = os.path.join(source, "f.txt") + with open(src_path, "wb") as fh: + fh.write(b"x") + + result, _ = run_client(source, dest, flags=["--iconv=,,,"], port=shared_server.port) + assert result.returncode != 0 + + +@pytest.mark.ci +def test_iconv_expanding_name_growth(shared_server): + """A long latin1 name whose UTF-8 encoding expands past the initial output + buffer exercises the E2BIG growth path in charset_convert (each high-bit + latin1 byte doubles in UTF-8), and must land unchanged on the destination.""" + source, dest = _make("growth") + name_bytes = b"a" * 40 + bytes(range(0x80, 0x80 + 40)) + b".txt" + _place_bytes(source, name_bytes, data=b"growth\n") + + result, _ = run_client( + source, dest, flags=["--iconv=iso-8859-1,utf-8"], port=shared_server.port + ) + assert result.returncode == 0, (result.stderr or result.stdout)[:400] + + assert os.path.exists(_dest_file(source, dest, name_bytes)) + + +def test_iconv_symlink_path_and_target(shared_server): + """A latin1-named symlink pointing at a latin1-named target survives the + transfer: both the link name and the link target are wire-converted and + re-decoded on the destination (-l preserves links).""" + source, dest = _make("symlink") + target = b"target\xe9.dat" + _place_bytes(source, target, data=b"t\n") + os.symlink(target, os.path.join(os.fsencode(source), b"link\xe9")) + + result, _ = run_client( + source, dest, flags=["--iconv=iso-8859-1,utf-8", "--links"], port=shared_server.port + ) + assert result.returncode == 0, (result.stderr or result.stdout)[:400] + + dst_target = _dest_file(source, dest, target) + dst_link = _dest_file(source, dest, b"link\xe9") + assert os.path.exists(dst_target), "dest latin1 target file missing" + assert os.path.islink(dst_link), "dest latin1 symlink missing" + assert os.readlink(dst_link) == target, "symlink target not preserved/decoded" + with open(dst_link, "rb") as fh: + assert fh.read() == b"t\n" + + +def test_iconv_hardlink_path_and_target(shared_server): + """A latin1-named hard-linked pair is preserved: -H transmits later group + members as a path+target link to the first member, so both the member name + and the target wire-convert (the two destination names must stay one + inode).""" + source, dest = _make("hardlink") + a = b"hl_a\xe9.txt" + b = b"hl_b\xe9.txt" + src_a = os.path.join(os.fsencode(source), a) + with open(src_a, "wb") as fh: + fh.write(b"shared\n") + os.link(src_a, os.path.join(os.fsencode(source), b)) + + result, _ = run_client( + source, dest, flags=["--iconv=iso-8859-1,utf-8", "--hard-links"], + port=shared_server.port, + ) + assert result.returncode == 0, (result.stderr or result.stdout)[:400] + + dst_a = _dest_file(source, dest, a) + dst_b = _dest_file(source, dest, b) + assert os.path.exists(dst_a) and os.path.exists(dst_b) + assert os.stat(dst_a).st_ino == os.stat(dst_b).st_ino, \ + "hard-link relationship not preserved across the transfer" + + +def test_iconv_delete_manifest_consistent(shared_server): + """Combining --iconv with --delete: the delete manifest's keep-set paths are + wire-converted on send and disk-converted on receive, so the receiver's + delete walker compares like with like and removes exactly the missing + latin1-named file (never a wrong-named mirror).""" + source, dest = _make("delete") + keep = b"keep\xe9.txt" + gone = b"gone\xe9.txt" + _place_bytes(source, keep, data=b"k\n") + _place_bytes(source, gone, data=b"g\n") + + with ServerManager() as server: + server.start(extra_args=["--allow-delete"]) + flags = ["--iconv=iso-8859-1,utf-8"] + result, _ = run_client(source, dest, flags=flags, port=server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:400] + assert os.path.exists(_dest_file(source, dest, keep)) + assert os.path.exists(_dest_file(source, dest, gone)) + + os.remove(os.path.join(os.fsencode(source), gone)) + result, _ = run_client( + source, dest, flags=flags + ["--delete"], port=server.port + ) + assert result.returncode == 0, (result.stderr or result.stdout)[:400] + assert os.path.exists(_dest_file(source, dest, keep)), "kept file deleted" + assert not os.path.exists(_dest_file(source, dest, gone)), \ + "missing file was not deleted" + + +def test_iconv_chunk_serialization_blob(shared_server): + """-s (chunk serialization) embeds paths and symlink targets inside the + serialized chunk blob rather than as separate frames; a latin1 name must + still wire-convert and re-decoded on the destination.""" + source, dest = _make("chunk") + name = b"\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9.txt" + _place_bytes(source, name, data=b"blob\n") + + result, _ = run_client( + source, dest, flags=["--iconv=iso-8859-1,utf-8", "-s"], port=shared_server.port + ) + assert result.returncode == 0, (result.stderr or result.stdout)[:400] + + assert os.path.exists(_dest_file(source, dest, name)) \ No newline at end of file diff --git a/tests/runner.c b/tests/runner.c index 5cf2d54..1492be0 100644 --- a/tests/runner.c +++ b/tests/runner.c @@ -14,6 +14,7 @@ #include "test_file_sendfile.h" #include "test_fuzz_smoke.h" #include "test_glob.h" +#include "test_iconv.h" #include "test_log.h" #include "test_metadata.h" #include "test_motd.h" @@ -59,6 +60,7 @@ int main() { RUN_TEST(test_protocol); RUN_TEST(test_metadata); RUN_TEST(test_glob); + RUN_TEST(test_iconv); RUN_TEST(test_file); RUN_TEST(test_trust_sender); RUN_TEST(test_delay_updates); diff --git a/tests/test_config.c b/tests/test_config.c index 49775d4..d5b8c4e 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -1565,6 +1565,110 @@ static void test_config_local_only_fields_not_serialized() { EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0); } +static void test_config_iconv_spec_wire_roundtrip() { + Config* send_cfg = config_create(); + EXPECT_NOT_NULL(send_cfg); + send_cfg->send_directory = str_dup("/src"); + send_cfg->receive_root_directory = str_dup("rel/path"); + send_cfg->iconv_spec = str_dup("utf-8,iso-8859-1"); + + int p[2]; + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + io_set_fds(p[0], p[1]); + io_set_bwlimit(0); + + pid_t pid = fork(); + if (pid == 0) { + close(p[1]); + io_set_fds(p[0], p[0]); + Config* recv_cfg = config_receive(p[0]); + bool ok = recv_cfg != NULL && recv_cfg->iconv_spec != NULL && + strcmp(recv_cfg->iconv_spec, "utf-8,iso-8859-1") == 0; + config_delete(recv_cfg); + close(p[0]); + _exit(ok ? 0 : 1); + } else { + close(p[0]); + io_set_fds(p[1], p[1]); + bool sent = config_send(p[1], send_cfg); + int status; + waitpid(pid, &status, 0); + close(p[1]); + config_delete(send_cfg); + EXPECT_TRUE(sent); + EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0); + } +} + +static void test_config_iconv_spec_empty_canonicalizes_to_null() { + Config* send_cfg = config_create(); + EXPECT_NOT_NULL(send_cfg); + send_cfg->send_directory = str_dup("/src"); + send_cfg->receive_root_directory = str_dup("/dst"); + /* iconv_spec left NULL -> serialized as "" -> received back as NULL. */ + + int p[2]; + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + io_set_fds(p[0], p[1]); + io_set_bwlimit(0); + + pid_t pid = fork(); + if (pid == 0) { + close(p[1]); + io_set_fds(p[0], p[0]); + Config* recv_cfg = config_receive(p[0]); + bool ok = recv_cfg != NULL && recv_cfg->iconv_spec == NULL; + config_delete(recv_cfg); + close(p[0]); + _exit(ok ? 0 : 1); + } else { + close(p[0]); + io_set_fds(p[1], p[1]); + bool sent = config_send(p[1], send_cfg); + int status; + waitpid(pid, &status, 0); + close(p[1]); + config_delete(send_cfg); + EXPECT_TRUE(sent); + EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0); + } +} + +static void test_config_receive_rejects_invalid_iconv_spec() { + Config* send_cfg = config_create(); + EXPECT_NOT_NULL(send_cfg); + send_cfg->send_directory = str_dup("/src"); + send_cfg->receive_root_directory = str_dup("/dst"); + send_cfg->iconv_spec = str_dup("no-such-charset,utf-8"); + + int p[2]; + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + io_set_fds(p[0], p[1]); + io_set_bwlimit(0); + + pid_t pid = fork(); + if (pid == 0) { + close(p[1]); + io_set_fds(p[0], p[0]); + /* A malformed/unsupported spec must be refused at the config handshake + (STATUS_ERROR makes config_send fail on the parent). */ + Config* recv_cfg = config_receive(p[0]); + config_delete(recv_cfg); + close(p[0]); + _exit(recv_cfg ? 1 : 0); + } else { + close(p[0]); + io_set_fds(p[1], p[1]); + bool sent = config_send(p[1], send_cfg); + int status; + waitpid(pid, &status, 0); + close(p[1]); + config_delete(send_cfg); + EXPECT_FALSE(sent); + EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0); + } +} + void test_config() { test_config_lifecycle(); test_config_ssh_dest(); @@ -1608,6 +1712,9 @@ void test_config() { test_config_module_wire_empty_canonicalizes_to_null(); test_config_daemon_auth_wire_roundtrip(); test_config_daemon_auth_wire_rejects_malformed(); + test_config_iconv_spec_wire_roundtrip(); + test_config_iconv_spec_empty_canonicalizes_to_null(); + test_config_receive_rejects_invalid_iconv_spec(); test_config_receive_with_validate_rejects(); } test_config_delete_timing_early_helper(); diff --git a/tests/test_iconv.c b/tests/test_iconv.c new file mode 100644 index 0000000..faee11b --- /dev/null +++ b/tests/test_iconv.c @@ -0,0 +1,217 @@ +#include "test_iconv.h" +#include "charset.h" +#include "protocol.h" +#include "test_utils.h" +#include "utils.h" +#include +#include +#include +#include +#include +#include + +/* --- CONVERT_SPEC parsing ------------------------------------------------ */ + +static void test_iconv_spec_parse_split() { + char* local = NULL; + char* remote = NULL; + EXPECT_EQ_INT(charset_spec_parse("utf-8,iso-8859-1", &local, &remote), 0); + EXPECT_EQ_STR(local, "utf-8"); + EXPECT_EQ_STR(remote, "iso-8859-1"); + free(local); + free(remote); +} + +static void test_iconv_spec_parse_single_defaults_to_local() { + char* local = NULL; + char* remote = NULL; + EXPECT_EQ_INT(charset_spec_parse("utf-8", &local, &remote), 0); + EXPECT_EQ_STR(local, "utf-8"); + EXPECT_EQ_STR(remote, "utf-8"); + free(local); + free(remote); +} + +static void test_iconv_spec_parse_garbage() { + char* local = NULL; + char* remote = NULL; + EXPECT_EQ_INT(charset_spec_parse(NULL, &local, &remote), -1); + EXPECT_EQ_INT(charset_spec_parse("", &local, &remote), -1); + EXPECT_EQ_INT(charset_spec_parse(",", &local, &remote), -1); + EXPECT_EQ_INT(charset_spec_parse("utf-8,", &local, &remote), -1); + EXPECT_EQ_INT(charset_spec_parse(",utf-8", &local, &remote), -1); +} + +static void test_iconv_spec_valid() { + EXPECT_TRUE(charset_spec_valid(NULL)); + EXPECT_TRUE(charset_spec_valid("utf-8")); + EXPECT_TRUE(charset_spec_valid("utf-8,iso-8859-1")); + EXPECT_TRUE(charset_spec_valid("iso-8859-1,ascii")); + EXPECT_FALSE(charset_spec_valid("no-such-charset,utf-8")); + EXPECT_FALSE(charset_spec_valid("utf-8,no-such-charset")); + EXPECT_FALSE(charset_spec_valid(",,,")); + EXPECT_FALSE(charset_spec_valid("utf-8,")); + /* A target charset whose conversion emits embedded NUL bytes would be + truncated by the C-string wire helpers; it must be rejected up front. */ + EXPECT_FALSE(charset_spec_valid("utf-8,utf-16")); + EXPECT_FALSE(charset_spec_valid("utf-16")); + EXPECT_FALSE(charset_spec_valid("iso-8859-1,utf-16")); +} + +/* --- one-shot conversion ------------------------------------------------ */ + +static void test_iconv_utf8_to_latin1() { + void* conv = charset_conversion_open("utf-8", "iso-8859-1"); + EXPECT_NOT_NULL(conv); + char* out = charset_convert(conv, "caf\xc3\xa9", NULL); + EXPECT_NOT_NULL(out); + EXPECT_EQ_INT(strcmp(out, "caf\xe9"), 0); + free(out); + charset_conversion_close(conv); +} + +static void test_iconv_latin1_to_utf8() { + void* conv = charset_conversion_open("iso-8859-1", "utf-8"); + EXPECT_NOT_NULL(conv); + char* out = charset_convert(conv, "caf\xe9", NULL); + EXPECT_NOT_NULL(out); + EXPECT_EQ_INT(strcmp(out, "caf\xc3\xa9"), 0); + free(out); + charset_conversion_close(conv); +} + +static void test_iconv_invalid_sequence_fails() { + int err = 0; + /* 0xff is not a valid UTF-8 sequence. */ + void* conv = charset_conversion_open("utf-8", "ascii"); + EXPECT_NOT_NULL(conv); + EXPECT_TRUE(charset_convert(conv, "bad\xff", &err) == NULL); + EXPECT_TRUE(err == EILSEQ || err == EINVAL); + charset_conversion_close(conv); +} + +static void test_iconv_unrepresentable_fails() { + /* "caf\xc3\xa9" (UTF-8 for cafe) has no ASCII representation. */ + void* conv = charset_conversion_open("utf-8", "ascii"); + EXPECT_NOT_NULL(conv); + EXPECT_TRUE(charset_convert(conv, "caf\xc3\xa9", NULL) == NULL); + charset_conversion_close(conv); +} + +/* A latin1 high-bit byte expands to two UTF-8 bytes. With exactly 16 high + * bytes the output is exactly cap = in_len + 16, so the final iconv call fills + * the buffer completely and a naive NUL-terminator write would overflow. */ +static void test_iconv_exact_fill_no_overflow() { + char name[64]; + strcpy(name, "dir/"); + int n = 4; + for (int i = 0; i < 16; i++) + name[n++] = (char)(0x80 + i); + name[n] = '\0'; + + void* conv = charset_conversion_open("iso-8859-1", "utf-8"); + EXPECT_NOT_NULL(conv); + char* out = charset_convert(conv, name, NULL); + EXPECT_NOT_NULL(out); + EXPECT_EQ_INT((int)strlen(out), n + 16); + charset_conversion_close(conv); + free(out); +} + +/* Many high-bit bytes force the output buffer past its initial cap, exercising + * the E2BIG growth path (input partially consumed/produced before the grow). */ +static void test_iconv_growth_expanding_name() { + char name[256]; + strcpy(name, "dir/"); + int n = 4; + for (int i = 0; i < 80; i++) + name[n++] = (char)(0x80 + (i % 0x80)); + name[n] = '\0'; + + void* conv = charset_conversion_open("iso-8859-1", "utf-8"); + EXPECT_NOT_NULL(conv); + char* out = charset_convert(conv, name, NULL); + EXPECT_NOT_NULL(out); + EXPECT_EQ_INT((int)strlen(out), n + 80); + charset_conversion_close(conv); + free(out); +} + +/* --- process-wide wire conversion ---------------------------------------- */ + +static void test_iconv_wire_sender_converts_local_to_remote() { + EXPECT_TRUE(charset_wire_init_sender("utf-8,iso-8859-1")); + char* wire = charset_wire_apply("caf\xc3\xa9"); + EXPECT_NOT_NULL(wire); + EXPECT_EQ_INT(strcmp(wire, "caf\xe9"), 0); + free(wire); + charset_wire_free(); +} + +static void test_iconv_wire_receiver_converts_remote_to_local() { + EXPECT_TRUE(charset_wire_init_receiver("utf-8,iso-8859-1", NULL)); + char* local = charset_wire_apply("caf\xe9"); + EXPECT_NOT_NULL(local); + EXPECT_EQ_INT(strcmp(local, "caf\xc3\xa9"), 0); + free(local); + charset_wire_free(); +} + +static void test_iconv_wire_disabled_passthrough() { + charset_wire_init_sender(NULL); + EXPECT_FALSE(charset_wire_active()); + char* out = charset_wire_apply("plain/name\xff"); + EXPECT_NOT_NULL(out); + EXPECT_EQ_INT(strcmp(out, "plain/name\xff"), 0); + free(out); + charset_wire_free(); +} + +static void test_iconv_wire_str_roundtrip() { + EXPECT_TRUE(charset_wire_init_sender("utf-8,iso-8859-1")); + int p[2]; + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + io_set_fds(p[0], p[1]); + io_set_bwlimit(0); + + pid_t pid = fork(); + if (pid == 0) { + close(p[1]); + io_set_fds(p[0], p[0]); + charset_wire_free(); + charset_wire_init_receiver("utf-8,iso-8859-1", NULL); + char* got = receive_wire_str(p[0]); + bool ok = got != NULL && strcmp(got, "caf\xc3\xa9") == 0; + free(got); + charset_wire_free(); + close(p[0]); + _exit(ok ? 0 : 1); + } else { + close(p[0]); + io_set_fds(p[1], p[1]); + bool sent = send_wire_str(p[1], "caf\xc3\xa9"); + int status; + waitpid(pid, &status, 0); + close(p[1]); + charset_wire_free(); + EXPECT_TRUE(sent); + EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0); + } +} + +void test_iconv() { + test_iconv_spec_parse_split(); + test_iconv_spec_parse_single_defaults_to_local(); + test_iconv_spec_parse_garbage(); + test_iconv_spec_valid(); + test_iconv_utf8_to_latin1(); + test_iconv_latin1_to_utf8(); + test_iconv_invalid_sequence_fails(); + test_iconv_unrepresentable_fails(); + test_iconv_exact_fill_no_overflow(); + test_iconv_growth_expanding_name(); + test_iconv_wire_sender_converts_local_to_remote(); + test_iconv_wire_receiver_converts_remote_to_local(); + test_iconv_wire_disabled_passthrough(); + test_iconv_wire_str_roundtrip(); +} \ No newline at end of file diff --git a/tests/test_iconv.h b/tests/test_iconv.h new file mode 100644 index 0000000..8d54ec6 --- /dev/null +++ b/tests/test_iconv.h @@ -0,0 +1,6 @@ +#ifndef TEST_ICONV_H +#define TEST_ICONV_H + +void test_iconv(void); + +#endif \ No newline at end of file