diff --git a/CMakeLists.txt b/CMakeLists.txt index fcc7cf4..727150b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.22) -project(FastFileTransfer VERSION 2.22.0) +project(FastFileTransfer VERSION 2.23.0) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_C_STANDARD 11) @@ -96,6 +96,7 @@ set(SHARED_SRCS src/shared/file_send.c src/shared/file_store.c src/shared/filter.c + src/shared/format.c src/shared/hardlink.c src/shared/identity.c src/shared/log.c @@ -213,6 +214,7 @@ set(TEST_SRCS tests/test_file.c tests/test_file_list.c tests/test_file_sendfile.c + tests/test_format.c tests/test_fuzz_smoke.c tests/test_glob.c tests/test_hardlink.c diff --git a/src/client/change_list.c b/src/client/change_list.c index 5cddaf0..dbe0ca1 100644 --- a/src/client/change_list.c +++ b/src/client/change_list.c @@ -7,21 +7,7 @@ #include #include #include - -/* Itemize code emitted for a transferred regular file. - * - * Layout (rsync-compatible 11-char item): `>f` marks a regular file that was - * transferred to the remote host; the trailing nine markers are, in order, - * c(hecksum) s(ize) t(ime) p(erms) o(wner) g(roup) u(ser/acl) a(ttrs) x(attrs). - * Every marker is `+` (FastSync does not compare each attribute on the - * receiving side, so a sent file is reported as fully updated). Files that - * are already up to date print no line at all, matching rsync's single -i - * which only itemizes changes. - * - * Because the scanner only yields regular-file transfer candidates, `>d` - * (directory) lines are never produced; directories are not transferred as - * items by FastSync. */ -#define ITEMIZE_SENT_FILE ">f+++++++++" +#include typedef struct { char* data; @@ -80,103 +66,14 @@ static bool strbuf_append(StrBuf* buf, const char* text) { return true; } -static bool strbuf_append_ull(StrBuf* buf, unsigned long long value) { - char digits[32]; - int written = snprintf(digits, sizeof(digits), "%llu", value); - if (written < 0 || (size_t)written >= sizeof(digits)) - return false; - return strbuf_append(buf, digits); -} - -static bool strbuf_append_longlong(StrBuf* buf, long long value) { - char digits[32]; - int written = snprintf(digits, sizeof(digits), "%lld", value); - if (written < 0 || (size_t)written >= sizeof(digits)) - return false; - return strbuf_append(buf, digits); -} - bool change_list_enabled(const Config* config) { return config != NULL && (config->itemize_changes || config->out_format != NULL || (config->log_file != NULL && config->log_file_format != NULL)); } -char* change_render_itemize(const ChangeEvent* event) { - if (event == NULL || event->decision != CHANGE_SENT) - return str_dup(""); - const char* code = event->is_directory ? ">d+++++++++" : ITEMIZE_SENT_FILE; - StrBuf line = {0}; - bool ok = strbuf_append(&line, code) && strbuf_append(&line, " ") && - strbuf_append(&line, event->path != NULL ? event->path : ""); - if (!ok) { - strbuf_free(&line); - return NULL; - } - return line.data; -} +/* ---- Itemize code ---- */ -static const char* leaf_name(const char* path) { - if (path == NULL) - return ""; - const char* slash = strrchr(path, '/'); - return slash != NULL && slash[1] != '\0' ? slash + 1 : path; -} - -char* change_render_format(const char* format, const ChangeEvent* event) { - if (format == NULL) - return NULL; - StrBuf line = {0}; - bool ok = true; - for (const char* p = format; *p != '\0' && ok;) { - if (*p != '%') { - ok = strbuf_append_char(&line, *p); - p++; - continue; - } - char token = p[1]; - if (token == '\0') { - ok = strbuf_append_char(&line, '%'); - break; - } - switch (token) { - case '%': - ok = strbuf_append_char(&line, '%'); - break; - case 'f': - ok = strbuf_append(&line, event->path != NULL ? event->path : ""); - break; - case 'n': - ok = strbuf_append(&line, leaf_name(event->path)); - break; - case 'l': - ok = strbuf_append_ull(&line, event->size); - break; - case 'b': - ok = strbuf_append_ull(&line, event->bytes_sent); - break; - case 'M': - ok = strbuf_append_longlong(&line, (long long)event->mtime_sec); - break; - default: - /* Unknown escape sequences are preserved verbatim. */ - ok = strbuf_append_char(&line, '%') && strbuf_append_char(&line, token); - break; - } - p += 2; - } - if (!ok) { - strbuf_free(&line); - return NULL; - } - if (line.data == NULL) { - line.data = str_dup(""); - if (!line.data) - return NULL; - } - return line.data; -} - -/* Format a mode as an `ls -l` permission string, e.g. `-rw-r--r--`. */ +/* Format the permission bits as an `ls -l` string, e.g. `-rw-r--r--`. */ static void mode_to_ls_string(mode_t mode, char out[11]) { out[0] = S_ISDIR(mode) ? 'd' : S_ISLNK(mode) ? 'l' @@ -198,29 +95,102 @@ static void mode_to_ls_string(mode_t mode, char out[11]) { out[10] = '\0'; } -char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime, - const char* path) { - char permission[11]; - mode_to_ls_string(mode, permission); - char date[32]; - struct tm broken_down; - if (localtime_r(&mtime, &broken_down) != NULL) { - if (strftime(date, sizeof(date), "%Y/%m/%d %H:%M:%S", &broken_down) == 0) - snprintf(date, sizeof(date), "?"); - } else { - snprintf(date, sizeof(date), "?"); +static char itemize_type_char(const ChangeEvent* event) { + if (event->is_directory) + return 'd'; + if (event->is_symlink) + return 'L'; + if (event->is_special) { + if (S_ISCHR(event->mode) || S_ISBLK(event->mode)) + return 'D'; + return 'S'; } + return 'f'; +} + +static bool times_match(const Config* config, const ChangeEvent* event) { + if (!event->dest.known || !event->dest.existed) + return false; + if (event->mtime_sec == event->dest.mtime_sec) + return event->mtime_nsec == event->dest.mtime_nsec; + long long delta = (long long)event->mtime_sec - (long long)event->dest.mtime_sec; + if (delta < 0) + delta = -delta; + return delta <= (long long)config->modify_window; +} + +/* Fill the 11-character itemize code (10 chars + NUL). `created` means the + * destination entry did not exist, so every attribute marker is `+`. */ +static void itemize_code(const Config* config, const ChangeEvent* event, char code[12]) { + bool known = event->dest.known; + bool created = !known || !event->dest.existed; + char update; + if (event->is_hardlink) + update = 'h'; + else if (created) + update = (event->is_directory || event->is_symlink || event->is_special) ? 'c' : '>'; + else + update = '>'; + code[0] = update; + code[1] = itemize_type_char(event); + if (created) { + for (int i = 0; i < 9; i++) + code[2 + i] = '+'; + code[11] = '\0'; + return; + } + bool size_diff = event->size != event->dest.size; + bool time_diff = !times_match(config, event); + bool perms_diff = (event->mode & 07777) != (event->dest.mode & 07777); + bool owner_diff = event->uid != (uid_t)event->dest.uid; + bool group_diff = event->gid != (gid_t)event->dest.gid; + code[2] = '.'; /* checksum: no destination digest available */ + code[3] = size_diff ? 's' : '.'; + code[4] = time_diff ? 't' : '.'; + code[5] = (config->preserve_perms && perms_diff) ? 'p' : '.'; + code[6] = (config->preserve_owner && owner_diff) ? 'o' : '.'; + code[7] = (config->preserve_group && group_diff) ? 'g' : '.'; + code[8] = '.'; /* reserved */ + code[9] = '.'; /* acl: not compared */ + code[10] = '.'; + code[11] = '\0'; +} + +char* change_render_itemize_code(const Config* config, const ChangeEvent* event) { + if (event == NULL || event->decision != CHANGE_SENT) + return str_dup(""); + char code[12]; + itemize_code(config, event, code); + return str_dup(code); +} + +/* rsync %n: the transfer-relative name, with a trailing slash for directories. */ +static bool append_name(StrBuf* buf, const ChangeEvent* event) { + if (!strbuf_append(buf, event->name != NULL ? event->name : "")) + return false; + if (event->is_directory && (event->name == NULL || event->name[0] == '\0' || + event->name[strlen(event->name) - 1] != '/')) + return strbuf_append_char(buf, '/'); + return true; +} + +/* rsync %L: " -> target" for a symlink, " => target" for a hard link, else "". */ +static bool append_link_suffix(StrBuf* buf, const ChangeEvent* event) { + if (event->is_symlink && event->symlink_target != NULL) + return strbuf_append(buf, " -> ") && strbuf_append(buf, event->symlink_target); + if (event->is_hardlink && event->hardlink_target != NULL) + return strbuf_append(buf, " => ") && strbuf_append(buf, event->hardlink_target); + return true; +} + +char* change_render_itemize(const Config* config, const ChangeEvent* event) { + if (event == NULL || event->decision != CHANGE_SENT) + return str_dup(""); + char code[12]; + itemize_code(config, event, code); StrBuf line = {0}; - char size_field[32]; - int written = snprintf(size_field, sizeof(size_field), "%llu", size); - if (written < 0 || (size_t)written >= sizeof(size_field)) { - strbuf_free(&line); - return NULL; - } - bool ok = strbuf_append(&line, permission) && strbuf_append_char(&line, ' ') && - strbuf_append(&line, size_field) && strbuf_append_char(&line, ' ') && - strbuf_append(&line, date) && strbuf_append_char(&line, ' ') && - strbuf_append(&line, path != NULL ? path : ""); + bool ok = strbuf_append(&line, code) && strbuf_append_char(&line, ' ') && + append_name(&line, event) && append_link_suffix(&line, event); if (!ok) { strbuf_free(&line); return NULL; @@ -228,6 +198,141 @@ char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime return line.data; } +/* ---- --out-format / --log-file-format ---- */ + +char* change_render_format(const char* format, const Config* config, const ChangeEvent* event) { + if (format == NULL || event == NULL) + return NULL; + StrBuf line = {0}; + bool ok = true; + for (const char* p = format; *p != '\0' && ok;) { + if (*p != '%') { + ok = strbuf_append_char(&line, *p); + p++; + continue; + } + char token = p[1]; + if (token == '\0') { + ok = strbuf_append_char(&line, '%'); + break; + } + switch (token) { + case '%': + ok = strbuf_append_char(&line, '%'); + break; + case 'i': { + char code[12]; + itemize_code(config, event, code); + ok = strbuf_append(&line, code); + break; + } + case 'f': + ok = strbuf_append(&line, event->path != NULL ? event->path : ""); + break; + case 'n': + ok = append_name(&line, event); + break; + case 'L': + ok = append_link_suffix(&line, event); + break; + case 'l': { + char digits[32]; + int written = snprintf(digits, sizeof(digits), "%llu", event->size); + ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits); + } break; + case 'b': { + char digits[32]; + int written = snprintf(digits, sizeof(digits), "%llu", event->bytes_sent); + ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits); + } break; + case 'M': { + char when[32]; + if (format_rsync_datetime(event->mtime_sec, true, when, sizeof(when))) + ok = strbuf_append(&line, when); + } break; + case 't': { + char when[32]; + if (format_rsync_datetime(time(NULL), false, when, sizeof(when))) + ok = strbuf_append(&line, when); + } break; + case 'o': + ok = strbuf_append(&line, "send"); + break; + case 'p': { + char digits[32]; + int written = snprintf(digits, sizeof(digits), "%ld", (long)getpid()); + ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits); + } break; + case 'B': { + char permission[11]; + mode_to_ls_string(event->mode, permission); + ok = strbuf_append(&line, permission + 1); + } break; + case 'U': { + char digits[32]; + int written = snprintf(digits, sizeof(digits), "%u", (unsigned)event->uid); + ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits); + } break; + case 'G': { + char digits[32]; + int written = snprintf(digits, sizeof(digits), "%u", (unsigned)event->gid); + ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits); + } break; + default: + /* Unknown escape sequences are preserved verbatim. */ + ok = strbuf_append_char(&line, '%') && strbuf_append_char(&line, token); + break; + } + p += 2; + } + if (!ok) { + strbuf_free(&line); + return NULL; + } + if (line.data == NULL) { + line.data = str_dup(""); + if (!line.data) + return NULL; + } + return line.data; +} + +/* ---- --list-only ---- */ + +char* change_render_list_line(const Config* config, const ChangeEvent* event) { + (void)config; + if (event == NULL) + return NULL; + char permission[11]; + mode_to_ls_string(event->mode, permission); + char date[32]; + if (!format_rsync_datetime(event->mtime_sec, false, date, sizeof(date))) + snprintf(date, sizeof(date), "?"); + StrBuf line = {0}; + char size_field[40]; + char grouped[32]; + if (!format_big_num(event->size, false, grouped, sizeof(grouped))) { + strbuf_free(&line); + return NULL; + } + int written = snprintf(size_field, sizeof(size_field), "%15s", grouped); + if (written < 0 || (size_t)written >= sizeof(size_field)) { + strbuf_free(&line); + return NULL; + } + const char* name = event->name != NULL && event->name[0] != '\0' ? event->name : "."; + bool ok = strbuf_append(&line, permission) && strbuf_append(&line, size_field) && + strbuf_append_char(&line, ' ') && strbuf_append(&line, date) && + strbuf_append_char(&line, ' ') && strbuf_append(&line, name); + if (!ok) { + strbuf_free(&line); + return NULL; + } + return line.data; +} + +/* ---- Event emission ---- */ + static void print_escaped_line(FILE* stream, const char* line, bool eight_bit_output) { char* escaped = output_escape(line, eight_bit_output); if (escaped != NULL) { @@ -247,15 +352,16 @@ void change_emit(const Config* config, const ChangeEvent* event) { bool to_stdout = config->itemize_changes || config->out_format != NULL; bool to_log = config->log_file != NULL && config->log_file_format != NULL; if (to_stdout) { - char* line = config->out_format != NULL ? change_render_format(config->out_format, event) - : change_render_itemize(event); + char* line = config->out_format != NULL + ? change_render_format(config->out_format, config, event) + : change_render_itemize(config, event); if (line != NULL) { print_escaped_line(stdout, line, config->eight_bit_output); free(line); } } if (to_log) { - char* line = change_render_format(config->log_file_format, event); + char* line = change_render_format(config->log_file_format, config, event); if (line != NULL) { print_escaped_line(config->log_file, line, config->eight_bit_output); free(line); @@ -266,9 +372,6 @@ void change_emit(const Config* config, const ChangeEvent* event) { static bool format_uses_mtime(const char* format) { if (format == NULL) return false; - /* Mirror change_render_format's tokenizer: "%%" is a literal percent (so - * "%%M" does NOT expand %M) and unknown "%X" escapes consume both chars. - * This keeps the optional stat() fallback below in step with the renderer. */ for (const char* p = format; *p != '\0';) { if (*p != '%') { p++; @@ -284,46 +387,136 @@ static bool format_uses_mtime(const char* format) { return false; } +/* Relative path of an entry below the transfer root (no leading slash). Uses + * the sender-side send_path override when present (bare-relative -R layout). */ +static char* relative_name(const Config* config, const File* file) { + const char* full = file_wire_path(file); + if (file->send_path != NULL) + return str_dup(full != NULL ? full : ""); + const char* root = config->send_directory; + if (root == NULL || full == NULL) + return str_dup(full != NULL ? full : ""); + size_t root_len = strlen(root); + while (root_len > 1 && root[root_len - 1] == '/') + root_len--; + if (strncmp(root, full, root_len) == 0) { + if (full[root_len] == '\0') + return str_dup(""); + if (full[root_len] == '/') + return str_dup(full + root_len + 1); + } + return str_dup(full); +} + +/* rsync %f long form: the source argument as typed (leading '/' removed, + * trailing '/' removed, leading "./" removed) joined to the relative name. */ +static char* display_name(const Config* config, const char* name) { + const char* root = config->send_directory; + if (root == NULL) + return str_dup(name != NULL ? name : ""); + const char* p = root; + while (*p == '/') + p++; + if (p[0] == '.' && p[1] == '/') + p += 2; + size_t root_len = strlen(p); + while (root_len > 0 && p[root_len - 1] == '/') + root_len--; + size_t name_len = name != NULL ? strlen(name) : 0; + if (root_len == 0 && name_len == 0) + return str_dup(""); + char* out = malloc(root_len + (root_len > 0 && name_len > 0 ? 1 : 0) + name_len + 1); + if (!out) + return NULL; + size_t offset = 0; + if (root_len > 0) { + memcpy(out, p, root_len); + offset = root_len; + } + if (root_len > 0 && name_len > 0) + out[offset++] = '/'; + if (name_len > 0) + memcpy(out + offset, name, name_len); + out[offset + name_len] = '\0'; + return out; +} + +static void fill_event_from_file(const Config* config, const File* file, ChangeEvent* event, + char** name_out, char** path_out) { + char* name = relative_name(config, file); + char* path = display_name(config, name); + event->name = name; + event->path = path; + *name_out = name; + *path_out = path; + if (file->metadata != NULL) { + event->mtime_sec = file->metadata->mtime_sec; + event->mtime_nsec = file->metadata->mtime_nsec; + event->mode = file->metadata->mode; + event->uid = file->metadata->uid; + event->gid = file->metadata->gid; + } else if (format_uses_mtime(config->out_format) || format_uses_mtime(config->log_file_format)) { + struct stat st; + if (file->path != NULL && stat(file->path, &st) == 0) { + event->mtime_sec = st.st_mtime; + event->mtime_nsec = st.st_mtim.tv_nsec; + } + } +} + void change_emit_file_sent(const Config* config, const File* file) { if (file == NULL || !change_list_enabled(config)) return; ChangeEvent event; memset(&event, 0, sizeof(event)); - /* The displayed path is the one transmitted (with -R + --files-from this is - the bare relative destination path); the metadata fallback below still - stats the local absolute path. */ - event.path = file_wire_path(file); event.decision = CHANGE_SENT; event.is_directory = false; + event.is_symlink = false; + event.is_special = false; + event.is_hardlink = false; event.size = file->data != NULL ? file->data->size : 0; - /* FastSync has no wire-byte counter yet, so %b reports the source length - * that had to be delivered (always equal to %l); the actual bytes written - * to the socket (compressed/delta) are not measured. */ - event.bytes_sent = event.size; - if (file->metadata != NULL) { - event.mtime_sec = file->metadata->mtime_sec; - } else if (format_uses_mtime(config->out_format) || format_uses_mtime(config->log_file_format)) { - /* Best-effort fallback for %M when no metadata was captured (no -M): the - * path is stat()ed just to fill the field, and any failure leaves 0. */ - struct stat st; - if (file->path != NULL && stat(file->path, &st) == 0) - event.mtime_sec = st.st_mtime; + event.dest = file->dest_state; + if (file->is_symlink) { + event.is_symlink = true; + event.symlink_target = file->symlink_target; + event.size = file->symlink_target != NULL ? strlen(file->symlink_target) : 0; + event.bytes_sent = 0; + } else if (file->is_special) { + event.is_special = true; + event.bytes_sent = 0; + } else if (file->link_group != 0 && !file->link_first) { + event.is_hardlink = true; + event.hardlink_target = file->hardlink_target; + event.bytes_sent = 0; + } else { + /* Literal payload bytes delivered; compressed/delta wire bytes are not + * separately counted. */ + event.bytes_sent = event.size; } - change_emit(config, &event); + char* name = NULL; + char* path = NULL; + fill_event_from_file(config, file, &event, &name, &path); + if (name != NULL && path != NULL) + change_emit(config, &event); + free(name); + free(path); } -/* Build and emit a CHANGE_SENT event for an explicit directory entry (-d). */ void change_emit_dir_sent(const Config* config, const File* file) { if (file == NULL || !change_list_enabled(config)) return; ChangeEvent event; memset(&event, 0, sizeof(event)); - event.path = file_wire_path(file); event.decision = CHANGE_SENT; event.is_directory = true; event.size = 0; event.bytes_sent = 0; - if (file->metadata != NULL) - event.mtime_sec = file->metadata->mtime_sec; - change_emit(config, &event); + event.dest = file->dest_state; + char* name = NULL; + char* path = NULL; + fill_event_from_file(config, file, &event, &name, &path); + if (name != NULL && path != NULL) + change_emit(config, &event); + free(name); + free(path); } diff --git a/src/client/change_list.h b/src/client/change_list.h index 9434026..874b9fe 100644 --- a/src/client/change_list.h +++ b/src/client/change_list.h @@ -3,6 +3,7 @@ #include "config.h" #include "file_types.h" +#include "format.h" #include #include #include @@ -26,42 +27,52 @@ typedef enum { } ChangeDecision; typedef struct { - const char* path; /* full source path */ + const char* path; /* long-form display path (rsync %f) */ + const char* name; /* transfer-relative path (rsync %n), no trailing slash */ ChangeDecision decision; bool is_directory; - unsigned long long size; /* source file length in bytes */ - /* The number of bytes reported for a sent file. FastSync has no wire-byte - * counter, so this is always the source length (== size / %l); actual - * post-compression/delta bytes on the wire are not counted. */ - unsigned long long bytes_sent; - time_t mtime_sec; /* 0 when unknown */ + bool is_symlink; + bool is_special; + bool is_hardlink; /* a hard-link sibling (linked, no data sent) */ + const char* symlink_target; + const char* hardlink_target; + unsigned long long size; /* source file length in bytes */ + unsigned long long bytes_sent; /* literal data bytes actually transferred */ + time_t mtime_sec; + long mtime_nsec; + mode_t mode; + uid_t uid; + gid_t gid; + /* Receiver-reported pre-transfer destination state (OutputDestState.known is + * false when no report was requested/received). */ + OutputDestState dest; } ChangeEvent; /* True when any output mode is active and per-file events matter. */ bool change_list_enabled(const Config* config); -/* Render the rsync-style itemize line for a transferred file: - * `>f+++++++++ ` - * The 11-char code is `>f` (regular file transferred to the remote host) - * followed by c/s/t/p/o/g/u/a/x markers that are all `+` (value will be set - * / differs) because FastSync does not separately compare checksums, size, - * mtime, perms, owner, group, uid, acl, or xattr on the receiving side, so a - * sent file is reported as fully updated. Up-to-date files print no line - * (rsync single `-i` only shows changes). Caller frees the result. */ -char* change_render_itemize(const ChangeEvent* event); +/* Render the rsync-style itemize line for a transferred item + * (`%i %n%L`): `>f+++++++++ sub/b.txt`. Caller frees the result. */ +char* change_render_itemize(const Config* config, const ChangeEvent* event); -/* Expand an --out-format/--log-file-format template. Tokens: - * %f full source path %b "bytes sent" == the source length (%l); - * %n leaf (base) name actual post-compression/delta wire bytes - * %l file length in bytes are not counted - * %M mtime in whole seconds %% a literal percent sign +/* Render only the 11-character itemize code (rsync %i). Caller frees. */ +char* change_render_itemize_code(const Config* config, const ChangeEvent* event); + +/* Expand an --out-format/--log-file-format template. Supported tokens: + * %i itemize code %n transfer-relative name (dir: trailing /) + * %f long display path %l file length in bytes + * %b bytes actually sent %M mtime (YYYY/MM/DD-HH:MM:SS) + * %t current time %o operation ("send"/"del.") + * %p pid %B permission bits without the type char + * %U uid %G gid + * %L " -> target" / " => target" %% a literal percent sign * Unknown %X sequences are preserved verbatim. Caller frees the result. */ -char* change_render_format(const char* format, const ChangeEvent* event); +char* change_render_format(const char* format, const Config* config, const ChangeEvent* event); /* Render one --list-only long-listing entry: - * `-rw-r--r-- 12 2026/09/06 10:00:00 ` + * `-rw-r--r-- 12 2026/09/06 10:00:00 sub/b.txt` * (ls -l style columns; mtime in the local time zone). Caller frees it. */ -char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime, const char* path); +char* change_render_list_line(const Config* config, const ChangeEvent* event); /* Emit an event to every active destination: * stdout: --itemize-changes line, or the --out-format expansion when set; diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 00bc37e..1110c56 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -334,7 +334,8 @@ static void apply_output_buffering(const Config* config) { } #endif -static int read_patterns_from_file(const char* filepath, char*** patterns, int* count); +static int read_patterns_from_file(const char* filepath, char*** patterns, int* count, + Config* config, char sign, const char* optname); static int parse_debug_flags(const char* value, Config* config) { if (!value || value[0] == '\0' || value[0] == ',' || value[strlen(value) - 1] == ',' || @@ -568,6 +569,27 @@ static int config_add_filter(Config* config, const char* rule) { return 0; } +/* Compile one --exclude/--include pattern into the SAME ordered filter rule + * list used by --filter/-f: `--exclude P` becomes the rule "- P" and + * `--include P` becomes "+ P", appended in command-line order. This is what + * makes rsync's first-match-wins semantics hold across a mixed sequence such as + * `--include='*.txt' --exclude='*'`. Returns 0 on success, -1 on error. */ +static int config_add_selection_rule(Config* config, char sign, const char* pattern, + const char* optname) { + size_t len = strlen(pattern); + char* rule = malloc(len + 3); + if (!rule) { + log_message(LOG_LEVEL_ERROR, "memory allocation failed for %s", optname); + return -1; + } + rule[0] = sign; + rule[1] = ' '; + memcpy(rule + 2, pattern, len + 1); + int rc = config_add_filter(config, rule); + free(rule); + return rc; +} + static int parse_skip_compress(Config* config, const char* value) { char* list = str_dup(value); if (!list) @@ -1323,7 +1345,8 @@ static bool cli_handle_ssh_and_pattern_options(CliParseCtx* ctx) { } if (strncmp(arg, "--exclude=", 10) == 0) { if (config_add_pattern(&config->exclude_patterns, &config->exclude_count, arg + 10, - "--exclude") != 0) + "--exclude") != 0 || + config_add_selection_rule(config, '-', arg + 10, "--exclude") != 0) ctx->exit_code = -1; return true; } @@ -1334,13 +1357,15 @@ static bool cli_handle_ssh_and_pattern_options(CliParseCtx* ctx) { return true; } if (config_add_pattern(&config->exclude_patterns, &config->exclude_count, ctx->argv[++ctx->i], - "--exclude") != 0) + "--exclude") != 0 || + config_add_selection_rule(config, '-', ctx->argv[ctx->i], "--exclude") != 0) ctx->exit_code = -1; return true; } if (strncmp(arg, "--include=", 10) == 0) { if (config_add_pattern(&config->include_patterns, &config->include_count, arg + 10, - "--include") != 0) + "--include") != 0 || + config_add_selection_rule(config, '+', arg + 10, "--include") != 0) ctx->exit_code = -1; return true; } @@ -1351,7 +1376,8 @@ static bool cli_handle_ssh_and_pattern_options(CliParseCtx* ctx) { return true; } if (config_add_pattern(&config->include_patterns, &config->include_count, ctx->argv[++ctx->i], - "--include") != 0) + "--include") != 0 || + config_add_selection_rule(config, '+', ctx->argv[ctx->i], "--include") != 0) ctx->exit_code = -1; return true; } @@ -1634,7 +1660,8 @@ static bool cli_handle_filter_options(CliParseCtx* ctx) { Config* config = ctx->config; const char* arg = ctx->argv[ctx->i]; if (strncmp(arg, "--exclude-from=", 15) == 0) { - if (read_patterns_from_file(arg + 15, &config->exclude_patterns, &config->exclude_count) != 0) + if (read_patterns_from_file(arg + 15, &config->exclude_patterns, &config->exclude_count, config, + '-', "--exclude-from") != 0) ctx->exit_code = -1; return true; } @@ -1645,12 +1672,13 @@ static bool cli_handle_filter_options(CliParseCtx* ctx) { return true; } if (read_patterns_from_file(ctx->argv[++ctx->i], &config->exclude_patterns, - &config->exclude_count) != 0) + &config->exclude_count, config, '-', "--exclude-from") != 0) ctx->exit_code = -1; return true; } if (strncmp(arg, "--include-from=", 15) == 0) { - if (read_patterns_from_file(arg + 15, &config->include_patterns, &config->include_count) != 0) + if (read_patterns_from_file(arg + 15, &config->include_patterns, &config->include_count, config, + '+', "--include-from") != 0) ctx->exit_code = -1; return true; } @@ -1661,7 +1689,7 @@ static bool cli_handle_filter_options(CliParseCtx* ctx) { return true; } if (read_patterns_from_file(ctx->argv[++ctx->i], &config->include_patterns, - &config->include_count) != 0) + &config->include_count, config, '+', "--include-from") != 0) ctx->exit_code = -1; return true; } @@ -2088,6 +2116,10 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool * --no-xattrs/--no-acls negation) so the sender's wire gate always matches * the flags the receiver will recompute from the received config. */ config->use_xattrs = config->preserve_acls || config->preserve_xattrs; + /* Output parity: -i/--itemize-changes and --out-format need the pre-transfer + * destination snapshot (new vs modified and which attributes differ), so ask + * the receiver to report it on every per-file check. This is a wire field. */ + config->report_dest_info = config->itemize_changes || config->out_format != NULL; return 0; } @@ -2284,7 +2316,8 @@ done: return result; } -static int read_patterns_from_file(const char* filepath, char*** patterns, int* count) { +static int read_patterns_from_file(const char* filepath, char*** patterns, int* count, + Config* config, char sign, const char* optname) { FILE* fp = fopen(filepath, "r"); if (!fp) { char* escaped = output_escape(filepath, false); @@ -2326,7 +2359,8 @@ static int read_patterns_from_file(const char* filepath, char*** patterns, int* p[--len] = '\0'; if (len == 0) continue; - if (config_add_pattern(patterns, count, p, "pattern file") != 0) { + if (config_add_pattern(patterns, count, p, "pattern file") != 0 || + config_add_selection_rule(config, sign, p, optname) != 0) { free(line); fclose(fp); return -1; diff --git a/src/client/client_send.c b/src/client/client_send.c index 8b0cf1f..01f2390 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -11,6 +11,7 @@ #include "file.h" #include "file_list.h" #include "filter.h" +#include "format.h" #include "hardlink.h" #include "metadata.h" #include "motd.h" @@ -69,32 +70,61 @@ static int progress_thread_fn(void* arg); static const char* display_bytes(unsigned long long bytes, bool human_readable, char* buffer, size_t buffer_size) { - if (human_readable && format_human_bytes(bytes, buffer, buffer_size)) + if (human_readable && format_human_size_decimal(bytes, buffer, buffer_size)) return buffer; snprintf(buffer, buffer_size, "%.1f MB", (double)bytes / (double)BYTES_PER_MIB); return buffer; } -/* Print the canonical `--stats` line. Shared by the single-threaded and - multithreaded send paths so both honor --stats, --human-readable and --quiet - identically; `start` marks the beginning of the transfer for the rate. */ +/* rsync byte count: human-readable decimal when -h was given, otherwise a + * comma-grouped integer (rsync's big_num in the C locale). */ +static const char* stats_bytes(const Config* config, unsigned long long bytes, char* buffer, + size_t buffer_size) { + if (!format_big_num(bytes, config->human_readable, buffer, buffer_size)) + snprintf(buffer, buffer_size, "%llu", bytes); + return buffer; +} + +/* Print the rsync `--stats` block on stdout. FastSync is a push sender, so a + few receiver-only counters (matched data, file-list bytes, deletion count) + are not observable and are reported as 0; the labels and layout match rsync + 3.4.1. Shared by the single-threaded and multithreaded send paths. */ static void report_transfer_stats(const Config* config, int total_files, unsigned long long total_bytes, time_t start) { if (!config->stats || config->quiet) return; double elapsed = difftime(time(NULL), start); - double rate = elapsed > 0.0 ? (double)total_bytes / ((double)BYTES_PER_MIB * elapsed) : 0.0; + double rate = elapsed > 0.0 ? (double)total_bytes / elapsed : 0.0; + char total_buffer[32]; + char rate_buffer[32]; + char human_rate[32]; + const char* total = stats_bytes(config, total_bytes, total_buffer, sizeof(total_buffer)); + const char* rate_str = rate_buffer; if (config->human_readable) { - char total_buffer[32]; - char rate_buffer[32]; - fprintf(stderr, "Stats: %d files, %s, %s/s\n", total_files, - display_bytes(total_bytes, true, total_buffer, sizeof(total_buffer)), - display_bytes((unsigned long long)(rate * (double)BYTES_PER_MIB), true, rate_buffer, - sizeof(rate_buffer))); + if (!format_human_size_decimal((unsigned long long)rate, human_rate, sizeof(human_rate))) + snprintf(human_rate, sizeof(human_rate), "0"); + rate_str = human_rate; } else { - fprintf(stderr, "Stats: %d files, %.1f MB, %.1f MB/s\n", total_files, - (double)total_bytes / (double)BYTES_PER_MIB, rate); + snprintf(rate_buffer, sizeof(rate_buffer), "%.2f", rate); } + printf("\n"); + printf("Number of files: %d\n", total_files); + printf("Number of created files: %d\n", total_files); + printf("Number of deleted files: 0\n"); + printf("Number of regular files transferred: %d\n", total_files); + printf("Total file size: %s bytes\n", total); + printf("Total transferred file size: %s bytes\n", total); + printf("Literal data: %s bytes\n", total); + printf("Matched data: 0 bytes\n"); + printf("File list size: 0\n"); + printf("File list generation time: 0.000 seconds\n"); + printf("File list transfer time: 0.000 seconds\n"); + printf("Total bytes sent: %s\n", total); + printf("Total bytes received: 0\n"); + printf("\n"); + printf("sent %s bytes received 0 bytes %s bytes/sec\n", total, rate_str); + printf("total size is %s speedup is %.2f\n", total, 1.0); + fflush(stdout); } /* Compiled scanner inputs that are shared read-only across scanner instances @@ -146,10 +176,16 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann options->preserve_xattrs = config->preserve_xattrs; options->preserve_acls = config->preserve_acls; options->chunk_size = config->chunk_size; - options->exclude_patterns = config->exclude_patterns; - options->exclude_count = config->exclude_count; - options->include_patterns = config->include_patterns; - options->include_count = config->include_count; + /* --exclude/--include are compiled, in command-line order, into the SAME + * ordered filter rule list as --filter/-f (see config_add_selection_rule), so + * the legacy per-kind arrays are deliberately NOT passed to the scanner: + * doing so would re-apply them with the old "excludes first, then includes as + * a mandatory whitelist" precedence and defeat rsync's first-match-wins + * ordering. The arrays remain populated purely for the Config API surface. */ + options->exclude_patterns = NULL; + options->exclude_count = 0; + options->include_patterns = NULL; + options->include_count = 0; options->max_size = config->max_size; options->min_size = config->min_size; options->max_depth = config->max_depth; @@ -467,7 +503,7 @@ static void receive_daemon_motd(Client* client, const Config* config) { static Client* connect_transfer_client(const Config* config) { if (config->transport == TRANSPORT_SSH) { if (config->use_sendfile) { - log_message(LOG_LEVEL_ERROR, "-f/--sendfile is not supported with SSH transport"); + log_message(LOG_LEVEL_ERROR, "--sendfile is not supported with SSH transport"); return NULL; } return client_connect_ssh(config->ssh_destination, config->ssh_port, @@ -782,30 +818,53 @@ static int send_dry_run_manifest(const Config* config) { } typedef struct { - char* path; + char* name; /* transfer-relative name ("" == the source root) */ mode_t mode; unsigned long long size; time_t mtime; + long mtime_nsec; + bool is_dir; + bool is_symlink; + char* link_target; } ListEntry; static void list_entries_destroy(ListEntry* entries, size_t count) { if (entries == NULL) return; - for (size_t i = 0; i < count; i++) - free(entries[i].path); + for (size_t i = 0; i < count; i++) { + free(entries[i].name); + free(entries[i].link_target); + } free(entries); } static int compare_list_entries(const void* left, const void* right) { const ListEntry* a = (const ListEntry*)left; const ListEntry* b = (const ListEntry*)right; - return strcmp(a->path, b->path); + return strcmp(a->name, b->name); } -/* --list-only: print an ls-style listing of the files that WOULD be +/* Relative path of an entry below `root` ("" for the root itself). Mirrors + * change_list's relative_name for list-only rendering. */ +static char* list_relative_name(const char* root, const char* full) { + if (root == NULL || full == NULL) + return str_dup(full != NULL ? full : ""); + size_t root_len = strlen(root); + while (root_len > 1 && root[root_len - 1] == '/') + root_len--; + if (strncmp(root, full, root_len) == 0) { + if (full[root_len] == '\0') + return str_dup(""); + if (full[root_len] == '/') + return str_dup(full + root_len + 1); + } + return str_dup(full); +} + +/* --list-only: print an ls-style listing of the entries that WOULD be * transferred and exit without contacting the server or writing anything. - * Directory lines are not printed because the scanner only yields regular - * transfer candidates. Returns 0 on success, 1 on error. */ + * Names are transfer-relative (rsync prints `a.txt`, `sub/b.txt`, `.`) and + * directory entries are included. Returns 0 on success, 1 on error. */ static int send_list_only(const Config* config) { int skipped = 0; if (!files_from_list_check(config, NULL, &skipped)) @@ -814,6 +873,7 @@ static int send_list_only(const Config* config) { if (!prepare_scanner(config, 0, &prepared)) return 1; prepared.options.use_metadata = true; /* capture mode + mtime for the listing */ + prepared.options.list_dirs = true; DirectoryScanner* scanner = directory_scanner_create_with_options(config->send_directory, &prepared.options); if (!scanner) { @@ -823,9 +883,31 @@ static int send_list_only(const Config* config) { ListEntry* entries = NULL; size_t count = 0; size_t capacity = 0; - Chunk* chunk; bool oom = false; - while ((chunk = directory_scanner_next(scanner)) != NULL) { + + /* rsync lists the source root itself (as "."). Only when the source is a + * directory and no --files-from subset is in effect. */ + if (config->files_from_set == NULL && config->send_directory != NULL) { + struct stat st; + if (stat(config->send_directory, &st) == 0 && S_ISDIR(st.st_mode)) { + capacity = 64; + entries = calloc(capacity, sizeof(ListEntry)); + if (entries == NULL) { + oom = true; + } else { + entries[0].name = str_dup(""); + entries[0].mode = st.st_mode; + entries[0].mtime = st.st_mtime; + entries[0].mtime_nsec = st.st_mtim.tv_nsec; + entries[0].size = (unsigned long long)st.st_size; + entries[0].is_dir = true; + count = 1; + } + } + } + + Chunk* chunk; + while (!oom && (chunk = directory_scanner_next(scanner)) != NULL) { for (int i = 0; i < chunk->element_count; i++) { File* f = chunk->items[i]; if (f == NULL) @@ -842,34 +924,47 @@ static int send_list_only(const Config* config) { break; } entries = grown; + memset(entries + capacity, 0, (new_capacity - capacity) * sizeof(ListEntry)); capacity = new_capacity; } - char* path = str_dup(file_wire_path(f)); - if (!path) { + char* name = list_relative_name(config->send_directory, file_wire_path(f)); + if (!name) { oom = true; break; } mode_t mode = 0; time_t mtime = 0; + long mtime_nsec = 0; if (f->metadata != NULL) { mode = f->metadata->mode; mtime = f->metadata->mtime_sec; + mtime_nsec = f->metadata->mtime_nsec; } else { struct stat st; - if (stat(f->path, &st) == 0) { + if (lstat(f->path, &st) == 0) { mode = st.st_mode; mtime = st.st_mtime; + mtime_nsec = st.st_mtim.tv_nsec; } } - entries[count].path = path; + entries[count].name = name; entries[count].mode = mode; entries[count].mtime = mtime; - entries[count].size = f->data != NULL ? f->data->size : 0; + entries[count].mtime_nsec = mtime_nsec; + if (f->is_symlink) + entries[count].size = f->symlink_target != NULL ? strlen(f->symlink_target) : 0; + else if (f->is_dir) { + struct stat dir_st; + entries[count].size = stat(f->path, &dir_st) == 0 ? (unsigned long long)dir_st.st_size : 0; + } else + entries[count].size = f->data != NULL ? f->data->size : 0; + entries[count].is_dir = f->is_dir; + entries[count].is_symlink = f->is_symlink; + entries[count].link_target = + f->is_symlink && f->symlink_target ? str_dup(f->symlink_target) : NULL; count++; } chunk_destroy(chunk); - if (oom) - break; } bool failed = oom || directory_scanner_failed(scanner) || directory_scanner_had_io_error(scanner); directory_scanner_destroy(scanner); @@ -883,8 +978,18 @@ static int send_list_only(const Config* config) { if (count > 1) qsort(entries, count, sizeof(ListEntry), compare_list_entries); for (size_t i = 0; i < count; i++) { - char* line = change_render_list_line(entries[i].mode, entries[i].size, entries[i].mtime, - entries[i].path); + ChangeEvent event; + memset(&event, 0, sizeof(event)); + event.name = entries[i].name; + event.path = entries[i].name; + event.mode = entries[i].mode; + event.size = entries[i].size; + event.mtime_sec = entries[i].mtime; + event.mtime_nsec = entries[i].mtime_nsec; + event.is_directory = entries[i].is_dir; + event.is_symlink = entries[i].is_symlink; + event.symlink_target = entries[i].link_target; + char* line = change_render_list_line(config, &event); if (line != NULL) { char* escaped = output_escape(line, config->eight_bit_output); printf("%s\n", escaped != NULL ? escaped : line); @@ -1052,6 +1157,19 @@ static int incremental_check(Client* client, File* file, const Config* config, Status s; if (!receive_status(client->file_descriptor, &s)) return -1; + /* Output parity: when dest-info reporting is negotiated the receiver sends + * the pre-transfer destination snapshot BEFORE its ordinary verdict. Consume + * it here so the following status read stays in sync. */ + if (config->report_dest_info) { + if (s != STATUS_DEST_INFO || + !format_dest_state_receive(client->file_descriptor, &file->dest_state)) { + log_message(LOG_LEVEL_ERROR, "Unexpected reply to the destination-state report"); + send_status(client->file_descriptor, STATUS_ERROR); + return -1; + } + if (!receive_status(client->file_descriptor, &s)) + return -1; + } if (s == STATUS_ERROR) { log_server_rejection("Server reported error for file"); return -1; diff --git a/src/client/scanner.c b/src/client/scanner.c index 2614d1c..dc3135c 100644 --- a/src/client/scanner.c +++ b/src/client/scanner.c @@ -134,6 +134,26 @@ bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entr return !one_file_system || entry_device == root_device; } +/* Build a payload-less directory File carrying the captured metadata (when + * requested). Used by -x mount-point emission and --list-only directory + * entries. Returns NULL on allocation failure. */ +static File* scanner_build_dir_file(const char* path, const struct stat* stats, + const ScannerOptions* options) { + File* dir = file_create(path); + if (dir == NULL) + return NULL; + dir->is_dir = true; + if (options->use_metadata) { + dir->metadata = + file_metadata_create(dir->path, stats, options->preserve_atimes, options->preserve_crtimes); + if (!dir->metadata) { + file_destroy(dir); + return NULL; + } + } + return dir; +} + /* Relative path of an on-disk path below `root`. The transfer root may be * given with a trailing slash; the returned rel path never has one and is "" * for the root itself. A root of "/" is handled (its children start at "/"). @@ -1044,9 +1064,31 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) { free(rel_copy); if (!scanner_same_filesystem(scanner->options.one_file_system, scanner->root_dev, stats.st_dev)) { + /* rsync's -x/--one-file-system emits the mount-point directory entry + itself (so the destination gets an empty directory) but does NOT + descend into it. Build a payload-less directory File and hand it to + the caller; never enqueue it for traversal. */ + File* mount = scanner_build_dir_file(cur_path, &stats, &scanner->options); + if (mount == NULL || !array_list_add(chunk_data, mount)) { + file_destroy(mount); + free(cur_path); + scanner->failed = true; + break; + } free(cur_path); continue; } + /* --list-only: list directory entries too (rsync prints them), even + though a real transfer never sends them explicitly. */ + if (scanner->options.list_dirs) { + File* dir = scanner_build_dir_file(cur_path, &stats, &scanner->options); + if (dir == NULL || !array_list_add(chunk_data, dir)) { + file_destroy(dir); + free(cur_path); + scanner->failed = true; + break; + } + } int next_depth = scanner->current_depth + 1; if (scanner->options.max_depth <= 0 || next_depth < scanner->options.max_depth) { DirEntry* de = dir_entry_create(cur_path, next_depth, scanner->current_node); @@ -1386,7 +1428,28 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo if (is_dir) { free(rel); if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) { + /* -x/--one-file-system: emit the mount-point directory entry (empty) but + do not descend into it (see the sequential scanner for the same rule). */ + File* mount = file_create(cur_path); free(cur_path); + if (mount == NULL) { + ps->failed = true; + return; + } + mount->is_dir = true; + if (options->use_metadata) { + mount->metadata = file_metadata_create(mount->path, &st, options->preserve_atimes, + options->preserve_crtimes); + if (!mount->metadata) { + file_destroy(mount); + ps->failed = true; + return; + } + } + if (!array_list_add(root_files, mount)) { + file_destroy(mount); + ps->failed = true; + } return; } if (!array_list_add(subdirs, cur_path)) { diff --git a/src/client/scanner.h b/src/client/scanner.h index 51200e2..925228b 100644 --- a/src/client/scanner.h +++ b/src/client/scanner.h @@ -65,6 +65,10 @@ typedef struct { bool per_dir_filters; /* -F: read .rsync-filter per directory */ bool dirs; /* -d/--dirs: transfer dir entries, no recursion */ bool relative; /* -R/--relative (dest rel paths, with --files-from) */ + /* --list-only: emit an is_dir File for every traversed directory (the listing + * includes directory entries, matching rsync). Client-only; never set on a + * real transfer, which relies on implicit parent creation. */ + bool list_dirs; /* --prune-empty-dirs (long only): in --dirs mode an empty source directory's explicit entry is omitted from the transfer file list (so nothing is created at the destination and it can be pruned by --delete); explicitly diff --git a/src/client/usage.c b/src/client/usage.c index b2bd9b4..7a1ea24 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -157,7 +157,8 @@ void print_usage(void) { printf(" --chunk-serialization Enable chunk serialization (long form only)\n"); printf(" -s, --secluded-args Protect-args compatibility option (no effect; remote\n"); printf(" SSH argv is already built injection-safe)\n"); - printf(" --sendfile Enable sendfile zero-copy (TCP only; long form only)\n"); + printf(" --sendfile Enable sendfile zero-copy (TCP only; long form only;\n"); + printf(" -f is bound to --filter, not --sendfile)\n"); printf(" --compress-choice Compression algorithm (default: zstd)\n"); printf(" --zc Alias for --compress-choice\n"); printf(" -v, --verbose Enable debug logging\n"); diff --git a/src/shared/config.c b/src/shared/config.c index fa1b79b..f7b72b8 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -1128,6 +1128,7 @@ CONFIG_DEFINE_SEND(send_daemon_auth, CONFIG_WIRE_DAEMON_AUTH_FIELDS) CONFIG_DEFINE_SEND(send_iconv_spec, CONFIG_WIRE_ICONV_FIELDS) CONFIG_DEFINE_SEND(send_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS) CONFIG_DEFINE_SEND(send_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS) +CONFIG_DEFINE_SEND(send_output_options, CONFIG_WIRE_OUTPUT_FIELDS) CONFIG_DEFINE_RECV(receive_core_fields, CONFIG_WIRE_CORE_FIELDS) CONFIG_DEFINE_RECV(receive_delta_fields, CONFIG_WIRE_DELTA_FIELDS) @@ -1146,6 +1147,7 @@ CONFIG_DEFINE_RECV(receive_daemon_auth, CONFIG_WIRE_DAEMON_AUTH_FIELDS) CONFIG_DEFINE_RECV(receive_iconv_spec, CONFIG_WIRE_ICONV_FIELDS) CONFIG_DEFINE_RECV(receive_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS) CONFIG_DEFINE_RECV(receive_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS) +CONFIG_DEFINE_RECV(receive_output_options, CONFIG_WIRE_OUTPUT_FIELDS) #undef XSEND #undef XRECV @@ -1262,7 +1264,8 @@ bool config_send_wire_block(int file_descriptor, const Config* config) { send_daemon_module(file_descriptor, config) && send_daemon_auth(file_descriptor, config) && send_iconv_spec(file_descriptor, config) && send_privilege_options(file_descriptor, config) && - send_copy_as_options(file_descriptor, config); + send_copy_as_options(file_descriptor, config) && + send_output_options(file_descriptor, config); } bool config_send(int file_descriptor, const Config* config) { @@ -1332,7 +1335,8 @@ Config* config_receive_with_validate(int file_descriptor, ConfigValidateFunc val !receive_daemon_auth(file_descriptor, config, &budget) || !receive_iconv_spec(file_descriptor, config, &budget) || !receive_privilege_options(file_descriptor, config, &budget) || - !receive_copy_as_options(file_descriptor, config, &budget)) + !receive_copy_as_options(file_descriptor, config, &budget) || + !receive_output_options(file_descriptor, config, &budget)) goto error; if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 && strcmp(config->compress_choice, "none") != 0) { diff --git a/src/shared/config.h b/src/shared/config.h index 6dfb1c7..5d8dda8 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -76,7 +76,7 @@ typedef struct { typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode; /* =========================================================================== - * Config wire-field table (single source of truth for protocol 2.22.0). + * Config wire-field table (single source of truth for protocol 2.23.0). * * Every field below crosses the wire. The table is the ONLY place a * serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare @@ -241,6 +241,13 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF X(copy_as_uid, int32_t, 0, COPY_AS_ID) \ X(copy_as_gid, int32_t, 0, COPY_AS_ID) +/* Output-parity wave (protocol 2.23.0). report_dest_info tells the receiver to + * answer every per-file STATUS_CHECK with a STATUS_DEST_INFO snapshot of the + * pre-transfer destination entry (see protocol.h). It is set by the client + * only when -i/--itemize-changes or --out-format asks for per-file change + * output; the transfer decision itself is unchanged. */ +#define CONFIG_WIRE_OUTPUT_FIELDS(X) X(report_dest_info, bool, false, BOOL) + /* All serialized fields, in exact wire order. Concatenating the per-segment * lists here is what keeps the declaration order = the wire order. */ #define CONFIG_WIRE_FIELDS(X) \ @@ -261,7 +268,8 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF CONFIG_WIRE_DAEMON_AUTH_FIELDS(X) \ CONFIG_WIRE_ICONV_FIELDS(X) \ CONFIG_WIRE_PRIVILEGE_FIELDS(X) \ - CONFIG_WIRE_COPY_AS_FIELDS(X) + CONFIG_WIRE_COPY_AS_FIELDS(X) \ + CONFIG_WIRE_OUTPUT_FIELDS(X) typedef struct Config { /* -j/--threads=N: number of parallel scanner worker threads for the -m @@ -849,8 +857,22 @@ typedef struct Config { * version before parsing anything else) is what keeps a 2.22 client and a 2.21 * server from ever reaching that state. The fixed-width FileMetadata layout is * UNCHANGED: the receiver still gates attribute application on use_metadata, - * which is now DERIVED from these attributes by config_derived_use_metadata(). */ -#define PROTOCOL_VERSION "2.22.0" + * which is now DERIVED from these attributes by config_derived_use_metadata(). + * + * Output-Parity Wave: 2.22.0 -> 2.23.0. + * + * WHY the bump, grounded in the wire: -i/--itemize-changes and --out-format + * must compare the source against the PRE-TRANSFER destination entry (new vs + * modified, and which of size/time/perms/owner/group differ), but FastSync's + * push sender never sees the destination. The receiver therefore answers a + * per-file STATUS_CHECK with a new STATUS_DEST_INFO frame (a fixed-width + * snapshot of the old entry) before its ordinary verdict when the config frame + * carries the new report_dest_info bool appended after the --copy-as block. + * This is both a config-frame layout change (one trailing bool) and a frame + * sequence change (the new status), so any peer that did not parse them would + * desynchronize; the strict same-version handshake keeps a 2.23 client and a + * 2.22 server from ever reaching that state. */ +#define PROTOCOL_VERSION "2.23.0" #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) /* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */ #define MAX_BASIS_DIRS 64 diff --git a/src/shared/file_receive.c b/src/shared/file_receive.c index 39cd387..fe37a72 100644 --- a/src/shared/file_receive.c +++ b/src/shared/file_receive.c @@ -19,6 +19,7 @@ #include "delay_updates.h" #include "delta.h" #include "file.h" +#include "format.h" #include "identity.h" #include "log.h" #include "metadata.h" @@ -1858,6 +1859,33 @@ static IncrementalCheckOutcome incremental_check_open_destination(IncrementalChe return INCREMENTAL_CONTINUE; } +/* Output parity (protocol 2.23.0): when the wire config asked for it, report a + snapshot of the pre-transfer destination entry BEFORE the ordinary verdict so + the sender can render rsync-accurate -i/--out-format columns. A missing + destination is reported explicitly (existed=false) rather than omitted, so + the sender can distinguish "new" from "unknown". */ +static IncrementalCheckOutcome incremental_check_report_dest_info(IncrementalCheckState* state) { + if (!state->config->report_dest_info) + return INCREMENTAL_CONTINUE; + OutputDestState info; + memset(&info, 0, sizeof(info)); + info.known = true; + info.existed = state->has_old_file; + if (state->has_old_file) { + info.size = (unsigned long long)state->old_st.st_size; + info.mtime_sec = (long long)state->old_st.st_mtime; +#ifdef __linux__ + info.mtime_nsec = state->old_st.st_mtim.tv_nsec; +#endif + info.mode = (uint32_t)state->old_st.st_mode; + info.uid = (int32_t)state->old_st.st_uid; + info.gid = (int32_t)state->old_st.st_gid; + } + if (!send_status(state->fd, STATUS_DEST_INFO) || !format_dest_state_send(state->fd, &info)) + return INCREMENTAL_ERROR; + return INCREMENTAL_CONTINUE; +} + /* Metadata-only (and, when --checksum forces it, content) up-to-date decision. Loads the old contents only when a checksum comparison or delta needs them. */ static IncrementalCheckOutcome incremental_check_quick_skip(IncrementalCheckState* state, @@ -2302,6 +2330,10 @@ File* receive_incremental_check_ex(int fd, const Config* config, bool* skipped, if (outcome == INCREMENTAL_ERROR) goto done; + outcome = incremental_check_report_dest_info(&state); + if (outcome == INCREMENTAL_ERROR) + goto done; + outcome = incremental_check_quick_skip(&state, &try_delta); if (outcome == INCREMENTAL_ERROR) goto done; diff --git a/src/shared/file_types.h b/src/shared/file_types.h index 9564d82..53ad1db 100644 --- a/src/shared/file_types.h +++ b/src/shared/file_types.h @@ -2,6 +2,7 @@ #define FILE_TYPES_H #include "data.h" +#include "format.h" #include "xattr.h" #include #include @@ -86,6 +87,12 @@ typedef struct { * Receiver: parsed off the wire, attached here, and applied fd-relative on * the written file. NULL/0 == the file carries no xattrs. */ FileXattrList* xattrs; + /* Sender-side output-parity state (never serialized): the receiver-reported + * pre-transfer destination snapshot for this entry, filled by the per-file + * STATUS_CHECK exchange when report_dest_info is set. `known` is false when + * no report was requested/received, in which case -i/--out-format treats the + * entry conservatively as newly created. */ + OutputDestState dest_state; } File; /* The path that should be sent on the wire and used for the receiver-side diff --git a/src/shared/format.c b/src/shared/format.c new file mode 100644 index 0000000..d690e1b --- /dev/null +++ b/src/shared/format.c @@ -0,0 +1,103 @@ +#include "format.h" +#include "protocol.h" +#include +#include + +bool format_human_size_decimal(unsigned long long bytes, char* buffer, size_t buffer_size) { + if (!buffer || buffer_size == 0) + return false; + if (bytes < 1000ULL) { + int written = snprintf(buffer, buffer_size, "%llu", bytes); + return written >= 0 && (size_t)written < buffer_size; + } + static const char units[] = "KMGTPE"; + double value = (double)bytes; + size_t divisions = 0; + while (value >= 1000.0 && divisions < sizeof(units) - 1) { + value /= 1000.0; + divisions++; + } + int written = snprintf(buffer, buffer_size, "%.2f%c", value, units[divisions - 1]); + return written >= 0 && (size_t)written < buffer_size; +} + +bool format_big_num(unsigned long long value, bool human_readable, char* buffer, + size_t buffer_size) { + if (human_readable) + return format_human_size_decimal(value, buffer, buffer_size); + char digits[32]; + int written = snprintf(digits, sizeof(digits), "%llu", value); + if (written < 0 || (size_t)written >= sizeof(digits)) + return false; + size_t len = (size_t)written; + size_t separators = len > 1 ? (len - 1) / 3 : 0; + size_t total = len + separators; + if (total + 1 > buffer_size) + return false; + size_t out = total; + buffer[out] = '\0'; + size_t digits_since_sep = 0; + for (size_t i = len; i > 0; i--) { + buffer[--out] = digits[i - 1]; + digits_since_sep++; + if (digits_since_sep == 3 && i > 1) { + buffer[--out] = ','; + digits_since_sep = 0; + } + } + return true; +} + +bool format_rsync_datetime(time_t when, bool dash, char* buffer, size_t buffer_size) { + if (!buffer || buffer_size == 0) + return false; + struct tm broken_down; + if (localtime_r(&when, &broken_down) == NULL) + return false; + const char* format = dash ? "%Y/%m/%d-%H:%M:%S" : "%Y/%m/%d %H:%M:%S"; + return strftime(buffer, buffer_size, format, &broken_down) != 0; +} + +bool format_dest_state_send(int fd, const OutputDestState* state) { + if (!state) + return false; + int32_t has_old = state->existed ? 1 : 0; + uint64_t size = (uint64_t)state->size; + int64_t mtime = (int64_t)state->mtime_sec; + int64_t mtime_nsec = state->mtime_nsec; + uint32_t mode = state->mode; + int32_t uid = state->uid; + int32_t gid = state->gid; + return send_n_data(fd, &has_old, sizeof(has_old)) && send_n_data(fd, &size, sizeof(size)) && + send_n_data(fd, &mtime, sizeof(mtime)) && + send_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) && send_n_data(fd, &mode, sizeof(mode)) && + send_n_data(fd, &uid, sizeof(uid)) && send_n_data(fd, &gid, sizeof(gid)); +} + +bool format_dest_state_receive(int fd, OutputDestState* state) { + if (!state) + return false; + int32_t has_old = 0; + uint64_t size = 0; + int64_t mtime = 0; + int64_t mtime_nsec = 0; + uint32_t mode = 0; + int32_t uid = 0; + int32_t gid = 0; + if (!receive_n_data(fd, &has_old, sizeof(has_old)) || !receive_n_data(fd, &size, sizeof(size)) || + !receive_n_data(fd, &mtime, sizeof(mtime)) || + !receive_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) || + !receive_n_data(fd, &mode, sizeof(mode)) || !receive_n_data(fd, &uid, sizeof(uid)) || + !receive_n_data(fd, &gid, sizeof(gid))) + return false; + memset(state, 0, sizeof(*state)); + state->known = true; + state->existed = has_old != 0; + state->size = size; + state->mtime_sec = mtime; + state->mtime_nsec = mtime_nsec; + state->mode = mode; + state->uid = uid; + state->gid = gid; + return true; +} diff --git a/src/shared/format.h b/src/shared/format.h new file mode 100644 index 0000000..7f4fc13 --- /dev/null +++ b/src/shared/format.h @@ -0,0 +1,59 @@ +#ifndef FORMAT_H +#define FORMAT_H + +#include +#include +#include +#include + +/* Low-level output-formatting primitives shared by the change-event model + * (change_list.c) and the transfer driver (client_send.c). + * + * The functions here are pure/string-level except for the STATUS_DEST_INFO + * codec, which lets the receiver report the pre-transfer destination entry so + * the sender can render rsync-accurate --itemize-changes / --out-format + * columns (see protocol.h). */ + +/* Pre-transfer destination snapshot, reported by the receiver when the wire + * config carries report_dest_info. `known` distinguishes "no report was + * requested/received" from "the destination did not exist" (`existed == false` + * with `known == true`). */ +typedef struct { + bool known; + bool existed; + unsigned long long size; + long long mtime_sec; + long long mtime_nsec; + uint32_t mode; + int32_t uid; + int32_t gid; +} OutputDestState; + +/* rsync's -h/--human-readable size (decimal, base 1000): integers below 1000 + * print verbatim; larger values use the largest unit that keeps the value + * below 1000 (K/M/G/T/P/E) with exactly two decimals, so 1500000 -> "1.50M" + * and 999999 -> "1000.00K" (matching rsync's human_num). Returns false when + * the buffer is too small (nothing is written). */ +bool format_human_size_decimal(unsigned long long bytes, char* buffer, size_t buffer_size); + +/* rsync's general number formatting (big_num). When `human_readable` is true + * this is format_human_size_decimal; otherwise the integer is rendered with a + * ',' thousands separator every three digits (rsync's separator in the C + * locale). Returns false on an undersized buffer. */ +bool format_big_num(unsigned long long value, bool human_readable, char* buffer, + size_t buffer_size); + +/* rsync's %M/%t timestamp. When `dash` is true the separator between the date + * and the time is '-' (the %M form: "YYYY/MM/DD-HH:MM:SS"); otherwise it is a + * space (the %t form: "YYYY/MM/DD HH:MM:SS"). Local time. Returns false on a + * bad time or an undersized buffer. */ +bool format_rsync_datetime(time_t when, bool dash, char* buffer, size_t buffer_size); + +/* Fixed-width STATUS_DEST_INFO record codec (int32 has_old, uint64 size, + * int64 mtime, int64 mtime_nsec, uint32 mode, int32 uid, int32 gid). The + * status frame itself is sent/received by the caller. Returns false on I/O + * failure. */ +bool format_dest_state_send(int fd, const OutputDestState* state); +bool format_dest_state_receive(int fd, OutputDestState* state); + +#endif diff --git a/src/shared/protocol.c b/src/shared/protocol.c index c5b7991..2a0cfbb 100644 --- a/src/shared/protocol.c +++ b/src/shared/protocol.c @@ -479,6 +479,8 @@ static const char* status_to_string(Status status) { return "ERROR_DETAIL"; case STATUS_DRY_RUN_TRANSFER: return "DRY_RUN_TRANSFER"; + case STATUS_DEST_INFO: + return "DEST_INFO"; default: return "UNKNOWN"; } diff --git a/src/shared/protocol.h b/src/shared/protocol.h index 4c2491d..3cb6225 100644 --- a/src/shared/protocol.h +++ b/src/shared/protocol.h @@ -155,7 +155,18 @@ enum NET_STATUS { * (the receiver reads none in dry-run). STATUS_OK keeps its meaning in this * path ("already up to date / nothing to do"). Appended after * STATUS_ERROR_DETAIL so no existing status is renumbered. */ - STATUS_DRY_RUN_TRANSFER + STATUS_DRY_RUN_TRANSFER, + /* Destination-state report for output parity (protocol 2.23.0). When the + * wire config carries report_dest_info=true, the receiver answers every + * per-file STATUS_CHECK request with STATUS_DEST_INFO FIRST, followed by a + * fixed record describing the pre-transfer destination entry + * (int32 has_old; uint64 size; int64 mtime; int64 mtime_nsec; uint32 mode; + * int32 uid; int32 gid). The ordinary STATUS_OK/STATUS_NEXT/... verdict + * follows, so the sender can render rsync-accurate -i/--out-format columns + * (new vs modified, and which of size/time/perms/owner/group differ) without + * changing the transfer decision itself. Appended after + * STATUS_DRY_RUN_TRANSFER so no existing status is renumbered. */ + STATUS_DEST_INFO }; void io_set_fds(int read_fd, int write_fd); diff --git a/tests/integration/test_fault_injection.py b/tests/integration/test_fault_injection.py index df2f607..7c7127f 100644 --- a/tests/integration/test_fault_injection.py +++ b/tests/integration/test_fault_injection.py @@ -36,7 +36,7 @@ from common import ( # noqa: E402 verify_transfer, ) -PROTOCOL_VERSION = b"2.22.0" +PROTOCOL_VERSION = b"2.23.0" STATUS_MANIFEST = 5 STATUS_OK = 0 diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index cf2f46f..ff83d86 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -325,7 +325,7 @@ class TestDryRun: result, dur = run_client(SOURCE_DIR, DEST_DIR, flags=["-h", "--dry-run"]) assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" assert "Total:" in result.stdout - assert "KB" in result.stdout + assert any(unit in result.stdout for unit in ("K", "M", "G")) def test_dry_run(self): clean_dir(DEST_DIR) @@ -1070,10 +1070,12 @@ class TestExclude: class TestInclude: def test_include_single(self, shared_server): + # rsync first-match-wins: an --include alone is NOT a whitelist, so the + # selector must pair it with --exclude '*' (the common idiom). clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, - flags=["--include", "binary.bin"], + flags=["--include", "binary.bin", "--exclude", "*"], port=shared_server.port, ) if result.returncode != 0: @@ -1086,13 +1088,14 @@ class TestInclude: clean_dir(DEST_DIR) result, dur = run_client( SOURCE_DIR, DEST_DIR, - flags=["--include", "*.bin"], + flags=["--include", "*.bin", "--exclude", "*"], port=shared_server.port, ) if result.returncode != 0: pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}") received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) assert os.path.exists(os.path.join(received, "binary.bin")), "binary.bin should be included" + assert not os.path.exists(os.path.join(received, "small.txt")), "small.txt should not be included" class TestSizeFilters: @@ -1646,8 +1649,8 @@ class TestDelete: port=shared_server.port, ) assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" - assert "Stats:" in result.stderr - assert "KB" in result.stderr + assert "Number of files:" in result.stdout + assert "Total file size:" in result.stdout def test_human_readable_stats_multithreaded(self, shared_server): # The multithreaded sender shares the single-threaded --stats format, @@ -1659,9 +1662,8 @@ class TestDelete: port=shared_server.port, ) assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" - assert "Stats:" in result.stderr - assert "KB" in result.stderr - assert "/s" in result.stderr + assert "Number of files:" in result.stdout + assert "bytes/sec" in result.stdout def test_human_readable_progress_multithreaded(self, shared_server): clean_dir(DEST_DIR) @@ -1673,7 +1675,6 @@ class TestDelete: assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" output = result.stdout + result.stderr assert "Sent " in output - assert "KB" in output assert "Done." in output @@ -2164,7 +2165,8 @@ class TestListOnly: result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only"]) assert result.returncode == 0, f"list-only failed: {result.stderr[:200]}" for full_path in _source_files(): - assert full_path in result.stdout, f"list-only omitted {full_path}" + rel = os.path.relpath(full_path, SOURCE_DIR) + assert rel in result.stdout, f"list-only omitted {rel}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) assert not os.path.exists(received), "list-only wrote to the destination" @@ -2180,7 +2182,8 @@ class TestListOnly: result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only", "--threads"]) assert result.returncode == 0, f"list-only -m failed: {result.stderr[:200]}" for full_path in _source_files(): - assert full_path in result.stdout, f"list-only -m omitted {full_path}" + rel = os.path.relpath(full_path, SOURCE_DIR) + assert rel in result.stdout, f"list-only -m omitted {rel}" received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) assert not os.path.exists(received), "list-only -m wrote to the destination" @@ -2193,7 +2196,7 @@ class TestItemizeChanges: result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--preserve", "-i"], port=shared_server.port) assert result.returncode == 0, f"itemize sync failed: {result.stderr[:200]}" - sent_lines = {">f+++++++++ " + p for p in _source_files()} + sent_lines = {">f+++++++++ " + os.path.relpath(p, SOURCE_DIR) for p in _source_files()} assert sent_lines <= set(result.stdout.splitlines()), ( f"missing itemize lines; got {result.stdout[:500]}" ) @@ -2214,7 +2217,7 @@ class TestItemizeChanges: result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--preserve", "-i", "--threads"], port=shared_server.port) assert result.returncode == 0, f"itemize -m sync failed: {result.stderr[:200]}" - sent_lines = {">f+++++++++ " + p for p in _source_files()} + sent_lines = {">f+++++++++ " + os.path.relpath(p, SOURCE_DIR) for p in _source_files()} assert sent_lines <= set(result.stdout.splitlines()), ( f"missing itemize lines in -m mode; got {result.stdout[:500]}" ) @@ -2249,7 +2252,9 @@ class TestItemizeChanges: port=shared_server.port) assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}" itemized = [line for line in result.stdout.splitlines() if line.startswith(">f")] - assert itemized == [">f+++++++++ " + changed], ( + # The content and mtime both changed, so the itemize compares the + # destination snapshot: size and time columns are set. + assert itemized == [">f.st...... changed.txt"], ( f"expected exactly one itemize line for {changed}, got {itemized}" ) received = get_dest_received_dir(dest, source) @@ -2265,7 +2270,11 @@ class TestOutFormat: result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--out-format=%f %l"], port=shared_server.port) assert result.returncode == 0, f"out-format sync failed: {result.stderr[:200]}" - expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()} + # %f is rsync's long display path: the source argument normalized + # (leading '/' stripped) joined to the transfer-relative name. + prefix = SOURCE_DIR.lstrip(os.sep) + expected = {f"{os.path.join(prefix, os.path.relpath(p, SOURCE_DIR))} {os.path.getsize(p)}" + for p in _source_files()} got = set(result.stdout.splitlines()) assert expected <= got, f"out-format lines missing: expected {len(expected)} got {len(got)}" @@ -2274,7 +2283,9 @@ class TestOutFormat: result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--out-format=%f %l", "--threads"], port=shared_server.port) assert result.returncode == 0, f"out-format -m sync failed: {result.stderr[:200]}" - expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()} + prefix = SOURCE_DIR.lstrip(os.sep) + expected = {f"{os.path.join(prefix, os.path.relpath(p, SOURCE_DIR))} {os.path.getsize(p)}" + for p in _source_files()} got = set(result.stdout.splitlines()) assert expected <= got, f"out-format -m lines missing: {result.stdout[:500]}" @@ -2296,7 +2307,9 @@ class TestLogFileFormat: assert os.path.exists(log_path), "--log-file created no log" with open(log_path, encoding="utf-8", errors="replace") as fh: content = fh.read() - expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()} + prefix = SOURCE_DIR.lstrip(os.sep) + expected = {f"{os.path.join(prefix, os.path.relpath(p, SOURCE_DIR))} {os.path.getsize(p)}" + for p in _source_files()} for line in expected: assert line in content, f"log file missing {line!r}" @@ -2322,7 +2335,8 @@ class TestLogFileFormat: assert os.path.exists(log_path), "--log-file created no log" with open(log_path, encoding="utf-8", errors="replace") as fh: content = fh.read() - expected = {f"{os.path.join(source, rel)} {len(data)}" for rel, data in files.items()} + prefix = os.path.abspath(source).lstrip(os.sep) + expected = {f"{os.path.join(prefix, rel)} {len(data)}" for rel, data in files.items()} for line in expected: assert line in content, f"log file (--threads) missing {line!r}" diff --git a/tests/integration/test_output_parity.py b/tests/integration/test_output_parity.py new file mode 100644 index 0000000..0843ca2 --- /dev/null +++ b/tests/integration/test_output_parity.py @@ -0,0 +1,282 @@ +"""Output-parity tests (#291 selection/output, #292 output formatting). + +These tests exercise rsync-style selection ordering and output formatting. The +differential tests run the SAME transfer with real ``rsync 3.4.1`` and with +fastsync and compare stdout, so they are skipped when rsync is unavailable. +""" +import os +import shutil +import subprocess +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(__file__)) +from common import TEST_DATA_DIR, run_client, clean_dir, get_dest_received_dir + +RSYNC = shutil.which("rsync") +requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed") + + +def _rsync(args): + env = dict(os.environ, LC_ALL="C") + return subprocess.run( + [RSYNC] + args, capture_output=True, text=True, env=env, timeout=120 + ) + + +def _make_selection_tree(root): + clean_dir(root) + os.makedirs(os.path.join(root, "sub")) + with open(os.path.join(root, "a.txt"), "wb") as fh: + fh.write(b"top text\n") + with open(os.path.join(root, "b.log"), "wb") as fh: + fh.write(b"log data\n") + with open(os.path.join(root, "sub", "c.txt"), "wb") as fh: + fh.write(b"nested text\n") + with open(os.path.join(root, "sub", "d.log"), "wb") as fh: + fh.write(b"nested log\n") + + +class TestSelectionOrdering: + """#291: --include/--exclude compile into one ordered rule list.""" + + @pytest.mark.ci + def test_include_then_exclude_keeps_only_matching(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "out_inc_src") + dest = os.path.join(TEST_DATA_DIR, "out_inc_dst") + _make_selection_tree(source) + clean_dir(dest) + result, _ = run_client( + source, dest, + flags=["--preserve", "--include=*.txt", "--exclude=*"], + port=shared_server.port, + ) + assert result.returncode == 0, f"include/exclude failed: {result.stderr[:300]}" + received = get_dest_received_dir(dest, source) + assert os.path.exists(os.path.join(received, "a.txt")) + # `*` also excludes the directory, so nothing below sub/ is sent. + assert not os.path.exists(os.path.join(received, "b.log")) + assert not os.path.exists(os.path.join(received, "sub", "c.txt")) + + @pytest.mark.ci + def test_include_dirs_then_files_idiom(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "out_inc2_src") + dest = os.path.join(TEST_DATA_DIR, "out_inc2_dst") + _make_selection_tree(source) + clean_dir(dest) + result, _ = run_client( + source, dest, + flags=["--preserve", "--include=*/", "--include=*.txt", "--exclude=*"], + port=shared_server.port, + ) + assert result.returncode == 0, f"include/exclude failed: {result.stderr[:300]}" + received = get_dest_received_dir(dest, source) + assert os.path.exists(os.path.join(received, "a.txt")) + assert os.path.exists(os.path.join(received, "sub", "c.txt")) + assert not os.path.exists(os.path.join(received, "b.log")) + assert not os.path.exists(os.path.join(received, "sub", "d.log")) + + @requires_rsync + def test_include_idiom_matches_rsync_selection(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "out_inc3_src") + dest = os.path.join(TEST_DATA_DIR, "out_inc3_dst") + rdst = os.path.join(TEST_DATA_DIR, "out_inc3_rdst") + _make_selection_tree(source) + clean_dir(dest) + clean_dir(rdst) + flags = ["--include=*/", "--include=*.txt", "--exclude=*"] + rsync_result = _rsync(["-a"] + flags + [source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + result, _ = run_client(source, dest, flags=["--preserve"] + flags, + port=shared_server.port) + assert result.returncode == 0 + received = get_dest_received_dir(dest, source) + assert os.path.exists(os.path.join(received, "a.txt")) + assert os.path.exists(os.path.join(received, "sub", "c.txt")) + assert not os.path.exists(os.path.join(received, "b.log")) + # rsync -a src/ dst/ writes directly into dst/ + assert os.path.exists(os.path.join(rdst, "a.txt")) + assert os.path.exists(os.path.join(rdst, "sub", "c.txt")) + assert not os.path.exists(os.path.join(rdst, "b.log")) + + +class TestOneFileSystem: + """#291: -x emits the mount-point directory but not its contents.""" + + def test_one_file_system_emits_mount_point_dir(self, shared_server): + local = os.stat(".") + shm = "/dev/shm" + try: + shm_stat = os.stat(shm) + except OSError: + pytest.skip("/dev/shm not available") + if shm_stat.st_dev == local.st_dev: + pytest.skip("no cross-device filesystem available") + + source = os.path.join(TEST_DATA_DIR, "out_ofs_src") + dest = os.path.join(TEST_DATA_DIR, "out_ofs_dst") + clean_dir(source) + clean_dir(dest) + os.makedirs(os.path.join(source, "nested")) + os.makedirs(os.path.join(shm, "fastsync_ofs_probe"), exist_ok=True) + with open(os.path.join(source, "keep.txt"), "wb") as fh: + fh.write(b"keep\n") + with open(os.path.join(shm, "fastsync_ofs_probe", "inside.txt"), "wb") as fh: + fh.write(b"cross\n") + link = os.path.join(source, "nested", "link") + try: + os.symlink(os.path.join(shm, "fastsync_ofs_probe"), link) + except OSError: + pytest.skip("cannot create symlink") + + try: + result, _ = run_client( + source, dest, + flags=["--preserve", "--copy-links", "-x"], + port=shared_server.port, + ) + assert result.returncode == 0, f"-x failed: {result.stderr[:300]}" + received = get_dest_received_dir(dest, source) + assert os.path.exists(os.path.join(received, "keep.txt")) + # The mount-point directory entry is created but its contents are not. + assert os.path.isdir(os.path.join(received, "nested", "link")) + assert not os.path.exists(os.path.join(received, "nested", "link", "inside.txt")) + finally: + shutil.rmtree(os.path.join(shm, "fastsync_ofs_probe"), ignore_errors=True) + + +def _make_output_tree(root): + clean_dir(root) + os.makedirs(os.path.join(root, "sub")) + with open(os.path.join(root, "a.txt"), "wb") as fh: + fh.write(b"hello\n") + with open(os.path.join(root, "sub", "b.txt"), "wb") as fh: + fh.write("wörld\n".encode("utf-8")) + os.symlink("a.txt", os.path.join(root, "link")) + + +class TestItemizeParity: + """#292: -i output matches rsync 3.4.1 for the cases fastsync can observe.""" + + @requires_rsync + @pytest.mark.ci + def test_itemize_first_transfer_matches_rsync(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "out_item_src") + dest = os.path.join(TEST_DATA_DIR, "out_item_dst") + rdst = os.path.join(TEST_DATA_DIR, "out_item_rdst") + _make_output_tree(source) + clean_dir(dest) + clean_dir(rdst) + rsync_result = _rsync(["-a", "-i", source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + rsync_lines = sorted( + line for line in rsync_result.stdout.splitlines() + if line.startswith(">f") or line.startswith("cL") + ) + result, _ = run_client(source, dest, flags=["-a", "-i"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + fast_lines = sorted( + line for line in result.stdout.splitlines() + if line.startswith(">f") or line.startswith("cL") + ) + assert fast_lines == rsync_lines, f"rsync={rsync_lines} fastsync={fast_lines}" + + @requires_rsync + @pytest.mark.ci + def test_itemize_modified_file_matches_rsync(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "out_item2_src") + dest = os.path.join(TEST_DATA_DIR, "out_item2_dst") + rdst = os.path.join(TEST_DATA_DIR, "out_item2_rdst") + _make_output_tree(source) + clean_dir(dest) + clean_dir(rdst) + seed = run_client(source, dest, flags=["-a"], port=shared_server.port) + assert seed[0].returncode == 0, seed[0].stderr[:300] + assert _rsync(["-a", source + "/", rdst + "/"]).returncode == 0 + + with open(os.path.join(source, "a.txt"), "wb") as fh: + fh.write(b"hello changed and longer\n") + # Pin the source mtime so rsync's `t` column is deterministic (a write + # that lands in the same whole second as the seed would not show `t`). + os.utime(os.path.join(source, "a.txt"), (1000000000, 1000000000)) + + rsync_result = _rsync(["-a", "-i", source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + rsync_lines = sorted( + line for line in rsync_result.stdout.splitlines() if line.startswith(">f") + ) + result, _ = run_client(source, dest, + flags=["-a", "-i", "--incremental"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + fast_lines = sorted( + line for line in result.stdout.splitlines() if line.startswith(">f") + ) + assert fast_lines == rsync_lines, f"rsync={rsync_lines} fastsync={fast_lines}" + + +class TestOutFormatParity: + @requires_rsync + @pytest.mark.ci + def test_out_format_n_l_matches_rsync(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "out_fmt_src") + dest = os.path.join(TEST_DATA_DIR, "out_fmt_dst") + rdst = os.path.join(TEST_DATA_DIR, "out_fmt_rdst") + _make_output_tree(source) + clean_dir(dest) + clean_dir(rdst) + fmt = "%n %l" + rsync_result = _rsync(["-a", "--out-format=" + fmt, source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + rsync_lines = sorted( + line for line in rsync_result.stdout.splitlines() + if line and not line.split(" ", 1)[0].endswith("/") + ) + result, _ = run_client(source, dest, + flags=["-a", "--out-format=" + fmt], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + fast_lines = sorted( + line for line in result.stdout.splitlines() + if line and not line.split(" ", 1)[0].endswith("/") + ) + assert fast_lines == rsync_lines, f"rsync={rsync_lines} fastsync={fast_lines}" + + @requires_rsync + @pytest.mark.ci + def test_out_format_M_datetime_shape(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "out_M_src") + dest = os.path.join(TEST_DATA_DIR, "out_M_dst") + _make_output_tree(source) + clean_dir(dest) + result, _ = run_client(source, dest, + flags=["-a", "--out-format=%M %f"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + import re + pattern = re.compile(r"^\d{4}/\d{2}/\d{2}-\d{2}:\d{2}:\d{2} ") + for line in result.stdout.splitlines(): + if line: + assert pattern.match(line), f"bad %M format: {line!r}" + + +class TestListOnlyParity: + @requires_rsync + @pytest.mark.ci + def test_list_only_matches_rsync(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "out_list_src") + dest = os.path.join(TEST_DATA_DIR, "out_list_dst") + _make_output_tree(source) + clean_dir(dest) + rsync_result = _rsync(["-r", "--list-only", source + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + rsync_lines = sorted(rsync_result.stdout.splitlines()) + result, _ = run_client(source, dest, flags=["--list-only", "-l"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + fast_lines = sorted(result.stdout.splitlines()) + assert fast_lines == rsync_lines, ( + f"rsync={rsync_lines}\nfastsync={fast_lines}" + ) diff --git a/tests/integration/test_preflight.py b/tests/integration/test_preflight.py index f72af11..d6b601a 100644 --- a/tests/integration/test_preflight.py +++ b/tests/integration/test_preflight.py @@ -94,14 +94,14 @@ def _seed_protocol_source(source): class TestProtocol: @pytest.mark.ci def test_protocol_current_version_accepted(self, shared_server): - """--protocol=2.22.0 (the current PROTOCOL_VERSION) is accepted and the + """--protocol=2.23.0 (the current PROTOCOL_VERSION) is accepted and the transfer completes normally.""" source = os.path.join(TEST_DATA_DIR, "proto_ok_src") dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst") shutil.rmtree(dest, ignore_errors=True) os.makedirs(dest) _seed_protocol_source(source) - result, _ = run_client(source, dest, flags=["--protocol=2.22.0"], + result, _ = run_client(source, dest, flags=["--protocol=2.23.0"], port=shared_server.port) assert result.returncode == 0, \ f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}" @@ -118,8 +118,8 @@ class TestProtocol: shutil.rmtree(dest, ignore_errors=True) os.makedirs(dest) _seed_protocol_source(source) - for bad in ("2.21.0", "2.20.0", "2.19.0", "2.18.0", "2.17.0", "2.15.0", "2.16.0", "216", - "31"): + for bad in ("2.22.0", "2.21.0", "2.20.0", "2.19.0", "2.18.0", "2.17.0", "2.15.0", "2.16.0", + "216", "31"): result, _ = run_client(source, dest, flags=[f"--protocol={bad}"], port=shared_server.port) assert result.returncode != 0, f"--protocol={bad} should be rejected" diff --git a/tests/runner.c b/tests/runner.c index b5a9f3f..05f583e 100644 --- a/tests/runner.c +++ b/tests/runner.c @@ -15,6 +15,7 @@ #include "test_file.h" #include "test_file_list.h" #include "test_file_sendfile.h" +#include "test_format.h" #include "test_fuzz_smoke.h" #include "test_glob.h" #include "test_hardlink.h" @@ -58,6 +59,7 @@ int main() { RUN_TEST(test_chunk); RUN_TEST(test_batch); RUN_TEST(test_change_list); + RUN_TEST(test_format); RUN_TEST(test_config); RUN_TEST(test_credentials); RUN_TEST(test_compression); diff --git a/tests/test_change_list.c b/tests/test_change_list.c index a5fe016..1cf91e0 100644 --- a/tests/test_change_list.c +++ b/tests/test_change_list.c @@ -1,5 +1,6 @@ #include "test_change_list.h" #include "change_list.h" +#include "config.h" #include "test_utils.h" #include "utils.h" #include @@ -9,64 +10,150 @@ static ChangeEvent sample_event(void) { ChangeEvent event; memset(&event, 0, sizeof(event)); - event.path = "/srv/root/sub/file.txt"; + event.path = "src/sub/file.txt"; + event.name = "sub/file.txt"; event.decision = CHANGE_SENT; event.is_directory = false; event.size = 12345; event.bytes_sent = 999; event.mtime_sec = 1700000000; + event.mtime_nsec = 0; + event.mode = 0100644; + event.uid = 1000; + event.gid = 1000; return event; } +/* Expected %M expansion computed independently with localtime_r. */ +static void expected_mtime(time_t when, char out[32]) { + struct tm broken_down; + localtime_r(&when, &broken_down); + strftime(out, 32, "%Y/%m/%d-%H:%M:%S", &broken_down); +} + static void test_format_tokens() { ChangeEvent event = sample_event(); - char* line = change_render_format("%f %n %l %b %M %%", &event); + Config* config = config_create(); + char when[32]; + expected_mtime(event.mtime_sec, when); + char* line = change_render_format("%f %n %l %b %M %%", config, &event); EXPECT_NOT_NULL(line); - EXPECT_EQ_STR(line, "/srv/root/sub/file.txt file.txt 12345 999 1700000000 %"); + char expected[256]; + snprintf(expected, sizeof(expected), "src/sub/file.txt sub/file.txt 12345 999 %s %%", when); + EXPECT_EQ_STR(line, expected); free(line); + config_delete(config); } static void test_format_unknown_tokens_preserved() { ChangeEvent event = sample_event(); - char* line = change_render_format("x%q=%f%z", &event); + Config* config = config_create(); + char* line = change_render_format("x%q=%f%z", config, &event); EXPECT_NOT_NULL(line); - EXPECT_EQ_STR(line, "x%q=/srv/root/sub/file.txt%z"); + EXPECT_EQ_STR(line, "x%q=src/sub/file.txt%z"); free(line); + config_delete(config); } -static void test_format_leaf_name() { +static void test_format_directory_name_has_trailing_slash() { ChangeEvent event = sample_event(); - event.path = "bare.txt"; - char* line = change_render_format("%n|%f", &event); + event.is_directory = true; + event.path = "src/sub"; + event.name = "sub"; + Config* config = config_create(); + char* line = change_render_format("%n|%f", config, &event); EXPECT_NOT_NULL(line); - EXPECT_EQ_STR(line, "bare.txt|bare.txt"); + EXPECT_EQ_STR(line, "sub/|src/sub"); free(line); + config_delete(config); } static void test_render_itemize_sent_file() { ChangeEvent event = sample_event(); - char* line = change_render_itemize(&event); + Config* config = config_create(); + char* line = change_render_itemize(config, &event); EXPECT_NOT_NULL(line); - EXPECT_EQ_STR(line, ">f+++++++++ /srv/root/sub/file.txt"); + EXPECT_EQ_STR(line, ">f+++++++++ sub/file.txt"); free(line); + config_delete(config); +} + +static void test_render_itemize_directory() { + ChangeEvent event = sample_event(); + event.is_directory = true; + event.path = "src/sub"; + event.name = "sub"; + Config* config = config_create(); + char* line = change_render_itemize(config, &event); + EXPECT_NOT_NULL(line); + EXPECT_EQ_STR(line, "cd+++++++++ sub/"); + free(line); + config_delete(config); +} + +static void test_render_itemize_symlink() { + ChangeEvent event = sample_event(); + event.is_symlink = true; + event.path = "src/link"; + event.name = "link"; + event.symlink_target = "a.txt"; + Config* config = config_create(); + char* line = change_render_itemize(config, &event); + EXPECT_NOT_NULL(line); + EXPECT_EQ_STR(line, "cL+++++++++ link -> a.txt"); + free(line); + config_delete(config); +} + +static void test_render_itemize_compares_destination() { + ChangeEvent event = sample_event(); + Config* config = config_create(); + config->preserve_perms = true; + config->preserve_owner = true; + config->preserve_group = true; + event.dest.known = true; + event.dest.existed = true; + event.dest.size = 1; + event.dest.mtime_sec = 1700000000; + event.dest.mtime_nsec = 0; + event.dest.mode = 0100600; + event.dest.uid = 1; + event.dest.gid = 2; + char* line = change_render_itemize(config, &event); + EXPECT_NOT_NULL(line); + /* size, perms, owner and group differ; time matches. */ + EXPECT_EQ_STR(line, ">f.s.pog... sub/file.txt"); + free(line); + config_delete(config); } static void test_render_itemize_up_to_date_is_empty() { ChangeEvent event = sample_event(); + Config* config = config_create(); event.decision = CHANGE_UP_TO_DATE; - char* line = change_render_itemize(&event); + char* line = change_render_itemize(config, &event); EXPECT_NOT_NULL(line); EXPECT_EQ_STR(line, ""); free(line); + config_delete(config); } static void test_render_list_line() { - char* line = change_render_list_line(0100644, 4096, 1700000000, "/srv/x.txt"); + ChangeEvent event; + memset(&event, 0, sizeof(event)); + Config* config = config_create(); + event.name = "sub/x.txt"; + event.path = "sub/x.txt"; + event.mode = 0100644; + event.size = 4096; + event.mtime_sec = 1700000000; + char* line = change_render_list_line(config, &event); EXPECT_NOT_NULL(line); EXPECT_TRUE(strncmp(line, "-rw-r--r--", 10) == 0); - EXPECT_TRUE(strstr(line, "4096") != NULL); - EXPECT_TRUE(strstr(line, "/srv/x.txt") != NULL); + EXPECT_TRUE(strstr(line, "4,096") != NULL); + EXPECT_TRUE(strstr(line, "sub/x.txt") != NULL); free(line); + config_delete(config); } static void test_change_list_enabled() { @@ -93,8 +180,11 @@ static void test_change_list_enabled() { void test_change_list() { test_format_tokens(); test_format_unknown_tokens_preserved(); - test_format_leaf_name(); + test_format_directory_name_has_trailing_slash(); test_render_itemize_sent_file(); + test_render_itemize_directory(); + test_render_itemize_symlink(); + test_render_itemize_compares_destination(); test_render_itemize_up_to_date_is_empty(); test_render_list_line(); test_change_list_enabled(); diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index 60dd58a..a185b10 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -317,7 +317,7 @@ static void test_parse_args_protocol_accept_current() { Config* cfg = valid_client_config(); EXPECT_NOT_NULL(cfg); char* argv_equals[] = {"fastsync", "--source-dir", "/src", - "--dest-dir", "/dst", "--protocol=2.22.0"}; + "--dest-dir", "/dst", "--protocol=2.23.0"}; int positional_args[2]; int positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 6, argv_equals, positional_args, &positional_count), 0); @@ -327,7 +327,7 @@ static void test_parse_args_protocol_accept_current() { cfg = valid_client_config(); EXPECT_NOT_NULL(cfg); char* argv_space[] = {"fastsync", "--source-dir", "/src", "--dest-dir", - "/dst", "--protocol", "2.22.0"}; + "/dst", "--protocol", "2.23.0"}; positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 7, argv_space, positional_args, &positional_count), 0); EXPECT_EQ_STR(cfg->version, PROTOCOL_VERSION); @@ -338,8 +338,8 @@ static void test_parse_args_protocol_accept_current() { * failure (parse_args simply stores it; validate_config rejects it up front). */ static void test_parse_args_protocol_rejects_other_versions() { static const char* const bad_versions[] = {"2.17", "2.16", "2.15.0", "2.16.0", "2.17.0", - "2.18.0", "2.19.0", "2.20.0", "2.21.0", "216", - "31", "abc", ""}; + "2.18.0", "2.19.0", "2.20.0", "2.21.0", "2.22.0", + "216", "31", "abc", ""}; for (size_t i = 0; i < sizeof(bad_versions) / sizeof(bad_versions[0]); i++) { Config* cfg = valid_client_config(); EXPECT_NOT_NULL(cfg); @@ -3729,6 +3729,12 @@ static void test_parse_args_inline_equals_forms() { EXPECT_EQ_STR(cfg->exclude_patterns[0], "*.log"); EXPECT_EQ_INT(cfg->include_count, 1); EXPECT_EQ_STR(cfg->include_patterns[0], "*.txt"); + /* The same patterns are compiled, in command-line order, into the shared + * ordered --filter rule list (rsync first-match-wins). */ + EXPECT_NOT_NULL(cfg->filters); + EXPECT_EQ_INT(cfg->filters->size, 2); + EXPECT_EQ_STR((char*)cfg->filters->items[0], "- *.log"); + EXPECT_EQ_STR((char*)cfg->filters->items[1], "+ *.txt"); config_delete(cfg); const char* list_path = "cli_inline_patterns.txt"; @@ -3776,6 +3782,41 @@ static void test_parse_args_inline_equals_forms() { config_delete(cfg); } +/* --exclude/--include compile into the SAME ordered filter list as --filter, so + * rsync's first-match-wins semantics hold: the common `--include='*.txt' + * --exclude='*'` idiom keeps the .txt files and drops the rest, and an + * --include rule with no matching exclude is not a mandatory whitelist. */ +static void test_parse_args_include_exclude_order() { + Config* cfg = config_create(); + int positional_args[2]; + int positional_count = 0; + char* argv[] = {"fastsync", "--include=*.txt", "--exclude=*", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_NOT_NULL(cfg->filters); + EXPECT_EQ_INT(cfg->filters->size, 2); + EXPECT_EQ_STR((char*)cfg->filters->items[0], "+ *.txt"); + EXPECT_EQ_STR((char*)cfg->filters->items[1], "- *"); + /* The order is reversible on the command line and the list follows it. */ + Config* cfg2 = config_create(); + positional_count = 0; + char* argv2[] = {"fastsync", "--exclude=*", "--include=*.txt", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg2, 5, argv2, positional_args, &positional_count), 0); + EXPECT_EQ_INT(cfg2->filters->size, 2); + EXPECT_EQ_STR((char*)cfg2->filters->items[0], "- *"); + EXPECT_EQ_STR((char*)cfg2->filters->items[1], "+ *.txt"); + /* --filter and --exclude/--include interleave in command-line order. */ + Config* cfg3 = config_create(); + positional_count = 0; + char* argv3[] = {"fastsync", "--filter=- *.tmp", "--include=*.txt", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg3, 5, argv3, positional_args, &positional_count), 0); + EXPECT_EQ_INT(cfg3->filters->size, 2); + EXPECT_EQ_STR((char*)cfg3->filters->items[0], "- *.tmp"); + EXPECT_EQ_STR((char*)cfg3->filters->items[1], "+ *.txt"); + config_delete(cfg); + config_delete(cfg2); + config_delete(cfg3); +} + /* OPT_NOOP compatibility flags (-s/--secluded-args, -r/--recursive) must never * swallow the next argv: `fastsync -s SRC DST` keeps both positionals. */ static void test_parse_args_noop_does_not_consume_argv() { @@ -4003,6 +4044,7 @@ void test_client_cli() { test_parse_args_short_clustering(); test_parse_args_attached_short_values(); test_parse_args_inline_equals_forms(); + test_parse_args_include_exclude_order(); test_parse_args_noop_does_not_consume_argv(); test_parse_args_backup_copy_links_shorts(); test_parse_args_rejects_unsupported_short(); diff --git a/tests/test_config.c b/tests/test_config.c index 3b6026e..f16a542 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -2665,14 +2665,13 @@ static void golden_config_populate(Config* c) { c->copy_as_gid = 222; } -/* The pinned golden frame (protocol 2.22.0). The values below are the only +/* The pinned golden frame (protocol 2.23.0). The values below are the only * thing that ties the generated table to the historical wire format; update - * them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.22.0 - * preserve-attribute split appends four serialized bools - * (preserve_perms/times/owner/group) to CONFIG_WIRE_METADATA_TIMES_FIELDS after - * omit_link_times. */ -#define GOLDEN_WIRE_LEN 653 -#define GOLDEN_WIRE_HASH 95530566005420798ULL + * them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.23.0 + * output-parity wave appends one serialized bool (report_dest_info) to the end + * of the frame, after the --copy-as block. */ +#define GOLDEN_WIRE_LEN 657 +#define GOLDEN_WIRE_HASH 4633069702262438591ULL static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) { unsigned long long h = 1469598103934665603ULL; @@ -2754,7 +2753,7 @@ static unsigned long long capture_wire_hash(const Config* cfg, size_t* out_len) return h; } -/* Byte-for-byte wire compatibility guard (protocol 2.22.0). The expected hash +/* Byte-for-byte wire compatibility guard (protocol 2.23.0). The expected hash * pins the pre-X-macro byte stream; the refactor MUST NOT change it. */ static void test_config_wire_golden() { if (is_running_under_valgrind()) diff --git a/tests/test_format.c b/tests/test_format.c new file mode 100644 index 0000000..9b8bd35 --- /dev/null +++ b/tests/test_format.c @@ -0,0 +1,94 @@ +#include "test_format.h" +#include "format.h" +#include "test_utils.h" +#include +#include +#include +#include +#include + +static void expect_big_num(unsigned long long value, bool human, const char* expected) { + char buffer[64]; + EXPECT_TRUE(format_big_num(value, human, buffer, sizeof(buffer))); + EXPECT_EQ_STR(buffer, expected); +} + +static void test_human_size_decimal() { + /* Values below 1000 print verbatim; larger values use the largest unit that + * keeps the value below 1000 and exactly two decimals (rsync human_num). */ + expect_big_num(0, true, "0"); + expect_big_num(999, true, "999"); + expect_big_num(1000, true, "1.00K"); + expect_big_num(1500, true, "1.50K"); + expect_big_num(9999, true, "10.00K"); + expect_big_num(999999, true, "1000.00K"); + expect_big_num(1000000, true, "1.00M"); + expect_big_num(1500000, true, "1.50M"); +} + +static void test_big_num_grouping() { + /* Non-human numbers are comma-grouped every three digits (rsync big_num). */ + expect_big_num(0, false, "0"); + expect_big_num(1, false, "1"); + expect_big_num(999, false, "999"); + expect_big_num(1000, false, "1,000"); + expect_big_num(4096, false, "4,096"); + expect_big_num(1234567, false, "1,234,567"); + expect_big_num(1000000000ULL, false, "1,000,000,000"); +} + +static void test_datetime_format() { + char buffer[32]; + time_t when = 1700000000; + EXPECT_TRUE(format_rsync_datetime(when, true, buffer, sizeof(buffer))); + /* %M shape: YYYY/MM/DD-HH:MM:SS */ + EXPECT_EQ_INT(strlen(buffer), 19); + EXPECT_EQ_INT(buffer[4], '/'); + EXPECT_EQ_INT(buffer[7], '/'); + EXPECT_EQ_INT(buffer[10], '-'); + EXPECT_EQ_INT(buffer[13], ':'); + EXPECT_EQ_INT(buffer[16], ':'); + + char space_form[32]; + EXPECT_TRUE(format_rsync_datetime(when, false, space_form, sizeof(space_form))); + EXPECT_EQ_INT(space_form[10], ' '); +} + +static void test_dest_state_roundtrip() { + /* The wire codec is exercised over a socketpair so the real send/receive + * primitives run. */ + int fds[2]; + if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) != 0) + return; + OutputDestState out; + memset(&out, 0, sizeof(out)); + out.known = true; + out.existed = true; + out.size = 123456789ULL; + out.mtime_sec = 1700000000; + out.mtime_nsec = 123456789; + out.mode = 0100644; + out.uid = 1000; + out.gid = 1000; + OutputDestState in; + memset(&in, 0, sizeof(in)); + EXPECT_TRUE(format_dest_state_send(fds[0], &out)); + EXPECT_TRUE(format_dest_state_receive(fds[1], &in)); + EXPECT_TRUE(in.known); + EXPECT_TRUE(in.existed); + EXPECT_TRUE(in.size == out.size); + EXPECT_TRUE(in.mtime_sec == out.mtime_sec); + EXPECT_TRUE(in.mtime_nsec == out.mtime_nsec); + EXPECT_TRUE(in.mode == out.mode); + EXPECT_TRUE(in.uid == out.uid); + EXPECT_TRUE(in.gid == out.gid); + close(fds[0]); + close(fds[1]); +} + +void test_format(void) { + test_human_size_decimal(); + test_big_num_grouping(); + test_datetime_format(); + test_dest_state_roundtrip(); +} diff --git a/tests/test_format.h b/tests/test_format.h new file mode 100644 index 0000000..41c5933 --- /dev/null +++ b/tests/test_format.h @@ -0,0 +1,6 @@ +#ifndef TEST_FORMAT_H +#define TEST_FORMAT_H + +void test_format(void); + +#endif diff --git a/tests/test_fuzz_smoke.c b/tests/test_fuzz_smoke.c index 22efd50..7d030dd 100644 --- a/tests/test_fuzz_smoke.c +++ b/tests/test_fuzz_smoke.c @@ -18,6 +18,9 @@ /* P8 config-frame tail: super_mode (4) + copy-as presence (4) + uid (4) + gid (4). */ #define P8_TAIL_BYTES 16 +/* Protocol 2.23.0 appends one trailing bool (report_dest_info) AFTER the P8 + * tail, so the P8 fields sit this many bytes before the end of the frame. */ +#define OUTPUT_TAIL_BYTES 4 /* Smoke test for chunk_deserialize fuzz target */ static void test_fuzz_chunk_deserialize() { @@ -334,31 +337,31 @@ static void test_fuzz_config_receive_p8_tail() { /* super_mode outside the 0..2 tri-state is refused. */ memcpy(mut, frame, len); - put_i32(mut, len - P8_TAIL_BYTES, 99); + put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, 99); EXPECT_FALSE(receive_config_frame(mut, len)); - put_i32(mut, len - P8_TAIL_BYTES, -1); + put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, -1); EXPECT_FALSE(receive_config_frame(mut, len)); /* A negative (sentinel) and an extreme copy-as uid/gid are refused. */ memcpy(mut, frame, len); - put_i32(mut, len - P8_TAIL_BYTES, SUPER_MODE_AUTO); - put_i32(mut, len - P8_TAIL_BYTES + 4, 1); - put_i32(mut, len - P8_TAIL_BYTES + 8, -1); - put_i32(mut, len - P8_TAIL_BYTES + 12, 0); + put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, SUPER_MODE_AUTO); + put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 4, 1); + put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 8, -1); + put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 12, 0); EXPECT_FALSE(receive_config_frame(mut, len)); - put_i32(mut, len - P8_TAIL_BYTES + 8, 0); - put_i32(mut, len - P8_TAIL_BYTES + 12, INT32_MIN); + put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 8, 0); + put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 12, INT32_MIN); EXPECT_FALSE(receive_config_frame(mut, len)); /* A presence int that is not a wire bool is refused. */ memcpy(mut, frame, len); - put_i32(mut, len - P8_TAIL_BYTES, SUPER_MODE_AUTO); - put_i32(mut, len - P8_TAIL_BYTES + 4, 2); + put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, SUPER_MODE_AUTO); + put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 4, 2); EXPECT_FALSE(receive_config_frame(mut, len)); /* Truncating anywhere inside the P8 tail is refused. */ EXPECT_FALSE(receive_config_frame(frame, len - 2)); - EXPECT_FALSE(receive_config_frame(frame, len - P8_TAIL_BYTES)); + EXPECT_FALSE(receive_config_frame(frame, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES)); free(mut); free(frame); diff --git a/tests/test_scanner.c b/tests/test_scanner.c index 9c3bcb6..2bc9980 100644 --- a/tests/test_scanner.c +++ b/tests/test_scanner.c @@ -644,17 +644,19 @@ static void test_scanner_one_file_system_cross_device() { EXPECT_EQ_INT(seq_off_rc, 0); EXPECT_TRUE(seq_off_found); EXPECT_EQ_INT(seq_off_total, 2); - /* Sequential: with -x the cross-device subtree is dropped, keep.txt remains. */ + /* Sequential: with -x the cross-device subtree is not descended into, but + * rsync-compatible behavior still emits the mount-point directory entry as an + * empty directory File, so keep.txt plus that entry are present. */ EXPECT_EQ_INT(seq_on_rc, 0); EXPECT_FALSE(seq_on_found); - EXPECT_EQ_INT(seq_on_total, 1); + EXPECT_EQ_INT(seq_on_total, 2); /* Parallel: same behavior, worker path (depth > 1). */ EXPECT_EQ_INT(par_off_rc, 0); EXPECT_TRUE(par_off_found); EXPECT_EQ_INT(par_off_total, 2); EXPECT_EQ_INT(par_on_rc, 0); EXPECT_FALSE(par_on_found); - EXPECT_EQ_INT(par_on_total, 1); + EXPECT_EQ_INT(par_on_total, 2); } /* Collect emitted file paths (relative to `root`) from a sequential scan. @@ -884,6 +886,35 @@ static void test_filter_rules(bool parallel) { free_paths(paths, count); filter_rule_list_free(base); + /* The common include idiom (the exact rule order the CLI compiles from + * --include='*.txt' --exclude='*'): only .txt files survive. */ + const char* idiom[] = {"+ *.txt", "- *"}; + base = filter_base_build(idiom, 2, false, err, sizeof(err)); + EXPECT_NOT_NULL(base); + options.base_filters = base; + rc = parallel ? collect_files_parallel(root, &options, &paths, &count) + : collect_files(root, &options, &paths, &count); + EXPECT_EQ_INT(rc, 0); + EXPECT_EQ_INT(count, 2); + EXPECT_TRUE(has_path(paths, count, "a.txt")); + EXPECT_TRUE(has_path(paths, count, "c.txt")); + EXPECT_FALSE(has_path(paths, count, "b.tmp")); + free_paths(paths, count); + filter_rule_list_free(base); + + /* An include rule alone is NOT a mandatory whitelist (rsync semantics): only + * the matching file is affected, everything else is still transferred. */ + const char* include_alone[] = {"+ *.txt"}; + base = filter_base_build(include_alone, 1, false, err, sizeof(err)); + EXPECT_NOT_NULL(base); + options.base_filters = base; + rc = parallel ? collect_files_parallel(root, &options, &paths, &count) + : collect_files(root, &options, &paths, &count); + EXPECT_EQ_INT(rc, 0); + EXPECT_EQ_INT(count, 3); + free_paths(paths, count); + filter_rule_list_free(base); + unlink("test_scan_filter/a.txt"); unlink("test_scan_filter/b.tmp"); unlink("test_scan_filter/c.txt");