feat(output): rsync 3.4.1 selection and output parity (#291, #292)

#291:
- Compile --exclude/--include/--exclude-from/--include-from into the SAME
  ordered rule list as --filter/-f (first match wins), so the common
  `--include='*.txt' --exclude='*'` idiom and include-alone semantics match
  rsync. The legacy per-kind scanner arrays are no longer applied.
- -x/--one-file-system emits the cross-device mount-point directory entry
  (empty) instead of dropping it, in both the sequential and parallel scanners.
- Stop passing the legacy arrays to the scanner; document -f is --filter.

#292:
- New src/shared/format.c/.h: rsync "big_num" (comma-grouped integers) and
  decimal -h human sizes, %M/%t timestamp, and the STATUS_DEST_INFO codec.
- Receiver answers each STATUS_CHECK with a pre-transfer destination snapshot
  (new report_dest_info wire field + STATUS_DEST_INFO, PROTOCOL_VERSION
  2.23.0) so the sender can render true itemize columns.
- Itemize now emits rsync-correct update/type chars and c/s/t/p/o/g columns
  for files, dirs, symlinks and hard links, comparing size/time/perms/owner/
  group against the reported destination.
- --out-format gains %i %n %f %l %b %M %t %o %p %B %U %G %L; %f is the
  relative display path, %M the YYYY/MM/DD-HH:MM:SS form, %b the literal
  bytes sent.
- --list-only prints transfer-relative names, directory entries and ls-style
  grouped sizes.
- --stats prints rsync's multi-line block on stdout; -h uses decimal units.

Tests: unit tests for the filter ordering, format primitives, itemize
columns; integration + differential tests against real rsync 3.4.1 for
itemize/out-format/list-only/selection and -x. Golden wire len/hash and
protocol version strings updated for 2.23.0.
This commit is contained in:
2026-09-15 21:57:39 +02:00
parent 23552e823d
commit 84827ca617
28 changed files with 1529 additions and 300 deletions
+3 -1
View File
@@ -1,6 +1,6 @@
cmake_minimum_required(VERSION 3.22) cmake_minimum_required(VERSION 3.22)
project(FastFileTransfer VERSION 2.22.0) project(FastFileTransfer VERSION 2.23.0)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD 11)
@@ -96,6 +96,7 @@ set(SHARED_SRCS
src/shared/file_send.c src/shared/file_send.c
src/shared/file_store.c src/shared/file_store.c
src/shared/filter.c src/shared/filter.c
src/shared/format.c
src/shared/hardlink.c src/shared/hardlink.c
src/shared/identity.c src/shared/identity.c
src/shared/log.c src/shared/log.c
@@ -213,6 +214,7 @@ set(TEST_SRCS
tests/test_file.c tests/test_file.c
tests/test_file_list.c tests/test_file_list.c
tests/test_file_sendfile.c tests/test_file_sendfile.c
tests/test_format.c
tests/test_fuzz_smoke.c tests/test_fuzz_smoke.c
tests/test_glob.c tests/test_glob.c
tests/test_hardlink.c tests/test_hardlink.c
+345 -152
View File
@@ -7,21 +7,7 @@
#include <string.h> #include <string.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <time.h> #include <time.h>
#include <unistd.h>
/* Itemize code emitted for a transferred regular file.
*
* Layout (rsync-compatible 11-char item): `>f` marks a regular file that was
* transferred to the remote host; the trailing nine markers are, in order,
* c(hecksum) s(ize) t(ime) p(erms) o(wner) g(roup) u(ser/acl) a(ttrs) x(attrs).
* Every marker is `+` (FastSync does not compare each attribute on the
* receiving side, so a sent file is reported as fully updated). Files that
* are already up to date print no line at all, matching rsync's single -i
* which only itemizes changes.
*
* Because the scanner only yields regular-file transfer candidates, `>d`
* (directory) lines are never produced; directories are not transferred as
* items by FastSync. */
#define ITEMIZE_SENT_FILE ">f+++++++++"
typedef struct { typedef struct {
char* data; char* data;
@@ -80,103 +66,14 @@ static bool strbuf_append(StrBuf* buf, const char* text) {
return true; return true;
} }
static bool strbuf_append_ull(StrBuf* buf, unsigned long long value) {
char digits[32];
int written = snprintf(digits, sizeof(digits), "%llu", value);
if (written < 0 || (size_t)written >= sizeof(digits))
return false;
return strbuf_append(buf, digits);
}
static bool strbuf_append_longlong(StrBuf* buf, long long value) {
char digits[32];
int written = snprintf(digits, sizeof(digits), "%lld", value);
if (written < 0 || (size_t)written >= sizeof(digits))
return false;
return strbuf_append(buf, digits);
}
bool change_list_enabled(const Config* config) { bool change_list_enabled(const Config* config) {
return config != NULL && (config->itemize_changes || config->out_format != NULL || return config != NULL && (config->itemize_changes || config->out_format != NULL ||
(config->log_file != NULL && config->log_file_format != NULL)); (config->log_file != NULL && config->log_file_format != NULL));
} }
char* change_render_itemize(const ChangeEvent* event) { /* ---- Itemize code ---- */
if (event == NULL || event->decision != CHANGE_SENT)
return str_dup("");
const char* code = event->is_directory ? ">d+++++++++" : ITEMIZE_SENT_FILE;
StrBuf line = {0};
bool ok = strbuf_append(&line, code) && strbuf_append(&line, " ") &&
strbuf_append(&line, event->path != NULL ? event->path : "");
if (!ok) {
strbuf_free(&line);
return NULL;
}
return line.data;
}
static const char* leaf_name(const char* path) { /* Format the permission bits as an `ls -l` string, e.g. `-rw-r--r--`. */
if (path == NULL)
return "";
const char* slash = strrchr(path, '/');
return slash != NULL && slash[1] != '\0' ? slash + 1 : path;
}
char* change_render_format(const char* format, const ChangeEvent* event) {
if (format == NULL)
return NULL;
StrBuf line = {0};
bool ok = true;
for (const char* p = format; *p != '\0' && ok;) {
if (*p != '%') {
ok = strbuf_append_char(&line, *p);
p++;
continue;
}
char token = p[1];
if (token == '\0') {
ok = strbuf_append_char(&line, '%');
break;
}
switch (token) {
case '%':
ok = strbuf_append_char(&line, '%');
break;
case 'f':
ok = strbuf_append(&line, event->path != NULL ? event->path : "");
break;
case 'n':
ok = strbuf_append(&line, leaf_name(event->path));
break;
case 'l':
ok = strbuf_append_ull(&line, event->size);
break;
case 'b':
ok = strbuf_append_ull(&line, event->bytes_sent);
break;
case 'M':
ok = strbuf_append_longlong(&line, (long long)event->mtime_sec);
break;
default:
/* Unknown escape sequences are preserved verbatim. */
ok = strbuf_append_char(&line, '%') && strbuf_append_char(&line, token);
break;
}
p += 2;
}
if (!ok) {
strbuf_free(&line);
return NULL;
}
if (line.data == NULL) {
line.data = str_dup("");
if (!line.data)
return NULL;
}
return line.data;
}
/* Format a mode as an `ls -l` permission string, e.g. `-rw-r--r--`. */
static void mode_to_ls_string(mode_t mode, char out[11]) { static void mode_to_ls_string(mode_t mode, char out[11]) {
out[0] = S_ISDIR(mode) ? 'd' out[0] = S_ISDIR(mode) ? 'd'
: S_ISLNK(mode) ? 'l' : S_ISLNK(mode) ? 'l'
@@ -198,29 +95,102 @@ static void mode_to_ls_string(mode_t mode, char out[11]) {
out[10] = '\0'; out[10] = '\0';
} }
char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime, static char itemize_type_char(const ChangeEvent* event) {
const char* path) { if (event->is_directory)
char permission[11]; return 'd';
mode_to_ls_string(mode, permission); if (event->is_symlink)
char date[32]; return 'L';
struct tm broken_down; if (event->is_special) {
if (localtime_r(&mtime, &broken_down) != NULL) { if (S_ISCHR(event->mode) || S_ISBLK(event->mode))
if (strftime(date, sizeof(date), "%Y/%m/%d %H:%M:%S", &broken_down) == 0) return 'D';
snprintf(date, sizeof(date), "?"); return 'S';
} else {
snprintf(date, sizeof(date), "?");
} }
return 'f';
}
static bool times_match(const Config* config, const ChangeEvent* event) {
if (!event->dest.known || !event->dest.existed)
return false;
if (event->mtime_sec == event->dest.mtime_sec)
return event->mtime_nsec == event->dest.mtime_nsec;
long long delta = (long long)event->mtime_sec - (long long)event->dest.mtime_sec;
if (delta < 0)
delta = -delta;
return delta <= (long long)config->modify_window;
}
/* Fill the 11-character itemize code (10 chars + NUL). `created` means the
* destination entry did not exist, so every attribute marker is `+`. */
static void itemize_code(const Config* config, const ChangeEvent* event, char code[12]) {
bool known = event->dest.known;
bool created = !known || !event->dest.existed;
char update;
if (event->is_hardlink)
update = 'h';
else if (created)
update = (event->is_directory || event->is_symlink || event->is_special) ? 'c' : '>';
else
update = '>';
code[0] = update;
code[1] = itemize_type_char(event);
if (created) {
for (int i = 0; i < 9; i++)
code[2 + i] = '+';
code[11] = '\0';
return;
}
bool size_diff = event->size != event->dest.size;
bool time_diff = !times_match(config, event);
bool perms_diff = (event->mode & 07777) != (event->dest.mode & 07777);
bool owner_diff = event->uid != (uid_t)event->dest.uid;
bool group_diff = event->gid != (gid_t)event->dest.gid;
code[2] = '.'; /* checksum: no destination digest available */
code[3] = size_diff ? 's' : '.';
code[4] = time_diff ? 't' : '.';
code[5] = (config->preserve_perms && perms_diff) ? 'p' : '.';
code[6] = (config->preserve_owner && owner_diff) ? 'o' : '.';
code[7] = (config->preserve_group && group_diff) ? 'g' : '.';
code[8] = '.'; /* reserved */
code[9] = '.'; /* acl: not compared */
code[10] = '.';
code[11] = '\0';
}
char* change_render_itemize_code(const Config* config, const ChangeEvent* event) {
if (event == NULL || event->decision != CHANGE_SENT)
return str_dup("");
char code[12];
itemize_code(config, event, code);
return str_dup(code);
}
/* rsync %n: the transfer-relative name, with a trailing slash for directories. */
static bool append_name(StrBuf* buf, const ChangeEvent* event) {
if (!strbuf_append(buf, event->name != NULL ? event->name : ""))
return false;
if (event->is_directory && (event->name == NULL || event->name[0] == '\0' ||
event->name[strlen(event->name) - 1] != '/'))
return strbuf_append_char(buf, '/');
return true;
}
/* rsync %L: " -> target" for a symlink, " => target" for a hard link, else "". */
static bool append_link_suffix(StrBuf* buf, const ChangeEvent* event) {
if (event->is_symlink && event->symlink_target != NULL)
return strbuf_append(buf, " -> ") && strbuf_append(buf, event->symlink_target);
if (event->is_hardlink && event->hardlink_target != NULL)
return strbuf_append(buf, " => ") && strbuf_append(buf, event->hardlink_target);
return true;
}
char* change_render_itemize(const Config* config, const ChangeEvent* event) {
if (event == NULL || event->decision != CHANGE_SENT)
return str_dup("");
char code[12];
itemize_code(config, event, code);
StrBuf line = {0}; StrBuf line = {0};
char size_field[32]; bool ok = strbuf_append(&line, code) && strbuf_append_char(&line, ' ') &&
int written = snprintf(size_field, sizeof(size_field), "%llu", size); append_name(&line, event) && append_link_suffix(&line, event);
if (written < 0 || (size_t)written >= sizeof(size_field)) {
strbuf_free(&line);
return NULL;
}
bool ok = strbuf_append(&line, permission) && strbuf_append_char(&line, ' ') &&
strbuf_append(&line, size_field) && strbuf_append_char(&line, ' ') &&
strbuf_append(&line, date) && strbuf_append_char(&line, ' ') &&
strbuf_append(&line, path != NULL ? path : "");
if (!ok) { if (!ok) {
strbuf_free(&line); strbuf_free(&line);
return NULL; return NULL;
@@ -228,6 +198,141 @@ char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime
return line.data; return line.data;
} }
/* ---- --out-format / --log-file-format ---- */
char* change_render_format(const char* format, const Config* config, const ChangeEvent* event) {
if (format == NULL || event == NULL)
return NULL;
StrBuf line = {0};
bool ok = true;
for (const char* p = format; *p != '\0' && ok;) {
if (*p != '%') {
ok = strbuf_append_char(&line, *p);
p++;
continue;
}
char token = p[1];
if (token == '\0') {
ok = strbuf_append_char(&line, '%');
break;
}
switch (token) {
case '%':
ok = strbuf_append_char(&line, '%');
break;
case 'i': {
char code[12];
itemize_code(config, event, code);
ok = strbuf_append(&line, code);
break;
}
case 'f':
ok = strbuf_append(&line, event->path != NULL ? event->path : "");
break;
case 'n':
ok = append_name(&line, event);
break;
case 'L':
ok = append_link_suffix(&line, event);
break;
case 'l': {
char digits[32];
int written = snprintf(digits, sizeof(digits), "%llu", event->size);
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
} break;
case 'b': {
char digits[32];
int written = snprintf(digits, sizeof(digits), "%llu", event->bytes_sent);
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
} break;
case 'M': {
char when[32];
if (format_rsync_datetime(event->mtime_sec, true, when, sizeof(when)))
ok = strbuf_append(&line, when);
} break;
case 't': {
char when[32];
if (format_rsync_datetime(time(NULL), false, when, sizeof(when)))
ok = strbuf_append(&line, when);
} break;
case 'o':
ok = strbuf_append(&line, "send");
break;
case 'p': {
char digits[32];
int written = snprintf(digits, sizeof(digits), "%ld", (long)getpid());
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
} break;
case 'B': {
char permission[11];
mode_to_ls_string(event->mode, permission);
ok = strbuf_append(&line, permission + 1);
} break;
case 'U': {
char digits[32];
int written = snprintf(digits, sizeof(digits), "%u", (unsigned)event->uid);
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
} break;
case 'G': {
char digits[32];
int written = snprintf(digits, sizeof(digits), "%u", (unsigned)event->gid);
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
} break;
default:
/* Unknown escape sequences are preserved verbatim. */
ok = strbuf_append_char(&line, '%') && strbuf_append_char(&line, token);
break;
}
p += 2;
}
if (!ok) {
strbuf_free(&line);
return NULL;
}
if (line.data == NULL) {
line.data = str_dup("");
if (!line.data)
return NULL;
}
return line.data;
}
/* ---- --list-only ---- */
char* change_render_list_line(const Config* config, const ChangeEvent* event) {
(void)config;
if (event == NULL)
return NULL;
char permission[11];
mode_to_ls_string(event->mode, permission);
char date[32];
if (!format_rsync_datetime(event->mtime_sec, false, date, sizeof(date)))
snprintf(date, sizeof(date), "?");
StrBuf line = {0};
char size_field[40];
char grouped[32];
if (!format_big_num(event->size, false, grouped, sizeof(grouped))) {
strbuf_free(&line);
return NULL;
}
int written = snprintf(size_field, sizeof(size_field), "%15s", grouped);
if (written < 0 || (size_t)written >= sizeof(size_field)) {
strbuf_free(&line);
return NULL;
}
const char* name = event->name != NULL && event->name[0] != '\0' ? event->name : ".";
bool ok = strbuf_append(&line, permission) && strbuf_append(&line, size_field) &&
strbuf_append_char(&line, ' ') && strbuf_append(&line, date) &&
strbuf_append_char(&line, ' ') && strbuf_append(&line, name);
if (!ok) {
strbuf_free(&line);
return NULL;
}
return line.data;
}
/* ---- Event emission ---- */
static void print_escaped_line(FILE* stream, const char* line, bool eight_bit_output) { static void print_escaped_line(FILE* stream, const char* line, bool eight_bit_output) {
char* escaped = output_escape(line, eight_bit_output); char* escaped = output_escape(line, eight_bit_output);
if (escaped != NULL) { if (escaped != NULL) {
@@ -247,15 +352,16 @@ void change_emit(const Config* config, const ChangeEvent* event) {
bool to_stdout = config->itemize_changes || config->out_format != NULL; bool to_stdout = config->itemize_changes || config->out_format != NULL;
bool to_log = config->log_file != NULL && config->log_file_format != NULL; bool to_log = config->log_file != NULL && config->log_file_format != NULL;
if (to_stdout) { if (to_stdout) {
char* line = config->out_format != NULL ? change_render_format(config->out_format, event) char* line = config->out_format != NULL
: change_render_itemize(event); ? change_render_format(config->out_format, config, event)
: change_render_itemize(config, event);
if (line != NULL) { if (line != NULL) {
print_escaped_line(stdout, line, config->eight_bit_output); print_escaped_line(stdout, line, config->eight_bit_output);
free(line); free(line);
} }
} }
if (to_log) { if (to_log) {
char* line = change_render_format(config->log_file_format, event); char* line = change_render_format(config->log_file_format, config, event);
if (line != NULL) { if (line != NULL) {
print_escaped_line(config->log_file, line, config->eight_bit_output); print_escaped_line(config->log_file, line, config->eight_bit_output);
free(line); free(line);
@@ -266,9 +372,6 @@ void change_emit(const Config* config, const ChangeEvent* event) {
static bool format_uses_mtime(const char* format) { static bool format_uses_mtime(const char* format) {
if (format == NULL) if (format == NULL)
return false; return false;
/* Mirror change_render_format's tokenizer: "%%" is a literal percent (so
* "%%M" does NOT expand %M) and unknown "%X" escapes consume both chars.
* This keeps the optional stat() fallback below in step with the renderer. */
for (const char* p = format; *p != '\0';) { for (const char* p = format; *p != '\0';) {
if (*p != '%') { if (*p != '%') {
p++; p++;
@@ -284,46 +387,136 @@ static bool format_uses_mtime(const char* format) {
return false; return false;
} }
/* Relative path of an entry below the transfer root (no leading slash). Uses
* the sender-side send_path override when present (bare-relative -R layout). */
static char* relative_name(const Config* config, const File* file) {
const char* full = file_wire_path(file);
if (file->send_path != NULL)
return str_dup(full != NULL ? full : "");
const char* root = config->send_directory;
if (root == NULL || full == NULL)
return str_dup(full != NULL ? full : "");
size_t root_len = strlen(root);
while (root_len > 1 && root[root_len - 1] == '/')
root_len--;
if (strncmp(root, full, root_len) == 0) {
if (full[root_len] == '\0')
return str_dup("");
if (full[root_len] == '/')
return str_dup(full + root_len + 1);
}
return str_dup(full);
}
/* rsync %f long form: the source argument as typed (leading '/' removed,
* trailing '/' removed, leading "./" removed) joined to the relative name. */
static char* display_name(const Config* config, const char* name) {
const char* root = config->send_directory;
if (root == NULL)
return str_dup(name != NULL ? name : "");
const char* p = root;
while (*p == '/')
p++;
if (p[0] == '.' && p[1] == '/')
p += 2;
size_t root_len = strlen(p);
while (root_len > 0 && p[root_len - 1] == '/')
root_len--;
size_t name_len = name != NULL ? strlen(name) : 0;
if (root_len == 0 && name_len == 0)
return str_dup("");
char* out = malloc(root_len + (root_len > 0 && name_len > 0 ? 1 : 0) + name_len + 1);
if (!out)
return NULL;
size_t offset = 0;
if (root_len > 0) {
memcpy(out, p, root_len);
offset = root_len;
}
if (root_len > 0 && name_len > 0)
out[offset++] = '/';
if (name_len > 0)
memcpy(out + offset, name, name_len);
out[offset + name_len] = '\0';
return out;
}
static void fill_event_from_file(const Config* config, const File* file, ChangeEvent* event,
char** name_out, char** path_out) {
char* name = relative_name(config, file);
char* path = display_name(config, name);
event->name = name;
event->path = path;
*name_out = name;
*path_out = path;
if (file->metadata != NULL) {
event->mtime_sec = file->metadata->mtime_sec;
event->mtime_nsec = file->metadata->mtime_nsec;
event->mode = file->metadata->mode;
event->uid = file->metadata->uid;
event->gid = file->metadata->gid;
} else if (format_uses_mtime(config->out_format) || format_uses_mtime(config->log_file_format)) {
struct stat st;
if (file->path != NULL && stat(file->path, &st) == 0) {
event->mtime_sec = st.st_mtime;
event->mtime_nsec = st.st_mtim.tv_nsec;
}
}
}
void change_emit_file_sent(const Config* config, const File* file) { void change_emit_file_sent(const Config* config, const File* file) {
if (file == NULL || !change_list_enabled(config)) if (file == NULL || !change_list_enabled(config))
return; return;
ChangeEvent event; ChangeEvent event;
memset(&event, 0, sizeof(event)); memset(&event, 0, sizeof(event));
/* The displayed path is the one transmitted (with -R + --files-from this is
the bare relative destination path); the metadata fallback below still
stats the local absolute path. */
event.path = file_wire_path(file);
event.decision = CHANGE_SENT; event.decision = CHANGE_SENT;
event.is_directory = false; event.is_directory = false;
event.is_symlink = false;
event.is_special = false;
event.is_hardlink = false;
event.size = file->data != NULL ? file->data->size : 0; event.size = file->data != NULL ? file->data->size : 0;
/* FastSync has no wire-byte counter yet, so %b reports the source length event.dest = file->dest_state;
* that had to be delivered (always equal to %l); the actual bytes written if (file->is_symlink) {
* to the socket (compressed/delta) are not measured. */ event.is_symlink = true;
event.symlink_target = file->symlink_target;
event.size = file->symlink_target != NULL ? strlen(file->symlink_target) : 0;
event.bytes_sent = 0;
} else if (file->is_special) {
event.is_special = true;
event.bytes_sent = 0;
} else if (file->link_group != 0 && !file->link_first) {
event.is_hardlink = true;
event.hardlink_target = file->hardlink_target;
event.bytes_sent = 0;
} else {
/* Literal payload bytes delivered; compressed/delta wire bytes are not
* separately counted. */
event.bytes_sent = event.size; event.bytes_sent = event.size;
if (file->metadata != NULL) {
event.mtime_sec = file->metadata->mtime_sec;
} else if (format_uses_mtime(config->out_format) || format_uses_mtime(config->log_file_format)) {
/* Best-effort fallback for %M when no metadata was captured (no -M): the
* path is stat()ed just to fill the field, and any failure leaves 0. */
struct stat st;
if (file->path != NULL && stat(file->path, &st) == 0)
event.mtime_sec = st.st_mtime;
} }
char* name = NULL;
char* path = NULL;
fill_event_from_file(config, file, &event, &name, &path);
if (name != NULL && path != NULL)
change_emit(config, &event); change_emit(config, &event);
free(name);
free(path);
} }
/* Build and emit a CHANGE_SENT event for an explicit directory entry (-d). */
void change_emit_dir_sent(const Config* config, const File* file) { void change_emit_dir_sent(const Config* config, const File* file) {
if (file == NULL || !change_list_enabled(config)) if (file == NULL || !change_list_enabled(config))
return; return;
ChangeEvent event; ChangeEvent event;
memset(&event, 0, sizeof(event)); memset(&event, 0, sizeof(event));
event.path = file_wire_path(file);
event.decision = CHANGE_SENT; event.decision = CHANGE_SENT;
event.is_directory = true; event.is_directory = true;
event.size = 0; event.size = 0;
event.bytes_sent = 0; event.bytes_sent = 0;
if (file->metadata != NULL) event.dest = file->dest_state;
event.mtime_sec = file->metadata->mtime_sec; char* name = NULL;
char* path = NULL;
fill_event_from_file(config, file, &event, &name, &path);
if (name != NULL && path != NULL)
change_emit(config, &event); change_emit(config, &event);
free(name);
free(path);
} }
+34 -23
View File
@@ -3,6 +3,7 @@
#include "config.h" #include "config.h"
#include "file_types.h" #include "file_types.h"
#include "format.h"
#include <stdbool.h> #include <stdbool.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <time.h> #include <time.h>
@@ -26,42 +27,52 @@ typedef enum {
} ChangeDecision; } ChangeDecision;
typedef struct { typedef struct {
const char* path; /* full source path */ const char* path; /* long-form display path (rsync %f) */
const char* name; /* transfer-relative path (rsync %n), no trailing slash */
ChangeDecision decision; ChangeDecision decision;
bool is_directory; bool is_directory;
bool is_symlink;
bool is_special;
bool is_hardlink; /* a hard-link sibling (linked, no data sent) */
const char* symlink_target;
const char* hardlink_target;
unsigned long long size; /* source file length in bytes */ unsigned long long size; /* source file length in bytes */
/* The number of bytes reported for a sent file. FastSync has no wire-byte unsigned long long bytes_sent; /* literal data bytes actually transferred */
* counter, so this is always the source length (== size / %l); actual time_t mtime_sec;
* post-compression/delta bytes on the wire are not counted. */ long mtime_nsec;
unsigned long long bytes_sent; mode_t mode;
time_t mtime_sec; /* 0 when unknown */ uid_t uid;
gid_t gid;
/* Receiver-reported pre-transfer destination state (OutputDestState.known is
* false when no report was requested/received). */
OutputDestState dest;
} ChangeEvent; } ChangeEvent;
/* True when any output mode is active and per-file events matter. */ /* True when any output mode is active and per-file events matter. */
bool change_list_enabled(const Config* config); bool change_list_enabled(const Config* config);
/* Render the rsync-style itemize line for a transferred file: /* Render the rsync-style itemize line for a transferred item
* `>f+++++++++ <path>` * (`%i %n%L`): `>f+++++++++ sub/b.txt`. Caller frees the result. */
* The 11-char code is `>f` (regular file transferred to the remote host) char* change_render_itemize(const Config* config, const ChangeEvent* event);
* followed by c/s/t/p/o/g/u/a/x markers that are all `+` (value will be set
* / differs) because FastSync does not separately compare checksums, size,
* mtime, perms, owner, group, uid, acl, or xattr on the receiving side, so a
* sent file is reported as fully updated. Up-to-date files print no line
* (rsync single `-i` only shows changes). Caller frees the result. */
char* change_render_itemize(const ChangeEvent* event);
/* Expand an --out-format/--log-file-format template. Tokens: /* Render only the 11-character itemize code (rsync %i). Caller frees. */
* %f full source path %b "bytes sent" == the source length (%l); char* change_render_itemize_code(const Config* config, const ChangeEvent* event);
* %n leaf (base) name actual post-compression/delta wire bytes
* %l file length in bytes are not counted /* Expand an --out-format/--log-file-format template. Supported tokens:
* %M mtime in whole seconds %% a literal percent sign * %i itemize code %n transfer-relative name (dir: trailing /)
* %f long display path %l file length in bytes
* %b bytes actually sent %M mtime (YYYY/MM/DD-HH:MM:SS)
* %t current time %o operation ("send"/"del.")
* %p pid %B permission bits without the type char
* %U uid %G gid
* %L " -> target" / " => target" %% a literal percent sign
* Unknown %X sequences are preserved verbatim. Caller frees the result. */ * Unknown %X sequences are preserved verbatim. Caller frees the result. */
char* change_render_format(const char* format, const ChangeEvent* event); char* change_render_format(const char* format, const Config* config, const ChangeEvent* event);
/* Render one --list-only long-listing entry: /* Render one --list-only long-listing entry:
* `-rw-r--r-- 12 2026/09/06 10:00:00 <path>` * `-rw-r--r-- 12 2026/09/06 10:00:00 sub/b.txt`
* (ls -l style columns; mtime in the local time zone). Caller frees it. */ * (ls -l style columns; mtime in the local time zone). Caller frees it. */
char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime, const char* path); char* change_render_list_line(const Config* config, const ChangeEvent* event);
/* Emit an event to every active destination: /* Emit an event to every active destination:
* stdout: --itemize-changes line, or the --out-format expansion when set; * stdout: --itemize-changes line, or the --out-format expansion when set;
+45 -11
View File
@@ -334,7 +334,8 @@ static void apply_output_buffering(const Config* config) {
} }
#endif #endif
static int read_patterns_from_file(const char* filepath, char*** patterns, int* count); static int read_patterns_from_file(const char* filepath, char*** patterns, int* count,
Config* config, char sign, const char* optname);
static int parse_debug_flags(const char* value, Config* config) { static int parse_debug_flags(const char* value, Config* config) {
if (!value || value[0] == '\0' || value[0] == ',' || value[strlen(value) - 1] == ',' || if (!value || value[0] == '\0' || value[0] == ',' || value[strlen(value) - 1] == ',' ||
@@ -568,6 +569,27 @@ static int config_add_filter(Config* config, const char* rule) {
return 0; return 0;
} }
/* Compile one --exclude/--include pattern into the SAME ordered filter rule
* list used by --filter/-f: `--exclude P` becomes the rule "- P" and
* `--include P` becomes "+ P", appended in command-line order. This is what
* makes rsync's first-match-wins semantics hold across a mixed sequence such as
* `--include='*.txt' --exclude='*'`. Returns 0 on success, -1 on error. */
static int config_add_selection_rule(Config* config, char sign, const char* pattern,
const char* optname) {
size_t len = strlen(pattern);
char* rule = malloc(len + 3);
if (!rule) {
log_message(LOG_LEVEL_ERROR, "memory allocation failed for %s", optname);
return -1;
}
rule[0] = sign;
rule[1] = ' ';
memcpy(rule + 2, pattern, len + 1);
int rc = config_add_filter(config, rule);
free(rule);
return rc;
}
static int parse_skip_compress(Config* config, const char* value) { static int parse_skip_compress(Config* config, const char* value) {
char* list = str_dup(value); char* list = str_dup(value);
if (!list) if (!list)
@@ -1323,7 +1345,8 @@ static bool cli_handle_ssh_and_pattern_options(CliParseCtx* ctx) {
} }
if (strncmp(arg, "--exclude=", 10) == 0) { if (strncmp(arg, "--exclude=", 10) == 0) {
if (config_add_pattern(&config->exclude_patterns, &config->exclude_count, arg + 10, if (config_add_pattern(&config->exclude_patterns, &config->exclude_count, arg + 10,
"--exclude") != 0) "--exclude") != 0 ||
config_add_selection_rule(config, '-', arg + 10, "--exclude") != 0)
ctx->exit_code = -1; ctx->exit_code = -1;
return true; return true;
} }
@@ -1334,13 +1357,15 @@ static bool cli_handle_ssh_and_pattern_options(CliParseCtx* ctx) {
return true; return true;
} }
if (config_add_pattern(&config->exclude_patterns, &config->exclude_count, ctx->argv[++ctx->i], if (config_add_pattern(&config->exclude_patterns, &config->exclude_count, ctx->argv[++ctx->i],
"--exclude") != 0) "--exclude") != 0 ||
config_add_selection_rule(config, '-', ctx->argv[ctx->i], "--exclude") != 0)
ctx->exit_code = -1; ctx->exit_code = -1;
return true; return true;
} }
if (strncmp(arg, "--include=", 10) == 0) { if (strncmp(arg, "--include=", 10) == 0) {
if (config_add_pattern(&config->include_patterns, &config->include_count, arg + 10, if (config_add_pattern(&config->include_patterns, &config->include_count, arg + 10,
"--include") != 0) "--include") != 0 ||
config_add_selection_rule(config, '+', arg + 10, "--include") != 0)
ctx->exit_code = -1; ctx->exit_code = -1;
return true; return true;
} }
@@ -1351,7 +1376,8 @@ static bool cli_handle_ssh_and_pattern_options(CliParseCtx* ctx) {
return true; return true;
} }
if (config_add_pattern(&config->include_patterns, &config->include_count, ctx->argv[++ctx->i], if (config_add_pattern(&config->include_patterns, &config->include_count, ctx->argv[++ctx->i],
"--include") != 0) "--include") != 0 ||
config_add_selection_rule(config, '+', ctx->argv[ctx->i], "--include") != 0)
ctx->exit_code = -1; ctx->exit_code = -1;
return true; return true;
} }
@@ -1634,7 +1660,8 @@ static bool cli_handle_filter_options(CliParseCtx* ctx) {
Config* config = ctx->config; Config* config = ctx->config;
const char* arg = ctx->argv[ctx->i]; const char* arg = ctx->argv[ctx->i];
if (strncmp(arg, "--exclude-from=", 15) == 0) { if (strncmp(arg, "--exclude-from=", 15) == 0) {
if (read_patterns_from_file(arg + 15, &config->exclude_patterns, &config->exclude_count) != 0) if (read_patterns_from_file(arg + 15, &config->exclude_patterns, &config->exclude_count, config,
'-', "--exclude-from") != 0)
ctx->exit_code = -1; ctx->exit_code = -1;
return true; return true;
} }
@@ -1645,12 +1672,13 @@ static bool cli_handle_filter_options(CliParseCtx* ctx) {
return true; return true;
} }
if (read_patterns_from_file(ctx->argv[++ctx->i], &config->exclude_patterns, if (read_patterns_from_file(ctx->argv[++ctx->i], &config->exclude_patterns,
&config->exclude_count) != 0) &config->exclude_count, config, '-', "--exclude-from") != 0)
ctx->exit_code = -1; ctx->exit_code = -1;
return true; return true;
} }
if (strncmp(arg, "--include-from=", 15) == 0) { if (strncmp(arg, "--include-from=", 15) == 0) {
if (read_patterns_from_file(arg + 15, &config->include_patterns, &config->include_count) != 0) if (read_patterns_from_file(arg + 15, &config->include_patterns, &config->include_count, config,
'+', "--include-from") != 0)
ctx->exit_code = -1; ctx->exit_code = -1;
return true; return true;
} }
@@ -1661,7 +1689,7 @@ static bool cli_handle_filter_options(CliParseCtx* ctx) {
return true; return true;
} }
if (read_patterns_from_file(ctx->argv[++ctx->i], &config->include_patterns, if (read_patterns_from_file(ctx->argv[++ctx->i], &config->include_patterns,
&config->include_count) != 0) &config->include_count, config, '+', "--include-from") != 0)
ctx->exit_code = -1; ctx->exit_code = -1;
return true; return true;
} }
@@ -2088,6 +2116,10 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool
* --no-xattrs/--no-acls negation) so the sender's wire gate always matches * --no-xattrs/--no-acls negation) so the sender's wire gate always matches
* the flags the receiver will recompute from the received config. */ * the flags the receiver will recompute from the received config. */
config->use_xattrs = config->preserve_acls || config->preserve_xattrs; config->use_xattrs = config->preserve_acls || config->preserve_xattrs;
/* Output parity: -i/--itemize-changes and --out-format need the pre-transfer
* destination snapshot (new vs modified and which attributes differ), so ask
* the receiver to report it on every per-file check. This is a wire field. */
config->report_dest_info = config->itemize_changes || config->out_format != NULL;
return 0; return 0;
} }
@@ -2284,7 +2316,8 @@ done:
return result; return result;
} }
static int read_patterns_from_file(const char* filepath, char*** patterns, int* count) { static int read_patterns_from_file(const char* filepath, char*** patterns, int* count,
Config* config, char sign, const char* optname) {
FILE* fp = fopen(filepath, "r"); FILE* fp = fopen(filepath, "r");
if (!fp) { if (!fp) {
char* escaped = output_escape(filepath, false); char* escaped = output_escape(filepath, false);
@@ -2326,7 +2359,8 @@ static int read_patterns_from_file(const char* filepath, char*** patterns, int*
p[--len] = '\0'; p[--len] = '\0';
if (len == 0) if (len == 0)
continue; continue;
if (config_add_pattern(patterns, count, p, "pattern file") != 0) { if (config_add_pattern(patterns, count, p, "pattern file") != 0 ||
config_add_selection_rule(config, sign, p, optname) != 0) {
free(line); free(line);
fclose(fp); fclose(fp);
return -1; return -1;
+152 -34
View File
@@ -11,6 +11,7 @@
#include "file.h" #include "file.h"
#include "file_list.h" #include "file_list.h"
#include "filter.h" #include "filter.h"
#include "format.h"
#include "hardlink.h" #include "hardlink.h"
#include "metadata.h" #include "metadata.h"
#include "motd.h" #include "motd.h"
@@ -69,32 +70,61 @@ static int progress_thread_fn(void* arg);
static const char* display_bytes(unsigned long long bytes, bool human_readable, char* buffer, static const char* display_bytes(unsigned long long bytes, bool human_readable, char* buffer,
size_t buffer_size) { size_t buffer_size) {
if (human_readable && format_human_bytes(bytes, buffer, buffer_size)) if (human_readable && format_human_size_decimal(bytes, buffer, buffer_size))
return buffer; return buffer;
snprintf(buffer, buffer_size, "%.1f MB", (double)bytes / (double)BYTES_PER_MIB); snprintf(buffer, buffer_size, "%.1f MB", (double)bytes / (double)BYTES_PER_MIB);
return buffer; return buffer;
} }
/* Print the canonical `--stats` line. Shared by the single-threaded and /* rsync byte count: human-readable decimal when -h was given, otherwise a
multithreaded send paths so both honor --stats, --human-readable and --quiet * comma-grouped integer (rsync's big_num in the C locale). */
identically; `start` marks the beginning of the transfer for the rate. */ static const char* stats_bytes(const Config* config, unsigned long long bytes, char* buffer,
size_t buffer_size) {
if (!format_big_num(bytes, config->human_readable, buffer, buffer_size))
snprintf(buffer, buffer_size, "%llu", bytes);
return buffer;
}
/* Print the rsync `--stats` block on stdout. FastSync is a push sender, so a
few receiver-only counters (matched data, file-list bytes, deletion count)
are not observable and are reported as 0; the labels and layout match rsync
3.4.1. Shared by the single-threaded and multithreaded send paths. */
static void report_transfer_stats(const Config* config, int total_files, static void report_transfer_stats(const Config* config, int total_files,
unsigned long long total_bytes, time_t start) { unsigned long long total_bytes, time_t start) {
if (!config->stats || config->quiet) if (!config->stats || config->quiet)
return; return;
double elapsed = difftime(time(NULL), start); double elapsed = difftime(time(NULL), start);
double rate = elapsed > 0.0 ? (double)total_bytes / ((double)BYTES_PER_MIB * elapsed) : 0.0; double rate = elapsed > 0.0 ? (double)total_bytes / elapsed : 0.0;
if (config->human_readable) {
char total_buffer[32]; char total_buffer[32];
char rate_buffer[32]; char rate_buffer[32];
fprintf(stderr, "Stats: %d files, %s, %s/s\n", total_files, char human_rate[32];
display_bytes(total_bytes, true, total_buffer, sizeof(total_buffer)), const char* total = stats_bytes(config, total_bytes, total_buffer, sizeof(total_buffer));
display_bytes((unsigned long long)(rate * (double)BYTES_PER_MIB), true, rate_buffer, const char* rate_str = rate_buffer;
sizeof(rate_buffer))); if (config->human_readable) {
if (!format_human_size_decimal((unsigned long long)rate, human_rate, sizeof(human_rate)))
snprintf(human_rate, sizeof(human_rate), "0");
rate_str = human_rate;
} else { } else {
fprintf(stderr, "Stats: %d files, %.1f MB, %.1f MB/s\n", total_files, snprintf(rate_buffer, sizeof(rate_buffer), "%.2f", rate);
(double)total_bytes / (double)BYTES_PER_MIB, rate);
} }
printf("\n");
printf("Number of files: %d\n", total_files);
printf("Number of created files: %d\n", total_files);
printf("Number of deleted files: 0\n");
printf("Number of regular files transferred: %d\n", total_files);
printf("Total file size: %s bytes\n", total);
printf("Total transferred file size: %s bytes\n", total);
printf("Literal data: %s bytes\n", total);
printf("Matched data: 0 bytes\n");
printf("File list size: 0\n");
printf("File list generation time: 0.000 seconds\n");
printf("File list transfer time: 0.000 seconds\n");
printf("Total bytes sent: %s\n", total);
printf("Total bytes received: 0\n");
printf("\n");
printf("sent %s bytes received 0 bytes %s bytes/sec\n", total, rate_str);
printf("total size is %s speedup is %.2f\n", total, 1.0);
fflush(stdout);
} }
/* Compiled scanner inputs that are shared read-only across scanner instances /* Compiled scanner inputs that are shared read-only across scanner instances
@@ -146,10 +176,16 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann
options->preserve_xattrs = config->preserve_xattrs; options->preserve_xattrs = config->preserve_xattrs;
options->preserve_acls = config->preserve_acls; options->preserve_acls = config->preserve_acls;
options->chunk_size = config->chunk_size; options->chunk_size = config->chunk_size;
options->exclude_patterns = config->exclude_patterns; /* --exclude/--include are compiled, in command-line order, into the SAME
options->exclude_count = config->exclude_count; * ordered filter rule list as --filter/-f (see config_add_selection_rule), so
options->include_patterns = config->include_patterns; * the legacy per-kind arrays are deliberately NOT passed to the scanner:
options->include_count = config->include_count; * doing so would re-apply them with the old "excludes first, then includes as
* a mandatory whitelist" precedence and defeat rsync's first-match-wins
* ordering. The arrays remain populated purely for the Config API surface. */
options->exclude_patterns = NULL;
options->exclude_count = 0;
options->include_patterns = NULL;
options->include_count = 0;
options->max_size = config->max_size; options->max_size = config->max_size;
options->min_size = config->min_size; options->min_size = config->min_size;
options->max_depth = config->max_depth; options->max_depth = config->max_depth;
@@ -467,7 +503,7 @@ static void receive_daemon_motd(Client* client, const Config* config) {
static Client* connect_transfer_client(const Config* config) { static Client* connect_transfer_client(const Config* config) {
if (config->transport == TRANSPORT_SSH) { if (config->transport == TRANSPORT_SSH) {
if (config->use_sendfile) { if (config->use_sendfile) {
log_message(LOG_LEVEL_ERROR, "-f/--sendfile is not supported with SSH transport"); log_message(LOG_LEVEL_ERROR, "--sendfile is not supported with SSH transport");
return NULL; return NULL;
} }
return client_connect_ssh(config->ssh_destination, config->ssh_port, return client_connect_ssh(config->ssh_destination, config->ssh_port,
@@ -782,30 +818,53 @@ static int send_dry_run_manifest(const Config* config) {
} }
typedef struct { typedef struct {
char* path; char* name; /* transfer-relative name ("" == the source root) */
mode_t mode; mode_t mode;
unsigned long long size; unsigned long long size;
time_t mtime; time_t mtime;
long mtime_nsec;
bool is_dir;
bool is_symlink;
char* link_target;
} ListEntry; } ListEntry;
static void list_entries_destroy(ListEntry* entries, size_t count) { static void list_entries_destroy(ListEntry* entries, size_t count) {
if (entries == NULL) if (entries == NULL)
return; return;
for (size_t i = 0; i < count; i++) for (size_t i = 0; i < count; i++) {
free(entries[i].path); free(entries[i].name);
free(entries[i].link_target);
}
free(entries); free(entries);
} }
static int compare_list_entries(const void* left, const void* right) { static int compare_list_entries(const void* left, const void* right) {
const ListEntry* a = (const ListEntry*)left; const ListEntry* a = (const ListEntry*)left;
const ListEntry* b = (const ListEntry*)right; const ListEntry* b = (const ListEntry*)right;
return strcmp(a->path, b->path); return strcmp(a->name, b->name);
} }
/* --list-only: print an ls-style listing of the files that WOULD be /* Relative path of an entry below `root` ("" for the root itself). Mirrors
* change_list's relative_name for list-only rendering. */
static char* list_relative_name(const char* root, const char* full) {
if (root == NULL || full == NULL)
return str_dup(full != NULL ? full : "");
size_t root_len = strlen(root);
while (root_len > 1 && root[root_len - 1] == '/')
root_len--;
if (strncmp(root, full, root_len) == 0) {
if (full[root_len] == '\0')
return str_dup("");
if (full[root_len] == '/')
return str_dup(full + root_len + 1);
}
return str_dup(full);
}
/* --list-only: print an ls-style listing of the entries that WOULD be
* transferred and exit without contacting the server or writing anything. * transferred and exit without contacting the server or writing anything.
* Directory lines are not printed because the scanner only yields regular * Names are transfer-relative (rsync prints `a.txt`, `sub/b.txt`, `.`) and
* transfer candidates. Returns 0 on success, 1 on error. */ * directory entries are included. Returns 0 on success, 1 on error. */
static int send_list_only(const Config* config) { static int send_list_only(const Config* config) {
int skipped = 0; int skipped = 0;
if (!files_from_list_check(config, NULL, &skipped)) if (!files_from_list_check(config, NULL, &skipped))
@@ -814,6 +873,7 @@ static int send_list_only(const Config* config) {
if (!prepare_scanner(config, 0, &prepared)) if (!prepare_scanner(config, 0, &prepared))
return 1; return 1;
prepared.options.use_metadata = true; /* capture mode + mtime for the listing */ prepared.options.use_metadata = true; /* capture mode + mtime for the listing */
prepared.options.list_dirs = true;
DirectoryScanner* scanner = DirectoryScanner* scanner =
directory_scanner_create_with_options(config->send_directory, &prepared.options); directory_scanner_create_with_options(config->send_directory, &prepared.options);
if (!scanner) { if (!scanner) {
@@ -823,9 +883,31 @@ static int send_list_only(const Config* config) {
ListEntry* entries = NULL; ListEntry* entries = NULL;
size_t count = 0; size_t count = 0;
size_t capacity = 0; size_t capacity = 0;
Chunk* chunk;
bool oom = false; bool oom = false;
while ((chunk = directory_scanner_next(scanner)) != NULL) {
/* rsync lists the source root itself (as "."). Only when the source is a
* directory and no --files-from subset is in effect. */
if (config->files_from_set == NULL && config->send_directory != NULL) {
struct stat st;
if (stat(config->send_directory, &st) == 0 && S_ISDIR(st.st_mode)) {
capacity = 64;
entries = calloc(capacity, sizeof(ListEntry));
if (entries == NULL) {
oom = true;
} else {
entries[0].name = str_dup("");
entries[0].mode = st.st_mode;
entries[0].mtime = st.st_mtime;
entries[0].mtime_nsec = st.st_mtim.tv_nsec;
entries[0].size = (unsigned long long)st.st_size;
entries[0].is_dir = true;
count = 1;
}
}
}
Chunk* chunk;
while (!oom && (chunk = directory_scanner_next(scanner)) != NULL) {
for (int i = 0; i < chunk->element_count; i++) { for (int i = 0; i < chunk->element_count; i++) {
File* f = chunk->items[i]; File* f = chunk->items[i];
if (f == NULL) if (f == NULL)
@@ -842,34 +924,47 @@ static int send_list_only(const Config* config) {
break; break;
} }
entries = grown; entries = grown;
memset(entries + capacity, 0, (new_capacity - capacity) * sizeof(ListEntry));
capacity = new_capacity; capacity = new_capacity;
} }
char* path = str_dup(file_wire_path(f)); char* name = list_relative_name(config->send_directory, file_wire_path(f));
if (!path) { if (!name) {
oom = true; oom = true;
break; break;
} }
mode_t mode = 0; mode_t mode = 0;
time_t mtime = 0; time_t mtime = 0;
long mtime_nsec = 0;
if (f->metadata != NULL) { if (f->metadata != NULL) {
mode = f->metadata->mode; mode = f->metadata->mode;
mtime = f->metadata->mtime_sec; mtime = f->metadata->mtime_sec;
mtime_nsec = f->metadata->mtime_nsec;
} else { } else {
struct stat st; struct stat st;
if (stat(f->path, &st) == 0) { if (lstat(f->path, &st) == 0) {
mode = st.st_mode; mode = st.st_mode;
mtime = st.st_mtime; mtime = st.st_mtime;
mtime_nsec = st.st_mtim.tv_nsec;
} }
} }
entries[count].path = path; entries[count].name = name;
entries[count].mode = mode; entries[count].mode = mode;
entries[count].mtime = mtime; entries[count].mtime = mtime;
entries[count].mtime_nsec = mtime_nsec;
if (f->is_symlink)
entries[count].size = f->symlink_target != NULL ? strlen(f->symlink_target) : 0;
else if (f->is_dir) {
struct stat dir_st;
entries[count].size = stat(f->path, &dir_st) == 0 ? (unsigned long long)dir_st.st_size : 0;
} else
entries[count].size = f->data != NULL ? f->data->size : 0; entries[count].size = f->data != NULL ? f->data->size : 0;
entries[count].is_dir = f->is_dir;
entries[count].is_symlink = f->is_symlink;
entries[count].link_target =
f->is_symlink && f->symlink_target ? str_dup(f->symlink_target) : NULL;
count++; count++;
} }
chunk_destroy(chunk); chunk_destroy(chunk);
if (oom)
break;
} }
bool failed = oom || directory_scanner_failed(scanner) || directory_scanner_had_io_error(scanner); bool failed = oom || directory_scanner_failed(scanner) || directory_scanner_had_io_error(scanner);
directory_scanner_destroy(scanner); directory_scanner_destroy(scanner);
@@ -883,8 +978,18 @@ static int send_list_only(const Config* config) {
if (count > 1) if (count > 1)
qsort(entries, count, sizeof(ListEntry), compare_list_entries); qsort(entries, count, sizeof(ListEntry), compare_list_entries);
for (size_t i = 0; i < count; i++) { for (size_t i = 0; i < count; i++) {
char* line = change_render_list_line(entries[i].mode, entries[i].size, entries[i].mtime, ChangeEvent event;
entries[i].path); memset(&event, 0, sizeof(event));
event.name = entries[i].name;
event.path = entries[i].name;
event.mode = entries[i].mode;
event.size = entries[i].size;
event.mtime_sec = entries[i].mtime;
event.mtime_nsec = entries[i].mtime_nsec;
event.is_directory = entries[i].is_dir;
event.is_symlink = entries[i].is_symlink;
event.symlink_target = entries[i].link_target;
char* line = change_render_list_line(config, &event);
if (line != NULL) { if (line != NULL) {
char* escaped = output_escape(line, config->eight_bit_output); char* escaped = output_escape(line, config->eight_bit_output);
printf("%s\n", escaped != NULL ? escaped : line); printf("%s\n", escaped != NULL ? escaped : line);
@@ -1052,6 +1157,19 @@ static int incremental_check(Client* client, File* file, const Config* config,
Status s; Status s;
if (!receive_status(client->file_descriptor, &s)) if (!receive_status(client->file_descriptor, &s))
return -1; return -1;
/* Output parity: when dest-info reporting is negotiated the receiver sends
* the pre-transfer destination snapshot BEFORE its ordinary verdict. Consume
* it here so the following status read stays in sync. */
if (config->report_dest_info) {
if (s != STATUS_DEST_INFO ||
!format_dest_state_receive(client->file_descriptor, &file->dest_state)) {
log_message(LOG_LEVEL_ERROR, "Unexpected reply to the destination-state report");
send_status(client->file_descriptor, STATUS_ERROR);
return -1;
}
if (!receive_status(client->file_descriptor, &s))
return -1;
}
if (s == STATUS_ERROR) { if (s == STATUS_ERROR) {
log_server_rejection("Server reported error for file"); log_server_rejection("Server reported error for file");
return -1; return -1;
+63
View File
@@ -134,6 +134,26 @@ bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entr
return !one_file_system || entry_device == root_device; return !one_file_system || entry_device == root_device;
} }
/* Build a payload-less directory File carrying the captured metadata (when
* requested). Used by -x mount-point emission and --list-only directory
* entries. Returns NULL on allocation failure. */
static File* scanner_build_dir_file(const char* path, const struct stat* stats,
const ScannerOptions* options) {
File* dir = file_create(path);
if (dir == NULL)
return NULL;
dir->is_dir = true;
if (options->use_metadata) {
dir->metadata =
file_metadata_create(dir->path, stats, options->preserve_atimes, options->preserve_crtimes);
if (!dir->metadata) {
file_destroy(dir);
return NULL;
}
}
return dir;
}
/* Relative path of an on-disk path below `root`. The transfer root may be /* Relative path of an on-disk path below `root`. The transfer root may be
* given with a trailing slash; the returned rel path never has one and is "" * given with a trailing slash; the returned rel path never has one and is ""
* for the root itself. A root of "/" is handled (its children start at "/"). * for the root itself. A root of "/" is handled (its children start at "/").
@@ -1044,9 +1064,31 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
free(rel_copy); free(rel_copy);
if (!scanner_same_filesystem(scanner->options.one_file_system, scanner->root_dev, if (!scanner_same_filesystem(scanner->options.one_file_system, scanner->root_dev,
stats.st_dev)) { stats.st_dev)) {
/* rsync's -x/--one-file-system emits the mount-point directory entry
itself (so the destination gets an empty directory) but does NOT
descend into it. Build a payload-less directory File and hand it to
the caller; never enqueue it for traversal. */
File* mount = scanner_build_dir_file(cur_path, &stats, &scanner->options);
if (mount == NULL || !array_list_add(chunk_data, mount)) {
file_destroy(mount);
free(cur_path);
scanner->failed = true;
break;
}
free(cur_path); free(cur_path);
continue; continue;
} }
/* --list-only: list directory entries too (rsync prints them), even
though a real transfer never sends them explicitly. */
if (scanner->options.list_dirs) {
File* dir = scanner_build_dir_file(cur_path, &stats, &scanner->options);
if (dir == NULL || !array_list_add(chunk_data, dir)) {
file_destroy(dir);
free(cur_path);
scanner->failed = true;
break;
}
}
int next_depth = scanner->current_depth + 1; int next_depth = scanner->current_depth + 1;
if (scanner->options.max_depth <= 0 || next_depth < scanner->options.max_depth) { if (scanner->options.max_depth <= 0 || next_depth < scanner->options.max_depth) {
DirEntry* de = dir_entry_create(cur_path, next_depth, scanner->current_node); DirEntry* de = dir_entry_create(cur_path, next_depth, scanner->current_node);
@@ -1386,7 +1428,28 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
if (is_dir) { if (is_dir) {
free(rel); free(rel);
if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) { if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) {
/* -x/--one-file-system: emit the mount-point directory entry (empty) but
do not descend into it (see the sequential scanner for the same rule). */
File* mount = file_create(cur_path);
free(cur_path); free(cur_path);
if (mount == NULL) {
ps->failed = true;
return;
}
mount->is_dir = true;
if (options->use_metadata) {
mount->metadata = file_metadata_create(mount->path, &st, options->preserve_atimes,
options->preserve_crtimes);
if (!mount->metadata) {
file_destroy(mount);
ps->failed = true;
return;
}
}
if (!array_list_add(root_files, mount)) {
file_destroy(mount);
ps->failed = true;
}
return; return;
} }
if (!array_list_add(subdirs, cur_path)) { if (!array_list_add(subdirs, cur_path)) {
+4
View File
@@ -65,6 +65,10 @@ typedef struct {
bool per_dir_filters; /* -F: read .rsync-filter per directory */ bool per_dir_filters; /* -F: read .rsync-filter per directory */
bool dirs; /* -d/--dirs: transfer dir entries, no recursion */ bool dirs; /* -d/--dirs: transfer dir entries, no recursion */
bool relative; /* -R/--relative (dest rel paths, with --files-from) */ bool relative; /* -R/--relative (dest rel paths, with --files-from) */
/* --list-only: emit an is_dir File for every traversed directory (the listing
* includes directory entries, matching rsync). Client-only; never set on a
* real transfer, which relies on implicit parent creation. */
bool list_dirs;
/* --prune-empty-dirs (long only): in --dirs mode an empty source directory's /* --prune-empty-dirs (long only): in --dirs mode an empty source directory's
explicit entry is omitted from the transfer file list (so nothing is explicit entry is omitted from the transfer file list (so nothing is
created at the destination and it can be pruned by --delete); explicitly created at the destination and it can be pruned by --delete); explicitly
+2 -1
View File
@@ -157,7 +157,8 @@ void print_usage(void) {
printf(" --chunk-serialization Enable chunk serialization (long form only)\n"); printf(" --chunk-serialization Enable chunk serialization (long form only)\n");
printf(" -s, --secluded-args Protect-args compatibility option (no effect; remote\n"); printf(" -s, --secluded-args Protect-args compatibility option (no effect; remote\n");
printf(" SSH argv is already built injection-safe)\n"); printf(" SSH argv is already built injection-safe)\n");
printf(" --sendfile Enable sendfile zero-copy (TCP only; long form only)\n"); printf(" --sendfile Enable sendfile zero-copy (TCP only; long form only;\n");
printf(" -f is bound to --filter, not --sendfile)\n");
printf(" --compress-choice <alg> Compression algorithm (default: zstd)\n"); printf(" --compress-choice <alg> Compression algorithm (default: zstd)\n");
printf(" --zc <alg> Alias for --compress-choice\n"); printf(" --zc <alg> Alias for --compress-choice\n");
printf(" -v, --verbose Enable debug logging\n"); printf(" -v, --verbose Enable debug logging\n");
+6 -2
View File
@@ -1128,6 +1128,7 @@ CONFIG_DEFINE_SEND(send_daemon_auth, CONFIG_WIRE_DAEMON_AUTH_FIELDS)
CONFIG_DEFINE_SEND(send_iconv_spec, CONFIG_WIRE_ICONV_FIELDS) CONFIG_DEFINE_SEND(send_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
CONFIG_DEFINE_SEND(send_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS) CONFIG_DEFINE_SEND(send_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
CONFIG_DEFINE_SEND(send_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS) CONFIG_DEFINE_SEND(send_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
CONFIG_DEFINE_SEND(send_output_options, CONFIG_WIRE_OUTPUT_FIELDS)
CONFIG_DEFINE_RECV(receive_core_fields, CONFIG_WIRE_CORE_FIELDS) CONFIG_DEFINE_RECV(receive_core_fields, CONFIG_WIRE_CORE_FIELDS)
CONFIG_DEFINE_RECV(receive_delta_fields, CONFIG_WIRE_DELTA_FIELDS) CONFIG_DEFINE_RECV(receive_delta_fields, CONFIG_WIRE_DELTA_FIELDS)
@@ -1146,6 +1147,7 @@ CONFIG_DEFINE_RECV(receive_daemon_auth, CONFIG_WIRE_DAEMON_AUTH_FIELDS)
CONFIG_DEFINE_RECV(receive_iconv_spec, CONFIG_WIRE_ICONV_FIELDS) CONFIG_DEFINE_RECV(receive_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
CONFIG_DEFINE_RECV(receive_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS) CONFIG_DEFINE_RECV(receive_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
CONFIG_DEFINE_RECV(receive_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS) CONFIG_DEFINE_RECV(receive_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
CONFIG_DEFINE_RECV(receive_output_options, CONFIG_WIRE_OUTPUT_FIELDS)
#undef XSEND #undef XSEND
#undef XRECV #undef XRECV
@@ -1262,7 +1264,8 @@ bool config_send_wire_block(int file_descriptor, const Config* config) {
send_daemon_module(file_descriptor, config) && send_daemon_auth(file_descriptor, config) && send_daemon_module(file_descriptor, config) && send_daemon_auth(file_descriptor, config) &&
send_iconv_spec(file_descriptor, config) && send_iconv_spec(file_descriptor, config) &&
send_privilege_options(file_descriptor, config) && send_privilege_options(file_descriptor, config) &&
send_copy_as_options(file_descriptor, config); send_copy_as_options(file_descriptor, config) &&
send_output_options(file_descriptor, config);
} }
bool config_send(int file_descriptor, const Config* config) { bool config_send(int file_descriptor, const Config* config) {
@@ -1332,7 +1335,8 @@ Config* config_receive_with_validate(int file_descriptor, ConfigValidateFunc val
!receive_daemon_auth(file_descriptor, config, &budget) || !receive_daemon_auth(file_descriptor, config, &budget) ||
!receive_iconv_spec(file_descriptor, config, &budget) || !receive_iconv_spec(file_descriptor, config, &budget) ||
!receive_privilege_options(file_descriptor, config, &budget) || !receive_privilege_options(file_descriptor, config, &budget) ||
!receive_copy_as_options(file_descriptor, config, &budget)) !receive_copy_as_options(file_descriptor, config, &budget) ||
!receive_output_options(file_descriptor, config, &budget))
goto error; goto error;
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 && if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
strcmp(config->compress_choice, "none") != 0) { strcmp(config->compress_choice, "none") != 0) {
+26 -4
View File
@@ -76,7 +76,7 @@ typedef struct {
typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode; typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode;
/* =========================================================================== /* ===========================================================================
* Config wire-field table (single source of truth for protocol 2.22.0). * Config wire-field table (single source of truth for protocol 2.23.0).
* *
* Every field below crosses the wire. The table is the ONLY place a * Every field below crosses the wire. The table is the ONLY place a
* serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare * serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare
@@ -241,6 +241,13 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
X(copy_as_uid, int32_t, 0, COPY_AS_ID) \ X(copy_as_uid, int32_t, 0, COPY_AS_ID) \
X(copy_as_gid, int32_t, 0, COPY_AS_ID) X(copy_as_gid, int32_t, 0, COPY_AS_ID)
/* Output-parity wave (protocol 2.23.0). report_dest_info tells the receiver to
* answer every per-file STATUS_CHECK with a STATUS_DEST_INFO snapshot of the
* pre-transfer destination entry (see protocol.h). It is set by the client
* only when -i/--itemize-changes or --out-format asks for per-file change
* output; the transfer decision itself is unchanged. */
#define CONFIG_WIRE_OUTPUT_FIELDS(X) X(report_dest_info, bool, false, BOOL)
/* All serialized fields, in exact wire order. Concatenating the per-segment /* All serialized fields, in exact wire order. Concatenating the per-segment
* lists here is what keeps the declaration order = the wire order. */ * lists here is what keeps the declaration order = the wire order. */
#define CONFIG_WIRE_FIELDS(X) \ #define CONFIG_WIRE_FIELDS(X) \
@@ -261,7 +268,8 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
CONFIG_WIRE_DAEMON_AUTH_FIELDS(X) \ CONFIG_WIRE_DAEMON_AUTH_FIELDS(X) \
CONFIG_WIRE_ICONV_FIELDS(X) \ CONFIG_WIRE_ICONV_FIELDS(X) \
CONFIG_WIRE_PRIVILEGE_FIELDS(X) \ CONFIG_WIRE_PRIVILEGE_FIELDS(X) \
CONFIG_WIRE_COPY_AS_FIELDS(X) CONFIG_WIRE_COPY_AS_FIELDS(X) \
CONFIG_WIRE_OUTPUT_FIELDS(X)
typedef struct Config { typedef struct Config {
/* -j/--threads=N: number of parallel scanner worker threads for the -m /* -j/--threads=N: number of parallel scanner worker threads for the -m
@@ -849,8 +857,22 @@ typedef struct Config {
* version before parsing anything else) is what keeps a 2.22 client and a 2.21 * version before parsing anything else) is what keeps a 2.22 client and a 2.21
* server from ever reaching that state. The fixed-width FileMetadata layout is * server from ever reaching that state. The fixed-width FileMetadata layout is
* UNCHANGED: the receiver still gates attribute application on use_metadata, * UNCHANGED: the receiver still gates attribute application on use_metadata,
* which is now DERIVED from these attributes by config_derived_use_metadata(). */ * which is now DERIVED from these attributes by config_derived_use_metadata().
#define PROTOCOL_VERSION "2.22.0" *
* Output-Parity Wave: 2.22.0 -> 2.23.0.
*
* WHY the bump, grounded in the wire: -i/--itemize-changes and --out-format
* must compare the source against the PRE-TRANSFER destination entry (new vs
* modified, and which of size/time/perms/owner/group differ), but FastSync's
* push sender never sees the destination. The receiver therefore answers a
* per-file STATUS_CHECK with a new STATUS_DEST_INFO frame (a fixed-width
* snapshot of the old entry) before its ordinary verdict when the config frame
* carries the new report_dest_info bool appended after the --copy-as block.
* This is both a config-frame layout change (one trailing bool) and a frame
* sequence change (the new status), so any peer that did not parse them would
* desynchronize; the strict same-version handshake keeps a 2.23 client and a
* 2.22 server from ever reaching that state. */
#define PROTOCOL_VERSION "2.23.0"
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */ /* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
#define MAX_BASIS_DIRS 64 #define MAX_BASIS_DIRS 64
+32
View File
@@ -19,6 +19,7 @@
#include "delay_updates.h" #include "delay_updates.h"
#include "delta.h" #include "delta.h"
#include "file.h" #include "file.h"
#include "format.h"
#include "identity.h" #include "identity.h"
#include "log.h" #include "log.h"
#include "metadata.h" #include "metadata.h"
@@ -1858,6 +1859,33 @@ static IncrementalCheckOutcome incremental_check_open_destination(IncrementalChe
return INCREMENTAL_CONTINUE; return INCREMENTAL_CONTINUE;
} }
/* Output parity (protocol 2.23.0): when the wire config asked for it, report a
snapshot of the pre-transfer destination entry BEFORE the ordinary verdict so
the sender can render rsync-accurate -i/--out-format columns. A missing
destination is reported explicitly (existed=false) rather than omitted, so
the sender can distinguish "new" from "unknown". */
static IncrementalCheckOutcome incremental_check_report_dest_info(IncrementalCheckState* state) {
if (!state->config->report_dest_info)
return INCREMENTAL_CONTINUE;
OutputDestState info;
memset(&info, 0, sizeof(info));
info.known = true;
info.existed = state->has_old_file;
if (state->has_old_file) {
info.size = (unsigned long long)state->old_st.st_size;
info.mtime_sec = (long long)state->old_st.st_mtime;
#ifdef __linux__
info.mtime_nsec = state->old_st.st_mtim.tv_nsec;
#endif
info.mode = (uint32_t)state->old_st.st_mode;
info.uid = (int32_t)state->old_st.st_uid;
info.gid = (int32_t)state->old_st.st_gid;
}
if (!send_status(state->fd, STATUS_DEST_INFO) || !format_dest_state_send(state->fd, &info))
return INCREMENTAL_ERROR;
return INCREMENTAL_CONTINUE;
}
/* Metadata-only (and, when --checksum forces it, content) up-to-date decision. /* Metadata-only (and, when --checksum forces it, content) up-to-date decision.
Loads the old contents only when a checksum comparison or delta needs them. */ Loads the old contents only when a checksum comparison or delta needs them. */
static IncrementalCheckOutcome incremental_check_quick_skip(IncrementalCheckState* state, static IncrementalCheckOutcome incremental_check_quick_skip(IncrementalCheckState* state,
@@ -2302,6 +2330,10 @@ File* receive_incremental_check_ex(int fd, const Config* config, bool* skipped,
if (outcome == INCREMENTAL_ERROR) if (outcome == INCREMENTAL_ERROR)
goto done; goto done;
outcome = incremental_check_report_dest_info(&state);
if (outcome == INCREMENTAL_ERROR)
goto done;
outcome = incremental_check_quick_skip(&state, &try_delta); outcome = incremental_check_quick_skip(&state, &try_delta);
if (outcome == INCREMENTAL_ERROR) if (outcome == INCREMENTAL_ERROR)
goto done; goto done;
+7
View File
@@ -2,6 +2,7 @@
#define FILE_TYPES_H #define FILE_TYPES_H
#include "data.h" #include "data.h"
#include "format.h"
#include "xattr.h" #include "xattr.h"
#include <stdbool.h> #include <stdbool.h>
#include <sys/stat.h> #include <sys/stat.h>
@@ -86,6 +87,12 @@ typedef struct {
* Receiver: parsed off the wire, attached here, and applied fd-relative on * Receiver: parsed off the wire, attached here, and applied fd-relative on
* the written file. NULL/0 == the file carries no xattrs. */ * the written file. NULL/0 == the file carries no xattrs. */
FileXattrList* xattrs; FileXattrList* xattrs;
/* Sender-side output-parity state (never serialized): the receiver-reported
* pre-transfer destination snapshot for this entry, filled by the per-file
* STATUS_CHECK exchange when report_dest_info is set. `known` is false when
* no report was requested/received, in which case -i/--out-format treats the
* entry conservatively as newly created. */
OutputDestState dest_state;
} File; } File;
/* The path that should be sent on the wire and used for the receiver-side /* The path that should be sent on the wire and used for the receiver-side
+103
View File
@@ -0,0 +1,103 @@
#include "format.h"
#include "protocol.h"
#include <stdio.h>
#include <string.h>
bool format_human_size_decimal(unsigned long long bytes, char* buffer, size_t buffer_size) {
if (!buffer || buffer_size == 0)
return false;
if (bytes < 1000ULL) {
int written = snprintf(buffer, buffer_size, "%llu", bytes);
return written >= 0 && (size_t)written < buffer_size;
}
static const char units[] = "KMGTPE";
double value = (double)bytes;
size_t divisions = 0;
while (value >= 1000.0 && divisions < sizeof(units) - 1) {
value /= 1000.0;
divisions++;
}
int written = snprintf(buffer, buffer_size, "%.2f%c", value, units[divisions - 1]);
return written >= 0 && (size_t)written < buffer_size;
}
bool format_big_num(unsigned long long value, bool human_readable, char* buffer,
size_t buffer_size) {
if (human_readable)
return format_human_size_decimal(value, buffer, buffer_size);
char digits[32];
int written = snprintf(digits, sizeof(digits), "%llu", value);
if (written < 0 || (size_t)written >= sizeof(digits))
return false;
size_t len = (size_t)written;
size_t separators = len > 1 ? (len - 1) / 3 : 0;
size_t total = len + separators;
if (total + 1 > buffer_size)
return false;
size_t out = total;
buffer[out] = '\0';
size_t digits_since_sep = 0;
for (size_t i = len; i > 0; i--) {
buffer[--out] = digits[i - 1];
digits_since_sep++;
if (digits_since_sep == 3 && i > 1) {
buffer[--out] = ',';
digits_since_sep = 0;
}
}
return true;
}
bool format_rsync_datetime(time_t when, bool dash, char* buffer, size_t buffer_size) {
if (!buffer || buffer_size == 0)
return false;
struct tm broken_down;
if (localtime_r(&when, &broken_down) == NULL)
return false;
const char* format = dash ? "%Y/%m/%d-%H:%M:%S" : "%Y/%m/%d %H:%M:%S";
return strftime(buffer, buffer_size, format, &broken_down) != 0;
}
bool format_dest_state_send(int fd, const OutputDestState* state) {
if (!state)
return false;
int32_t has_old = state->existed ? 1 : 0;
uint64_t size = (uint64_t)state->size;
int64_t mtime = (int64_t)state->mtime_sec;
int64_t mtime_nsec = state->mtime_nsec;
uint32_t mode = state->mode;
int32_t uid = state->uid;
int32_t gid = state->gid;
return send_n_data(fd, &has_old, sizeof(has_old)) && send_n_data(fd, &size, sizeof(size)) &&
send_n_data(fd, &mtime, sizeof(mtime)) &&
send_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) && send_n_data(fd, &mode, sizeof(mode)) &&
send_n_data(fd, &uid, sizeof(uid)) && send_n_data(fd, &gid, sizeof(gid));
}
bool format_dest_state_receive(int fd, OutputDestState* state) {
if (!state)
return false;
int32_t has_old = 0;
uint64_t size = 0;
int64_t mtime = 0;
int64_t mtime_nsec = 0;
uint32_t mode = 0;
int32_t uid = 0;
int32_t gid = 0;
if (!receive_n_data(fd, &has_old, sizeof(has_old)) || !receive_n_data(fd, &size, sizeof(size)) ||
!receive_n_data(fd, &mtime, sizeof(mtime)) ||
!receive_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) ||
!receive_n_data(fd, &mode, sizeof(mode)) || !receive_n_data(fd, &uid, sizeof(uid)) ||
!receive_n_data(fd, &gid, sizeof(gid)))
return false;
memset(state, 0, sizeof(*state));
state->known = true;
state->existed = has_old != 0;
state->size = size;
state->mtime_sec = mtime;
state->mtime_nsec = mtime_nsec;
state->mode = mode;
state->uid = uid;
state->gid = gid;
return true;
}
+59
View File
@@ -0,0 +1,59 @@
#ifndef FORMAT_H
#define FORMAT_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <time.h>
/* Low-level output-formatting primitives shared by the change-event model
* (change_list.c) and the transfer driver (client_send.c).
*
* The functions here are pure/string-level except for the STATUS_DEST_INFO
* codec, which lets the receiver report the pre-transfer destination entry so
* the sender can render rsync-accurate --itemize-changes / --out-format
* columns (see protocol.h). */
/* Pre-transfer destination snapshot, reported by the receiver when the wire
* config carries report_dest_info. `known` distinguishes "no report was
* requested/received" from "the destination did not exist" (`existed == false`
* with `known == true`). */
typedef struct {
bool known;
bool existed;
unsigned long long size;
long long mtime_sec;
long long mtime_nsec;
uint32_t mode;
int32_t uid;
int32_t gid;
} OutputDestState;
/* rsync's -h/--human-readable size (decimal, base 1000): integers below 1000
* print verbatim; larger values use the largest unit that keeps the value
* below 1000 (K/M/G/T/P/E) with exactly two decimals, so 1500000 -> "1.50M"
* and 999999 -> "1000.00K" (matching rsync's human_num). Returns false when
* the buffer is too small (nothing is written). */
bool format_human_size_decimal(unsigned long long bytes, char* buffer, size_t buffer_size);
/* rsync's general number formatting (big_num). When `human_readable` is true
* this is format_human_size_decimal; otherwise the integer is rendered with a
* ',' thousands separator every three digits (rsync's separator in the C
* locale). Returns false on an undersized buffer. */
bool format_big_num(unsigned long long value, bool human_readable, char* buffer,
size_t buffer_size);
/* rsync's %M/%t timestamp. When `dash` is true the separator between the date
* and the time is '-' (the %M form: "YYYY/MM/DD-HH:MM:SS"); otherwise it is a
* space (the %t form: "YYYY/MM/DD HH:MM:SS"). Local time. Returns false on a
* bad time or an undersized buffer. */
bool format_rsync_datetime(time_t when, bool dash, char* buffer, size_t buffer_size);
/* Fixed-width STATUS_DEST_INFO record codec (int32 has_old, uint64 size,
* int64 mtime, int64 mtime_nsec, uint32 mode, int32 uid, int32 gid). The
* status frame itself is sent/received by the caller. Returns false on I/O
* failure. */
bool format_dest_state_send(int fd, const OutputDestState* state);
bool format_dest_state_receive(int fd, OutputDestState* state);
#endif
+2
View File
@@ -479,6 +479,8 @@ static const char* status_to_string(Status status) {
return "ERROR_DETAIL"; return "ERROR_DETAIL";
case STATUS_DRY_RUN_TRANSFER: case STATUS_DRY_RUN_TRANSFER:
return "DRY_RUN_TRANSFER"; return "DRY_RUN_TRANSFER";
case STATUS_DEST_INFO:
return "DEST_INFO";
default: default:
return "UNKNOWN"; return "UNKNOWN";
} }
+12 -1
View File
@@ -155,7 +155,18 @@ enum NET_STATUS {
* (the receiver reads none in dry-run). STATUS_OK keeps its meaning in this * (the receiver reads none in dry-run). STATUS_OK keeps its meaning in this
* path ("already up to date / nothing to do"). Appended after * path ("already up to date / nothing to do"). Appended after
* STATUS_ERROR_DETAIL so no existing status is renumbered. */ * STATUS_ERROR_DETAIL so no existing status is renumbered. */
STATUS_DRY_RUN_TRANSFER STATUS_DRY_RUN_TRANSFER,
/* Destination-state report for output parity (protocol 2.23.0). When the
* wire config carries report_dest_info=true, the receiver answers every
* per-file STATUS_CHECK request with STATUS_DEST_INFO FIRST, followed by a
* fixed record describing the pre-transfer destination entry
* (int32 has_old; uint64 size; int64 mtime; int64 mtime_nsec; uint32 mode;
* int32 uid; int32 gid). The ordinary STATUS_OK/STATUS_NEXT/... verdict
* follows, so the sender can render rsync-accurate -i/--out-format columns
* (new vs modified, and which of size/time/perms/owner/group differ) without
* changing the transfer decision itself. Appended after
* STATUS_DRY_RUN_TRANSFER so no existing status is renumbered. */
STATUS_DEST_INFO
}; };
void io_set_fds(int read_fd, int write_fd); void io_set_fds(int read_fd, int write_fd);
+1 -1
View File
@@ -36,7 +36,7 @@ from common import ( # noqa: E402
verify_transfer, verify_transfer,
) )
PROTOCOL_VERSION = b"2.22.0" PROTOCOL_VERSION = b"2.23.0"
STATUS_MANIFEST = 5 STATUS_MANIFEST = 5
STATUS_OK = 0 STATUS_OK = 0
+32 -18
View File
@@ -325,7 +325,7 @@ class TestDryRun:
result, dur = run_client(SOURCE_DIR, DEST_DIR, flags=["-h", "--dry-run"]) result, dur = run_client(SOURCE_DIR, DEST_DIR, flags=["-h", "--dry-run"])
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
assert "Total:" in result.stdout assert "Total:" in result.stdout
assert "KB" in result.stdout assert any(unit in result.stdout for unit in ("K", "M", "G"))
def test_dry_run(self): def test_dry_run(self):
clean_dir(DEST_DIR) clean_dir(DEST_DIR)
@@ -1070,10 +1070,12 @@ class TestExclude:
class TestInclude: class TestInclude:
def test_include_single(self, shared_server): def test_include_single(self, shared_server):
# rsync first-match-wins: an --include alone is NOT a whitelist, so the
# selector must pair it with --exclude '*' (the common idiom).
clean_dir(DEST_DIR) clean_dir(DEST_DIR)
result, dur = run_client( result, dur = run_client(
SOURCE_DIR, DEST_DIR, SOURCE_DIR, DEST_DIR,
flags=["--include", "binary.bin"], flags=["--include", "binary.bin", "--exclude", "*"],
port=shared_server.port, port=shared_server.port,
) )
if result.returncode != 0: if result.returncode != 0:
@@ -1086,13 +1088,14 @@ class TestInclude:
clean_dir(DEST_DIR) clean_dir(DEST_DIR)
result, dur = run_client( result, dur = run_client(
SOURCE_DIR, DEST_DIR, SOURCE_DIR, DEST_DIR,
flags=["--include", "*.bin"], flags=["--include", "*.bin", "--exclude", "*"],
port=shared_server.port, port=shared_server.port,
) )
if result.returncode != 0: if result.returncode != 0:
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}") pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
assert os.path.exists(os.path.join(received, "binary.bin")), "binary.bin should be included" assert os.path.exists(os.path.join(received, "binary.bin")), "binary.bin should be included"
assert not os.path.exists(os.path.join(received, "small.txt")), "small.txt should not be included"
class TestSizeFilters: class TestSizeFilters:
@@ -1646,8 +1649,8 @@ class TestDelete:
port=shared_server.port, port=shared_server.port,
) )
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
assert "Stats:" in result.stderr assert "Number of files:" in result.stdout
assert "KB" in result.stderr assert "Total file size:" in result.stdout
def test_human_readable_stats_multithreaded(self, shared_server): def test_human_readable_stats_multithreaded(self, shared_server):
# The multithreaded sender shares the single-threaded --stats format, # The multithreaded sender shares the single-threaded --stats format,
@@ -1659,9 +1662,8 @@ class TestDelete:
port=shared_server.port, port=shared_server.port,
) )
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
assert "Stats:" in result.stderr assert "Number of files:" in result.stdout
assert "KB" in result.stderr assert "bytes/sec" in result.stdout
assert "/s" in result.stderr
def test_human_readable_progress_multithreaded(self, shared_server): def test_human_readable_progress_multithreaded(self, shared_server):
clean_dir(DEST_DIR) clean_dir(DEST_DIR)
@@ -1673,7 +1675,6 @@ class TestDelete:
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}" assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
output = result.stdout + result.stderr output = result.stdout + result.stderr
assert "Sent " in output assert "Sent " in output
assert "KB" in output
assert "Done." in output assert "Done." in output
@@ -2164,7 +2165,8 @@ class TestListOnly:
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only"]) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only"])
assert result.returncode == 0, f"list-only failed: {result.stderr[:200]}" assert result.returncode == 0, f"list-only failed: {result.stderr[:200]}"
for full_path in _source_files(): for full_path in _source_files():
assert full_path in result.stdout, f"list-only omitted {full_path}" rel = os.path.relpath(full_path, SOURCE_DIR)
assert rel in result.stdout, f"list-only omitted {rel}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
assert not os.path.exists(received), "list-only wrote to the destination" assert not os.path.exists(received), "list-only wrote to the destination"
@@ -2180,7 +2182,8 @@ class TestListOnly:
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only", "--threads"]) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only", "--threads"])
assert result.returncode == 0, f"list-only -m failed: {result.stderr[:200]}" assert result.returncode == 0, f"list-only -m failed: {result.stderr[:200]}"
for full_path in _source_files(): for full_path in _source_files():
assert full_path in result.stdout, f"list-only -m omitted {full_path}" rel = os.path.relpath(full_path, SOURCE_DIR)
assert rel in result.stdout, f"list-only -m omitted {rel}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
assert not os.path.exists(received), "list-only -m wrote to the destination" assert not os.path.exists(received), "list-only -m wrote to the destination"
@@ -2193,7 +2196,7 @@ class TestItemizeChanges:
result, _ = run_client(SOURCE_DIR, DEST_DIR, result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["--preserve", "-i"], port=shared_server.port) flags=["--preserve", "-i"], port=shared_server.port)
assert result.returncode == 0, f"itemize sync failed: {result.stderr[:200]}" assert result.returncode == 0, f"itemize sync failed: {result.stderr[:200]}"
sent_lines = {">f+++++++++ " + p for p in _source_files()} sent_lines = {">f+++++++++ " + os.path.relpath(p, SOURCE_DIR) for p in _source_files()}
assert sent_lines <= set(result.stdout.splitlines()), ( assert sent_lines <= set(result.stdout.splitlines()), (
f"missing itemize lines; got {result.stdout[:500]}" f"missing itemize lines; got {result.stdout[:500]}"
) )
@@ -2214,7 +2217,7 @@ class TestItemizeChanges:
result, _ = run_client(SOURCE_DIR, DEST_DIR, result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["--preserve", "-i", "--threads"], port=shared_server.port) flags=["--preserve", "-i", "--threads"], port=shared_server.port)
assert result.returncode == 0, f"itemize -m sync failed: {result.stderr[:200]}" assert result.returncode == 0, f"itemize -m sync failed: {result.stderr[:200]}"
sent_lines = {">f+++++++++ " + p for p in _source_files()} sent_lines = {">f+++++++++ " + os.path.relpath(p, SOURCE_DIR) for p in _source_files()}
assert sent_lines <= set(result.stdout.splitlines()), ( assert sent_lines <= set(result.stdout.splitlines()), (
f"missing itemize lines in -m mode; got {result.stdout[:500]}" f"missing itemize lines in -m mode; got {result.stdout[:500]}"
) )
@@ -2249,7 +2252,9 @@ class TestItemizeChanges:
port=shared_server.port) port=shared_server.port)
assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}" assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}"
itemized = [line for line in result.stdout.splitlines() if line.startswith(">f")] itemized = [line for line in result.stdout.splitlines() if line.startswith(">f")]
assert itemized == [">f+++++++++ " + changed], ( # The content and mtime both changed, so the itemize compares the
# destination snapshot: size and time columns are set.
assert itemized == [">f.st...... changed.txt"], (
f"expected exactly one itemize line for {changed}, got {itemized}" f"expected exactly one itemize line for {changed}, got {itemized}"
) )
received = get_dest_received_dir(dest, source) received = get_dest_received_dir(dest, source)
@@ -2265,7 +2270,11 @@ class TestOutFormat:
result, _ = run_client(SOURCE_DIR, DEST_DIR, result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["--out-format=%f %l"], port=shared_server.port) flags=["--out-format=%f %l"], port=shared_server.port)
assert result.returncode == 0, f"out-format sync failed: {result.stderr[:200]}" assert result.returncode == 0, f"out-format sync failed: {result.stderr[:200]}"
expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()} # %f is rsync's long display path: the source argument normalized
# (leading '/' stripped) joined to the transfer-relative name.
prefix = SOURCE_DIR.lstrip(os.sep)
expected = {f"{os.path.join(prefix, os.path.relpath(p, SOURCE_DIR))} {os.path.getsize(p)}"
for p in _source_files()}
got = set(result.stdout.splitlines()) got = set(result.stdout.splitlines())
assert expected <= got, f"out-format lines missing: expected {len(expected)} got {len(got)}" assert expected <= got, f"out-format lines missing: expected {len(expected)} got {len(got)}"
@@ -2274,7 +2283,9 @@ class TestOutFormat:
result, _ = run_client(SOURCE_DIR, DEST_DIR, result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["--out-format=%f %l", "--threads"], port=shared_server.port) flags=["--out-format=%f %l", "--threads"], port=shared_server.port)
assert result.returncode == 0, f"out-format -m sync failed: {result.stderr[:200]}" assert result.returncode == 0, f"out-format -m sync failed: {result.stderr[:200]}"
expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()} prefix = SOURCE_DIR.lstrip(os.sep)
expected = {f"{os.path.join(prefix, os.path.relpath(p, SOURCE_DIR))} {os.path.getsize(p)}"
for p in _source_files()}
got = set(result.stdout.splitlines()) got = set(result.stdout.splitlines())
assert expected <= got, f"out-format -m lines missing: {result.stdout[:500]}" assert expected <= got, f"out-format -m lines missing: {result.stdout[:500]}"
@@ -2296,7 +2307,9 @@ class TestLogFileFormat:
assert os.path.exists(log_path), "--log-file created no log" assert os.path.exists(log_path), "--log-file created no log"
with open(log_path, encoding="utf-8", errors="replace") as fh: with open(log_path, encoding="utf-8", errors="replace") as fh:
content = fh.read() content = fh.read()
expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()} prefix = SOURCE_DIR.lstrip(os.sep)
expected = {f"{os.path.join(prefix, os.path.relpath(p, SOURCE_DIR))} {os.path.getsize(p)}"
for p in _source_files()}
for line in expected: for line in expected:
assert line in content, f"log file missing {line!r}" assert line in content, f"log file missing {line!r}"
@@ -2322,7 +2335,8 @@ class TestLogFileFormat:
assert os.path.exists(log_path), "--log-file created no log" assert os.path.exists(log_path), "--log-file created no log"
with open(log_path, encoding="utf-8", errors="replace") as fh: with open(log_path, encoding="utf-8", errors="replace") as fh:
content = fh.read() content = fh.read()
expected = {f"{os.path.join(source, rel)} {len(data)}" for rel, data in files.items()} prefix = os.path.abspath(source).lstrip(os.sep)
expected = {f"{os.path.join(prefix, rel)} {len(data)}" for rel, data in files.items()}
for line in expected: for line in expected:
assert line in content, f"log file (--threads) missing {line!r}" assert line in content, f"log file (--threads) missing {line!r}"
+282
View File
@@ -0,0 +1,282 @@
"""Output-parity tests (#291 selection/output, #292 output formatting).
These tests exercise rsync-style selection ordering and output formatting. The
differential tests run the SAME transfer with real ``rsync 3.4.1`` and with
fastsync and compare stdout, so they are skipped when rsync is unavailable.
"""
import os
import shutil
import subprocess
import sys
import pytest
sys.path.insert(0, os.path.dirname(__file__))
from common import TEST_DATA_DIR, run_client, clean_dir, get_dest_received_dir
RSYNC = shutil.which("rsync")
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
def _rsync(args):
env = dict(os.environ, LC_ALL="C")
return subprocess.run(
[RSYNC] + args, capture_output=True, text=True, env=env, timeout=120
)
def _make_selection_tree(root):
clean_dir(root)
os.makedirs(os.path.join(root, "sub"))
with open(os.path.join(root, "a.txt"), "wb") as fh:
fh.write(b"top text\n")
with open(os.path.join(root, "b.log"), "wb") as fh:
fh.write(b"log data\n")
with open(os.path.join(root, "sub", "c.txt"), "wb") as fh:
fh.write(b"nested text\n")
with open(os.path.join(root, "sub", "d.log"), "wb") as fh:
fh.write(b"nested log\n")
class TestSelectionOrdering:
"""#291: --include/--exclude compile into one ordered rule list."""
@pytest.mark.ci
def test_include_then_exclude_keeps_only_matching(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "out_inc_src")
dest = os.path.join(TEST_DATA_DIR, "out_inc_dst")
_make_selection_tree(source)
clean_dir(dest)
result, _ = run_client(
source, dest,
flags=["--preserve", "--include=*.txt", "--exclude=*"],
port=shared_server.port,
)
assert result.returncode == 0, f"include/exclude failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
assert os.path.exists(os.path.join(received, "a.txt"))
# `*` also excludes the directory, so nothing below sub/ is sent.
assert not os.path.exists(os.path.join(received, "b.log"))
assert not os.path.exists(os.path.join(received, "sub", "c.txt"))
@pytest.mark.ci
def test_include_dirs_then_files_idiom(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "out_inc2_src")
dest = os.path.join(TEST_DATA_DIR, "out_inc2_dst")
_make_selection_tree(source)
clean_dir(dest)
result, _ = run_client(
source, dest,
flags=["--preserve", "--include=*/", "--include=*.txt", "--exclude=*"],
port=shared_server.port,
)
assert result.returncode == 0, f"include/exclude failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
assert os.path.exists(os.path.join(received, "a.txt"))
assert os.path.exists(os.path.join(received, "sub", "c.txt"))
assert not os.path.exists(os.path.join(received, "b.log"))
assert not os.path.exists(os.path.join(received, "sub", "d.log"))
@requires_rsync
def test_include_idiom_matches_rsync_selection(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "out_inc3_src")
dest = os.path.join(TEST_DATA_DIR, "out_inc3_dst")
rdst = os.path.join(TEST_DATA_DIR, "out_inc3_rdst")
_make_selection_tree(source)
clean_dir(dest)
clean_dir(rdst)
flags = ["--include=*/", "--include=*.txt", "--exclude=*"]
rsync_result = _rsync(["-a"] + flags + [source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
result, _ = run_client(source, dest, flags=["--preserve"] + flags,
port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(dest, source)
assert os.path.exists(os.path.join(received, "a.txt"))
assert os.path.exists(os.path.join(received, "sub", "c.txt"))
assert not os.path.exists(os.path.join(received, "b.log"))
# rsync -a src/ dst/ writes directly into dst/
assert os.path.exists(os.path.join(rdst, "a.txt"))
assert os.path.exists(os.path.join(rdst, "sub", "c.txt"))
assert not os.path.exists(os.path.join(rdst, "b.log"))
class TestOneFileSystem:
"""#291: -x emits the mount-point directory but not its contents."""
def test_one_file_system_emits_mount_point_dir(self, shared_server):
local = os.stat(".")
shm = "/dev/shm"
try:
shm_stat = os.stat(shm)
except OSError:
pytest.skip("/dev/shm not available")
if shm_stat.st_dev == local.st_dev:
pytest.skip("no cross-device filesystem available")
source = os.path.join(TEST_DATA_DIR, "out_ofs_src")
dest = os.path.join(TEST_DATA_DIR, "out_ofs_dst")
clean_dir(source)
clean_dir(dest)
os.makedirs(os.path.join(source, "nested"))
os.makedirs(os.path.join(shm, "fastsync_ofs_probe"), exist_ok=True)
with open(os.path.join(source, "keep.txt"), "wb") as fh:
fh.write(b"keep\n")
with open(os.path.join(shm, "fastsync_ofs_probe", "inside.txt"), "wb") as fh:
fh.write(b"cross\n")
link = os.path.join(source, "nested", "link")
try:
os.symlink(os.path.join(shm, "fastsync_ofs_probe"), link)
except OSError:
pytest.skip("cannot create symlink")
try:
result, _ = run_client(
source, dest,
flags=["--preserve", "--copy-links", "-x"],
port=shared_server.port,
)
assert result.returncode == 0, f"-x failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
assert os.path.exists(os.path.join(received, "keep.txt"))
# The mount-point directory entry is created but its contents are not.
assert os.path.isdir(os.path.join(received, "nested", "link"))
assert not os.path.exists(os.path.join(received, "nested", "link", "inside.txt"))
finally:
shutil.rmtree(os.path.join(shm, "fastsync_ofs_probe"), ignore_errors=True)
def _make_output_tree(root):
clean_dir(root)
os.makedirs(os.path.join(root, "sub"))
with open(os.path.join(root, "a.txt"), "wb") as fh:
fh.write(b"hello\n")
with open(os.path.join(root, "sub", "b.txt"), "wb") as fh:
fh.write("wörld\n".encode("utf-8"))
os.symlink("a.txt", os.path.join(root, "link"))
class TestItemizeParity:
"""#292: -i output matches rsync 3.4.1 for the cases fastsync can observe."""
@requires_rsync
@pytest.mark.ci
def test_itemize_first_transfer_matches_rsync(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "out_item_src")
dest = os.path.join(TEST_DATA_DIR, "out_item_dst")
rdst = os.path.join(TEST_DATA_DIR, "out_item_rdst")
_make_output_tree(source)
clean_dir(dest)
clean_dir(rdst)
rsync_result = _rsync(["-a", "-i", source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
rsync_lines = sorted(
line for line in rsync_result.stdout.splitlines()
if line.startswith(">f") or line.startswith("cL")
)
result, _ = run_client(source, dest, flags=["-a", "-i"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
fast_lines = sorted(
line for line in result.stdout.splitlines()
if line.startswith(">f") or line.startswith("cL")
)
assert fast_lines == rsync_lines, f"rsync={rsync_lines} fastsync={fast_lines}"
@requires_rsync
@pytest.mark.ci
def test_itemize_modified_file_matches_rsync(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "out_item2_src")
dest = os.path.join(TEST_DATA_DIR, "out_item2_dst")
rdst = os.path.join(TEST_DATA_DIR, "out_item2_rdst")
_make_output_tree(source)
clean_dir(dest)
clean_dir(rdst)
seed = run_client(source, dest, flags=["-a"], port=shared_server.port)
assert seed[0].returncode == 0, seed[0].stderr[:300]
assert _rsync(["-a", source + "/", rdst + "/"]).returncode == 0
with open(os.path.join(source, "a.txt"), "wb") as fh:
fh.write(b"hello changed and longer\n")
# Pin the source mtime so rsync's `t` column is deterministic (a write
# that lands in the same whole second as the seed would not show `t`).
os.utime(os.path.join(source, "a.txt"), (1000000000, 1000000000))
rsync_result = _rsync(["-a", "-i", source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
rsync_lines = sorted(
line for line in rsync_result.stdout.splitlines() if line.startswith(">f")
)
result, _ = run_client(source, dest,
flags=["-a", "-i", "--incremental"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
fast_lines = sorted(
line for line in result.stdout.splitlines() if line.startswith(">f")
)
assert fast_lines == rsync_lines, f"rsync={rsync_lines} fastsync={fast_lines}"
class TestOutFormatParity:
@requires_rsync
@pytest.mark.ci
def test_out_format_n_l_matches_rsync(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "out_fmt_src")
dest = os.path.join(TEST_DATA_DIR, "out_fmt_dst")
rdst = os.path.join(TEST_DATA_DIR, "out_fmt_rdst")
_make_output_tree(source)
clean_dir(dest)
clean_dir(rdst)
fmt = "%n %l"
rsync_result = _rsync(["-a", "--out-format=" + fmt, source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
rsync_lines = sorted(
line for line in rsync_result.stdout.splitlines()
if line and not line.split(" ", 1)[0].endswith("/")
)
result, _ = run_client(source, dest,
flags=["-a", "--out-format=" + fmt],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
fast_lines = sorted(
line for line in result.stdout.splitlines()
if line and not line.split(" ", 1)[0].endswith("/")
)
assert fast_lines == rsync_lines, f"rsync={rsync_lines} fastsync={fast_lines}"
@requires_rsync
@pytest.mark.ci
def test_out_format_M_datetime_shape(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "out_M_src")
dest = os.path.join(TEST_DATA_DIR, "out_M_dst")
_make_output_tree(source)
clean_dir(dest)
result, _ = run_client(source, dest,
flags=["-a", "--out-format=%M %f"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
import re
pattern = re.compile(r"^\d{4}/\d{2}/\d{2}-\d{2}:\d{2}:\d{2} ")
for line in result.stdout.splitlines():
if line:
assert pattern.match(line), f"bad %M format: {line!r}"
class TestListOnlyParity:
@requires_rsync
@pytest.mark.ci
def test_list_only_matches_rsync(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "out_list_src")
dest = os.path.join(TEST_DATA_DIR, "out_list_dst")
_make_output_tree(source)
clean_dir(dest)
rsync_result = _rsync(["-r", "--list-only", source + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
rsync_lines = sorted(rsync_result.stdout.splitlines())
result, _ = run_client(source, dest, flags=["--list-only", "-l"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
fast_lines = sorted(result.stdout.splitlines())
assert fast_lines == rsync_lines, (
f"rsync={rsync_lines}\nfastsync={fast_lines}"
)
+4 -4
View File
@@ -94,14 +94,14 @@ def _seed_protocol_source(source):
class TestProtocol: class TestProtocol:
@pytest.mark.ci @pytest.mark.ci
def test_protocol_current_version_accepted(self, shared_server): def test_protocol_current_version_accepted(self, shared_server):
"""--protocol=2.22.0 (the current PROTOCOL_VERSION) is accepted and the """--protocol=2.23.0 (the current PROTOCOL_VERSION) is accepted and the
transfer completes normally.""" transfer completes normally."""
source = os.path.join(TEST_DATA_DIR, "proto_ok_src") source = os.path.join(TEST_DATA_DIR, "proto_ok_src")
dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst") dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst")
shutil.rmtree(dest, ignore_errors=True) shutil.rmtree(dest, ignore_errors=True)
os.makedirs(dest) os.makedirs(dest)
_seed_protocol_source(source) _seed_protocol_source(source)
result, _ = run_client(source, dest, flags=["--protocol=2.22.0"], result, _ = run_client(source, dest, flags=["--protocol=2.23.0"],
port=shared_server.port) port=shared_server.port)
assert result.returncode == 0, \ assert result.returncode == 0, \
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}" f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
@@ -118,8 +118,8 @@ class TestProtocol:
shutil.rmtree(dest, ignore_errors=True) shutil.rmtree(dest, ignore_errors=True)
os.makedirs(dest) os.makedirs(dest)
_seed_protocol_source(source) _seed_protocol_source(source)
for bad in ("2.21.0", "2.20.0", "2.19.0", "2.18.0", "2.17.0", "2.15.0", "2.16.0", "216", for bad in ("2.22.0", "2.21.0", "2.20.0", "2.19.0", "2.18.0", "2.17.0", "2.15.0", "2.16.0",
"31"): "216", "31"):
result, _ = run_client(source, dest, flags=[f"--protocol={bad}"], result, _ = run_client(source, dest, flags=[f"--protocol={bad}"],
port=shared_server.port) port=shared_server.port)
assert result.returncode != 0, f"--protocol={bad} should be rejected" assert result.returncode != 0, f"--protocol={bad} should be rejected"
+2
View File
@@ -15,6 +15,7 @@
#include "test_file.h" #include "test_file.h"
#include "test_file_list.h" #include "test_file_list.h"
#include "test_file_sendfile.h" #include "test_file_sendfile.h"
#include "test_format.h"
#include "test_fuzz_smoke.h" #include "test_fuzz_smoke.h"
#include "test_glob.h" #include "test_glob.h"
#include "test_hardlink.h" #include "test_hardlink.h"
@@ -58,6 +59,7 @@ int main() {
RUN_TEST(test_chunk); RUN_TEST(test_chunk);
RUN_TEST(test_batch); RUN_TEST(test_batch);
RUN_TEST(test_change_list); RUN_TEST(test_change_list);
RUN_TEST(test_format);
RUN_TEST(test_config); RUN_TEST(test_config);
RUN_TEST(test_credentials); RUN_TEST(test_credentials);
RUN_TEST(test_compression); RUN_TEST(test_compression);
+106 -16
View File
@@ -1,5 +1,6 @@
#include "test_change_list.h" #include "test_change_list.h"
#include "change_list.h" #include "change_list.h"
#include "config.h"
#include "test_utils.h" #include "test_utils.h"
#include "utils.h" #include "utils.h"
#include <stdlib.h> #include <stdlib.h>
@@ -9,64 +10,150 @@
static ChangeEvent sample_event(void) { static ChangeEvent sample_event(void) {
ChangeEvent event; ChangeEvent event;
memset(&event, 0, sizeof(event)); memset(&event, 0, sizeof(event));
event.path = "/srv/root/sub/file.txt"; event.path = "src/sub/file.txt";
event.name = "sub/file.txt";
event.decision = CHANGE_SENT; event.decision = CHANGE_SENT;
event.is_directory = false; event.is_directory = false;
event.size = 12345; event.size = 12345;
event.bytes_sent = 999; event.bytes_sent = 999;
event.mtime_sec = 1700000000; event.mtime_sec = 1700000000;
event.mtime_nsec = 0;
event.mode = 0100644;
event.uid = 1000;
event.gid = 1000;
return event; return event;
} }
/* Expected %M expansion computed independently with localtime_r. */
static void expected_mtime(time_t when, char out[32]) {
struct tm broken_down;
localtime_r(&when, &broken_down);
strftime(out, 32, "%Y/%m/%d-%H:%M:%S", &broken_down);
}
static void test_format_tokens() { static void test_format_tokens() {
ChangeEvent event = sample_event(); ChangeEvent event = sample_event();
char* line = change_render_format("%f %n %l %b %M %%", &event); Config* config = config_create();
char when[32];
expected_mtime(event.mtime_sec, when);
char* line = change_render_format("%f %n %l %b %M %%", config, &event);
EXPECT_NOT_NULL(line); EXPECT_NOT_NULL(line);
EXPECT_EQ_STR(line, "/srv/root/sub/file.txt file.txt 12345 999 1700000000 %"); char expected[256];
snprintf(expected, sizeof(expected), "src/sub/file.txt sub/file.txt 12345 999 %s %%", when);
EXPECT_EQ_STR(line, expected);
free(line); free(line);
config_delete(config);
} }
static void test_format_unknown_tokens_preserved() { static void test_format_unknown_tokens_preserved() {
ChangeEvent event = sample_event(); ChangeEvent event = sample_event();
char* line = change_render_format("x%q=%f%z", &event); Config* config = config_create();
char* line = change_render_format("x%q=%f%z", config, &event);
EXPECT_NOT_NULL(line); EXPECT_NOT_NULL(line);
EXPECT_EQ_STR(line, "x%q=/srv/root/sub/file.txt%z"); EXPECT_EQ_STR(line, "x%q=src/sub/file.txt%z");
free(line); free(line);
config_delete(config);
} }
static void test_format_leaf_name() { static void test_format_directory_name_has_trailing_slash() {
ChangeEvent event = sample_event(); ChangeEvent event = sample_event();
event.path = "bare.txt"; event.is_directory = true;
char* line = change_render_format("%n|%f", &event); event.path = "src/sub";
event.name = "sub";
Config* config = config_create();
char* line = change_render_format("%n|%f", config, &event);
EXPECT_NOT_NULL(line); EXPECT_NOT_NULL(line);
EXPECT_EQ_STR(line, "bare.txt|bare.txt"); EXPECT_EQ_STR(line, "sub/|src/sub");
free(line); free(line);
config_delete(config);
} }
static void test_render_itemize_sent_file() { static void test_render_itemize_sent_file() {
ChangeEvent event = sample_event(); ChangeEvent event = sample_event();
char* line = change_render_itemize(&event); Config* config = config_create();
char* line = change_render_itemize(config, &event);
EXPECT_NOT_NULL(line); EXPECT_NOT_NULL(line);
EXPECT_EQ_STR(line, ">f+++++++++ /srv/root/sub/file.txt"); EXPECT_EQ_STR(line, ">f+++++++++ sub/file.txt");
free(line); free(line);
config_delete(config);
}
static void test_render_itemize_directory() {
ChangeEvent event = sample_event();
event.is_directory = true;
event.path = "src/sub";
event.name = "sub";
Config* config = config_create();
char* line = change_render_itemize(config, &event);
EXPECT_NOT_NULL(line);
EXPECT_EQ_STR(line, "cd+++++++++ sub/");
free(line);
config_delete(config);
}
static void test_render_itemize_symlink() {
ChangeEvent event = sample_event();
event.is_symlink = true;
event.path = "src/link";
event.name = "link";
event.symlink_target = "a.txt";
Config* config = config_create();
char* line = change_render_itemize(config, &event);
EXPECT_NOT_NULL(line);
EXPECT_EQ_STR(line, "cL+++++++++ link -> a.txt");
free(line);
config_delete(config);
}
static void test_render_itemize_compares_destination() {
ChangeEvent event = sample_event();
Config* config = config_create();
config->preserve_perms = true;
config->preserve_owner = true;
config->preserve_group = true;
event.dest.known = true;
event.dest.existed = true;
event.dest.size = 1;
event.dest.mtime_sec = 1700000000;
event.dest.mtime_nsec = 0;
event.dest.mode = 0100600;
event.dest.uid = 1;
event.dest.gid = 2;
char* line = change_render_itemize(config, &event);
EXPECT_NOT_NULL(line);
/* size, perms, owner and group differ; time matches. */
EXPECT_EQ_STR(line, ">f.s.pog... sub/file.txt");
free(line);
config_delete(config);
} }
static void test_render_itemize_up_to_date_is_empty() { static void test_render_itemize_up_to_date_is_empty() {
ChangeEvent event = sample_event(); ChangeEvent event = sample_event();
Config* config = config_create();
event.decision = CHANGE_UP_TO_DATE; event.decision = CHANGE_UP_TO_DATE;
char* line = change_render_itemize(&event); char* line = change_render_itemize(config, &event);
EXPECT_NOT_NULL(line); EXPECT_NOT_NULL(line);
EXPECT_EQ_STR(line, ""); EXPECT_EQ_STR(line, "");
free(line); free(line);
config_delete(config);
} }
static void test_render_list_line() { static void test_render_list_line() {
char* line = change_render_list_line(0100644, 4096, 1700000000, "/srv/x.txt"); ChangeEvent event;
memset(&event, 0, sizeof(event));
Config* config = config_create();
event.name = "sub/x.txt";
event.path = "sub/x.txt";
event.mode = 0100644;
event.size = 4096;
event.mtime_sec = 1700000000;
char* line = change_render_list_line(config, &event);
EXPECT_NOT_NULL(line); EXPECT_NOT_NULL(line);
EXPECT_TRUE(strncmp(line, "-rw-r--r--", 10) == 0); EXPECT_TRUE(strncmp(line, "-rw-r--r--", 10) == 0);
EXPECT_TRUE(strstr(line, "4096") != NULL); EXPECT_TRUE(strstr(line, "4,096") != NULL);
EXPECT_TRUE(strstr(line, "/srv/x.txt") != NULL); EXPECT_TRUE(strstr(line, "sub/x.txt") != NULL);
free(line); free(line);
config_delete(config);
} }
static void test_change_list_enabled() { static void test_change_list_enabled() {
@@ -93,8 +180,11 @@ static void test_change_list_enabled() {
void test_change_list() { void test_change_list() {
test_format_tokens(); test_format_tokens();
test_format_unknown_tokens_preserved(); test_format_unknown_tokens_preserved();
test_format_leaf_name(); test_format_directory_name_has_trailing_slash();
test_render_itemize_sent_file(); test_render_itemize_sent_file();
test_render_itemize_directory();
test_render_itemize_symlink();
test_render_itemize_compares_destination();
test_render_itemize_up_to_date_is_empty(); test_render_itemize_up_to_date_is_empty();
test_render_list_line(); test_render_list_line();
test_change_list_enabled(); test_change_list_enabled();
+46 -4
View File
@@ -317,7 +317,7 @@ static void test_parse_args_protocol_accept_current() {
Config* cfg = valid_client_config(); Config* cfg = valid_client_config();
EXPECT_NOT_NULL(cfg); EXPECT_NOT_NULL(cfg);
char* argv_equals[] = {"fastsync", "--source-dir", "/src", char* argv_equals[] = {"fastsync", "--source-dir", "/src",
"--dest-dir", "/dst", "--protocol=2.22.0"}; "--dest-dir", "/dst", "--protocol=2.23.0"};
int positional_args[2]; int positional_args[2];
int positional_count = 0; int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 6, argv_equals, positional_args, &positional_count), 0); EXPECT_EQ_INT(parse_args(cfg, 6, argv_equals, positional_args, &positional_count), 0);
@@ -327,7 +327,7 @@ static void test_parse_args_protocol_accept_current() {
cfg = valid_client_config(); cfg = valid_client_config();
EXPECT_NOT_NULL(cfg); EXPECT_NOT_NULL(cfg);
char* argv_space[] = {"fastsync", "--source-dir", "/src", "--dest-dir", char* argv_space[] = {"fastsync", "--source-dir", "/src", "--dest-dir",
"/dst", "--protocol", "2.22.0"}; "/dst", "--protocol", "2.23.0"};
positional_count = 0; positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 7, argv_space, positional_args, &positional_count), 0); EXPECT_EQ_INT(parse_args(cfg, 7, argv_space, positional_args, &positional_count), 0);
EXPECT_EQ_STR(cfg->version, PROTOCOL_VERSION); EXPECT_EQ_STR(cfg->version, PROTOCOL_VERSION);
@@ -338,8 +338,8 @@ static void test_parse_args_protocol_accept_current() {
* failure (parse_args simply stores it; validate_config rejects it up front). */ * failure (parse_args simply stores it; validate_config rejects it up front). */
static void test_parse_args_protocol_rejects_other_versions() { static void test_parse_args_protocol_rejects_other_versions() {
static const char* const bad_versions[] = {"2.17", "2.16", "2.15.0", "2.16.0", "2.17.0", static const char* const bad_versions[] = {"2.17", "2.16", "2.15.0", "2.16.0", "2.17.0",
"2.18.0", "2.19.0", "2.20.0", "2.21.0", "216", "2.18.0", "2.19.0", "2.20.0", "2.21.0", "2.22.0",
"31", "abc", ""}; "216", "31", "abc", ""};
for (size_t i = 0; i < sizeof(bad_versions) / sizeof(bad_versions[0]); i++) { for (size_t i = 0; i < sizeof(bad_versions) / sizeof(bad_versions[0]); i++) {
Config* cfg = valid_client_config(); Config* cfg = valid_client_config();
EXPECT_NOT_NULL(cfg); EXPECT_NOT_NULL(cfg);
@@ -3729,6 +3729,12 @@ static void test_parse_args_inline_equals_forms() {
EXPECT_EQ_STR(cfg->exclude_patterns[0], "*.log"); EXPECT_EQ_STR(cfg->exclude_patterns[0], "*.log");
EXPECT_EQ_INT(cfg->include_count, 1); EXPECT_EQ_INT(cfg->include_count, 1);
EXPECT_EQ_STR(cfg->include_patterns[0], "*.txt"); EXPECT_EQ_STR(cfg->include_patterns[0], "*.txt");
/* The same patterns are compiled, in command-line order, into the shared
* ordered --filter rule list (rsync first-match-wins). */
EXPECT_NOT_NULL(cfg->filters);
EXPECT_EQ_INT(cfg->filters->size, 2);
EXPECT_EQ_STR((char*)cfg->filters->items[0], "- *.log");
EXPECT_EQ_STR((char*)cfg->filters->items[1], "+ *.txt");
config_delete(cfg); config_delete(cfg);
const char* list_path = "cli_inline_patterns.txt"; const char* list_path = "cli_inline_patterns.txt";
@@ -3776,6 +3782,41 @@ static void test_parse_args_inline_equals_forms() {
config_delete(cfg); config_delete(cfg);
} }
/* --exclude/--include compile into the SAME ordered filter list as --filter, so
* rsync's first-match-wins semantics hold: the common `--include='*.txt'
* --exclude='*'` idiom keeps the .txt files and drops the rest, and an
* --include rule with no matching exclude is not a mandatory whitelist. */
static void test_parse_args_include_exclude_order() {
Config* cfg = config_create();
int positional_args[2];
int positional_count = 0;
char* argv[] = {"fastsync", "--include=*.txt", "--exclude=*", "/src", "/dst"};
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
EXPECT_NOT_NULL(cfg->filters);
EXPECT_EQ_INT(cfg->filters->size, 2);
EXPECT_EQ_STR((char*)cfg->filters->items[0], "+ *.txt");
EXPECT_EQ_STR((char*)cfg->filters->items[1], "- *");
/* The order is reversible on the command line and the list follows it. */
Config* cfg2 = config_create();
positional_count = 0;
char* argv2[] = {"fastsync", "--exclude=*", "--include=*.txt", "/src", "/dst"};
EXPECT_EQ_INT(parse_args(cfg2, 5, argv2, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg2->filters->size, 2);
EXPECT_EQ_STR((char*)cfg2->filters->items[0], "- *");
EXPECT_EQ_STR((char*)cfg2->filters->items[1], "+ *.txt");
/* --filter and --exclude/--include interleave in command-line order. */
Config* cfg3 = config_create();
positional_count = 0;
char* argv3[] = {"fastsync", "--filter=- *.tmp", "--include=*.txt", "/src", "/dst"};
EXPECT_EQ_INT(parse_args(cfg3, 5, argv3, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg3->filters->size, 2);
EXPECT_EQ_STR((char*)cfg3->filters->items[0], "- *.tmp");
EXPECT_EQ_STR((char*)cfg3->filters->items[1], "+ *.txt");
config_delete(cfg);
config_delete(cfg2);
config_delete(cfg3);
}
/* OPT_NOOP compatibility flags (-s/--secluded-args, -r/--recursive) must never /* OPT_NOOP compatibility flags (-s/--secluded-args, -r/--recursive) must never
* swallow the next argv: `fastsync -s SRC DST` keeps both positionals. */ * swallow the next argv: `fastsync -s SRC DST` keeps both positionals. */
static void test_parse_args_noop_does_not_consume_argv() { static void test_parse_args_noop_does_not_consume_argv() {
@@ -4003,6 +4044,7 @@ void test_client_cli() {
test_parse_args_short_clustering(); test_parse_args_short_clustering();
test_parse_args_attached_short_values(); test_parse_args_attached_short_values();
test_parse_args_inline_equals_forms(); test_parse_args_inline_equals_forms();
test_parse_args_include_exclude_order();
test_parse_args_noop_does_not_consume_argv(); test_parse_args_noop_does_not_consume_argv();
test_parse_args_backup_copy_links_shorts(); test_parse_args_backup_copy_links_shorts();
test_parse_args_rejects_unsupported_short(); test_parse_args_rejects_unsupported_short();
+7 -8
View File
@@ -2665,14 +2665,13 @@ static void golden_config_populate(Config* c) {
c->copy_as_gid = 222; c->copy_as_gid = 222;
} }
/* The pinned golden frame (protocol 2.22.0). The values below are the only /* The pinned golden frame (protocol 2.23.0). The values below are the only
* thing that ties the generated table to the historical wire format; update * thing that ties the generated table to the historical wire format; update
* them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.22.0 * them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.23.0
* preserve-attribute split appends four serialized bools * output-parity wave appends one serialized bool (report_dest_info) to the end
* (preserve_perms/times/owner/group) to CONFIG_WIRE_METADATA_TIMES_FIELDS after * of the frame, after the --copy-as block. */
* omit_link_times. */ #define GOLDEN_WIRE_LEN 657
#define GOLDEN_WIRE_LEN 653 #define GOLDEN_WIRE_HASH 4633069702262438591ULL
#define GOLDEN_WIRE_HASH 95530566005420798ULL
static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) { static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) {
unsigned long long h = 1469598103934665603ULL; unsigned long long h = 1469598103934665603ULL;
@@ -2754,7 +2753,7 @@ static unsigned long long capture_wire_hash(const Config* cfg, size_t* out_len)
return h; return h;
} }
/* Byte-for-byte wire compatibility guard (protocol 2.22.0). The expected hash /* Byte-for-byte wire compatibility guard (protocol 2.23.0). The expected hash
* pins the pre-X-macro byte stream; the refactor MUST NOT change it. */ * pins the pre-X-macro byte stream; the refactor MUST NOT change it. */
static void test_config_wire_golden() { static void test_config_wire_golden() {
if (is_running_under_valgrind()) if (is_running_under_valgrind())
+94
View File
@@ -0,0 +1,94 @@
#include "test_format.h"
#include "format.h"
#include "test_utils.h"
#include <stdio.h>
#include <string.h>
#include <sys/socket.h>
#include <time.h>
#include <unistd.h>
static void expect_big_num(unsigned long long value, bool human, const char* expected) {
char buffer[64];
EXPECT_TRUE(format_big_num(value, human, buffer, sizeof(buffer)));
EXPECT_EQ_STR(buffer, expected);
}
static void test_human_size_decimal() {
/* Values below 1000 print verbatim; larger values use the largest unit that
* keeps the value below 1000 and exactly two decimals (rsync human_num). */
expect_big_num(0, true, "0");
expect_big_num(999, true, "999");
expect_big_num(1000, true, "1.00K");
expect_big_num(1500, true, "1.50K");
expect_big_num(9999, true, "10.00K");
expect_big_num(999999, true, "1000.00K");
expect_big_num(1000000, true, "1.00M");
expect_big_num(1500000, true, "1.50M");
}
static void test_big_num_grouping() {
/* Non-human numbers are comma-grouped every three digits (rsync big_num). */
expect_big_num(0, false, "0");
expect_big_num(1, false, "1");
expect_big_num(999, false, "999");
expect_big_num(1000, false, "1,000");
expect_big_num(4096, false, "4,096");
expect_big_num(1234567, false, "1,234,567");
expect_big_num(1000000000ULL, false, "1,000,000,000");
}
static void test_datetime_format() {
char buffer[32];
time_t when = 1700000000;
EXPECT_TRUE(format_rsync_datetime(when, true, buffer, sizeof(buffer)));
/* %M shape: YYYY/MM/DD-HH:MM:SS */
EXPECT_EQ_INT(strlen(buffer), 19);
EXPECT_EQ_INT(buffer[4], '/');
EXPECT_EQ_INT(buffer[7], '/');
EXPECT_EQ_INT(buffer[10], '-');
EXPECT_EQ_INT(buffer[13], ':');
EXPECT_EQ_INT(buffer[16], ':');
char space_form[32];
EXPECT_TRUE(format_rsync_datetime(when, false, space_form, sizeof(space_form)));
EXPECT_EQ_INT(space_form[10], ' ');
}
static void test_dest_state_roundtrip() {
/* The wire codec is exercised over a socketpair so the real send/receive
* primitives run. */
int fds[2];
if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) != 0)
return;
OutputDestState out;
memset(&out, 0, sizeof(out));
out.known = true;
out.existed = true;
out.size = 123456789ULL;
out.mtime_sec = 1700000000;
out.mtime_nsec = 123456789;
out.mode = 0100644;
out.uid = 1000;
out.gid = 1000;
OutputDestState in;
memset(&in, 0, sizeof(in));
EXPECT_TRUE(format_dest_state_send(fds[0], &out));
EXPECT_TRUE(format_dest_state_receive(fds[1], &in));
EXPECT_TRUE(in.known);
EXPECT_TRUE(in.existed);
EXPECT_TRUE(in.size == out.size);
EXPECT_TRUE(in.mtime_sec == out.mtime_sec);
EXPECT_TRUE(in.mtime_nsec == out.mtime_nsec);
EXPECT_TRUE(in.mode == out.mode);
EXPECT_TRUE(in.uid == out.uid);
EXPECT_TRUE(in.gid == out.gid);
close(fds[0]);
close(fds[1]);
}
void test_format(void) {
test_human_size_decimal();
test_big_num_grouping();
test_datetime_format();
test_dest_state_roundtrip();
}
+6
View File
@@ -0,0 +1,6 @@
#ifndef TEST_FORMAT_H
#define TEST_FORMAT_H
void test_format(void);
#endif
+14 -11
View File
@@ -18,6 +18,9 @@
/* P8 config-frame tail: super_mode (4) + copy-as presence (4) + uid (4) + gid (4). */ /* P8 config-frame tail: super_mode (4) + copy-as presence (4) + uid (4) + gid (4). */
#define P8_TAIL_BYTES 16 #define P8_TAIL_BYTES 16
/* Protocol 2.23.0 appends one trailing bool (report_dest_info) AFTER the P8
* tail, so the P8 fields sit this many bytes before the end of the frame. */
#define OUTPUT_TAIL_BYTES 4
/* Smoke test for chunk_deserialize fuzz target */ /* Smoke test for chunk_deserialize fuzz target */
static void test_fuzz_chunk_deserialize() { static void test_fuzz_chunk_deserialize() {
@@ -334,31 +337,31 @@ static void test_fuzz_config_receive_p8_tail() {
/* super_mode outside the 0..2 tri-state is refused. */ /* super_mode outside the 0..2 tri-state is refused. */
memcpy(mut, frame, len); memcpy(mut, frame, len);
put_i32(mut, len - P8_TAIL_BYTES, 99); put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, 99);
EXPECT_FALSE(receive_config_frame(mut, len)); EXPECT_FALSE(receive_config_frame(mut, len));
put_i32(mut, len - P8_TAIL_BYTES, -1); put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, -1);
EXPECT_FALSE(receive_config_frame(mut, len)); EXPECT_FALSE(receive_config_frame(mut, len));
/* A negative (sentinel) and an extreme copy-as uid/gid are refused. */ /* A negative (sentinel) and an extreme copy-as uid/gid are refused. */
memcpy(mut, frame, len); memcpy(mut, frame, len);
put_i32(mut, len - P8_TAIL_BYTES, SUPER_MODE_AUTO); put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, SUPER_MODE_AUTO);
put_i32(mut, len - P8_TAIL_BYTES + 4, 1); put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 4, 1);
put_i32(mut, len - P8_TAIL_BYTES + 8, -1); put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 8, -1);
put_i32(mut, len - P8_TAIL_BYTES + 12, 0); put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 12, 0);
EXPECT_FALSE(receive_config_frame(mut, len)); EXPECT_FALSE(receive_config_frame(mut, len));
put_i32(mut, len - P8_TAIL_BYTES + 8, 0); put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 8, 0);
put_i32(mut, len - P8_TAIL_BYTES + 12, INT32_MIN); put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 12, INT32_MIN);
EXPECT_FALSE(receive_config_frame(mut, len)); EXPECT_FALSE(receive_config_frame(mut, len));
/* A presence int that is not a wire bool is refused. */ /* A presence int that is not a wire bool is refused. */
memcpy(mut, frame, len); memcpy(mut, frame, len);
put_i32(mut, len - P8_TAIL_BYTES, SUPER_MODE_AUTO); put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES, SUPER_MODE_AUTO);
put_i32(mut, len - P8_TAIL_BYTES + 4, 2); put_i32(mut, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES + 4, 2);
EXPECT_FALSE(receive_config_frame(mut, len)); EXPECT_FALSE(receive_config_frame(mut, len));
/* Truncating anywhere inside the P8 tail is refused. */ /* Truncating anywhere inside the P8 tail is refused. */
EXPECT_FALSE(receive_config_frame(frame, len - 2)); EXPECT_FALSE(receive_config_frame(frame, len - 2));
EXPECT_FALSE(receive_config_frame(frame, len - P8_TAIL_BYTES)); EXPECT_FALSE(receive_config_frame(frame, len - OUTPUT_TAIL_BYTES - P8_TAIL_BYTES));
free(mut); free(mut);
free(frame); free(frame);
+34 -3
View File
@@ -644,17 +644,19 @@ static void test_scanner_one_file_system_cross_device() {
EXPECT_EQ_INT(seq_off_rc, 0); EXPECT_EQ_INT(seq_off_rc, 0);
EXPECT_TRUE(seq_off_found); EXPECT_TRUE(seq_off_found);
EXPECT_EQ_INT(seq_off_total, 2); EXPECT_EQ_INT(seq_off_total, 2);
/* Sequential: with -x the cross-device subtree is dropped, keep.txt remains. */ /* Sequential: with -x the cross-device subtree is not descended into, but
* rsync-compatible behavior still emits the mount-point directory entry as an
* empty directory File, so keep.txt plus that entry are present. */
EXPECT_EQ_INT(seq_on_rc, 0); EXPECT_EQ_INT(seq_on_rc, 0);
EXPECT_FALSE(seq_on_found); EXPECT_FALSE(seq_on_found);
EXPECT_EQ_INT(seq_on_total, 1); EXPECT_EQ_INT(seq_on_total, 2);
/* Parallel: same behavior, worker path (depth > 1). */ /* Parallel: same behavior, worker path (depth > 1). */
EXPECT_EQ_INT(par_off_rc, 0); EXPECT_EQ_INT(par_off_rc, 0);
EXPECT_TRUE(par_off_found); EXPECT_TRUE(par_off_found);
EXPECT_EQ_INT(par_off_total, 2); EXPECT_EQ_INT(par_off_total, 2);
EXPECT_EQ_INT(par_on_rc, 0); EXPECT_EQ_INT(par_on_rc, 0);
EXPECT_FALSE(par_on_found); EXPECT_FALSE(par_on_found);
EXPECT_EQ_INT(par_on_total, 1); EXPECT_EQ_INT(par_on_total, 2);
} }
/* Collect emitted file paths (relative to `root`) from a sequential scan. /* Collect emitted file paths (relative to `root`) from a sequential scan.
@@ -884,6 +886,35 @@ static void test_filter_rules(bool parallel) {
free_paths(paths, count); free_paths(paths, count);
filter_rule_list_free(base); filter_rule_list_free(base);
/* The common include idiom (the exact rule order the CLI compiles from
* --include='*.txt' --exclude='*'): only .txt files survive. */
const char* idiom[] = {"+ *.txt", "- *"};
base = filter_base_build(idiom, 2, false, err, sizeof(err));
EXPECT_NOT_NULL(base);
options.base_filters = base;
rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 2);
EXPECT_TRUE(has_path(paths, count, "a.txt"));
EXPECT_TRUE(has_path(paths, count, "c.txt"));
EXPECT_FALSE(has_path(paths, count, "b.tmp"));
free_paths(paths, count);
filter_rule_list_free(base);
/* An include rule alone is NOT a mandatory whitelist (rsync semantics): only
* the matching file is affected, everything else is still transferred. */
const char* include_alone[] = {"+ *.txt"};
base = filter_base_build(include_alone, 1, false, err, sizeof(err));
EXPECT_NOT_NULL(base);
options.base_filters = base;
rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
: collect_files(root, &options, &paths, &count);
EXPECT_EQ_INT(rc, 0);
EXPECT_EQ_INT(count, 3);
free_paths(paths, count);
filter_rule_list_free(base);
unlink("test_scan_filter/a.txt"); unlink("test_scan_filter/a.txt");
unlink("test_scan_filter/b.tmp"); unlink("test_scan_filter/b.tmp");
unlink("test_scan_filter/c.txt"); unlink("test_scan_filter/c.txt");