Merge feat/p2-itemize-output: structured change output (-i, --list-only, --out-format, --log-file-format)

Client-only per-file event/format model (change_list.c). Reviewed
(c-review APPROVE WITH NITS, all fixed); PR #263.

tests/integration/test_features.py

@@ -1225,3 +1225,183 @@ class TestTempDir:
                                port=shared_server.port)
         assert result.returncode != 0, "absolute --temp-dir was not rejected"
         assert not os.path.lexists(abs_escape), "file created outside the destination root"
+
+
+def _source_files():
+    """All source paths (absolute) that a transfer would send right now."""
+    return [
+        os.path.join(root, name)
+        for root, _dirs, names in os.walk(SOURCE_DIR)
+        for name in names
+    ]
+
+
+class TestListOnly:
+    """--list-only prints every transfer candidate and changes nothing."""
+
+    def test_list_only_prints_each_file_and_does_not_transfer(self):
+        clean_dir(DEST_DIR)
+        result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only"])
+        assert result.returncode == 0, f"list-only failed: {result.stderr[:200]}"
+        for full_path in _source_files():
+            assert full_path in result.stdout, f"list-only omitted {full_path}"
+        received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
+        assert not os.path.exists(received), "list-only wrote to the destination"
+
+    def test_list_only_with_dry_run_does_not_error(self):
+        clean_dir(DEST_DIR)
+        result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only", "--dry-run"])
+        assert result.returncode == 0, f"list-only -n failed: {result.stderr[:200]}"
+        received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
+        assert not os.path.exists(received)
+
+    def test_list_only_multithreaded(self):
+        clean_dir(DEST_DIR)
+        result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only", "-m"])
+        assert result.returncode == 0, f"list-only -m failed: {result.stderr[:200]}"
+        for full_path in _source_files():
+            assert full_path in result.stdout, f"list-only -m omitted {full_path}"
+        received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
+        assert not os.path.exists(received), "list-only -m wrote to the destination"
+
+
+class TestItemizeChanges:
+    """-i/--itemize-changes prints rsync-style lines only for files sent."""
+
+    def test_first_run_prints_sent_lines(self, shared_server):
+        clean_dir(DEST_DIR)
+        result, _ = run_client(SOURCE_DIR, DEST_DIR,
+                               flags=["-M", "-i"], port=shared_server.port)
+        assert result.returncode == 0, f"itemize sync failed: {result.stderr[:200]}"
+        sent_lines = {">f+++++++++ " + p for p in _source_files()}
+        assert sent_lines <= set(result.stdout.splitlines()), (
+            f"missing itemize lines; got {result.stdout[:500]}"
+        )
+
+    def test_incremental_second_run_prints_no_line_for_unchanged(self, shared_server):
+        clean_dir(DEST_DIR)
+        result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
+        assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
+        result, _ = run_client(SOURCE_DIR, DEST_DIR,
+                               flags=["-M", "-i", "--incremental"],
+                               port=shared_server.port)
+        assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}"
+        itemized = [line for line in result.stdout.splitlines() if line and line[0] in ">.<c"]
+        assert itemized == [], f"unchanged files were itemized: {itemized[:5]}"
+
+    def test_multithreaded_emits_same_itemize_lines(self, shared_server):
+        clean_dir(DEST_DIR)
+        result, _ = run_client(SOURCE_DIR, DEST_DIR,
+                               flags=["-M", "-i", "-m"], port=shared_server.port)
+        assert result.returncode == 0, f"itemize -m sync failed: {result.stderr[:200]}"
+        sent_lines = {">f+++++++++ " + p for p in _source_files()}
+        assert sent_lines <= set(result.stdout.splitlines()), (
+            f"missing itemize lines in -m mode; got {result.stdout[:500]}"
+        )
+
+    def test_dry_run_with_itemize_does_not_error(self):
+        clean_dir(DEST_DIR)
+        result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-i", "--dry-run"])
+        assert result.returncode == 0, f"dry-run -i failed: {result.stderr[:200]}"
+
+    def test_changed_file_on_second_incremental_run_prints_exactly_one_line(self, shared_server):
+        """A changed file itemizes exactly once on an incremental rerun while
+        unchanged files print nothing (no double emission)."""
+        source = os.path.join(TEST_DATA_DIR, "itemize_change_src")
+        dest = os.path.join(TEST_DATA_DIR, "itemize_change_dst")
+        clean_dir(source)
+        clean_dir(dest)
+        changed = os.path.join(source, "changed.txt")
+        untouched = os.path.join(source, "untouched.txt")
+        with open(changed, "wb") as fh:
+            fh.write(b"original\n")
+        with open(untouched, "wb") as fh:
+            fh.write(b"stable\n")
+
+        result, _ = run_client(source, dest, flags=["-M"], port=shared_server.port)
+        assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
+
+        with open(changed, "wb") as fh:
+            fh.write(b"edited payload\n")
+
+        result, _ = run_client(source, dest,
+                               flags=["-M", "-i", "--incremental"],
+                               port=shared_server.port)
+        assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}"
+        itemized = [line for line in result.stdout.splitlines() if line.startswith(">f")]
+        assert itemized == [">f+++++++++ " + changed], (
+            f"expected exactly one itemize line for {changed}, got {itemized}"
+        )
+        received = get_dest_received_dir(dest, source)
+        assert _read_file(os.path.join(received, "changed.txt")) == b"edited payload\n"
+        assert _read_file(os.path.join(received, "untouched.txt")) == b"stable\n"
+
+
+class TestOutFormat:
+    """--out-format prints a line per transferred file using the template."""
+
+    def test_out_format_path_and_size(self, shared_server):
+        clean_dir(DEST_DIR)
+        result, _ = run_client(SOURCE_DIR, DEST_DIR,
+                               flags=["--out-format=%f %l"], port=shared_server.port)
+        assert result.returncode == 0, f"out-format sync failed: {result.stderr[:200]}"
+        expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()}
+        got = set(result.stdout.splitlines())
+        assert expected <= got, f"out-format lines missing: expected {len(expected)} got {len(got)}"
+
+    def test_out_format_multithreaded_matches_single(self, shared_server):
+        clean_dir(DEST_DIR)
+        result, _ = run_client(SOURCE_DIR, DEST_DIR,
+                               flags=["--out-format=%f %l", "-m"], port=shared_server.port)
+        assert result.returncode == 0, f"out-format -m sync failed: {result.stderr[:200]}"
+        expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()}
+        got = set(result.stdout.splitlines())
+        assert expected <= got, f"out-format -m lines missing: {result.stdout[:500]}"
+
+
+class TestLogFileFormat:
+    """--log-file plus --log-file-format writes per-file lines to the log."""
+
+    def test_log_file_format_writes_transferred_files(self, shared_server):
+        clean_dir(DEST_DIR)
+        log_path = os.path.join(TEST_DATA_DIR, "itemize_transfer.log")
+        if os.path.exists(log_path):
+            os.unlink(log_path)
+        result, _ = run_client(
+            SOURCE_DIR, DEST_DIR,
+            flags=["--log-file", log_path, "--log-file-format=%f %l"],
+            port=shared_server.port,
+        )
+        assert result.returncode == 0, f"log-file sync failed: {result.stderr[:200]}"
+        assert os.path.exists(log_path), "--log-file created no log"
+        with open(log_path, encoding="utf-8", errors="replace") as fh:
+            content = fh.read()
+        expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()}
+        for line in expected:
+            assert line in content, f"log file missing {line!r}"
+
+    def test_log_file_format_multithreaded_writes_transferred_files(self, shared_server):
+        source = os.path.join(TEST_DATA_DIR, "itemize_log_mt_src")
+        dest = os.path.join(TEST_DATA_DIR, "itemize_log_mt_dst")
+        clean_dir(source)
+        clean_dir(dest)
+        files = {"a.txt": b"alpha\n", "b.txt": b"beta\n"}
+        for rel, data in files.items():
+            with open(os.path.join(source, rel), "wb") as fh:
+                fh.write(data)
+        log_path = os.path.join(TEST_DATA_DIR, "itemize_mt.log")
+        if os.path.exists(log_path):
+            os.unlink(log_path)
+        result, _ = run_client(
+            source,
+            dest,
+            flags=["--log-file", log_path, "--log-file-format=%f %l", "-m"],
+            port=shared_server.port,
+        )
+        assert result.returncode == 0, f"log-file -m sync failed: {result.stderr[:200]}"
+        assert os.path.exists(log_path), "--log-file created no log"
+        with open(log_path, encoding="utf-8", errors="replace") as fh:
+            content = fh.read()
+        expected = {f"{os.path.join(source, rel)} {len(data)}" for rel, data in files.items()}
+        for line in expected:
+            assert line in content, f"log file (-m) missing {line!r}"

tests/runner.c

@@ -1,5 +1,6 @@
 #include "test_array_list.h"
 #include "test_chunk.h"
+#include "test_change_list.h"
 #include "test_client_cli.h"
 #include "test_compression.h"
 #include "test_config.h"
@@ -40,6 +41,7 @@ int main() {
   RUN_TEST(test_array_list);
   RUN_TEST(test_shared_utils);
   RUN_TEST(test_chunk);
+  RUN_TEST(test_change_list);
   RUN_TEST(test_config);
   RUN_TEST(test_compression);
   RUN_TEST(test_scanner);

tests/test_change_list.c

@@ -0,0 +1,101 @@
+#include "test_change_list.h"
+#include "change_list.h"
+#include "test_utils.h"
+#include "utils.h"
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+static ChangeEvent sample_event(void) {
+  ChangeEvent event;
+  memset(&event, 0, sizeof(event));
+  event.path = "/srv/root/sub/file.txt";
+  event.decision = CHANGE_SENT;
+  event.is_directory = false;
+  event.size = 12345;
+  event.bytes_sent = 999;
+  event.mtime_sec = 1700000000;
+  return event;
+}
+
+static void test_format_tokens() {
+  ChangeEvent event = sample_event();
+  char* line = change_render_format("%f %n %l %b %M %%", &event);
+  EXPECT_NOT_NULL(line);
+  EXPECT_EQ_STR(line, "/srv/root/sub/file.txt file.txt 12345 999 1700000000 %");
+  free(line);
+}
+
+static void test_format_unknown_tokens_preserved() {
+  ChangeEvent event = sample_event();
+  char* line = change_render_format("x%q=%f%z", &event);
+  EXPECT_NOT_NULL(line);
+  EXPECT_EQ_STR(line, "x%q=/srv/root/sub/file.txt%z");
+  free(line);
+}
+
+static void test_format_leaf_name() {
+  ChangeEvent event = sample_event();
+  event.path = "bare.txt";
+  char* line = change_render_format("%n|%f", &event);
+  EXPECT_NOT_NULL(line);
+  EXPECT_EQ_STR(line, "bare.txt|bare.txt");
+  free(line);
+}
+
+static void test_render_itemize_sent_file() {
+  ChangeEvent event = sample_event();
+  char* line = change_render_itemize(&event);
+  EXPECT_NOT_NULL(line);
+  EXPECT_EQ_STR(line, ">f+++++++++ /srv/root/sub/file.txt");
+  free(line);
+}
+
+static void test_render_itemize_up_to_date_is_empty() {
+  ChangeEvent event = sample_event();
+  event.decision = CHANGE_UP_TO_DATE;
+  char* line = change_render_itemize(&event);
+  EXPECT_NOT_NULL(line);
+  EXPECT_EQ_STR(line, "");
+  free(line);
+}
+
+static void test_render_list_line() {
+  char* line = change_render_list_line(0100644, 4096, 1700000000, "/srv/x.txt");
+  EXPECT_NOT_NULL(line);
+  EXPECT_TRUE(strncmp(line, "-rw-r--r--", 10) == 0);
+  EXPECT_TRUE(strstr(line, "4096") != NULL);
+  EXPECT_TRUE(strstr(line, "/srv/x.txt") != NULL);
+  free(line);
+}
+
+static void test_change_list_enabled() {
+  Config* config = config_create();
+  EXPECT_NOT_NULL(config);
+  EXPECT_FALSE(change_list_enabled(config));
+  config->itemize_changes = true;
+  EXPECT_TRUE(change_list_enabled(config));
+  config->itemize_changes = false;
+  config->out_format = str_dup("%f");
+  EXPECT_TRUE(change_list_enabled(config));
+  free(config->out_format);
+  config->out_format = NULL;
+  EXPECT_FALSE(change_list_enabled(config));
+  /* config_delete() closes log_file, so use a throwaway tmpfile. */
+  config->log_file = tmpfile();
+  EXPECT_NOT_NULL(config->log_file);
+  EXPECT_FALSE(change_list_enabled(config)); /* needs a format too */
+  config->log_file_format = str_dup("%n");
+  EXPECT_TRUE(change_list_enabled(config));
+  config_delete(config); /* closes config->log_file */
+}
+
+void test_change_list() {
+  test_format_tokens();
+  test_format_unknown_tokens_preserved();
+  test_format_leaf_name();
+  test_render_itemize_sent_file();
+  test_render_itemize_up_to_date_is_empty();
+  test_render_list_line();
+  test_change_list_enabled();
+}

tests/test_change_list.h

@@ -0,0 +1,6 @@
+#ifndef TEST_CHANGE_LIST_H
+#define TEST_CHANGE_LIST_H
+
+void test_change_list(void);
+
+#endif

tests/test_client_cli.c

@@ -602,10 +602,6 @@ static void test_parse_args_rejects_unimplemented_options() {
                                         "--xattrs",
                                         "-D",
                                         "--devices",
-                                        "-i",
-                                        "--itemize-changes",
-                                        "--out-format",
-                                        "--list-only",
                                         "--append",
                                         "--append-verify",
                                         "--delete-excluded",
@@ -1182,7 +1178,8 @@ static void test_parse_args_table_equals_string_and_int_options() {
 /* Options that take a separate value must report "missing argument", not the
  * generic "Unknown option", when they are the final argv entry. */
 static void test_parse_args_missing_argument_diagnostic() {
-  static const char* const options[] = {"--exclude", "--server-port", "--skip-compress", "-T"};
+  static const char* const options[] = {"--exclude", "--server-port", "--skip-compress",
+                                        "-T",        "--out-format",  "--log-file-format"};
 
   for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
     Config* cfg = config_create();
@@ -1208,6 +1205,66 @@ static void test_parse_args_missing_argument_diagnostic() {
   }
 }
 
+static void test_parse_args_itemize_changes() {
+  static const char* const flags[] = {"-i", "--itemize-changes"};
+  for (size_t i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
+    Config* cfg = config_create();
+    char* argv[] = {"fastsync", (char*)flags[i], "/src", "/dst"};
+    int positional_args[2];
+    int positional_count = 0;
+
+    EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
+    EXPECT_TRUE(cfg->itemize_changes);
+    EXPECT_EQ_INT(positional_count, 2);
+    config_delete(cfg);
+  }
+}
+
+static void test_parse_args_list_only() {
+  Config* cfg = config_create();
+  char* argv[] = {"fastsync", "--list-only", "/src", "/dst"};
+  int positional_args[2];
+  int positional_count = 0;
+
+  EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
+  EXPECT_TRUE(cfg->list_only);
+  config_delete(cfg);
+}
+
+static void test_parse_args_out_format() {
+  Config* cfg = config_create();
+  char* argv[] = {"fastsync", "--out-format=%f %l", "/src", "/dst"};
+  int positional_args[2];
+  int positional_count = 0;
+  EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
+  EXPECT_EQ_STR(cfg->out_format, "%f %l");
+  config_delete(cfg);
+
+  cfg = config_create();
+  char* separate_argv[] = {"fastsync", "--out-format", "%f %l", "/src", "/dst"};
+  positional_count = 0;
+  EXPECT_EQ_INT(parse_args(cfg, 5, separate_argv, positional_args, &positional_count), 0);
+  EXPECT_EQ_STR(cfg->out_format, "%f %l");
+  config_delete(cfg);
+}
+
+static void test_parse_args_log_file_format() {
+  Config* cfg = config_create();
+  char* argv[] = {"fastsync", "--log-file-format=%n %M", "/src", "/dst"};
+  int positional_args[2];
+  int positional_count = 0;
+  EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
+  EXPECT_EQ_STR(cfg->log_file_format, "%n %M");
+  config_delete(cfg);
+
+  cfg = config_create();
+  char* separate_argv[] = {"fastsync", "--log-file-format", "%n %M", "/src", "/dst"};
+  positional_count = 0;
+  EXPECT_EQ_INT(parse_args(cfg, 5, separate_argv, positional_args, &positional_count), 0);
+  EXPECT_EQ_STR(cfg->log_file_format, "%n %M");
+  config_delete(cfg);
+}
+
 void test_client_cli() {
   test_validate_config_required_paths();
   test_validate_config_incompatible_options();
@@ -1280,6 +1337,10 @@ void test_client_cli() {
   test_parse_args_table_equals_string_and_int_options();
   test_parse_args_missing_argument_diagnostic();
   test_parse_args_partial_progress();
+  test_parse_args_itemize_changes();
+  test_parse_args_list_only();
+  test_parse_args_out_format();
+  test_parse_args_log_file_format();
   test_parse_args_checksum_choice_aliases();
   test_parse_args_checksum_choice_requires_value();
   test_parse_args_temp_dir();