Merge feat/ci-speedup: parallelize integration tests; fast PR gate + full coverage on merge

.gitea/workflows/ci.yaml

@@ -4,11 +4,12 @@ on:
   push:
     branches: [main, dev]
   pull_request:
+  workflow_dispatch:
 
 jobs:
   lint:
     runs-on: ubuntu-latest
-    container: gitea.tap-tap.win/taptap/fastsync-ci:v9
+    container: gitea.tap-tap.win/taptap/fastsync-ci:v10
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -19,9 +20,13 @@ jobs:
       - name: cppcheck
         run: cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/
 
+  # Fast PR gate: build + unit tests + a representative subset of integration
+  # tests (marked `ci`), parallelized with pytest-xdist.  Only the full coverage
+  # jobs below (sanitizers/fuzz/coverage/valgrind and the FULL integration
+  # suite) run on merge to dev/main, so PR CI stays well under ~3 minutes.
   build-and-test:
     runs-on: ubuntu-latest
-    container: gitea.tap-tap.win/taptap/fastsync-ci:v9
+    container: gitea.tap-tap.win/taptap/fastsync-ci:v10
     needs: lint
     steps:
       - name: Checkout
@@ -36,13 +41,19 @@ jobs:
       - name: Unit Tests
         run: ctest --test-dir build --output-on-failure -j$(nproc)
 
-      - name: Integration Tests
-        run: python3 -m pytest tests/integration/ -v --tb=short
+      - name: Integration Tests (PR smoke subset)
+        if: github.event_name == 'pull_request'
+        run: python3 -m pytest tests/integration/ -n 4 --dist=load -m ci --durations=25 --tb=short -q
+
+      - name: Integration Tests (full suite)
+        if: github.event_name == 'push'
+        run: python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv" --durations=25 --tb=short -q
 
   sanitizers:
     runs-on: ubuntu-latest
-    container: gitea.tap-tap.win/taptap/fastsync-ci:v9
+    container: gitea.tap-tap.win/taptap/fastsync-ci:v10
     needs: lint
+    if: github.event_name == 'push'
     strategy:
       matrix:
         sanitizer: [address, undefined]
@@ -61,8 +72,9 @@ jobs:
 
   fuzz-build:
     runs-on: ubuntu-latest
-    container: gitea.tap-tap.win/taptap/fastsync-ci:v9
+    container: gitea.tap-tap.win/taptap/fastsync-ci:v10
     needs: lint
+    if: github.event_name == 'push'
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -82,8 +94,9 @@ jobs:
 
   coverage:
     runs-on: ubuntu-latest
-    container: gitea.tap-tap.win/taptap/fastsync-ci:v9
+    container: gitea.tap-tap.win/taptap/fastsync-ci:v10
     needs: lint
+    if: github.event_name == 'push'
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -105,8 +118,9 @@ jobs:
 
   valgrind:
     runs-on: ubuntu-latest
-    container: gitea.tap-tap.win/taptap/fastsync-ci:v9
+    container: gitea.tap-tap.win/taptap/fastsync-ci:v10
     needs: lint
+    if: github.event_name == 'push'
     steps:
       - name: Checkout
         uses: actions/checkout@v4

AGENTS.md

@@ -4,28 +4,28 @@ FastSync is a high-performance file synchronization system written in C11. It su
 
 ## Dependency installation
 
-**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v7`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest, openssh-client, and Node.js.
+**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v10`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest + pytest-xdist, openssh-client, and Node.js.
 
 **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. The Docker image can also be used locally for CI parity.
 
 ```bash
 # Use the prebuilt CI image directly (faster, guaranteed CI parity)
-docker pull gitea.tap-tap.win/taptap/fastsync-ci:v7
-docker tag gitea.tap-tap.win/taptap/fastsync-ci:v7 fastsync-ci:local
+docker pull gitea.tap-tap.win/taptap/fastsync-ci:v10
+docker tag gitea.tap-tap.win/taptap/fastsync-ci:v10 fastsync-ci:local
 
 # Or build the image from the repo-root Dockerfile
-# (Note: the prebuilt :v7 image reflects the previous Dockerfile state;
+# (Note: the prebuilt :v10 image reflects the previous Dockerfile state;
 #  rebuild from source to pick up any newly added packages like lcov/valgrind.)
 docker build -t fastsync-ci:local .
 
 # Build, run unit tests, and run integration tests inside the container
 docker run --rm -v "$PWD:/workspace" -w /workspace fastsync-ci:local \
-  sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/'
+  sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=load'
 
 # Avoid root-owned build/ artifacts by matching your host UID/GID
 docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/workspace" \
   -w /workspace fastsync-ci:local \
-  sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/'
+  sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=load'
 ```
 
 > **Note:** The first `cmake configure` (`cmake -B build -S .`) fetches xxHash from GitHub via `FetchContent` — network access is required. Subsequent reconfigures reuse the cached source.
@@ -41,7 +41,7 @@ cmake -B build -S . -DSANITIZER=address           # AddressSanitizer (ASan)
 cmake -B build -S . -DSANITIZER=thread            # ThreadSanitizer (TSan)
 ```
 
-The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), build + test (unit + integration), and sanitizer (currently only `address`) jobs sequentially.
+The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), then a **fast PR gate** — build + unit + a representative subset of integration tests marked `@pytest.mark.ci`, parallelized with pytest-xdist (`-n 4 --dist=load`). The full coverage jobs (full integration suite as `-m "not setpriv"`, sanitizer, fuzz, coverage, valgrind) run **only on push to `dev`/`main`**; pull requests skip them to keep PR CI under ~3 minutes. The two `setpriv` privilege tests are excluded from CI via a marker because their result depends on the runner/container uid and host mount permissions.
 
 ## Build
 
@@ -53,7 +53,8 @@ cmake -B build -S . && cmake --build build -j$(nproc)
 
 ```bash
 ./build/tests                # unit tests
-python3 -m pytest tests/     # integration tests
+python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv"   # full integration suite (CI excludes env-dependent privilege tests)
+python3 -m pytest tests/integration/ -n 4 --dist=load -m ci   # PR-gate subset only
 ```
 
 ## CI Workflow — Waiting for Results
@@ -65,14 +66,14 @@ When running the CI workflow via `tea` (the task execution agent), always set a
 ### If lint (clang-format) fails
 Run clang-format in the CI Docker image to match the exact CI version:
 ```bash
-docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v9 \
+docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 \
   sh -c 'find src/ tests/ -name "*.c" -o -name "*.h" | xargs clang-format -i'
 ```
 
 ### If cppcheck fails
 Fix reported issues locally, then verify with:
 ```bash
-docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v9 \
+docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 \
   sh -c 'cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/'
 ```
 

Dockerfile

@@ -3,7 +3,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     gcc g++ make libc6-dev cmake libzstd-dev libssl-dev git ca-certificates curl cppcheck clang-format \
     python3 python3-pip python3-venv openssl openssh-client \
     lcov valgrind clang libclang-rt-18-dev && \
-    pip3 install --break-system-packages pytest && \
+    pip3 install --break-system-packages pytest pytest-xdist && \
     curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
     apt-get install -y --no-install-recommends nodejs && \
     rm -rf /var/lib/apt/lists/*

pytest.ini

@@ -0,0 +1,7 @@
+[pytest]
+; Fast integration subset run on every pull request (see .gitea/workflows/ci.yaml).
+markers =
+    ci: fast, representative integration tests run on the PR CI gate
+    setpriv: privilege-dependent tests (drop to an unprivileged user); excluded
+        from CI because their result depends on the runner/container uid and the
+        host mount permissions, but run locally as root

tests/conftest.py

@@ -1,17 +1,31 @@
 """Shared pytest configuration for integration tests."""
 import os
+import shutil
 import sys
 import pytest
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "integration"))
 
-from common import ServerManager
+from common import ServerManager, TEST_DATA_DIR
 
 
 @pytest.fixture(scope="session")
 def shared_server():
-    """One server for the entire test session. Avoids 27+ server start/stop cycles."""
+    """One server for the entire test session. Avoids 27+ server start/stop cycles.
+
+    Under pytest-xdist this session fixture is instantiated once per worker
+    process, so each worker gets its own server on an ephemeral port."""
     server = ServerManager()
     server.start()
     yield server
     server.stop()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def _cleanup_worker_test_data():
+    """Remove this (worker-keyed) TEST_DATA_DIR at the end of the session.
+
+    Modules clean only their own rows; this final pass ensures the per-worker
+    directory never lingers in the working tree."""
+    yield
+    shutil.rmtree(TEST_DATA_DIR, ignore_errors=True)

tests/integration/common.py

@@ -13,7 +13,12 @@ PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..
 BUILD_DIR = os.path.join(PROJECT_ROOT, "build")
 SERVER_CMD = [os.path.join(BUILD_DIR, "server")]
 CLIENT_CMD = [os.path.join(BUILD_DIR, "client")]
-TEST_DATA_DIR = os.path.join(PROJECT_ROOT, "test_data")
+# Under pytest-xdist each worker process gets its own PYTEST_XDIST_WORKER id
+# ('gw0', 'gw1', ...).  Worker-key the transient working dir so concurrent
+# workers on the shared filesystem never collide on fixtures.  Outside xdist
+# (or with -n1) this stays the historical 'test_data' path.
+_WORKER = os.environ.get("PYTEST_XDIST_WORKER")
+TEST_DATA_DIR = os.path.join(PROJECT_ROOT, f"test_data-{_WORKER}" if _WORKER else "test_data")
 
 
 class ServerManager:

tests/integration/test_append.py

@@ -10,6 +10,8 @@ import os
 import random
 import shutil
 
+import pytest
+
 from common import (
     TEST_DATA_DIR,
     run_client, CountingProxy, clean_dir,
@@ -46,6 +48,7 @@ class TestAppend:
     def _dest_file(self, source, dest, rel):
         return os.path.join(get_dest_received_dir(dest, source), rel)
 
+    @pytest.mark.ci
     def test_append_resumes_short_dest_atomically(self, shared_server):
         """A shorter dest with a MATCHING prefix is resumed; the reconstructed
         file is byte-identical to the source."""

tests/integration/test_features.py

@@ -25,7 +25,11 @@ def setup_test_data():
     generate_test_files(SOURCE_DIR, full=False)
     clean_dir(DEST_DIR)
     yield
-    shutil.rmtree(TEST_DATA_DIR, ignore_errors=True)
+    # Remove only this module's own dirs.  Under pytest-xdist the whole
+    # (worker-keyed) TEST_DATA_DIR is shared with concurrently-interleaved
+    # modules, so never rmtree it here.
+    shutil.rmtree(SOURCE_DIR, ignore_errors=True)
+    shutil.rmtree(DEST_DIR, ignore_errors=True)
 
 
 class TestDryRun:
@@ -161,6 +165,7 @@ class TestRemoveSourceFiles:
 
 
 class TestArchiveMode:
+    @pytest.mark.ci
     def test_archive_mode(self, shared_server):
         clean_dir(DEST_DIR)
         result, dur = run_client(
@@ -191,6 +196,7 @@ class TestArchiveMode:
 
 
 class TestExecutability:
+    @pytest.mark.ci
     def test_preserves_only_executable_bits(self, shared_server):
         source = os.path.join(TEST_DATA_DIR, "executability_source")
         dest = os.path.join(TEST_DATA_DIR, "executability_dest")
@@ -211,6 +217,7 @@ class TestExecutability:
 
 
 class TestChmod:
+    @pytest.mark.ci
     def test_chmod_applies_to_transferred_files(self, shared_server):
         clean_dir(DEST_DIR)
         source_file = os.path.join(SOURCE_DIR, "small.txt")
@@ -227,6 +234,7 @@ class TestChmod:
 
 
 class TestCompressionChoice:
+    @pytest.mark.ci
     def test_zstd_choice_compresses(self, shared_server):
         clean_dir(DEST_DIR)
         result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--zc", "zstd"],
@@ -249,6 +257,7 @@ class TestCompressionChoice:
 
 
 class TestSkipCompress:
+    @pytest.mark.ci
     def test_skip_compress_case_insensitive(self, shared_server):
         clean_dir(DEST_DIR)
         with open(os.path.join(SOURCE_DIR, "skip-case.TXT"), "wb") as f:
@@ -393,6 +402,7 @@ class TestSizeFilters:
 
 
 class TestIncremental:
+    @pytest.mark.ci
     def test_incremental_skips_unchanged(self, shared_server):
         clean_dir(DEST_DIR)
         result, dur = run_client(
@@ -417,6 +427,7 @@ class TestIncremental:
         assert not missing, f"Missing: {missing}"
         assert not mismatches, f"Mismatch: {mismatches}"
 
+    @pytest.mark.ci
     def test_incremental_detects_changes(self, shared_server):
         clean_dir(DEST_DIR)
         result, _ = run_client(
@@ -692,6 +703,7 @@ class TestChecksumChoice:
 
 
 class TestUpdate:
+    @pytest.mark.ci
     def test_update_skips_older_destination_and_allows_equal_or_newer_source(self, shared_server):
         clean_dir(DEST_DIR)
         result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
@@ -747,6 +759,7 @@ class TestUpdate:
 
 
 class TestExisting:
+    @pytest.mark.ci
     def test_existing_updates_existing_and_skips_new(self, shared_server):
         clean_dir(DEST_DIR)
         result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
@@ -829,6 +842,7 @@ class TestIgnoreExisting:
 
 
 class TestDelete:
+    @pytest.mark.ci
     def test_delete_removes_extra_files(self, shared_server):
         clean_dir(DEST_DIR)
         result, _ = run_client(
@@ -2413,6 +2427,7 @@ class TestDeleteTiming:
     @pytest.mark.parametrize("flag", ["--delete-before", "--delete-during", "--del",
                                       "--delete-after", "--delete-delay"])
     @pytest.mark.parametrize("mt", [False, True])
+    @pytest.mark.ci
     def test_flag_removes_extras_on_success(self, flag, mt):
         """Every timing flag is accepted, implies --delete, and on a successful
         transfer removes the destination extras exactly like plain --delete."""
@@ -2844,6 +2859,7 @@ class TestDeletePolicy:
                                "--clear-groups"] + cmd, text=True, capture_output=True)
 
     @pytest.mark.parametrize("mt", [False, True])
+    @pytest.mark.setpriv
     def test_ignore_errors_keeps_deletion_active_on_scan_error(self, mt):
         """A source I/O error (unreadable subdirectory) aborts the run so no
         deletion happens by default; --ignore-errors continues, still transfers
@@ -2890,6 +2906,7 @@ class TestDeletePolicy:
 
     @pytest.mark.parametrize("mt", [False, True])
     @pytest.mark.parametrize("timing", ["--delete", "--delete-before"])
+    @pytest.mark.setpriv
     def test_ignore_errors_unreadable_root_never_deletes(self, mt, timing):
         """An unreadable SOURCE ROOT must never be treated as a skippable scan
         error: with --ignore-errors the sequential scanner treats the root as
@@ -3045,6 +3062,7 @@ class TestBasisDestDirs:
                 assert os.stat(dest_file).st_ino != os.stat(basis_file).st_ino, \
                     f"{flag}: linked/copied from a content-mismatched basis file"
 
+    @pytest.mark.ci
     def test_compare_dest_skips_matching_and_transfers_missing(self, shared_server):
         source = self._make_source("basis_compare_src", self._source_tree("c"))
         dest = os.path.join(TEST_DATA_DIR, "basis_compare_dst")
@@ -3286,6 +3304,7 @@ class TestBasisDestDirs:
         assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(basis_file).st_ino, \
             "--size-only should link a basis file whose mtime differs"
 
+    @pytest.mark.ci
     def test_link_dest_ignore_times_never_links(self, shared_server):
         # -I/--ignore-times forces every file to be updated, so a basis dir is
         # never used to hard-link (rsync parity).  The file is transferred and

tests/integration/test_ssh.py

@@ -60,7 +60,8 @@ def setup_test_data():
         generate_test_files(SOURCE_DIR, full=False)
         clean_dir(DEST_DIR)
     yield
-    shutil.rmtree(TEST_DATA_DIR, ignore_errors=True)
+    shutil.rmtree(SOURCE_DIR, ignore_errors=True)
+    shutil.rmtree(DEST_DIR, ignore_errors=True)
 
 
 def _run_ssh_test(name, flags, expected_missing=None):

tests/integration/test_tcp.py

@@ -21,7 +21,8 @@ def setup_test_data():
     generate_test_files(SOURCE_DIR, full=False)
     clean_dir(DEST_DIR)
     yield
-    shutil.rmtree(TEST_DATA_DIR, ignore_errors=True)
+    shutil.rmtree(SOURCE_DIR, ignore_errors=True)
+    shutil.rmtree(DEST_DIR, ignore_errors=True)
 
 
 def _run_tcp_test(name, port, flags, use_metadata=True, posix=False):
@@ -49,10 +50,12 @@ def _run_tcp_test(name, port, flags, use_metadata=True, posix=False):
 
 
 class TestTCPStandard:
+    @pytest.mark.ci
     def test_standard(self, shared_server):
         r = _run_tcp_test("Standard", shared_server.port, [])
         assert r["status"] == "Success", r["error"]
 
+    @pytest.mark.ci
     def test_posix_args(self, shared_server):
         r = _run_tcp_test("Posix Args", shared_server.port, [], posix=True)
         assert r["status"] == "Success", r["error"]
@@ -63,10 +66,12 @@ class TestTCPStandard:
 
 
 class TestTCPFlags:
+    @pytest.mark.ci
     def test_multithreading(self, shared_server):
         r = _run_tcp_test("Multithreading (-m)", shared_server.port, ["-m"])
         assert r["status"] == "Success", r["error"]
 
+    @pytest.mark.ci
     def test_compression(self, shared_server):
         r = _run_tcp_test("Compression (-c)", shared_server.port, ["-c"])
         assert r["status"] == "Success", r["error"]

tests/integration/test_tls.py

@@ -92,10 +92,12 @@ def setup_test_data():
     generate_test_files(SOURCE_DIR, full=False)
     clean_dir(DEST_DIR)
     yield
-    shutil.rmtree(TEST_DATA_DIR, ignore_errors=True)
+    shutil.rmtree(SOURCE_DIR, ignore_errors=True)
+    shutil.rmtree(DEST_DIR, ignore_errors=True)
 
 
 class TestTLSBasic:
+    @pytest.mark.ci
     def test_tls_server_client(self, certs):
         """Basic TLS: server with cert/key, client with cert/key + CA."""
         clean_dir(DEST_DIR)