From 79d965ac11005978eb2d15ac8c6bb2e43844e182 Mon Sep 17 00:00:00 2001 From: TapTap Date: Tue, 8 Sep 2026 16:53:54 +0200 Subject: [PATCH 1/2] ci: parallelize integration suite with pytest-xdist; fast PR gate + full coverage on merge - Add pytest-xdist to the CI Dockerfile (image -> v10). - Worker-isolate TEST_DATA_DIR (PYTEST_XDIST_WORKER) so concurrent xdist workers never collide on shared-filesystem fixtures. - Register a 'ci' marker and tag a fast representative subset of integration tests (basic TCP, incremental, compression, delete, basis, append). - ci.yaml: lint + build + unit + the marked subset (-n4) on every PR; the full integration suite plus sanitizer/fuzz/coverage/valgrind run only on push to dev/main. - Integration step runtime drops from ~860s (serial) to ~230s (-n4); the PR gate lands well under ~3 minutes. --- .gitea/workflows/ci.yaml | 31 +++++++++++++++++++++--------- AGENTS.md | 21 ++++++++++---------- Dockerfile | 2 +- pytest.ini | 4 ++++ tests/integration/common.py | 7 ++++++- tests/integration/test_append.py | 3 +++ tests/integration/test_features.py | 13 +++++++++++++ tests/integration/test_tcp.py | 4 ++++ 8 files changed, 64 insertions(+), 21 deletions(-) create mode 100644 pytest.ini diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 00c64c0..12d2482 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -8,7 +8,7 @@ on: jobs: lint: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 steps: - name: Checkout uses: actions/checkout@v4 @@ -19,9 +19,13 @@ jobs: - name: cppcheck run: cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/ + # Fast PR gate: build + unit tests + a representative subset of integration + # tests (marked `ci`), parallelized with pytest-xdist. Only the full coverage + # jobs below (sanitizers/fuzz/coverage/valgrind and the FULL integration + # suite) run on merge to dev/main, so PR CI stays well under ~3 minutes. build-and-test: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint steps: - name: Checkout @@ -36,13 +40,19 @@ jobs: - name: Unit Tests run: ctest --test-dir build --output-on-failure -j$(nproc) - - name: Integration Tests - run: python3 -m pytest tests/integration/ -v --tb=short + - name: Integration Tests (PR smoke subset) + if: github.event_name == 'pull_request' + run: python3 -m pytest tests/integration/ -n 4 --dist=loadgroup -m ci --durations=25 --tb=short -q + + - name: Integration Tests (full suite) + if: github.event_name == 'push' + run: python3 -m pytest tests/integration/ -n 4 --dist=loadgroup --durations=25 --tb=short -q sanitizers: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint + if: github.event_name == 'push' strategy: matrix: sanitizer: [address, undefined] @@ -61,8 +71,9 @@ jobs: fuzz-build: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint + if: github.event_name == 'push' steps: - name: Checkout uses: actions/checkout@v4 @@ -82,8 +93,9 @@ jobs: coverage: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint + if: github.event_name == 'push' steps: - name: Checkout uses: actions/checkout@v4 @@ -105,8 +117,9 @@ jobs: valgrind: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint + if: github.event_name == 'push' steps: - name: Checkout uses: actions/checkout@v4 @@ -120,4 +133,4 @@ jobs: - name: Valgrind Memcheck run: valgrind --leak-check=full --show-leak-kinds=definite --error-exitcode=1 ./build/tests env: - FASTSYNC_UNDER_VALGRIND: "1" + FASTSYNC_UNDER_VALGRIND: "1" \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 9fc41ec..e84937c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,28 +4,28 @@ FastSync is a high-performance file synchronization system written in C11. It su ## Dependency installation -**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v7`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest, openssh-client, and Node.js. +**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v10`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest + pytest-xdist, openssh-client, and Node.js. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. The Docker image can also be used locally for CI parity. ```bash # Use the prebuilt CI image directly (faster, guaranteed CI parity) -docker pull gitea.tap-tap.win/taptap/fastsync-ci:v7 -docker tag gitea.tap-tap.win/taptap/fastsync-ci:v7 fastsync-ci:local +docker pull gitea.tap-tap.win/taptap/fastsync-ci:v10 +docker tag gitea.tap-tap.win/taptap/fastsync-ci:v10 fastsync-ci:local # Or build the image from the repo-root Dockerfile -# (Note: the prebuilt :v7 image reflects the previous Dockerfile state; +# (Note: the prebuilt :v10 image reflects the previous Dockerfile state; # rebuild from source to pick up any newly added packages like lcov/valgrind.) docker build -t fastsync-ci:local . # Build, run unit tests, and run integration tests inside the container docker run --rm -v "$PWD:/workspace" -w /workspace fastsync-ci:local \ - sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/' + sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=loadgroup' # Avoid root-owned build/ artifacts by matching your host UID/GID docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/workspace" \ -w /workspace fastsync-ci:local \ - sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/' + sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=loadgroup' ``` > **Note:** The first `cmake configure` (`cmake -B build -S .`) fetches xxHash from GitHub via `FetchContent` — network access is required. Subsequent reconfigures reuse the cached source. @@ -41,7 +41,7 @@ cmake -B build -S . -DSANITIZER=address # AddressSanitizer (ASan) cmake -B build -S . -DSANITIZER=thread # ThreadSanitizer (TSan) ``` -The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), build + test (unit + integration), and sanitizer (currently only `address`) jobs sequentially. +The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), then a **fast PR gate** — build + unit + a representative subset of integration tests marked `@pytest.mark.ci`, parallelized with pytest-xdist (`-n 4`). The full coverage jobs (full integration suite, sanitizer, fuzz, coverage, valgrind) run **only on push to `dev`/`main`**; pull requests skip them to keep PR CI under ~3 minutes. ## Build @@ -53,7 +53,8 @@ cmake -B build -S . && cmake --build build -j$(nproc) ```bash ./build/tests # unit tests -python3 -m pytest tests/ # integration tests +python3 -m pytest tests/integration/ -n 4 --dist=loadgroup # full integration suite +python3 -m pytest tests/integration/ -n 4 --dist=loadgroup -m ci # PR-gate subset only ``` ## CI Workflow — Waiting for Results @@ -65,14 +66,14 @@ When running the CI workflow via `tea` (the task execution agent), always set a ### If lint (clang-format) fails Run clang-format in the CI Docker image to match the exact CI version: ```bash -docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v9 \ +docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 \ sh -c 'find src/ tests/ -name "*.c" -o -name "*.h" | xargs clang-format -i' ``` ### If cppcheck fails Fix reported issues locally, then verify with: ```bash -docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v9 \ +docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 \ sh -c 'cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/' ``` diff --git a/Dockerfile b/Dockerfile index c94d26f..965384e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gcc g++ make libc6-dev cmake libzstd-dev libssl-dev git ca-certificates curl cppcheck clang-format \ python3 python3-pip python3-venv openssl openssh-client \ lcov valgrind clang libclang-rt-18-dev && \ - pip3 install --break-system-packages pytest && \ + pip3 install --break-system-packages pytest pytest-xdist && \ curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \ apt-get install -y --no-install-recommends nodejs && \ rm -rf /var/lib/apt/lists/* diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..2f2ed93 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +; Fast integration subset run on every pull request (see .gitea/workflows/ci.yaml). +markers = + ci: fast, representative integration tests run on the PR CI gate \ No newline at end of file diff --git a/tests/integration/common.py b/tests/integration/common.py index 9fc93ad..c47e6dd 100644 --- a/tests/integration/common.py +++ b/tests/integration/common.py @@ -13,7 +13,12 @@ PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".. BUILD_DIR = os.path.join(PROJECT_ROOT, "build") SERVER_CMD = [os.path.join(BUILD_DIR, "server")] CLIENT_CMD = [os.path.join(BUILD_DIR, "client")] -TEST_DATA_DIR = os.path.join(PROJECT_ROOT, "test_data") +# Under pytest-xdist each worker process gets its own PYTEST_XDIST_WORKER id +# ('gw0', 'gw1', ...). Worker-key the transient working dir so concurrent +# workers on the shared filesystem never collide on fixtures. Outside xdist +# (or with -n1) this stays the historical 'test_data' path. +_WORKER = os.environ.get("PYTEST_XDIST_WORKER") +TEST_DATA_DIR = os.path.join(PROJECT_ROOT, f"test_data-{_WORKER}" if _WORKER else "test_data") class ServerManager: diff --git a/tests/integration/test_append.py b/tests/integration/test_append.py index be84a5a..c7897af 100644 --- a/tests/integration/test_append.py +++ b/tests/integration/test_append.py @@ -10,6 +10,8 @@ import os import random import shutil +import pytest + from common import ( TEST_DATA_DIR, run_client, CountingProxy, clean_dir, @@ -46,6 +48,7 @@ class TestAppend: def _dest_file(self, source, dest, rel): return os.path.join(get_dest_received_dir(dest, source), rel) + @pytest.mark.ci def test_append_resumes_short_dest_atomically(self, shared_server): """A shorter dest with a MATCHING prefix is resumed; the reconstructed file is byte-identical to the source.""" diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index d061799..00a07c1 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -161,6 +161,7 @@ class TestRemoveSourceFiles: class TestArchiveMode: + @pytest.mark.ci def test_archive_mode(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( @@ -191,6 +192,7 @@ class TestArchiveMode: class TestExecutability: + @pytest.mark.ci def test_preserves_only_executable_bits(self, shared_server): source = os.path.join(TEST_DATA_DIR, "executability_source") dest = os.path.join(TEST_DATA_DIR, "executability_dest") @@ -211,6 +213,7 @@ class TestExecutability: class TestChmod: + @pytest.mark.ci def test_chmod_applies_to_transferred_files(self, shared_server): clean_dir(DEST_DIR) source_file = os.path.join(SOURCE_DIR, "small.txt") @@ -227,6 +230,7 @@ class TestChmod: class TestCompressionChoice: + @pytest.mark.ci def test_zstd_choice_compresses(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--zc", "zstd"], @@ -249,6 +253,7 @@ class TestCompressionChoice: class TestSkipCompress: + @pytest.mark.ci def test_skip_compress_case_insensitive(self, shared_server): clean_dir(DEST_DIR) with open(os.path.join(SOURCE_DIR, "skip-case.TXT"), "wb") as f: @@ -393,6 +398,7 @@ class TestSizeFilters: class TestIncremental: + @pytest.mark.ci def test_incremental_skips_unchanged(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( @@ -417,6 +423,7 @@ class TestIncremental: assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" + @pytest.mark.ci def test_incremental_detects_changes(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client( @@ -692,6 +699,7 @@ class TestChecksumChoice: class TestUpdate: + @pytest.mark.ci def test_update_skips_older_destination_and_allows_equal_or_newer_source(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port) @@ -747,6 +755,7 @@ class TestUpdate: class TestExisting: + @pytest.mark.ci def test_existing_updates_existing_and_skips_new(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port) @@ -829,6 +838,7 @@ class TestIgnoreExisting: class TestDelete: + @pytest.mark.ci def test_delete_removes_extra_files(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client( @@ -2413,6 +2423,7 @@ class TestDeleteTiming: @pytest.mark.parametrize("flag", ["--delete-before", "--delete-during", "--del", "--delete-after", "--delete-delay"]) @pytest.mark.parametrize("mt", [False, True]) + @pytest.mark.ci def test_flag_removes_extras_on_success(self, flag, mt): """Every timing flag is accepted, implies --delete, and on a successful transfer removes the destination extras exactly like plain --delete.""" @@ -3045,6 +3056,7 @@ class TestBasisDestDirs: assert os.stat(dest_file).st_ino != os.stat(basis_file).st_ino, \ f"{flag}: linked/copied from a content-mismatched basis file" + @pytest.mark.ci def test_compare_dest_skips_matching_and_transfers_missing(self, shared_server): source = self._make_source("basis_compare_src", self._source_tree("c")) dest = os.path.join(TEST_DATA_DIR, "basis_compare_dst") @@ -3286,6 +3298,7 @@ class TestBasisDestDirs: assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(basis_file).st_ino, \ "--size-only should link a basis file whose mtime differs" + @pytest.mark.ci def test_link_dest_ignore_times_never_links(self, shared_server): # -I/--ignore-times forces every file to be updated, so a basis dir is # never used to hard-link (rsync parity). The file is transferred and diff --git a/tests/integration/test_tcp.py b/tests/integration/test_tcp.py index ab4f405..fdf1917 100644 --- a/tests/integration/test_tcp.py +++ b/tests/integration/test_tcp.py @@ -49,10 +49,12 @@ def _run_tcp_test(name, port, flags, use_metadata=True, posix=False): class TestTCPStandard: + @pytest.mark.ci def test_standard(self, shared_server): r = _run_tcp_test("Standard", shared_server.port, []) assert r["status"] == "Success", r["error"] + @pytest.mark.ci def test_posix_args(self, shared_server): r = _run_tcp_test("Posix Args", shared_server.port, [], posix=True) assert r["status"] == "Success", r["error"] @@ -63,10 +65,12 @@ class TestTCPStandard: class TestTCPFlags: + @pytest.mark.ci def test_multithreading(self, shared_server): r = _run_tcp_test("Multithreading (-m)", shared_server.port, ["-m"]) assert r["status"] == "Success", r["error"] + @pytest.mark.ci def test_compression(self, shared_server): r = _run_tcp_test("Compression (-c)", shared_server.port, ["-c"]) assert r["status"] == "Success", r["error"] From c90b07afcba6f476a70a6d964d52cac0601b4510 Mon Sep 17 00:00:00 2001 From: TapTap Date: Tue, 8 Sep 2026 17:25:56 +0200 Subject: [PATCH 2/2] =?UTF-8?q?ci:=20address=20review=20=E2=80=94=20use=20?= =?UTF-8?q?--dist=3Dload,=20per-module=20teardown,=20exclude=20setpriv?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace --dist=loadgroup (a no-op: it only groups by xdist_group marks) with --dist=load, and make module teardowns remove only their own SOURCE_DIR/DEST_DIR (never the shared worker-keyed TEST_DATA_DIR) so interleaved modules can't wipe each other's fixtures. Add a session-scoped cleanup of the per-worker dir. (loadfile grouped the whole test_features.py module onto one worker and slowed the full suite to 761s vs 224s with load.) - Mark the two setpriv privilege tests with a 'setpriv' marker and run the full merge suite as -m "not setpriv", so the dev/main gate can't go red on the env-dependent /root-traversal tests regardless of the runner uid. - Add a TLS basic test to the ci subset, add workflow_dispatch for on-demand full runs, and fix trailing newlines. - Measured: smoke -m ci = 28 passed/39s; full -n4 = 280 passed/11 skipped/1 xpassed in 224s (was 860s serial). --- .gitea/workflows/ci.yaml | 7 ++++--- AGENTS.md | 10 +++++----- pytest.ini | 5 ++++- tests/conftest.py | 18 ++++++++++++++++-- tests/integration/test_features.py | 8 +++++++- tests/integration/test_ssh.py | 3 ++- tests/integration/test_tcp.py | 3 ++- tests/integration/test_tls.py | 4 +++- 8 files changed, 43 insertions(+), 15 deletions(-) diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 12d2482..6fb81bb 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -4,6 +4,7 @@ on: push: branches: [main, dev] pull_request: + workflow_dispatch: jobs: lint: @@ -42,11 +43,11 @@ jobs: - name: Integration Tests (PR smoke subset) if: github.event_name == 'pull_request' - run: python3 -m pytest tests/integration/ -n 4 --dist=loadgroup -m ci --durations=25 --tb=short -q + run: python3 -m pytest tests/integration/ -n 4 --dist=load -m ci --durations=25 --tb=short -q - name: Integration Tests (full suite) if: github.event_name == 'push' - run: python3 -m pytest tests/integration/ -n 4 --dist=loadgroup --durations=25 --tb=short -q + run: python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv" --durations=25 --tb=short -q sanitizers: runs-on: ubuntu-latest @@ -133,4 +134,4 @@ jobs: - name: Valgrind Memcheck run: valgrind --leak-check=full --show-leak-kinds=definite --error-exitcode=1 ./build/tests env: - FASTSYNC_UNDER_VALGRIND: "1" \ No newline at end of file + FASTSYNC_UNDER_VALGRIND: "1" diff --git a/AGENTS.md b/AGENTS.md index e84937c..b2293ae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,12 +20,12 @@ docker build -t fastsync-ci:local . # Build, run unit tests, and run integration tests inside the container docker run --rm -v "$PWD:/workspace" -w /workspace fastsync-ci:local \ - sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=loadgroup' + sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=load' # Avoid root-owned build/ artifacts by matching your host UID/GID docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/workspace" \ -w /workspace fastsync-ci:local \ - sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=loadgroup' + sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=load' ``` > **Note:** The first `cmake configure` (`cmake -B build -S .`) fetches xxHash from GitHub via `FetchContent` — network access is required. Subsequent reconfigures reuse the cached source. @@ -41,7 +41,7 @@ cmake -B build -S . -DSANITIZER=address # AddressSanitizer (ASan) cmake -B build -S . -DSANITIZER=thread # ThreadSanitizer (TSan) ``` -The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), then a **fast PR gate** — build + unit + a representative subset of integration tests marked `@pytest.mark.ci`, parallelized with pytest-xdist (`-n 4`). The full coverage jobs (full integration suite, sanitizer, fuzz, coverage, valgrind) run **only on push to `dev`/`main`**; pull requests skip them to keep PR CI under ~3 minutes. +The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), then a **fast PR gate** — build + unit + a representative subset of integration tests marked `@pytest.mark.ci`, parallelized with pytest-xdist (`-n 4 --dist=load`). The full coverage jobs (full integration suite as `-m "not setpriv"`, sanitizer, fuzz, coverage, valgrind) run **only on push to `dev`/`main`**; pull requests skip them to keep PR CI under ~3 minutes. The two `setpriv` privilege tests are excluded from CI via a marker because their result depends on the runner/container uid and host mount permissions. ## Build @@ -53,8 +53,8 @@ cmake -B build -S . && cmake --build build -j$(nproc) ```bash ./build/tests # unit tests -python3 -m pytest tests/integration/ -n 4 --dist=loadgroup # full integration suite -python3 -m pytest tests/integration/ -n 4 --dist=loadgroup -m ci # PR-gate subset only +python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv" # full integration suite (CI excludes env-dependent privilege tests) +python3 -m pytest tests/integration/ -n 4 --dist=load -m ci # PR-gate subset only ``` ## CI Workflow — Waiting for Results diff --git a/pytest.ini b/pytest.ini index 2f2ed93..9047220 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,7 @@ [pytest] ; Fast integration subset run on every pull request (see .gitea/workflows/ci.yaml). markers = - ci: fast, representative integration tests run on the PR CI gate \ No newline at end of file + ci: fast, representative integration tests run on the PR CI gate + setpriv: privilege-dependent tests (drop to an unprivileged user); excluded + from CI because their result depends on the runner/container uid and the + host mount permissions, but run locally as root diff --git a/tests/conftest.py b/tests/conftest.py index 51d2ded..d4dd89d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,17 +1,31 @@ """Shared pytest configuration for integration tests.""" import os +import shutil import sys import pytest sys.path.insert(0, os.path.join(os.path.dirname(__file__), "integration")) -from common import ServerManager +from common import ServerManager, TEST_DATA_DIR @pytest.fixture(scope="session") def shared_server(): - """One server for the entire test session. Avoids 27+ server start/stop cycles.""" + """One server for the entire test session. Avoids 27+ server start/stop cycles. + + Under pytest-xdist this session fixture is instantiated once per worker + process, so each worker gets its own server on an ephemeral port.""" server = ServerManager() server.start() yield server server.stop() + + +@pytest.fixture(scope="session", autouse=True) +def _cleanup_worker_test_data(): + """Remove this (worker-keyed) TEST_DATA_DIR at the end of the session. + + Modules clean only their own rows; this final pass ensures the per-worker + directory never lingers in the working tree.""" + yield + shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index 00a07c1..4d8aedb 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -25,7 +25,11 @@ def setup_test_data(): generate_test_files(SOURCE_DIR, full=False) clean_dir(DEST_DIR) yield - shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) + # Remove only this module's own dirs. Under pytest-xdist the whole + # (worker-keyed) TEST_DATA_DIR is shared with concurrently-interleaved + # modules, so never rmtree it here. + shutil.rmtree(SOURCE_DIR, ignore_errors=True) + shutil.rmtree(DEST_DIR, ignore_errors=True) class TestDryRun: @@ -2855,6 +2859,7 @@ class TestDeletePolicy: "--clear-groups"] + cmd, text=True, capture_output=True) @pytest.mark.parametrize("mt", [False, True]) + @pytest.mark.setpriv def test_ignore_errors_keeps_deletion_active_on_scan_error(self, mt): """A source I/O error (unreadable subdirectory) aborts the run so no deletion happens by default; --ignore-errors continues, still transfers @@ -2901,6 +2906,7 @@ class TestDeletePolicy: @pytest.mark.parametrize("mt", [False, True]) @pytest.mark.parametrize("timing", ["--delete", "--delete-before"]) + @pytest.mark.setpriv def test_ignore_errors_unreadable_root_never_deletes(self, mt, timing): """An unreadable SOURCE ROOT must never be treated as a skippable scan error: with --ignore-errors the sequential scanner treats the root as diff --git a/tests/integration/test_ssh.py b/tests/integration/test_ssh.py index 7cf69b9..20b1ac6 100644 --- a/tests/integration/test_ssh.py +++ b/tests/integration/test_ssh.py @@ -60,7 +60,8 @@ def setup_test_data(): generate_test_files(SOURCE_DIR, full=False) clean_dir(DEST_DIR) yield - shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) + shutil.rmtree(SOURCE_DIR, ignore_errors=True) + shutil.rmtree(DEST_DIR, ignore_errors=True) def _run_ssh_test(name, flags, expected_missing=None): diff --git a/tests/integration/test_tcp.py b/tests/integration/test_tcp.py index fdf1917..68f5fa2 100644 --- a/tests/integration/test_tcp.py +++ b/tests/integration/test_tcp.py @@ -21,7 +21,8 @@ def setup_test_data(): generate_test_files(SOURCE_DIR, full=False) clean_dir(DEST_DIR) yield - shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) + shutil.rmtree(SOURCE_DIR, ignore_errors=True) + shutil.rmtree(DEST_DIR, ignore_errors=True) def _run_tcp_test(name, port, flags, use_metadata=True, posix=False): diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index 29074b2..bf82fc8 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -92,10 +92,12 @@ def setup_test_data(): generate_test_files(SOURCE_DIR, full=False) clean_dir(DEST_DIR) yield - shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) + shutil.rmtree(SOURCE_DIR, ignore_errors=True) + shutil.rmtree(DEST_DIR, ignore_errors=True) class TestTLSBasic: + @pytest.mark.ci def test_tls_server_client(self, certs): """Basic TLS: server with cert/key, client with cert/key + CA.""" clean_dir(DEST_DIR)