diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 00c64c0..6fb81bb 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -4,11 +4,12 @@ on: push: branches: [main, dev] pull_request: + workflow_dispatch: jobs: lint: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 steps: - name: Checkout uses: actions/checkout@v4 @@ -19,9 +20,13 @@ jobs: - name: cppcheck run: cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/ + # Fast PR gate: build + unit tests + a representative subset of integration + # tests (marked `ci`), parallelized with pytest-xdist. Only the full coverage + # jobs below (sanitizers/fuzz/coverage/valgrind and the FULL integration + # suite) run on merge to dev/main, so PR CI stays well under ~3 minutes. build-and-test: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint steps: - name: Checkout @@ -36,13 +41,19 @@ jobs: - name: Unit Tests run: ctest --test-dir build --output-on-failure -j$(nproc) - - name: Integration Tests - run: python3 -m pytest tests/integration/ -v --tb=short + - name: Integration Tests (PR smoke subset) + if: github.event_name == 'pull_request' + run: python3 -m pytest tests/integration/ -n 4 --dist=load -m ci --durations=25 --tb=short -q + + - name: Integration Tests (full suite) + if: github.event_name == 'push' + run: python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv" --durations=25 --tb=short -q sanitizers: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint + if: github.event_name == 'push' strategy: matrix: sanitizer: [address, undefined] @@ -61,8 +72,9 @@ jobs: fuzz-build: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint + if: github.event_name == 'push' steps: - name: Checkout uses: actions/checkout@v4 @@ -82,8 +94,9 @@ jobs: coverage: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint + if: github.event_name == 'push' steps: - name: Checkout uses: actions/checkout@v4 @@ -105,8 +118,9 @@ jobs: valgrind: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint + if: github.event_name == 'push' steps: - name: Checkout uses: actions/checkout@v4 diff --git a/AGENTS.md b/AGENTS.md index 9fc41ec..b2293ae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,28 +4,28 @@ FastSync is a high-performance file synchronization system written in C11. It su ## Dependency installation -**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v7`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest, openssh-client, and Node.js. +**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v10`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest + pytest-xdist, openssh-client, and Node.js. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. The Docker image can also be used locally for CI parity. ```bash # Use the prebuilt CI image directly (faster, guaranteed CI parity) -docker pull gitea.tap-tap.win/taptap/fastsync-ci:v7 -docker tag gitea.tap-tap.win/taptap/fastsync-ci:v7 fastsync-ci:local +docker pull gitea.tap-tap.win/taptap/fastsync-ci:v10 +docker tag gitea.tap-tap.win/taptap/fastsync-ci:v10 fastsync-ci:local # Or build the image from the repo-root Dockerfile -# (Note: the prebuilt :v7 image reflects the previous Dockerfile state; +# (Note: the prebuilt :v10 image reflects the previous Dockerfile state; # rebuild from source to pick up any newly added packages like lcov/valgrind.) docker build -t fastsync-ci:local . # Build, run unit tests, and run integration tests inside the container docker run --rm -v "$PWD:/workspace" -w /workspace fastsync-ci:local \ - sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/' + sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=load' # Avoid root-owned build/ artifacts by matching your host UID/GID docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/workspace" \ -w /workspace fastsync-ci:local \ - sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/' + sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=load' ``` > **Note:** The first `cmake configure` (`cmake -B build -S .`) fetches xxHash from GitHub via `FetchContent` — network access is required. Subsequent reconfigures reuse the cached source. @@ -41,7 +41,7 @@ cmake -B build -S . -DSANITIZER=address # AddressSanitizer (ASan) cmake -B build -S . -DSANITIZER=thread # ThreadSanitizer (TSan) ``` -The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), build + test (unit + integration), and sanitizer (currently only `address`) jobs sequentially. +The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), then a **fast PR gate** — build + unit + a representative subset of integration tests marked `@pytest.mark.ci`, parallelized with pytest-xdist (`-n 4 --dist=load`). The full coverage jobs (full integration suite as `-m "not setpriv"`, sanitizer, fuzz, coverage, valgrind) run **only on push to `dev`/`main`**; pull requests skip them to keep PR CI under ~3 minutes. The two `setpriv` privilege tests are excluded from CI via a marker because their result depends on the runner/container uid and host mount permissions. ## Build @@ -53,7 +53,8 @@ cmake -B build -S . && cmake --build build -j$(nproc) ```bash ./build/tests # unit tests -python3 -m pytest tests/ # integration tests +python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv" # full integration suite (CI excludes env-dependent privilege tests) +python3 -m pytest tests/integration/ -n 4 --dist=load -m ci # PR-gate subset only ``` ## CI Workflow — Waiting for Results @@ -65,14 +66,14 @@ When running the CI workflow via `tea` (the task execution agent), always set a ### If lint (clang-format) fails Run clang-format in the CI Docker image to match the exact CI version: ```bash -docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v9 \ +docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 \ sh -c 'find src/ tests/ -name "*.c" -o -name "*.h" | xargs clang-format -i' ``` ### If cppcheck fails Fix reported issues locally, then verify with: ```bash -docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v9 \ +docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 \ sh -c 'cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/' ``` diff --git a/Dockerfile b/Dockerfile index c94d26f..965384e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gcc g++ make libc6-dev cmake libzstd-dev libssl-dev git ca-certificates curl cppcheck clang-format \ python3 python3-pip python3-venv openssl openssh-client \ lcov valgrind clang libclang-rt-18-dev && \ - pip3 install --break-system-packages pytest && \ + pip3 install --break-system-packages pytest pytest-xdist && \ curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \ apt-get install -y --no-install-recommends nodejs && \ rm -rf /var/lib/apt/lists/* diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..9047220 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,7 @@ +[pytest] +; Fast integration subset run on every pull request (see .gitea/workflows/ci.yaml). +markers = + ci: fast, representative integration tests run on the PR CI gate + setpriv: privilege-dependent tests (drop to an unprivileged user); excluded + from CI because their result depends on the runner/container uid and the + host mount permissions, but run locally as root diff --git a/tests/conftest.py b/tests/conftest.py index 51d2ded..d4dd89d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,17 +1,31 @@ """Shared pytest configuration for integration tests.""" import os +import shutil import sys import pytest sys.path.insert(0, os.path.join(os.path.dirname(__file__), "integration")) -from common import ServerManager +from common import ServerManager, TEST_DATA_DIR @pytest.fixture(scope="session") def shared_server(): - """One server for the entire test session. Avoids 27+ server start/stop cycles.""" + """One server for the entire test session. Avoids 27+ server start/stop cycles. + + Under pytest-xdist this session fixture is instantiated once per worker + process, so each worker gets its own server on an ephemeral port.""" server = ServerManager() server.start() yield server server.stop() + + +@pytest.fixture(scope="session", autouse=True) +def _cleanup_worker_test_data(): + """Remove this (worker-keyed) TEST_DATA_DIR at the end of the session. + + Modules clean only their own rows; this final pass ensures the per-worker + directory never lingers in the working tree.""" + yield + shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) diff --git a/tests/integration/common.py b/tests/integration/common.py index 9fc93ad..c47e6dd 100644 --- a/tests/integration/common.py +++ b/tests/integration/common.py @@ -13,7 +13,12 @@ PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".. BUILD_DIR = os.path.join(PROJECT_ROOT, "build") SERVER_CMD = [os.path.join(BUILD_DIR, "server")] CLIENT_CMD = [os.path.join(BUILD_DIR, "client")] -TEST_DATA_DIR = os.path.join(PROJECT_ROOT, "test_data") +# Under pytest-xdist each worker process gets its own PYTEST_XDIST_WORKER id +# ('gw0', 'gw1', ...). Worker-key the transient working dir so concurrent +# workers on the shared filesystem never collide on fixtures. Outside xdist +# (or with -n1) this stays the historical 'test_data' path. +_WORKER = os.environ.get("PYTEST_XDIST_WORKER") +TEST_DATA_DIR = os.path.join(PROJECT_ROOT, f"test_data-{_WORKER}" if _WORKER else "test_data") class ServerManager: diff --git a/tests/integration/test_append.py b/tests/integration/test_append.py index be84a5a..c7897af 100644 --- a/tests/integration/test_append.py +++ b/tests/integration/test_append.py @@ -10,6 +10,8 @@ import os import random import shutil +import pytest + from common import ( TEST_DATA_DIR, run_client, CountingProxy, clean_dir, @@ -46,6 +48,7 @@ class TestAppend: def _dest_file(self, source, dest, rel): return os.path.join(get_dest_received_dir(dest, source), rel) + @pytest.mark.ci def test_append_resumes_short_dest_atomically(self, shared_server): """A shorter dest with a MATCHING prefix is resumed; the reconstructed file is byte-identical to the source.""" diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index d061799..4d8aedb 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -25,7 +25,11 @@ def setup_test_data(): generate_test_files(SOURCE_DIR, full=False) clean_dir(DEST_DIR) yield - shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) + # Remove only this module's own dirs. Under pytest-xdist the whole + # (worker-keyed) TEST_DATA_DIR is shared with concurrently-interleaved + # modules, so never rmtree it here. + shutil.rmtree(SOURCE_DIR, ignore_errors=True) + shutil.rmtree(DEST_DIR, ignore_errors=True) class TestDryRun: @@ -161,6 +165,7 @@ class TestRemoveSourceFiles: class TestArchiveMode: + @pytest.mark.ci def test_archive_mode(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( @@ -191,6 +196,7 @@ class TestArchiveMode: class TestExecutability: + @pytest.mark.ci def test_preserves_only_executable_bits(self, shared_server): source = os.path.join(TEST_DATA_DIR, "executability_source") dest = os.path.join(TEST_DATA_DIR, "executability_dest") @@ -211,6 +217,7 @@ class TestExecutability: class TestChmod: + @pytest.mark.ci def test_chmod_applies_to_transferred_files(self, shared_server): clean_dir(DEST_DIR) source_file = os.path.join(SOURCE_DIR, "small.txt") @@ -227,6 +234,7 @@ class TestChmod: class TestCompressionChoice: + @pytest.mark.ci def test_zstd_choice_compresses(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--zc", "zstd"], @@ -249,6 +257,7 @@ class TestCompressionChoice: class TestSkipCompress: + @pytest.mark.ci def test_skip_compress_case_insensitive(self, shared_server): clean_dir(DEST_DIR) with open(os.path.join(SOURCE_DIR, "skip-case.TXT"), "wb") as f: @@ -393,6 +402,7 @@ class TestSizeFilters: class TestIncremental: + @pytest.mark.ci def test_incremental_skips_unchanged(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( @@ -417,6 +427,7 @@ class TestIncremental: assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" + @pytest.mark.ci def test_incremental_detects_changes(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client( @@ -692,6 +703,7 @@ class TestChecksumChoice: class TestUpdate: + @pytest.mark.ci def test_update_skips_older_destination_and_allows_equal_or_newer_source(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port) @@ -747,6 +759,7 @@ class TestUpdate: class TestExisting: + @pytest.mark.ci def test_existing_updates_existing_and_skips_new(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port) @@ -829,6 +842,7 @@ class TestIgnoreExisting: class TestDelete: + @pytest.mark.ci def test_delete_removes_extra_files(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client( @@ -2413,6 +2427,7 @@ class TestDeleteTiming: @pytest.mark.parametrize("flag", ["--delete-before", "--delete-during", "--del", "--delete-after", "--delete-delay"]) @pytest.mark.parametrize("mt", [False, True]) + @pytest.mark.ci def test_flag_removes_extras_on_success(self, flag, mt): """Every timing flag is accepted, implies --delete, and on a successful transfer removes the destination extras exactly like plain --delete.""" @@ -2844,6 +2859,7 @@ class TestDeletePolicy: "--clear-groups"] + cmd, text=True, capture_output=True) @pytest.mark.parametrize("mt", [False, True]) + @pytest.mark.setpriv def test_ignore_errors_keeps_deletion_active_on_scan_error(self, mt): """A source I/O error (unreadable subdirectory) aborts the run so no deletion happens by default; --ignore-errors continues, still transfers @@ -2890,6 +2906,7 @@ class TestDeletePolicy: @pytest.mark.parametrize("mt", [False, True]) @pytest.mark.parametrize("timing", ["--delete", "--delete-before"]) + @pytest.mark.setpriv def test_ignore_errors_unreadable_root_never_deletes(self, mt, timing): """An unreadable SOURCE ROOT must never be treated as a skippable scan error: with --ignore-errors the sequential scanner treats the root as @@ -3045,6 +3062,7 @@ class TestBasisDestDirs: assert os.stat(dest_file).st_ino != os.stat(basis_file).st_ino, \ f"{flag}: linked/copied from a content-mismatched basis file" + @pytest.mark.ci def test_compare_dest_skips_matching_and_transfers_missing(self, shared_server): source = self._make_source("basis_compare_src", self._source_tree("c")) dest = os.path.join(TEST_DATA_DIR, "basis_compare_dst") @@ -3286,6 +3304,7 @@ class TestBasisDestDirs: assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(basis_file).st_ino, \ "--size-only should link a basis file whose mtime differs" + @pytest.mark.ci def test_link_dest_ignore_times_never_links(self, shared_server): # -I/--ignore-times forces every file to be updated, so a basis dir is # never used to hard-link (rsync parity). The file is transferred and diff --git a/tests/integration/test_ssh.py b/tests/integration/test_ssh.py index 7cf69b9..20b1ac6 100644 --- a/tests/integration/test_ssh.py +++ b/tests/integration/test_ssh.py @@ -60,7 +60,8 @@ def setup_test_data(): generate_test_files(SOURCE_DIR, full=False) clean_dir(DEST_DIR) yield - shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) + shutil.rmtree(SOURCE_DIR, ignore_errors=True) + shutil.rmtree(DEST_DIR, ignore_errors=True) def _run_ssh_test(name, flags, expected_missing=None): diff --git a/tests/integration/test_tcp.py b/tests/integration/test_tcp.py index ab4f405..68f5fa2 100644 --- a/tests/integration/test_tcp.py +++ b/tests/integration/test_tcp.py @@ -21,7 +21,8 @@ def setup_test_data(): generate_test_files(SOURCE_DIR, full=False) clean_dir(DEST_DIR) yield - shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) + shutil.rmtree(SOURCE_DIR, ignore_errors=True) + shutil.rmtree(DEST_DIR, ignore_errors=True) def _run_tcp_test(name, port, flags, use_metadata=True, posix=False): @@ -49,10 +50,12 @@ def _run_tcp_test(name, port, flags, use_metadata=True, posix=False): class TestTCPStandard: + @pytest.mark.ci def test_standard(self, shared_server): r = _run_tcp_test("Standard", shared_server.port, []) assert r["status"] == "Success", r["error"] + @pytest.mark.ci def test_posix_args(self, shared_server): r = _run_tcp_test("Posix Args", shared_server.port, [], posix=True) assert r["status"] == "Success", r["error"] @@ -63,10 +66,12 @@ class TestTCPStandard: class TestTCPFlags: + @pytest.mark.ci def test_multithreading(self, shared_server): r = _run_tcp_test("Multithreading (-m)", shared_server.port, ["-m"]) assert r["status"] == "Success", r["error"] + @pytest.mark.ci def test_compression(self, shared_server): r = _run_tcp_test("Compression (-c)", shared_server.port, ["-c"]) assert r["status"] == "Success", r["error"] diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index 29074b2..bf82fc8 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -92,10 +92,12 @@ def setup_test_data(): generate_test_files(SOURCE_DIR, full=False) clean_dir(DEST_DIR) yield - shutil.rmtree(TEST_DATA_DIR, ignore_errors=True) + shutil.rmtree(SOURCE_DIR, ignore_errors=True) + shutil.rmtree(DEST_DIR, ignore_errors=True) class TestTLSBasic: + @pytest.mark.ci def test_tls_server_client(self, certs): """Basic TLS: server with cert/key, client with cert/key + CA.""" clean_dir(DEST_DIR)