diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 00c64c0..12d2482 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -8,7 +8,7 @@ on: jobs: lint: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 steps: - name: Checkout uses: actions/checkout@v4 @@ -19,9 +19,13 @@ jobs: - name: cppcheck run: cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/ + # Fast PR gate: build + unit tests + a representative subset of integration + # tests (marked `ci`), parallelized with pytest-xdist. Only the full coverage + # jobs below (sanitizers/fuzz/coverage/valgrind and the FULL integration + # suite) run on merge to dev/main, so PR CI stays well under ~3 minutes. build-and-test: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint steps: - name: Checkout @@ -36,13 +40,19 @@ jobs: - name: Unit Tests run: ctest --test-dir build --output-on-failure -j$(nproc) - - name: Integration Tests - run: python3 -m pytest tests/integration/ -v --tb=short + - name: Integration Tests (PR smoke subset) + if: github.event_name == 'pull_request' + run: python3 -m pytest tests/integration/ -n 4 --dist=loadgroup -m ci --durations=25 --tb=short -q + + - name: Integration Tests (full suite) + if: github.event_name == 'push' + run: python3 -m pytest tests/integration/ -n 4 --dist=loadgroup --durations=25 --tb=short -q sanitizers: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint + if: github.event_name == 'push' strategy: matrix: sanitizer: [address, undefined] @@ -61,8 +71,9 @@ jobs: fuzz-build: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint + if: github.event_name == 'push' steps: - name: Checkout uses: actions/checkout@v4 @@ -82,8 +93,9 @@ jobs: coverage: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint + if: github.event_name == 'push' steps: - name: Checkout uses: actions/checkout@v4 @@ -105,8 +117,9 @@ jobs: valgrind: runs-on: ubuntu-latest - container: gitea.tap-tap.win/taptap/fastsync-ci:v9 + container: gitea.tap-tap.win/taptap/fastsync-ci:v10 needs: lint + if: github.event_name == 'push' steps: - name: Checkout uses: actions/checkout@v4 @@ -120,4 +133,4 @@ jobs: - name: Valgrind Memcheck run: valgrind --leak-check=full --show-leak-kinds=definite --error-exitcode=1 ./build/tests env: - FASTSYNC_UNDER_VALGRIND: "1" + FASTSYNC_UNDER_VALGRIND: "1" \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 9fc41ec..e84937c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,28 +4,28 @@ FastSync is a high-performance file synchronization system written in C11. It su ## Dependency installation -**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v7`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest, openssh-client, and Node.js. +**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v10`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest + pytest-xdist, openssh-client, and Node.js. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. The Docker image can also be used locally for CI parity. ```bash # Use the prebuilt CI image directly (faster, guaranteed CI parity) -docker pull gitea.tap-tap.win/taptap/fastsync-ci:v7 -docker tag gitea.tap-tap.win/taptap/fastsync-ci:v7 fastsync-ci:local +docker pull gitea.tap-tap.win/taptap/fastsync-ci:v10 +docker tag gitea.tap-tap.win/taptap/fastsync-ci:v10 fastsync-ci:local # Or build the image from the repo-root Dockerfile -# (Note: the prebuilt :v7 image reflects the previous Dockerfile state; +# (Note: the prebuilt :v10 image reflects the previous Dockerfile state; # rebuild from source to pick up any newly added packages like lcov/valgrind.) docker build -t fastsync-ci:local . # Build, run unit tests, and run integration tests inside the container docker run --rm -v "$PWD:/workspace" -w /workspace fastsync-ci:local \ - sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/' + sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=loadgroup' # Avoid root-owned build/ artifacts by matching your host UID/GID docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/workspace" \ -w /workspace fastsync-ci:local \ - sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/' + sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=loadgroup' ``` > **Note:** The first `cmake configure` (`cmake -B build -S .`) fetches xxHash from GitHub via `FetchContent` — network access is required. Subsequent reconfigures reuse the cached source. @@ -41,7 +41,7 @@ cmake -B build -S . -DSANITIZER=address # AddressSanitizer (ASan) cmake -B build -S . -DSANITIZER=thread # ThreadSanitizer (TSan) ``` -The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), build + test (unit + integration), and sanitizer (currently only `address`) jobs sequentially. +The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), then a **fast PR gate** — build + unit + a representative subset of integration tests marked `@pytest.mark.ci`, parallelized with pytest-xdist (`-n 4`). The full coverage jobs (full integration suite, sanitizer, fuzz, coverage, valgrind) run **only on push to `dev`/`main`**; pull requests skip them to keep PR CI under ~3 minutes. ## Build @@ -53,7 +53,8 @@ cmake -B build -S . && cmake --build build -j$(nproc) ```bash ./build/tests # unit tests -python3 -m pytest tests/ # integration tests +python3 -m pytest tests/integration/ -n 4 --dist=loadgroup # full integration suite +python3 -m pytest tests/integration/ -n 4 --dist=loadgroup -m ci # PR-gate subset only ``` ## CI Workflow — Waiting for Results @@ -65,14 +66,14 @@ When running the CI workflow via `tea` (the task execution agent), always set a ### If lint (clang-format) fails Run clang-format in the CI Docker image to match the exact CI version: ```bash -docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v9 \ +docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 \ sh -c 'find src/ tests/ -name "*.c" -o -name "*.h" | xargs clang-format -i' ``` ### If cppcheck fails Fix reported issues locally, then verify with: ```bash -docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v9 \ +docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 \ sh -c 'cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/' ``` diff --git a/Dockerfile b/Dockerfile index c94d26f..965384e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gcc g++ make libc6-dev cmake libzstd-dev libssl-dev git ca-certificates curl cppcheck clang-format \ python3 python3-pip python3-venv openssl openssh-client \ lcov valgrind clang libclang-rt-18-dev && \ - pip3 install --break-system-packages pytest && \ + pip3 install --break-system-packages pytest pytest-xdist && \ curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \ apt-get install -y --no-install-recommends nodejs && \ rm -rf /var/lib/apt/lists/* diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..2f2ed93 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +; Fast integration subset run on every pull request (see .gitea/workflows/ci.yaml). +markers = + ci: fast, representative integration tests run on the PR CI gate \ No newline at end of file diff --git a/tests/integration/common.py b/tests/integration/common.py index 9fc93ad..c47e6dd 100644 --- a/tests/integration/common.py +++ b/tests/integration/common.py @@ -13,7 +13,12 @@ PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".. BUILD_DIR = os.path.join(PROJECT_ROOT, "build") SERVER_CMD = [os.path.join(BUILD_DIR, "server")] CLIENT_CMD = [os.path.join(BUILD_DIR, "client")] -TEST_DATA_DIR = os.path.join(PROJECT_ROOT, "test_data") +# Under pytest-xdist each worker process gets its own PYTEST_XDIST_WORKER id +# ('gw0', 'gw1', ...). Worker-key the transient working dir so concurrent +# workers on the shared filesystem never collide on fixtures. Outside xdist +# (or with -n1) this stays the historical 'test_data' path. +_WORKER = os.environ.get("PYTEST_XDIST_WORKER") +TEST_DATA_DIR = os.path.join(PROJECT_ROOT, f"test_data-{_WORKER}" if _WORKER else "test_data") class ServerManager: diff --git a/tests/integration/test_append.py b/tests/integration/test_append.py index be84a5a..c7897af 100644 --- a/tests/integration/test_append.py +++ b/tests/integration/test_append.py @@ -10,6 +10,8 @@ import os import random import shutil +import pytest + from common import ( TEST_DATA_DIR, run_client, CountingProxy, clean_dir, @@ -46,6 +48,7 @@ class TestAppend: def _dest_file(self, source, dest, rel): return os.path.join(get_dest_received_dir(dest, source), rel) + @pytest.mark.ci def test_append_resumes_short_dest_atomically(self, shared_server): """A shorter dest with a MATCHING prefix is resumed; the reconstructed file is byte-identical to the source.""" diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index d061799..00a07c1 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -161,6 +161,7 @@ class TestRemoveSourceFiles: class TestArchiveMode: + @pytest.mark.ci def test_archive_mode(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( @@ -191,6 +192,7 @@ class TestArchiveMode: class TestExecutability: + @pytest.mark.ci def test_preserves_only_executable_bits(self, shared_server): source = os.path.join(TEST_DATA_DIR, "executability_source") dest = os.path.join(TEST_DATA_DIR, "executability_dest") @@ -211,6 +213,7 @@ class TestExecutability: class TestChmod: + @pytest.mark.ci def test_chmod_applies_to_transferred_files(self, shared_server): clean_dir(DEST_DIR) source_file = os.path.join(SOURCE_DIR, "small.txt") @@ -227,6 +230,7 @@ class TestChmod: class TestCompressionChoice: + @pytest.mark.ci def test_zstd_choice_compresses(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--zc", "zstd"], @@ -249,6 +253,7 @@ class TestCompressionChoice: class TestSkipCompress: + @pytest.mark.ci def test_skip_compress_case_insensitive(self, shared_server): clean_dir(DEST_DIR) with open(os.path.join(SOURCE_DIR, "skip-case.TXT"), "wb") as f: @@ -393,6 +398,7 @@ class TestSizeFilters: class TestIncremental: + @pytest.mark.ci def test_incremental_skips_unchanged(self, shared_server): clean_dir(DEST_DIR) result, dur = run_client( @@ -417,6 +423,7 @@ class TestIncremental: assert not missing, f"Missing: {missing}" assert not mismatches, f"Mismatch: {mismatches}" + @pytest.mark.ci def test_incremental_detects_changes(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client( @@ -692,6 +699,7 @@ class TestChecksumChoice: class TestUpdate: + @pytest.mark.ci def test_update_skips_older_destination_and_allows_equal_or_newer_source(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port) @@ -747,6 +755,7 @@ class TestUpdate: class TestExisting: + @pytest.mark.ci def test_existing_updates_existing_and_skips_new(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port) @@ -829,6 +838,7 @@ class TestIgnoreExisting: class TestDelete: + @pytest.mark.ci def test_delete_removes_extra_files(self, shared_server): clean_dir(DEST_DIR) result, _ = run_client( @@ -2413,6 +2423,7 @@ class TestDeleteTiming: @pytest.mark.parametrize("flag", ["--delete-before", "--delete-during", "--del", "--delete-after", "--delete-delay"]) @pytest.mark.parametrize("mt", [False, True]) + @pytest.mark.ci def test_flag_removes_extras_on_success(self, flag, mt): """Every timing flag is accepted, implies --delete, and on a successful transfer removes the destination extras exactly like plain --delete.""" @@ -3045,6 +3056,7 @@ class TestBasisDestDirs: assert os.stat(dest_file).st_ino != os.stat(basis_file).st_ino, \ f"{flag}: linked/copied from a content-mismatched basis file" + @pytest.mark.ci def test_compare_dest_skips_matching_and_transfers_missing(self, shared_server): source = self._make_source("basis_compare_src", self._source_tree("c")) dest = os.path.join(TEST_DATA_DIR, "basis_compare_dst") @@ -3286,6 +3298,7 @@ class TestBasisDestDirs: assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(basis_file).st_ino, \ "--size-only should link a basis file whose mtime differs" + @pytest.mark.ci def test_link_dest_ignore_times_never_links(self, shared_server): # -I/--ignore-times forces every file to be updated, so a basis dir is # never used to hard-link (rsync parity). The file is transferred and diff --git a/tests/integration/test_tcp.py b/tests/integration/test_tcp.py index ab4f405..fdf1917 100644 --- a/tests/integration/test_tcp.py +++ b/tests/integration/test_tcp.py @@ -49,10 +49,12 @@ def _run_tcp_test(name, port, flags, use_metadata=True, posix=False): class TestTCPStandard: + @pytest.mark.ci def test_standard(self, shared_server): r = _run_tcp_test("Standard", shared_server.port, []) assert r["status"] == "Success", r["error"] + @pytest.mark.ci def test_posix_args(self, shared_server): r = _run_tcp_test("Posix Args", shared_server.port, [], posix=True) assert r["status"] == "Success", r["error"] @@ -63,10 +65,12 @@ class TestTCPStandard: class TestTCPFlags: + @pytest.mark.ci def test_multithreading(self, shared_server): r = _run_tcp_test("Multithreading (-m)", shared_server.port, ["-m"]) assert r["status"] == "Success", r["error"] + @pytest.mark.ci def test_compression(self, shared_server): r = _run_tcp_test("Compression (-c)", shared_server.port, ["-c"]) assert r["status"] == "Success", r["error"]