"""Fault injection: the server must survive truncated / corrupted protocol frames and abrupt mid-frame disconnects, and keep serving later connections. These tests deliberately speak raw bytes to a real server process: * malformed frames before/inside the config handshake (oversized length headers, truncated string bodies, outright garbage), * a captured *valid* config frame replayed so the connection reaches the operation loop, followed by a partial ``STATUS_MANIFEST`` frame that is cut mid-body and dropped, and * a real client run relayed through a proxy that truncates the stream at a range of byte offsets and resets both ends. After every fault the server process is asserted alive and a subsequent ordinary transfer must complete and verify, proving the accept loop and per-connection children recovered cleanly. All interactions are bounded by short socket timeouts (no sleeps). """ import os import select import shutil import socket import struct import subprocess import sys import threading import pytest sys.path.insert(0, os.path.dirname(__file__)) from common import ( # noqa: E402 ServerManager, TEST_DATA_DIR, get_dest_received_dir, run_client, verify_transfer, ) PROTOCOL_VERSION = b"2.28.0" STATUS_MANIFEST = 5 STATUS_OK = 0 SOURCE_DIR = os.path.join(TEST_DATA_DIR, "fault_src") DEST_DIR = os.path.join(TEST_DATA_DIR, "fault_dst") @pytest.fixture(scope="module") def fault_server(): """A dedicated server so the aliveness assertions observe exactly the process these faults were sent to.""" server = ServerManager() server.start() yield server server.stop() @pytest.fixture(scope="module", autouse=True) def _seed_source(): if os.path.exists(SOURCE_DIR): shutil.rmtree(SOURCE_DIR) os.makedirs(os.path.join(SOURCE_DIR, "nested")) # The receiver rejects a destination root that does not exist, and the # capture fixture can run before any test that creates it, so create it # here (test order/distribution must not matter). os.makedirs(DEST_DIR, exist_ok=True) with open(os.path.join(SOURCE_DIR, "hello.txt"), "wb") as fh: fh.write(b"fault injection payload\n" * 64) with open(os.path.join(SOURCE_DIR, "nested", "deep.bin"), "wb") as fh: fh.write(bytes(range(256)) * 16) yield shutil.rmtree(SOURCE_DIR, ignore_errors=True) shutil.rmtree(DEST_DIR, ignore_errors=True) def _assert_alive(server): assert server._proc is not None, "server process missing" assert server._proc.poll() is None, ( f"server exited with {server._proc.returncode} after fault injection" ) def _recover(server, label): """Run one ordinary transfer and verify it end-to-end.""" shutil.rmtree(DEST_DIR, ignore_errors=True) os.makedirs(DEST_DIR) result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--preserve"], port=server.port) assert result.returncode == 0, ( f"{label}: recovery transfer failed rc={result.returncode}: " f"{(result.stderr or result.stdout)[:200]}" ) received = get_dest_received_dir(DEST_DIR, SOURCE_DIR) mismatches, missing = verify_transfer(SOURCE_DIR, received) assert not missing, f"{label}: recovery missing {missing}" assert not mismatches, f"{label}: recovery mismatch {mismatches}" def _abrupt_close(sock): """Force an RST instead of a graceful FIN, the nastier mid-frame drop.""" try: sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) except OSError: pass try: sock.close() except OSError: pass def _raw_connect(server): sock = socket.create_connection(("127.0.0.1", server.port), timeout=5) sock.settimeout(5) return sock def _recv_exact(sock, n): buf = b"" while len(buf) < n: chunk = sock.recv(n - len(buf)) if not chunk: return None buf += chunk return buf # --- faults before/inside the config handshake ----------------------------- CONFIG_HANDSHAKE_FAULTS = { "empty": b"", # Length header claims a 1 EiB string body that never arrives. "oversized_length": struct.pack("server connection and record the client's config frame (all client bytes forwarded before the server's first reply).""" def __init__(self, target_port): self.target = ("127.0.0.1", target_port) self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.listener.bind(("127.0.0.1", 0)) self.listener.listen(1) self.listener.settimeout(20) self.port = self.listener.getsockname()[1] self.config_frame = None def run(self, cmd): def serve(): try: client, _ = self.listener.accept() except OSError: return try: backend = socket.create_connection(self.target, timeout=10) except OSError: client.close() return client.settimeout(20) backend.settimeout(20) buf_c = bytearray() seen_server = False try: while True: ready, _, _ = select.select([client, backend], [], [], 20) if not ready: break done = False for sock in ready: data = sock.recv(65536) if not data: done = True continue if sock is client: buf_c += data backend.sendall(data) else: if not seen_server: seen_server = True self.config_frame = bytes(buf_c) client.sendall(data) if done: break except OSError: pass finally: client.close() backend.close() thread = threading.Thread(target=serve) thread.start() result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) thread.join(20) return result def close(self): try: self.listener.close() except OSError: pass @pytest.fixture(scope="module") def captured_config(fault_server): """Capture the config frame of one real client run through a relay.""" proxy = _CaptureProxy(fault_server.port) cmd = [ os.path.join(os.path.dirname(__file__), "..", "..", "build", "client"), "--source-dir", SOURCE_DIR, "--dest-dir", DEST_DIR, "--save-to-disk", "--server-port", str(proxy.port), ] try: result = proxy.run(cmd) assert result.returncode == 0, ( f"capture run failed rc={result.returncode}: " f"{(result.stderr or result.stdout)[:200]}" ) assert proxy.config_frame, "failed to capture the client config frame" yield proxy.config_frame finally: proxy.close() class TestTruncatedStatusFrame: def test_partial_manifest_frame_then_drop(self, fault_server, captured_config): sock = _raw_connect(fault_server) sock.sendall(captured_config) ack = _recv_exact(sock, 4) assert ack is not None, "server closed before the config ack" (status,) = struct.unpack("= self.max_client_bytes: break except (OSError, socket.timeout): pass for sock in (client, backend): try: sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) except OSError: pass try: sock.close() except OSError: pass thread = threading.Thread(target=serve) thread.start() try: subprocess.run(cmd, capture_output=True, text=True, timeout=30) finally: thread.join(20) self.listener.close() class TestAbruptMidTransferDisconnect: def test_client_stream_cut_at_offsets(self, fault_server, captured_config): """Cut the real client stream at offsets anchored to the config frame's actual size: mid-config, right after the config, and into the operation stream -- each followed by an RST of both ends.""" config_len = len(captured_config) cuts = sorted({max(1, config_len // 2), max(1, config_len - 1), config_len + 8, config_len + 256}) for cut in cuts: proxy = _TruncatingProxy(fault_server.port, cut) cmd = [ os.path.join(os.path.dirname(__file__), "..", "..", "build", "client"), "--source-dir", SOURCE_DIR, "--dest-dir", DEST_DIR, "--save-to-disk", "--server-port", str(proxy.port), ] # The client is expected to fail; what matters is the server survives. proxy.run(cmd) _assert_alive(fault_server) _recover(fault_server, "abrupt mid-transfer disconnects")