TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Merge branch 'parity2/f2' into parity/no-wire

Browse Source
This commit is contained in:
TapTap
2026-09-24 01:25:41 +02:00
parent 8843f1e3cf c387beb182
commit 90f2fb613f
21 changed files with 1758 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/integration/common.py
+6 -2
View File
@@ -34,7 +34,7 @@ class ServerManager:
self._proc = None
self._port = None
def start(self, extra_args=None):
def start(self, extra_args=None, env=None):
self.stop()
self._port = _find_free_port()
# Plain TCP is intentionally explicit in the server; integration tests
@@ -42,7 +42,11 @@ class ServerManager:
cmd = SERVER_CMD + ["-p", str(self._port), "--allow-unauthenticated"]
if extra_args:
cmd += extra_args
self._proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
proc_env = dict(os.environ)
if env:
proc_env.update(env)
self._proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
env=proc_env)
_wait_for_port(self._port, timeout=5)
def stop(self):
tests/integration/test_large_stream.py
+222
View File
@@ -0,0 +1,222 @@
"""#318: whole-file streaming above the receiver's 256 MiB ceiling.
The receiver's historical whole-file bound (``MAX_RECEIVE_WHOLE_FILE_SIZE``,
256 MiB) refused any single-file payload above it. The transfer engine now
streams such a payload (and the basis read/verify/hash) through a bounded buffer
and spools it to a temp file, so arbitrarily large single files transfer without
being materialized in memory.
To exercise the streaming path deterministically and quickly, these tests lower
the receiver bound with the test-only ``FASTSYNC_MAX_WHOLE_FILE_SIZE`` hook (it
can only lower, never raise, the protocol ceiling) and transfer a file a few
times larger than the lowered bound. A real >256 MiB transfer is covered once,
unmarked, so it runs in the full suite but not the fast PR gate.
"""
import hashlib
import os
import random
import shutil
import sys
import pytest
sys.path.insert(0, os.path.dirname(__file__))
from common import ( # noqa: E402
ServerManager,
TEST_DATA_DIR,
clean_dir,
get_dest_received_dir,
run_client,
)
LOW_BOUND = 1024 * 1024
FILE_SIZE = 3 * 1024 * 1024
OLD_MTIME = 1_500_000_000
@pytest.fixture(scope="module")
def small_bound_server():
"""A server whose whole-file streaming bound is 1 MiB."""
server = ServerManager()
server.start(extra_args=["--allow-super"],
env={"FASTSYNC_MAX_WHOLE_FILE_SIZE": str(LOW_BOUND)})
yield server
server.stop()
def _payload(n):
rng = random.Random(0xC0FFEE)
return rng.randbytes(n)
def _write(path, data, mtime=None):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "wb") as fh:
fh.write(data)
if mtime is not None:
os.utime(path, (mtime, mtime))
def _resolved(dest, source, rel):
return os.path.join(get_dest_received_dir(dest, source), rel)
def _no_spool_leftovers(dest):
leftovers = []
for root, _dirs, files in os.walk(dest):
leftovers += [os.path.join(root, f) for f in files if ".fastsync-spool." in f]
return leftovers
class TestStreamedWholeFile:
"""A file above the (lowered) bound transfers correctly in every mode."""
@pytest.mark.parametrize(
"flags",
[
["-a"],
["-a", "--incremental"],
["-a", "-z"],
["-a", "--incremental", "-z"],
["-a", "--threads", "--incremental"],
["-a", "--inplace"],
["-a", "--partial"],
],
)
def test_above_bound_transfers(self, small_bound_server, flags):
tag = "_".join(f.strip("-") for f in flags) or "default"
source = os.path.join(TEST_DATA_DIR, f"stream_src_{tag}")
dest = os.path.join(TEST_DATA_DIR, f"stream_dst_{tag}")
clean_dir(source)
clean_dir(dest)
data = _payload(FILE_SIZE)
_write(os.path.join(source, "big.bin"), data, OLD_MTIME)
if "--inplace" in flags:
# --inplace only matters when the destination already exists.
_write(_resolved(dest, source, "big.bin"), b"stale", OLD_MTIME)
result, _ = run_client(source, dest, flags=flags, port=small_bound_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
got = os.path.join(get_dest_received_dir(dest, source), "big.bin")
assert os.path.exists(got), "streamed file was not written"
with open(got, "rb") as fh:
assert fh.read() == data, "streamed file content mismatch"
assert _no_spool_leftovers(dest) == [], "a spool temp file leaked"
class TestStreamedBasis:
"""A basis above the bound is streamed, not refused (compare/copy/link)."""
def _seed(self, dest, source, data):
clean_dir(source)
clean_dir(dest)
_write(os.path.join(source, "big.bin"), data, OLD_MTIME)
# FastSync resolves a relative basis DIR against the destination and
# appends the transfer-relative name.
_write(os.path.join(dest, "basis", "big.bin"), data, OLD_MTIME)
def test_compare_dest_above_bound(self, small_bound_server):
source = os.path.join(TEST_DATA_DIR, "sbasis_cmp_src")
dest = os.path.join(TEST_DATA_DIR, "sbasis_cmp_dst")
data = _payload(FILE_SIZE)
self._seed(dest, source, data)
result, _ = run_client(source, dest,
flags=["-a", "--compare-dest=basis", "--incremental"],
port=small_bound_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
# compare-dest never copies: an already-present destination stays sparse.
assert not os.path.exists(_resolved(dest, source, "big.bin"))
def test_copy_dest_above_bound(self, small_bound_server):
source = os.path.join(TEST_DATA_DIR, "sbasis_cpy_src")
dest = os.path.join(TEST_DATA_DIR, "sbasis_cpy_dst")
data = _payload(FILE_SIZE)
self._seed(dest, source, data)
result, _ = run_client(source, dest,
flags=["-a", "--copy-dest=basis", "--incremental"],
port=small_bound_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
got = _resolved(dest, source, "big.bin")
assert os.path.exists(got)
with open(got, "rb") as fh:
assert fh.read() == data
assert os.stat(got).st_ino != os.stat(os.path.join(dest, "basis", "big.bin")).st_ino
assert _no_spool_leftovers(dest) == []
def test_link_dest_above_bound(self, small_bound_server):
source = os.path.join(TEST_DATA_DIR, "sbasis_lnk_src")
dest = os.path.join(TEST_DATA_DIR, "sbasis_lnk_dst")
data = _payload(FILE_SIZE)
self._seed(dest, source, data)
result, _ = run_client(source, dest,
flags=["-a", "--link-dest=basis", "--incremental"],
port=small_bound_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
got = _resolved(dest, source, "big.bin")
assert os.path.exists(got)
with open(got, "rb") as fh:
assert fh.read() == data
assert os.stat(got).st_ino == os.stat(os.path.join(dest, "basis", "big.bin")).st_ino
class TestFuzzyAboveBound:
"""-y/--fuzzy reuses a basis above the bound by streaming its signature."""
def test_fuzzy_oversized_sibling(self, small_bound_server):
source = os.path.join(TEST_DATA_DIR, "sfuzzy_src")
dest = os.path.join(TEST_DATA_DIR, "sfuzzy_dst")
clean_dir(source)
clean_dir(dest)
base = _payload(FILE_SIZE)
sibling = bytearray(base)
sibling[FILE_SIZE // 2:FILE_SIZE // 2 + 4096] = bytes(
(b + 1) % 256 for b in sibling[FILE_SIZE // 2:FILE_SIZE // 2 + 4096])
_write(os.path.join(source, "report_v2.txt"), base)
_write(os.path.join(get_dest_received_dir(dest, source), "report_v1.txt"), bytes(sibling))
result, _ = run_client(
source, dest,
flags=["-a", "--incremental", "--delta", "--fuzzy", "--stats"],
port=small_bound_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
got = _resolved(dest, source, "report_v2.txt")
with open(got, "rb") as fh:
assert fh.read() == base, "fuzzy reconstruction mismatch"
assert _no_spool_leftovers(dest) == []
class TestRealLargeFile:
"""A real >256 MiB transfer, run only in the full (non-PR-gate) suite."""
def test_real_300mib_transfer(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "real_large_src")
dest = os.path.join(TEST_DATA_DIR, "real_large_dst")
clean_dir(source)
clean_dir(dest)
n = 300 * 1024 * 1024
# Deterministic, compressible pattern written in bounded chunks.
chunk = bytes(range(256)) * 4096
digest = hashlib.sha256()
with open(os.path.join(source, "big.bin"), "wb") as fh:
written = 0
while written < n:
piece = chunk[: min(len(chunk), n - written)]
fh.write(piece)
digest.update(piece)
written += len(piece)
result, _ = run_client(source, dest, flags=["-a", "--incremental"],
port=shared_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
got = os.path.join(get_dest_received_dir(dest, source), "big.bin")
assert os.path.getsize(got) == n
got_digest = hashlib.sha256()
with open(got, "rb") as fh:
while True:
block = fh.read(1 << 20)
if not block:
break
got_digest.update(block)
assert got_digest.hexdigest() == digest.hexdigest()
shutil.rmtree(source, ignore_errors=True)
shutil.rmtree(dest, ignore_errors=True)
tests/test_file.c
+155
View File
@@ -5,6 +5,8 @@
#include "file.h"
#include "file_receive.h"
#include "data.h"
#include "compression.h"
#include "delta.h"
#include "config.h"
#include "charset.h"
#include "utils.h"
@@ -2465,7 +2467,160 @@ static void test_manifest_would_delete_protects_absolute_basis() {
free(extra);
}
/* #318: a whole-file payload above the streaming bound must be written to a
* spool temp file in bounded chunks, not materialized in memory. Exercises the
* raw and zstd-compressed paths and asserts the exact bytes land in the spool. */
static void test_file_receive_payload_streams(void) {
const char* dir = "test_file_stream_tmp";
char dest_path[512];
snprintf(dest_path, sizeof(dest_path), "%s/out.bin", dir);
mkdir(dir, 0777);
const unsigned long long stream_limit = 4096;
size_t size = 20000;
unsigned char* payload = malloc(size);
EXPECT_NOT_NULL(payload);
for (size_t i = 0; i < size; i++)
payload[i] = (unsigned char)((i * 7 + 3) & 0xff);
/* Raw (uncompressed) streamed payload. */
{
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
unsigned long long hdr = size;
EXPECT_TRUE(send_n_data(p[1], &hdr, sizeof(hdr)));
EXPECT_TRUE(send_n_data(p[1], payload, size));
Data* buffer = NULL;
char* spool = NULL;
unsigned long long out_size = 0;
EXPECT_TRUE(file_receive_payload(p[0], false, size, dest_path, stream_limit, &buffer, &spool,
&out_size));
EXPECT_NULL(buffer);
EXPECT_NOT_NULL(spool);
EXPECT_TRUE(out_size == size);
if (spool) {
FILE* fh = fopen(spool, "rb");
EXPECT_NOT_NULL(fh);
if (fh) {
unsigned char* got = malloc(size);
EXPECT_TRUE(fread(got, 1, size, fh) == size);
EXPECT_EQ_INT(memcmp(got, payload, size), 0);
free(got);
fclose(fh);
}
unlink(spool);
free(spool);
}
close(p[0]);
close(p[1]);
}
/* zstd-compressed payload whose logical size exceeds the bound: the frame is
* decompressed incrementally straight into the spool. */
{
unsigned char* copy = malloc(size);
EXPECT_NOT_NULL(copy);
memcpy(copy, payload, size);
Data* raw = data_create(copy, size); /* data_create takes ownership of copy */
Data* compressed = data_compress_codec(raw, COMPRESSION_ALGO_ZSTD, 3, 0);
data_destroy(raw);
EXPECT_NOT_NULL(compressed);
if (compressed) {
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
EXPECT_TRUE(send_data(p[1], compressed));
Data* buffer = NULL;
char* spool = NULL;
unsigned long long out_size = 0;
EXPECT_TRUE(file_receive_payload(p[0], true, size, dest_path, stream_limit, &buffer, &spool,
&out_size));
EXPECT_NULL(buffer);
EXPECT_NOT_NULL(spool);
EXPECT_TRUE(out_size == size);
if (spool) {
FILE* fh = fopen(spool, "rb");
EXPECT_NOT_NULL(fh);
if (fh) {
unsigned char* got = malloc(size);
EXPECT_TRUE(fread(got, 1, size, fh) == size);
EXPECT_EQ_INT(memcmp(got, payload, size), 0);
free(got);
fclose(fh);
}
unlink(spool);
free(spool);
}
close(p[0]);
close(p[1]);
data_destroy(compressed);
}
}
free(payload);
unlink(dest_path);
rmdir(dir);
}
/* #318: the fd-based delta helpers must match the in-memory ones and stream the
* reconstruction to a descriptor without allocating the whole output. */
static void test_delta_stream_helpers(void) {
const unsigned char basis[] = {0x11, 0x22, 0x33, 0x44};
const char* basis_path = "test_file_delta_basis.bin";
int bfd = open(basis_path, O_RDWR | O_CREAT | O_TRUNC, 0600);
EXPECT_TRUE(bfd >= 0);
EXPECT_TRUE(write(bfd, basis, sizeof(basis)) == (ssize_t)sizeof(basis));
EXPECT_TRUE(lseek(bfd, 0, SEEK_SET) == 0);
DeltaSignature* fd_sig = delta_signature_create_fd_seeded(bfd, sizeof(basis), 4, 0);
DeltaSignature* mem_sig = delta_signature_create_seeded(basis, sizeof(basis), 4, 0);
EXPECT_NOT_NULL(fd_sig);
EXPECT_NOT_NULL(mem_sig);
if (fd_sig && mem_sig) {
EXPECT_TRUE(fd_sig->block_count == mem_sig->block_count);
for (uint32_t i = 0; i < fd_sig->block_count && i < mem_sig->block_count; i++) {
EXPECT_TRUE(fd_sig->blocks[i].adler32 == mem_sig->blocks[i].adler32);
EXPECT_TRUE(fd_sig->blocks[i].xxhash == mem_sig->blocks[i].xxhash);
}
}
delta_signature_destroy(fd_sig);
delta_signature_destroy(mem_sig);
DeltaInstruction instrs[2];
instrs[0].type = DELTA_INSTR_BLOCK_MATCH;
instrs[0].match.block_index = 0;
instrs[0].match.block_offset = 0;
instrs[0].match.length = 4;
instrs[1].type = DELTA_INSTR_LITERAL;
instrs[1].literal.data = (uint8_t*)"XY";
instrs[1].literal.length = 2;
Delta delta;
delta.new_file_size = 6;
delta.instruction_count = 2;
delta.instructions = instrs;
delta.delta_size = 0;
int out[2];
EXPECT_EQ_INT(pipe(out), 0);
EXPECT_TRUE(delta_apply_to_fd(NULL, bfd, sizeof(basis), &delta, 4, out[1]));
close(out[1]);
unsigned char got[6] = {0};
size_t total = 0;
while (total < sizeof(got)) {
ssize_t n = read(out[0], got + total, sizeof(got) - total);
if (n <= 0)
break;
total += (size_t)n;
}
EXPECT_TRUE(total == sizeof(got));
EXPECT_TRUE(memcmp(got, "\x11\x22\x33\x44XY", 6) == 0);
close(out[0]);
close(bfd);
unlink(basis_path);
}
void test_file() {
test_file_receive_payload_streams();
test_delta_stream_helpers();
test_file_create();
test_file_special_rdev_valid();
test_file_destroy_null();