TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Merge remote-tracking branch 'origin/feat/max-alloc' into dev

Browse Source
This commit is contained in:
TapTap
2026-09-05 02:13:35 +02:00
parent 3304541337 638d953b1d
commit 90112a7585
26 changed files with 554 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_client_cli.c
+45 -1
View File
@@ -435,6 +435,48 @@ static void test_parse_args_rejects_invalid_modify_window() {
}
}
static void test_parse_args_max_alloc_sizes() {
const char* values[] = {"1", "4K", "2m", "3G", "1T", "1P", "1E", "512B"};
const unsigned long long expected[] = {1,
4ULL * 1024,
2ULL * 1024 * 1024,
3ULL * 1024 * 1024 * 1024,
1ULL * 1024 * 1024 * 1024 * 1024,
1ULL * 1024 * 1024 * 1024 * 1024 * 1024,
1ULL * 1024 * 1024 * 1024 * 1024 * 1024 * 1024,
512};
for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--max-alloc", (char*)values[i], "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->max_alloc == expected[i]);
config_delete(cfg);
}
Config* cfg = config_create();
char* argv[] = {"fastsync", "--max-alloc=8M", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->max_alloc == 8ULL * 1024 * 1024);
config_delete(cfg);
}
static void test_parse_args_rejects_invalid_max_alloc() {
const char* values[] = {"0", "-1", "+1", " 1", "1 ",
"1Z", "1K2", "1 K", "1\tK", "18446744073709551615K"};
for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--max-alloc", (char*)values[i], "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
}
static void test_parse_args_skip_compress() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--skip-compress=.ZIP, .GZ", "/src", "/dst"};
@@ -867,7 +909,7 @@ void test_client_cli() {
test_parse_args_invalid_server_port();
test_parse_args_invalid_compression_level();
test_parse_args_valid_compression_level();
test_parse_args_debug_flags();
test_parse_args_debug_flags();
test_parse_args_debug_help();
test_parse_args_debug_flags_validation();
test_parse_args_modify_window();
@@ -875,6 +917,8 @@ void test_client_cli() {
test_parse_args_skip_compress();
test_parse_args_empty_skip_compress();
test_parse_args_compression_threads();
test_parse_args_max_alloc_sizes();
test_parse_args_rejects_invalid_max_alloc();
test_parse_args_unknown_option();
test_parse_args_rejects_unimplemented_options();
test_parse_args_quiet();
tests/test_config.c
+9 -5
View File
@@ -95,6 +95,7 @@ static void test_pipeline_sender_lifecycle() {
EXPECT_EQ_INT(pcs->queue_loader->capacity, 15);
EXPECT_FALSE(pcs->scanner_done);
EXPECT_FALSE(pcs->loader_done);
EXPECT_EQ_INT((int)pcs->allocation_session.max_alloc, (int)cfg->max_alloc);
pipeline_context_sender_destroy(pcs);
}
@@ -131,7 +132,7 @@ static void test_config_send_receive() {
send_cfg->size_only = true;
send_cfg->compression_level = 5;
send_cfg->chunk_size = 1024;
send_cfg->eight_bit_output = true;
send_cfg->eight_bit_output = true;
send_cfg->modify_window = 4;
send_cfg->existing = true;
send_cfg->ignore_existing = true;
@@ -139,6 +140,7 @@ static void test_config_send_receive() {
send_cfg->skip_compress_count = 1;
send_cfg->skip_compress_suffixes = calloc(1, sizeof(char*));
send_cfg->skip_compress_suffixes[0] = str_dup(".zip");
send_cfg->max_alloc = MAX_SERVER_ALLOC + 1;
/* Use socketpair for bidirectional communication */
int p[2];
@@ -173,7 +175,7 @@ static void test_config_send_receive() {
ok = false;
if (recv_cfg->chunk_size != 1024)
ok = false;
if (!recv_cfg->use_executability)
if (!recv_cfg->use_executability)
ok = false;
if (!recv_cfg->size_only)
ok = false;
@@ -183,8 +185,6 @@ static void test_config_send_receive() {
ok = false;
if (recv_cfg->use_delta)
ok = false;
if (!recv_cfg->whole_file)
ok = false;
if (recv_cfg->modify_window != 4)
ok = false;
if (!recv_cfg->existing)
@@ -194,6 +194,8 @@ static void test_config_send_receive() {
if (!recv_cfg->skip_compress_set || recv_cfg->skip_compress_count != 1 ||
strcmp(recv_cfg->skip_compress_suffixes[0], ".zip") != 0)
ok = false;
if (recv_cfg->max_alloc != MAX_SERVER_ALLOC)
ok = false;
}
config_delete(recv_cfg);
close(p[0]);
@@ -223,7 +225,7 @@ static void test_config_send_receive_version_mismatch() {
Config* cfg = config_create();
EXPECT_NOT_NULL(cfg);
free(cfg->version);
cfg->version = str_dup("2.2.0");
cfg->version = str_dup("2.3.0");
cfg->send_directory = str_dup("/src");
cfg->receive_root_directory = str_dup("/dst");
@@ -267,6 +269,8 @@ static void test_config_receive_truncated() {
/* A valid prefix exercises cleanup after allocated wire strings and a
* partially received scalar field. */
EXPECT_TRUE(send_str(p[1], PROTOCOL_VERSION));
unsigned long long max_alloc = DEFAULT_MAX_ALLOC;
EXPECT_TRUE(send_n_data(p[1], &max_alloc, sizeof(max_alloc)));
EXPECT_TRUE(send_str(p[1], "/src"));
EXPECT_TRUE(send_str(p[1], "/dst"));
EXPECT_TRUE(send_int(p[1], 1));
tests/test_protocol.c
+233
View File
@@ -3,6 +3,60 @@
#include <limits.h>
#include <string.h>
#include <unistd.h>
#include <threads.h>
typedef struct {
ProtocolSession* session;
bool allocation_allowed;
} AllocationWorkerArg;
static int allocation_worker(void* arg) {
AllocationWorkerArg* worker = arg;
protocol_session_bind(worker->session);
void* allocation = protocol_alloc(8);
worker->allocation_allowed = allocation != NULL;
free(allocation);
protocol_session_unbind();
return thrd_success;
}
typedef struct {
ProtocolSession* session;
int read_fd;
bool released;
} AccountingWorkerArg;
typedef struct {
ProtocolSession* session;
atomic_int* ready;
atomic_bool* release;
bool received;
} ConcurrentAccountingWorkerArg;
static int accounting_worker(void* arg) {
AccountingWorkerArg* worker = arg;
protocol_session_bind(worker->session);
Data* data = protocol_receive_data_limited(worker->session, 8);
if (data) {
data_destroy(data);
worker->released = atomic_load(&worker->session->total_allocated_bytes) == 0;
}
protocol_session_unbind();
return data ? thrd_success : thrd_error;
}
static int concurrent_accounting_worker(void* arg) {
ConcurrentAccountingWorkerArg* worker = arg;
protocol_session_bind(worker->session);
Data* data = protocol_receive_data_limited(worker->session, 8);
worker->received = data != NULL;
atomic_fetch_add(worker->ready, 1);
while (!atomic_load(worker->release))
thrd_yield();
data_destroy(data);
protocol_session_unbind();
return thrd_success;
}
static void test_send_receive_n_data() {
int p[2];
@@ -187,6 +241,177 @@ static void test_receive_str_truncated() {
close(p[0]);
}
static void test_max_alloc_rejects_single_buffer() {
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
ProtocolSession session;
protocol_session_init(&session, p[0], p[1]);
protocol_session_set_max_alloc(&session, 4);
protocol_session_bind(&session);
char payload[8] = {0};
EXPECT_TRUE(write(p[1], &(size_t){sizeof(payload)}, sizeof(size_t)) == sizeof(size_t));
EXPECT_NULL(protocol_receive_str(&session));
protocol_session_unbind();
close(p[0]);
close(p[1]);
}
static void test_explicit_session_max_alloc_cannot_be_bypassed() {
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
ProtocolSession explicit_session;
ProtocolSession unrelated_session;
protocol_session_init(&explicit_session, p[0], p[1]);
protocol_session_init(&unrelated_session, p[0], p[1]);
protocol_session_set_max_alloc(&explicit_session, 4);
protocol_session_set_max_alloc(&unrelated_session, 64);
protocol_session_bind(&unrelated_session);
unsigned long long size = 8;
EXPECT_EQ_INT((int)write(p[1], &size, sizeof(size)), (int)sizeof(size));
EXPECT_EQ_INT((int)write(p[1], "12345678", 8), 8);
EXPECT_NULL(protocol_receive_data_limited(&explicit_session, 8));
EXPECT_EQ_INT((int)atomic_load(&explicit_session.total_allocated_bytes), 0);
protocol_session_unbind();
close(p[0]);
close(p[1]);
}
static void test_max_alloc_allows_configured_buffer() {
ProtocolSession session;
protocol_session_init(&session, -1, -1);
protocol_session_set_max_alloc(&session, 4);
protocol_session_bind(&session);
void* allowed = protocol_alloc(4);
const void* rejected = protocol_alloc(5);
EXPECT_NOT_NULL(allowed);
EXPECT_NULL(rejected);
free(allowed);
protocol_session_unbind();
}
static void test_max_alloc_is_bound_in_worker_threads() {
enum { WORKER_COUNT = 4 };
ProtocolSession sessions[WORKER_COUNT];
AllocationWorkerArg args[WORKER_COUNT] = {0};
thrd_t threads[WORKER_COUNT];
for (int i = 0; i < WORKER_COUNT; i++) {
protocol_session_init(&sessions[i], -1, -1);
protocol_session_set_max_alloc(&sessions[i], 4);
args[i].session = &sessions[i];
EXPECT_EQ_INT(thrd_create(&threads[i], allocation_worker, &args[i]), thrd_success);
}
for (int i = 0; i < WORKER_COUNT; i++) {
int result;
EXPECT_EQ_INT(thrd_join(threads[i], &result), thrd_success);
EXPECT_EQ_INT(result, thrd_success);
EXPECT_FALSE(args[i].allocation_allowed);
}
}
static void test_protocol_accounting_is_released_in_worker_threads() {
enum { WORKER_COUNT = 4 };
ProtocolSession sessions[WORKER_COUNT];
AccountingWorkerArg args[WORKER_COUNT] = {0};
thrd_t threads[WORKER_COUNT];
for (int i = 0; i < WORKER_COUNT; i++) {
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
protocol_session_init(&sessions[i], p[0], p[1]);
protocol_session_set_max_alloc(&sessions[i], 64);
unsigned long long size = 8;
EXPECT_EQ_INT((int)write(p[1], &size, sizeof(size)), (int)sizeof(size));
EXPECT_EQ_INT((int)write(p[1], "12345678", 8), 8);
close(p[1]);
args[i].session = &sessions[i];
args[i].read_fd = p[0];
EXPECT_EQ_INT(thrd_create(&threads[i], accounting_worker, &args[i]), thrd_success);
}
for (int i = 0; i < WORKER_COUNT; i++) {
int result;
EXPECT_EQ_INT(thrd_join(threads[i], &result), thrd_success);
EXPECT_EQ_INT(result, thrd_success);
EXPECT_TRUE(args[i].released);
EXPECT_EQ_INT((int)atomic_load(&sessions[i].total_allocated_bytes), 0);
close(args[i].read_fd);
}
}
static void test_protocol_accounting_reservation_is_atomic() {
enum { WORKER_COUNT = 8 };
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
ProtocolSession session;
protocol_session_init(&session, p[0], p[1]);
protocol_session_set_max_alloc(&session, 64);
const unsigned long long budget_before = MAX_SERVER_ALLOC - 8;
atomic_store(&session.total_allocated_bytes, budget_before);
for (int i = 0; i < WORKER_COUNT; i++) {
unsigned long long size = 8;
EXPECT_EQ_INT((int)write(p[1], &size, sizeof(size)), (int)sizeof(size));
EXPECT_EQ_INT((int)write(p[1], "12345678", 8), 8);
}
close(p[1]);
atomic_int ready;
atomic_bool release;
atomic_init(&ready, 0);
atomic_init(&release, false);
ConcurrentAccountingWorkerArg args[WORKER_COUNT] = {0};
thrd_t threads[WORKER_COUNT];
for (int i = 0; i < WORKER_COUNT; i++) {
args[i].session = &session;
args[i].ready = &ready;
args[i].release = &release;
EXPECT_EQ_INT(thrd_create(&threads[i], concurrent_accounting_worker, &args[i]), thrd_success);
}
while (atomic_load(&ready) != WORKER_COUNT)
thrd_yield();
bool budget_ok = atomic_load(&session.total_allocated_bytes) == budget_before + 8;
atomic_store(&release, true);
int received = 0;
for (int i = 0; i < WORKER_COUNT; i++) {
int result;
EXPECT_EQ_INT(thrd_join(threads[i], &result), thrd_success);
EXPECT_EQ_INT(result, thrd_success);
received += args[i].received ? 1 : 0;
}
EXPECT_EQ_INT(received, 1);
EXPECT_TRUE(budget_ok);
EXPECT_EQ_INT((int)atomic_load(&session.total_allocated_bytes), (int)budget_before);
close(p[0]);
}
static void test_protocol_string_accounting_is_transient() {
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
ProtocolSession session;
protocol_session_init(&session, p[0], p[1]);
protocol_session_set_max_alloc(&session, 64);
EXPECT_TRUE(protocol_send_str(&session, "temporary"));
char* received = protocol_receive_str(&session);
EXPECT_NOT_NULL(received);
EXPECT_EQ_STR(received, "temporary");
EXPECT_EQ_INT((int)atomic_load(&session.total_allocated_bytes), 0);
free(received);
close(p[0]);
close(p[1]);
}
static void test_protocol_accounting_release_does_not_underflow() {
ProtocolSession session;
protocol_session_init(&session, -1, -1);
atomic_store(&session.total_allocated_bytes, 4);
protocol_session_bind(&session);
protocol_release_memory(8);
EXPECT_EQ_INT((int)atomic_load(&session.total_allocated_bytes), 0);
protocol_release_memory(1);
EXPECT_EQ_INT((int)atomic_load(&session.total_allocated_bytes), 0);
protocol_session_unbind();
}
void test_protocol() {
test_send_receive_n_data();
test_send_receive_n_data_zero();
@@ -198,4 +423,12 @@ void test_protocol() {
test_send_receive_status();
test_receive_n_data_truncated();
test_receive_str_truncated();
test_max_alloc_rejects_single_buffer();
test_explicit_session_max_alloc_cannot_be_bypassed();
test_max_alloc_allows_configured_buffer();
test_max_alloc_is_bound_in_worker_threads();
test_protocol_accounting_is_released_in_worker_threads();
test_protocol_accounting_reservation_is_atomic();
test_protocol_string_accounting_is_transient();
test_protocol_accounting_release_does_not_underflow();
}
tests/test_robustness.c
+15
View File
@@ -109,6 +109,20 @@ static void test_delta_deserialize_garbage() {
data_destroy(d);
}
static void test_delta_deserialize_respects_max_alloc() {
unsigned char serialized[sizeof(uint64_t) + sizeof(uint32_t)] = {0};
Data data = {.data = serialized, .size = sizeof(serialized)};
ProtocolSession session;
protocol_session_init(&session, -1, -1);
protocol_session_set_max_alloc(&session, sizeof(Delta) - 1);
protocol_session_bind(&session);
const Delta* result = delta_deserialize(&data);
EXPECT_NULL(result);
protocol_session_unbind();
}
static void test_delta_signature_deserialize_truncated() {
char old_data[4096];
for (int i = 0; i < 4096; i++)
@@ -206,6 +220,7 @@ void test_robustness() {
test_delta_deserialize_truncated();
test_delta_deserialize_empty();
test_delta_deserialize_garbage();
test_delta_deserialize_respects_max_alloc();
test_delta_deserialize_truncated_instructions();
test_delta_signature_deserialize_truncated();
test_delta_apply_null();
tests/test_scanner.c
+1 -1
View File
@@ -396,7 +396,7 @@ static void test_parallel_scanner_root_chunks_without_workers() {
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0,
0, 0, false, false, false, false, false};
ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options);
ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options, NULL);
EXPECT_NOT_NULL(scanner);
int total_files = 0;