TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

perf(send,scanner): byte-bound sender queues; drop redundant stat

Browse Source
This commit is contained in:
TapTap
2026-09-13 04:06:30 +02:00
parent ddc71a7df5
commit 69fe7f3c9f
6 changed files with 389 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_multiprocessing.c
+119
View File
@@ -332,6 +332,123 @@ static void test_receiver_enqueue_byte_budget() {
config_delete(cfg);
}
/* Build a one-file chunk that appears to hold `bytes` of loaded payload by
handing it a real buffer of that size (the byte accounting counts only
in-memory `data->data`, mirroring sendfile's streamed chunks). */
static Chunk* make_loaded_chunk(const char* name, size_t bytes) {
File* file = file_create(name);
if (!file)
return NULL;
void* buffer = malloc(bytes > 0 ? bytes : 1);
if (!buffer) {
file_destroy(file);
return NULL;
}
file->data->data = buffer;
file->data->size = bytes;
File* items[1] = {file};
Chunk* chunk = chunk_create(items, 1);
if (!chunk)
file_destroy(file);
return chunk;
}
/* Only loaded (in-memory) payload is charged: a chunk whose files have no
buffer (e.g. sendfile streams the bytes from disk) accounts for zero. */
static void test_sender_chunk_bytes_accounting() {
EXPECT_EQ_INT((int)pipeline_context_sender_chunk_bytes(NULL), 0);
File* streamed = file_create("sender_account_streamed");
EXPECT_NOT_NULL(streamed);
streamed->data->size = 4096; /* declared size, but no in-memory buffer */
File* streamed_items[1] = {streamed};
Chunk* streamed_chunk = chunk_create(streamed_items, 1);
EXPECT_NOT_NULL(streamed_chunk);
EXPECT_EQ_INT((int)pipeline_context_sender_chunk_bytes(streamed_chunk), 0);
chunk_destroy(streamed_chunk);
Chunk* loaded = make_loaded_chunk("sender_account_loaded", 2000);
EXPECT_NOT_NULL(loaded);
EXPECT_EQ_INT((int)pipeline_context_sender_chunk_bytes(loaded), 2000);
chunk_destroy(loaded);
}
typedef struct {
PipelineContextSender* context;
Chunk* chunk;
atomic_bool* done;
atomic_bool* result;
} SenderByteBudgetArg;
static int sender_byte_budget_worker(void* arg) {
SenderByteBudgetArg* worker = arg;
bool ok = pipeline_context_sender_enqueue_chunk(worker->context, worker->chunk);
atomic_store(worker->result, ok);
atomic_store(worker->done, true);
return thrd_success;
}
/* The sender's loader stage must not buffer more loaded payload bytes ahead of
the network writer than the configured byte budget: an enqueue that would
exceed the budget blocks until the sender releases bytes. */
static void test_sender_enqueue_byte_budget() {
Config* cfg = config_create();
EXPECT_NOT_NULL(cfg);
free(cfg->version);
cfg->version = str_dup(PROTOCOL_VERSION);
cfg->send_directory = str_dup("/src");
cfg->receive_root_directory = str_dup("/dst");
Queue* q_scanner = queue_create(16, chunk_destroy);
Queue* q_loader = queue_create(16, chunk_destroy);
EXPECT_NOT_NULL(q_scanner);
EXPECT_NOT_NULL(q_loader);
PipelineContextSender* ctx = pipeline_context_sender_create(cfg, q_scanner, q_loader);
EXPECT_NOT_NULL(ctx);
pipeline_context_sender_set_queue_byte_limit(ctx, 3000);
Chunk* first = make_loaded_chunk("sender_budget_1", 2000);
EXPECT_NOT_NULL(first);
EXPECT_TRUE(pipeline_context_sender_enqueue_chunk(ctx, first));
EXPECT_EQ_INT((int)ctx->queued_bytes, 2000);
/* A second 2000-byte chunk would push the pipeline to 4000 > 3000 budget, so
its enqueue must block until the first chunk's bytes are released. */
Chunk* second = make_loaded_chunk("sender_budget_2", 2000);
EXPECT_NOT_NULL(second);
atomic_bool done;
atomic_bool result;
atomic_init(&done, false);
atomic_init(&result, false);
SenderByteBudgetArg arg = {ctx, second, &done, &result};
thrd_t enqueuer;
EXPECT_EQ_INT(thrd_create(&enqueuer, sender_byte_budget_worker, &arg), thrd_success);
/* Give a broken (unbounded) implementation every chance to enqueue. */
struct timespec wait = {0, 200 * 1000000L};
thrd_sleep(&wait, NULL);
EXPECT_FALSE(atomic_load(&done));
EXPECT_EQ_INT((int)ctx->queued_bytes, 2000); /* budget still honored */
/* Simulate the sender: dequeue + destroy the first chunk, then release its
bytes. Only the post-join state (below) is deterministic. */
Chunk* drained =
queue_dequeue_multithreaded(q_loader, &ctx->mutex_loader, &ctx->condition_not_empty_loader,
&ctx->condition_not_full_loader, &ctx->loader_done);
EXPECT_NOT_NULL(drained);
chunk_destroy(drained);
pipeline_context_sender_note_bytes_released(ctx, 2000);
EXPECT_EQ_INT(thrd_join(enqueuer, NULL), thrd_success);
EXPECT_TRUE(atomic_load(&done));
EXPECT_TRUE(atomic_load(&result));
EXPECT_EQ_INT((int)ctx->queued_bytes, 2000); /* second payload now in flight */
/* pipeline_context_sender_destroy frees the still-queued second chunk and
owns cfg/q_scanner/q_loader from here on. */
pipeline_context_sender_destroy(ctx);
}
void test_multiprocessing() {
test_sender_create_destroy();
test_receiver_create_destroy();
@@ -344,4 +461,6 @@ void test_multiprocessing() {
}
test_write_thread_done();
test_receiver_enqueue_byte_budget();
test_sender_chunk_bytes_accounting();
test_sender_enqueue_byte_budget();
}
tests/test_scanner.c
+134
View File
@@ -1369,6 +1369,139 @@ static void test_scanner_chunk_ownership() {
rmdir(dir);
}
/* The scanner derives entry type from a single lstat() for non-symlinks
* (regular files and directories) and only calls stat() to dereference real
* symlinks. Guard the regular-file/directory/symlink distinction across the
* default (symlinks skipped), --copy-links (dereferenced) and -l (carried)
* modes so the lstat/stat reuse cannot misclassify entries. */
static void test_scanner_entry_classification() {
const char* root = "test_scan_classify";
const char* sub = "test_scan_classify/sub";
const char* file = "test_scan_classify/file.txt";
const char* nested = "test_scan_classify/sub/nested.txt";
const char* link_file = "test_scan_classify/link_file";
const char* link_dir = "test_scan_classify/link_dir";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
create_test_file(file, "hello"); /* 5 bytes */
create_test_file(nested, "nested"); /* 6 bytes */
EXPECT_EQ_INT(symlink("file.txt", link_file), 0);
EXPECT_EQ_INT(symlink("sub", link_dir), 0);
/* Default: no link option -> symlinks are skipped entirely; regular files and
directories (descended, not emitted) are classified as before. */
{
ScannerOptions options = {0};
DirectoryScanner* scanner = directory_scanner_create_with_options(root, &options);
EXPECT_NOT_NULL(scanner);
size_t root_len = strlen(root);
bool file_ok = false, nested_ok = false, link_seen = false;
Chunk* chunk;
while ((chunk = directory_scanner_next(scanner)) != NULL) {
for (int i = 0; i < chunk->element_count; i++) {
const File* f = chunk->items[i];
const char* rel = f->path + root_len;
if (*rel == '/')
rel++;
if (strcmp(rel, "file.txt") == 0) {
file_ok = !f->is_dir && !f->is_symlink && f->data->size == 5;
} else if (strcmp(rel, "sub/nested.txt") == 0) {
nested_ok = !f->is_dir && !f->is_symlink && f->data->size == 6;
} else {
link_seen = true;
}
}
chunk_destroy(chunk);
}
EXPECT_FALSE(directory_scanner_failed(scanner));
EXPECT_TRUE(file_ok);
EXPECT_TRUE(nested_ok);
EXPECT_FALSE(link_seen);
directory_scanner_destroy(scanner);
}
/* --copy-links: symlinks are dereferenced. A link to a file becomes a
regular file with the referent's size; a link to a directory is traversed. */
{
ScannerOptions options = {0};
options.copy_links = true;
DirectoryScanner* scanner = directory_scanner_create_with_options(root, &options);
EXPECT_NOT_NULL(scanner);
size_t root_len = strlen(root);
bool file_ok = false, nested_ok = false, link_file_ok = false;
bool link_dir_nested_ok = false, symlink_leaked = false;
Chunk* chunk;
while ((chunk = directory_scanner_next(scanner)) != NULL) {
for (int i = 0; i < chunk->element_count; i++) {
const File* f = chunk->items[i];
const char* rel = f->path + root_len;
if (*rel == '/')
rel++;
if (f->is_symlink)
symlink_leaked = true;
if (strcmp(rel, "file.txt") == 0)
file_ok = !f->is_dir && f->data->size == 5;
else if (strcmp(rel, "sub/nested.txt") == 0)
nested_ok = !f->is_dir && f->data->size == 6;
else if (strcmp(rel, "link_file") == 0)
link_file_ok = !f->is_dir && f->data->size == 5;
else if (strcmp(rel, "link_dir/nested.txt") == 0)
link_dir_nested_ok = !f->is_dir && f->data->size == 6;
}
chunk_destroy(chunk);
}
EXPECT_FALSE(directory_scanner_failed(scanner));
EXPECT_TRUE(file_ok);
EXPECT_TRUE(nested_ok);
EXPECT_TRUE(link_file_ok);
EXPECT_TRUE(link_dir_nested_ok);
EXPECT_FALSE(symlink_leaked);
directory_scanner_destroy(scanner);
}
/* -l (--links): symlinks are carried through as symlinks, not dereferenced. */
{
ScannerOptions options = {0};
options.follow_symlinks = true;
DirectoryScanner* scanner = directory_scanner_create_with_options(root, &options);
EXPECT_NOT_NULL(scanner);
size_t root_len = strlen(root);
bool file_ok = false, link_file_ok = false, link_dir_ok = false, leaked_dir = false;
Chunk* chunk;
while ((chunk = directory_scanner_next(scanner)) != NULL) {
for (int i = 0; i < chunk->element_count; i++) {
const File* f = chunk->items[i];
const char* rel = f->path + root_len;
if (*rel == '/')
rel++;
if (strcmp(rel, "file.txt") == 0)
file_ok = !f->is_dir && !f->is_symlink && f->data->size == 5;
else if (strcmp(rel, "link_file") == 0)
link_file_ok = f->is_symlink && !f->is_dir;
else if (strcmp(rel, "link_dir") == 0)
link_dir_ok = f->is_symlink && !f->is_dir;
else if (strcmp(rel, "link_dir/nested.txt") == 0)
leaked_dir = true;
}
chunk_destroy(chunk);
}
EXPECT_FALSE(directory_scanner_failed(scanner));
EXPECT_TRUE(file_ok);
EXPECT_TRUE(link_file_ok);
EXPECT_TRUE(link_dir_ok);
EXPECT_FALSE(leaked_dir);
directory_scanner_destroy(scanner);
}
unlink(link_file);
unlink(link_dir);
unlink(nested);
unlink(file);
rmdir(sub);
rmdir(root);
}
void test_scanner() {
test_scanner_single_file();
test_scanner_multiple_files();
@@ -1406,4 +1539,5 @@ void test_scanner() {
test_files_from_relative_send_path();
test_scanner_captures_directory_times();
test_scanner_chunk_ownership();
test_scanner_entry_classification();
}