TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

perf(utils,file-list): index delete keep-set and --files-from lookups

Browse Source
This commit is contained in:
TapTap
2026-09-13 04:16:28 +02:00
parent ddc71a7df5
commit 1a83e284c4
8 changed files with 470 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/runner.c
+2
View File
@@ -12,6 +12,7 @@
#include "test_delay_updates.h"
#include "test_delta.h"
#include "test_file.h"
#include "test_file_list.h"
#include "test_file_sendfile.h"
#include "test_fuzz_smoke.h"
#include "test_glob.h"
@@ -67,6 +68,7 @@ int main() {
RUN_TEST(test_glob);
RUN_TEST(test_iconv);
RUN_TEST(test_file);
RUN_TEST(test_file_list);
RUN_TEST(test_trust_sender);
RUN_TEST(test_delay_updates);
RUN_TEST(test_file_sendfile);
tests/test_file_list.c
+123
View File
@@ -0,0 +1,123 @@
#include "test_file_list.h"
#include "file_list.h"
#include "test_utils.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
/* Reference implementation of the ORIGINAL file_list_affects linear scan. The
indexed implementation must agree with it on every query; this pins the
subtle semantics: empty entry == whole tree, exact match, rel under a listed
directory, and rel an ancestor directory of a listed entry. */
static bool reference_affects(const FileListSet* set, const char* rel) {
if (!set)
return true;
if (!rel)
return false;
for (int i = 0; i < set->count; i++) {
const char* entry = set->entries[i];
if (entry[0] == '\0')
return true;
if (strcmp(rel, entry) == 0)
return true;
size_t entry_len = strlen(entry);
if (strncmp(rel, entry, entry_len) == 0 && (rel[entry_len] == '/' || rel[entry_len] == '\0'))
return true;
size_t rel_len = strlen(rel);
if (strncmp(entry, rel, rel_len) == 0 && (entry[rel_len] == '/' || entry[rel_len] == '\0'))
return true;
}
return false;
}
static void write_list(const char* path, const char* bytes) {
FILE* fp = fopen(path, "wb");
EXPECT_NOT_NULL(fp);
size_t len = strlen(bytes);
EXPECT_EQ_INT((int)fwrite(bytes, 1, len, fp), (int)len);
fclose(fp);
}
static void check_queries(const FileListSet* set, const char* const* queries, int query_count) {
for (int i = 0; i < query_count; i++) {
bool expected = reference_affects(set, queries[i]);
bool actual = file_list_affects(set, queries[i]);
if (expected != actual) {
printf(" [FAIL] affects(\"%s\"): expected %d, got %d\n", queries[i], expected, actual);
current_test_failed = true;
return;
}
}
}
static void test_membership_matches_reference() {
const char* path = "test_file_list_case.txt";
char err[160];
/* Nested directories, an ancestor of a listed entry, an exact file, a
non-matching neighbor with the same prefix, and a literal '*'. */
write_list(path, "a\na/b\na/b/c\nab\nc.txt\nsub/b.bin\n*\n");
FileListSet* set = file_list_load(path, false, err, sizeof(err));
EXPECT_NOT_NULL(set);
const char* queries[] = {
"a", "a/b", "a/b/c", "a/b/c/d", "a/bx", "a/x", "ab",
"abc", "c.txt", "c.txt/x", "c", "sub", "sub/b.bin", "sub/b.bin/z",
"sub2", "*", "x", "", "a/b/cd", "/", "a/b/",
};
check_queries(set, queries, (int)(sizeof(queries) / sizeof(queries[0])));
EXPECT_TRUE(file_list_affects(set, "a/b/c/d"));
EXPECT_TRUE(file_list_affects(set, "a/bx")); /* under listed directory "a" */
EXPECT_TRUE(file_list_affects(set, "a/b/c"));
EXPECT_TRUE(file_list_affects(set, "a/x")); /* under listed directory "a" */
EXPECT_FALSE(file_list_affects(set, "abc")); /* component boundary: not "a" */
EXPECT_TRUE(file_list_affects(set, "sub"));
EXPECT_FALSE(file_list_affects(set, "sub2"));
file_list_destroy(set);
remove(path);
/* A single "." entry means the whole tree: every non-NULL query is true. */
write_list(path, ".\n");
set = file_list_load(path, false, err, sizeof(err));
EXPECT_NOT_NULL(set);
const char* root_queries[] = {"", "a", "a/b/c", "unrelated", "*", "/"};
for (int i = 0; i < (int)(sizeof(root_queries) / sizeof(root_queries[0])); i++)
EXPECT_TRUE(file_list_affects(set, root_queries[i]));
EXPECT_FALSE(file_list_affects(set, NULL));
check_queries(set, root_queries, (int)(sizeof(root_queries) / sizeof(root_queries[0])));
file_list_destroy(set);
remove(path);
/* Trailing slashes and "./" prefixes normalize away, so the query matches the
clean path (and not the raw spelling). */
write_list(path, "./dir/\ndir2/./x\n");
set = file_list_load(path, false, err, sizeof(err));
EXPECT_NOT_NULL(set);
EXPECT_TRUE(file_list_affects(set, "dir"));
EXPECT_TRUE(file_list_affects(set, "dir/x"));
EXPECT_TRUE(file_list_affects(set, "dir2/x"));
EXPECT_TRUE(file_list_affects(set, "dir2"));
EXPECT_TRUE(file_list_affects(set, "dir/")); /* boundary prefix of listed "dir" */
check_queries(set, (const char*[]){"dir", "dir/", "dir/x", "dir2", "dir2/x", "dir3"}, 6);
file_list_destroy(set);
remove(path);
/* An empty file yields an empty set: nothing is affected, and NULL set still
means "everything". */
write_list(path, "");
set = file_list_load(path, false, err, sizeof(err));
EXPECT_NOT_NULL(set);
EXPECT_EQ_INT(set->count, 0);
EXPECT_FALSE(file_list_affects(set, "a"));
EXPECT_FALSE(file_list_affects(set, ""));
check_queries(set, (const char*[]){"a", "a/b", ""}, 3);
file_list_destroy(set);
remove(path);
/* NULL set is the unrestricted case. */
EXPECT_TRUE(file_list_affects(NULL, "anything"));
EXPECT_TRUE(file_list_affects(NULL, NULL));
}
void test_file_list() {
test_membership_matches_reference();
}
tests/test_file_list.h
+6
View File
@@ -0,0 +1,6 @@
#ifndef TEST_FILE_LIST_H
#define TEST_FILE_LIST_H
void test_file_list();
#endif
tests/test_shared_utils.c
+38
View File
@@ -150,6 +150,43 @@ static void test_walker_removes_extras_keeps_manifest_and_protected() {
free(root);
}
static void test_walker_keeps_nested_manifest_dirs() {
/* The keep-set index must preserve deep content: a directory is protected
when its own name is a keep entry OR when kept content lives below it, and
an exact kept file survives while its siblings are removed. */
char* root = make_walk_root("nestedkeep");
EXPECT_NOT_NULL(root);
EXPECT_TRUE(write_file_at(root, "extra.txt", "extra"));
EXPECT_EQ_INT(make_subdir(root, "keepdir"), 0);
EXPECT_EQ_INT(make_subdir(root, "keepdir/deep"), 0);
EXPECT_TRUE(write_file_at(root, "keepdir/deep/keep.txt", "kept"));
EXPECT_TRUE(write_file_at(root, "keepdir/extra2.txt", "extra"));
EXPECT_EQ_INT(make_subdir(root, "dropdir"), 0);
EXPECT_EQ_INT(make_subdir(root, "keep2"), 0);
EXPECT_TRUE(write_file_at(root, "keep2/inner.txt", "kept"));
EXPECT_EQ_INT(make_subdir(root, "keep3"), 0);
const char* keeps[] = {"keepdir/deep/keep.txt", "keep2/inner.txt", "keep3"};
ArrayList* manifest = make_manifest_strings(keeps, 3);
EXPECT_NOT_NULL(manifest);
size_t deleted = 0;
DeleteWalkResult result = delete_extras_limited(root, manifest, 100000, NULL, 0, &deleted);
EXPECT_EQ_INT((int)result, (int)DELETE_WALK_OK);
EXPECT_FALSE(file_exists(root, "extra.txt"));
EXPECT_TRUE(file_exists(root, "keepdir/deep/keep.txt"));
EXPECT_FALSE(file_exists(root, "keepdir/extra2.txt"));
EXPECT_TRUE(dir_exists(root, "keepdir"));
EXPECT_TRUE(dir_exists(root, "keepdir/deep"));
EXPECT_FALSE(dir_exists(root, "dropdir"));
EXPECT_TRUE(dir_exists(root, "keep2"));
EXPECT_TRUE(file_exists(root, "keep2/inner.txt"));
EXPECT_TRUE(dir_exists(root, "keep3")); /* an exact directory keep entry survives */
EXPECT_EQ_INT((int)deleted, 3);
array_list_delete(manifest);
remove_walk_tree(root);
free(root);
}
static void test_walker_max_delete_exceeded_deletes_nothing() {
char* root = make_walk_root("maxdel");
EXPECT_NOT_NULL(root);
@@ -421,6 +458,7 @@ static void test_fd_peer_ip() {
void test_shared_utils() {
test_walker_removes_extras_keeps_manifest_and_protected();
test_walker_keeps_nested_manifest_dirs();
test_walker_max_delete_exceeded_deletes_nothing();
test_walker_max_delete_exact_bound_deletes();
test_walker_unlimited_deletes_all();