From 3d598040dd3e63b070d284339393fb308bf8efa9 Mon Sep 17 00:00:00 2001 From: TapTap Date: Tue, 21 Jul 2026 16:55:07 +0200 Subject: [PATCH] test: add transport module unit tests; docs: update README (#39, #119) --- README.md | 129 +++++++++++++++++++++++++++++++------ src/shared/protocol.h | 5 +- tests/runner.c | 6 ++ tests/test_protocol.c | 5 +- tests/test_transport_ssh.c | 18 ++++++ tests/test_transport_ssh.h | 6 ++ tests/test_transport_tcp.c | 43 +++++++++++++ tests/test_transport_tcp.h | 6 ++ tests/test_transport_tls.c | 24 +++++++ tests/test_transport_tls.h | 6 ++ 10 files changed, 226 insertions(+), 22 deletions(-) create mode 100644 tests/test_transport_ssh.c create mode 100644 tests/test_transport_ssh.h create mode 100644 tests/test_transport_tcp.c create mode 100644 tests/test_transport_tcp.h create mode 100644 tests/test_transport_tls.c create mode 100644 tests/test_transport_tls.h diff --git a/README.md b/README.md index 38682d8..353133c 100644 --- a/README.md +++ b/README.md @@ -5,17 +5,26 @@ A high-performance file synchronization system with SSH and TCP transport, TLS e ## Technical Overview 1. **Dual transport**: custom TCP client-server or SSH subprocess (rsync-style `user@host:/path`) -2. **TLS encryption**: OpenSSL-based TLS 1.2+ for encrypted TCP connections +2. **TLS encryption**: OpenSSL-based TLS 1.2+ for encrypted TCP connections with optional CA verification 3. **Chunked file transfer**: files grouped into configurable-size chunks (default ~10 MB) 4. **Streaming zstd compression** (levels 1–22) using `ZSTD_compressStream2` 5. **Multithreading**: producer-consumer pipeline with thread-safe queues (scanner → loader → sender) 6. **Incremental sync**: skip files unchanged since last transfer (compares size + mtime) -7. **Metadata preservation**: `mode`, `uid`, `gid`, `mtime` restored on disk when enabled -8. **`sendfile()` zero-copy** on TCP (~2× faster on loopback) -9. **SSH ControlMaster** for connection reuse across repeated invocations -10. **Bandwidth limiting**: token-bucket throttling (`--bwlimit`) -11. **`--delete`**: receiver removes files not present in sender manifest -12. **`--exclude` / `--include`**: glob-pattern filename filtering +7. **Batch incremental**: send incremental checks in batched groups for reduced round-trips +8. **Metadata preservation**: `mode`, `uid`, `gid`, `mtime` restored on disk when enabled +9. **`sendfile()` zero-copy** on TCP (~2× faster on loopback) +10. **SSH ControlMaster** for connection reuse across repeated invocations +11. **Bandwidth limiting**: token-bucket throttling (`--bwlimit`) +12. **`--delete`**: receiver removes files not present in sender manifest +13. **`--exclude` / `--include`**: glob-pattern filename filtering +14. **Path traversal protection**: `..` sequences in file paths are rejected automatically +15. **Connection limits**: server enforces maximum concurrent connections (default 100) +16. **Keep-alive**: periodic `STATUS_KEEPALIVE` messages detect stalled connections +17. **Abort handling**: `SIGINT` sends `STATUS_ABORT` for clean server-side teardown +18. **Atomic writes**: received files are written to a temporary name then atomically renamed +19. **Backup mode**: `--backup` preserves overwritten files with optional `--backup-dir` +20. **Log file**: `--log-file` redirects log output to a file instead of stderr +21. **Transfer statistics**: `--stats` prints summary of transferred bytes, files, and timing ## System Architecture @@ -25,20 +34,33 @@ A high-performance file synchronization system with SSH and TCP transport, TLS e - Streaming zstd compression with configurable level - Chunk serialization (compact binary format) or per-file transfer - Incremental transfer: sends file metadata to server, skips unchanged files +- Batch incremental: groups incremental checks to minimize round-trips - Manifests all sent paths when `--delete` is active - Sends via TCP `sendfile()` or SSH pipe - Optional progress display with throughput - Bandwidth limiting via token-bucket algorithm +- Configurable I/O and connection timeouts (`--timeout`, `--contimeout`) +- Quiet mode (`-q`/`--quiet`) suppresses all non-error output +- Backup overwritten files (`--backup`) with optional directory (`--backup-dir`) +- Transfer statistics summary (`--stats`) +- Maximum directory depth control (`--max-depth`) +- Log file output (`--log-file`) +- Configurable multithreaded queue size (`--queue-size`) +- Exclude patterns from file (`--exclude-from`) ### Server - TCP mode: listens on configurable port (default 8080); SSH mode: runs via `--stdio` -- TLS mode: wraps TCP connections with OpenSSL +- TLS mode: wraps TCP connections with OpenSSL with optional CA verification - Receives and reassembles files - Decompresses (streaming zstd), deserializes, restores metadata - Handles incremental checks: compares size + mtime against destination files +- Handles batch incremental checks for reduced round-trips - Processes `STATUS_MANIFEST` for `--delete`: walks destination tree, removes extras -- Per-connection concurrency via `fork()` +- Per-connection concurrency via `fork()` with configurable connection limit (default 100) - Thread pool for parallel processing +- Atomic writes: files written to `.tmp` path then atomically renamed on success +- Abort handling: cleanly shuts down on `STATUS_ABORT` from client +- Path traversal protection: rejects file paths containing `..` ## Protocol Details @@ -52,6 +74,11 @@ A high-performance file synchronization system with SSH and TCP transport, TLS e | `STATUS_CHUNK` | Following data is a serialized chunk | | `STATUS_MANIFEST` | Following data is a file manifest (for `--delete`) | | `STATUS_CHECK` | Incremental check: client sends file path + size + mtime, server responds with OK (skip) or NEXT (send) | +| `STATUS_CHECK_BATCH` | Batch incremental check: multiple file checks sent in one message | +| `STATUS_KEEPALIVE` | Keep-alive heartbeat to detect stalled connections | +| `STATUS_ABORT` | Abort signal: client interrupts, server cleans up and exits | +| `STATUS_DELTA_SIGNATURE` | Delta sync: following data is a file signature (rsync-style rolling hash) | +| `STATUS_DELTA_DATA` | Delta sync: following data is a delta patch for a file | ### Wire Format — Metadata @@ -59,12 +86,16 @@ When `use_metadata` is enabled (`-M`), each file entry carries a 4-byte `present ### Transfer Flow ``` -Config → (STATUS_NEXT | STATUS_CHUNK | STATUS_CHECK)* → [STATUS_MANIFEST] → STATUS_FINISHED → STATUS_OK +Config → (STATUS_NEXT | STATUS_CHUNK | STATUS_CHECK | STATUS_CHECK_BATCH)* → [STATUS_MANIFEST] → STATUS_FINISHED → STATUS_OK ``` +Keep-alive (`STATUS_KEEPALIVE`) may be sent at any point during the transfer. The receiver resets its inactivity timer on receipt. If no data arrives within the receive timeout, the connection is aborted. + +Abort (`STATUS_ABORT`) may be sent at any point. On receipt the server cleans up temporary files and exits the child process. + ### Protocol Version -`1.1.0` — server and client must match. Mismatch results in `STATUS_ERROR`. +`1.3.0` — server and client must match. Mismatch results in `STATUS_ERROR`. ## Command-Line Arguments @@ -83,15 +114,26 @@ Config → (STATUS_NEXT | STATUS_CHUNK | STATUS_CHECK)* → [STATUS_MANIFEST] | `-n, --dry-run` | Scan and print what would be transferred | | `-p ` | SSH port (default: 22) | | `-v, --verbose` | Enable debug logging | +| `-q, --quiet` | Suppress all non-error output | +| `--silent` | Alias for `--quiet` | | `--progress` | Show real-time transfer speed | | `--delete` | Delete files on receiver not present in source | | `--exclude ` | Exclude files matching glob pattern (repeatable) | +| `--exclude-from ` | Read exclude patterns from a file (one per line) | | `--include ` | Only transfer files matching glob pattern (repeatable, whitelist) | | `--max-size ` | Skip files larger than n bytes | | `--min-size ` | Skip files smaller than n bytes | | `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `-s`. | | `--bwlimit ` | Bandwidth limit in kilobytes per second | | `--chunk-size ` | Chunk size in bytes (default: 10485760) | +| `--timeout ` | I/O timeout in seconds (default: 30) | +| `--contimeout ` | Connection timeout in seconds (default: 10) | +| `--backup` | Backup existing destination files before overwriting | +| `--backup-dir ` | Target directory for backups (requires `--backup`) | +| `--stats` | Print transfer statistics at end (bytes, files, timing) | +| `--max-depth ` | Maximum directory depth to recurse (0 = unlimited, default: 0) | +| `--log-file ` | Write log messages to file instead of stderr | +| `--queue-size ` | Queue capacity for multithreaded mode (default: 100) | | `--source-dir ` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) | | `--dest-dir ` | Server destination directory (overrides `FASTSYNC_DEST_DIR`) | | `--save-to-disk` | Write received files to disk | @@ -122,6 +164,12 @@ Config → (STATUS_NEXT | STATUS_CHUNK | STATUS_CHECK)* → [STATUS_MANIFEST] | `FASTSYNC_SOURCE_DIR` | — | Source directory fallback | | `FASTSYNC_DEST_DIR` | — | Destination directory fallback | | `FASTSYNC_SAVE_TO_DISK` | `false` | Disk persistence fallback | +| `FASTSYNC_SSH_PORT` | `22` | Default SSH port | +| `FASTSYNC_SERVER_HOST` | `127.0.0.1` | Default server host | +| `FASTSYNC_SERVER_PORT` | `8080` | Default server port | +| `FASTSYNC_TLS_CERT` | — | Default TLS certificate path | +| `FASTSYNC_TLS_KEY` | — | Default TLS private key path | +| `FASTSYNC_TLS_CA` | — | Default TLS CA certificate path | ## Implementation Details @@ -129,21 +177,44 @@ Config → (STATUS_NEXT | STATUS_CHUNK | STATUS_CHECK)* → [STATUS_MANIFEST] 1. **Chunk** — collection of files (~10 MB total by default) 2. **File** — path, content (`Data`), optional `FileMetadata` pointer 3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec` -4. **Config** — runtime parameters (transported over wire, TLS settings excluded) +4. **Config** — runtime parameters (transported over wire, TLS settings excluded). Includes `timeout`, `contimeout`, `quiet`, `backup`, `backup_dir`, `stats`, `max_depth`, `log_file`, `queue_size`. 5. **Queue** — thread-safe bounded queue with condition variables -6. **DirectoryScanner** — recursive BFS traversal with exclude and include pattern support +6. **DirectoryScanner** — recursive BFS traversal with exclude and include pattern support, max-depth enforcement ### Key Algorithms -1. **File scanning** — BFS directory traversal; entries matched against exclude and include patterns +1. **File scanning** — BFS directory traversal; entries matched against exclude and include patterns, max-depth enforced 2. **Chunking** — files accumulated until `chunk_size` threshold, then flushed 3. **Compression** — streaming zstd via `ZSTD_compressStream2` / `ZSTD_decompressStream` -4. **Network protocol** — status-code-driven exchange with metadata packing -5. **Incremental check** — client sends `STATUS_CHECK` + path + size + mtime; server compares against destination +4. **Network protocol** — status-code-driven exchange with metadata packing, keep-alive, and abort support +5. **Incremental check** — client sends `STATUS_CHECK` + path + size + mtime; server compares against destination. Can be batched via `STATUS_CHECK_BATCH` for reduced round-trips. 6. **Bandwidth limiting** — token-bucket algorithm with `nanosleep` throttling on 64 KB write chunks 7. **Metadata restoration** — `chmod()`, `chown()`, `utimensat()` on the receiving side 8. **`--delete`** — sender tracks all sent paths; receiver walks destination tree and removes unlisted files/directories 9. **SSH transport** — `socketpair()` + `fork()` + `execvp("ssh", ...)` with `ControlMaster` and port support 10. **TLS transport** — OpenSSL `SSL_CTX` with TLS 1.2 minimum, optional CA verification, transparent `SSL_read`/`SSL_write` via `io_set_ssl()` +11. **Path traversal protection** — `has_path_traversal()` rejects any file path containing `..` components, preventing directory escape attacks +12. **Connection limiting** — server tracks active connections and rejects new ones beyond `max_connections` (default 100) +13. **Keep-alive** — idle connections receive periodic `STATUS_KEEPALIVE` to detect half-open TCP connections +14. **Abort handling** — `SIGINT` sets an abort flag; the next protocol operation sends `STATUS_ABORT` for clean server cleanup +15. **Atomic writes** — files are written to a `.tmp` suffix then atomically renamed via `rename()`, preventing partial files +16. **Backup** — before overwriting, existing files are moved to `--backup-dir` (or same directory with `~` suffix) preserving the original + +## Security Features + +### Path Traversal Protection +All received file paths are validated by `has_path_traversal()` before any disk operation. Any path containing `..` components is rejected with `STATUS_ERROR`, preventing directory escape attacks. + +### TLS Certificate Verification +When `--ca` is provided, the server performs mutual TLS verification (`SSL_VERIFY_PEER` with depth 4). Without `--ca`, TLS is still encrypted but peer certificates are not verified. + +### Connection Limits +The server enforces a maximum of 100 concurrent connections (configurable via `max_connections` in `Server`). When the limit is reached, new connections are immediately rejected and closed. + +### Abort Handling +If the client receives `SIGINT` (Ctrl+C) during a transfer, it sends `STATUS_ABORT` to the server. The server then cleans up temporary files and exits the child process, preventing incomplete files from remaining on disk. + +### Atomic Writes +Received files are written to a temporary path (suffixed with `.tmp`) and then atomically renamed to the final filename via `rename()`. This prevents partial or corrupted files from appearing at the destination if the transfer is interrupted. ## Build Requirements @@ -223,6 +294,21 @@ Place the `fastsync-server` binary in the remote `$PATH`. The client runs `ssh u # Bandwidth limit to 1 MB/s ./build/client --bwlimit 1024 /src user@host:/dst +# With timeouts, quiet mode, and stats +./build/client --timeout 60 --contimeout 15 --quiet --stats /src user@host:/dst + +# Backup overwritten files to a directory +./build/client --backup --backup-dir /backups /src user@host:/dst + +# Exclude patterns from file, limit depth +./build/client --exclude-from ignore.txt --max-depth 3 /src user@host:/dst + +# Custom queue size for multithreading +./build/client -m --queue-size 200 /src user@host:/dst + +# Log to file +./build/client --log-file /tmp/fastsync.log /src user@host:/dst + # All features ./build/client -a --progress --chunk-size 5242880 --exclude "*.log" --delete /src /dst ``` @@ -230,7 +316,9 @@ Place the `fastsync-server` binary in the remote `$PATH`. The client runs `ssh u ## Testing ```bash -# Unit tests (7 suites) +# Unit tests (18 suites — array_list, chunk, compression, config, data, delta, file, glob, +# metadata, property, protocol, queue, robustness, scanner, +# shared_utils, stress, transport_tcp, transport_ssh, transport_tls) ./build/tests # Integration + benchmark suite @@ -244,12 +332,15 @@ The benchmark prints throughput metrics, best configuration, and speedup vs rsyn 1. Chunk size (~10 MB default) balances memory and transfer efficiency 2. Compression level trades CPU for bandwidth 3. `sendfile()` bypasses userspace — ~2× faster on localhost for large files -4. Multithreading scales with core count +4. Multithreading scales with core count; `--queue-size` controls pipeline buffering 5. Metadata transfer adds negligible overhead (~24 bytes per file when enabled) 6. SSH socketpair buffer set to 1 MB for improved pipe throughput 7. SSH ControlMaster reuses connections across repeated invocations 8. Incremental sync eliminates redundant transfers entirely -9. Bandwidth limiting uses token-bucket with nanosleep for accurate throttling +9. Batch incremental reduces round-trips by grouping multiple checks into one message +10. Bandwidth limiting uses token-bucket with nanosleep for accurate throttling +11. Atomic writes add a single `rename()` per file — negligible overhead +12. Path traversal check is O(n) in path length with negligible cost ## Benchmark Results diff --git a/src/shared/protocol.h b/src/shared/protocol.h index a7854f1..113f14f 100644 --- a/src/shared/protocol.h +++ b/src/shared/protocol.h @@ -17,7 +17,10 @@ enum NET_STATUS { STATUS_MANIFEST, STATUS_CHECK, STATUS_DELTA_SIGNATURE, - STATUS_DELTA_DATA + STATUS_DELTA_DATA, + STATUS_KEEPALIVE, + STATUS_ABORT, + STATUS_CHECK_BATCH }; void io_set_fds(int read_fd, int write_fd); diff --git a/tests/runner.c b/tests/runner.c index 6ae6c12..112e4ab 100644 --- a/tests/runner.c +++ b/tests/runner.c @@ -14,6 +14,9 @@ #include "test_scanner.h" #include "test_shared_utils.h" #include "test_stress.h" +#include "test_transport_tcp.h" +#include "test_transport_ssh.h" +#include "test_transport_tls.h" #include "test_utils.h" #include @@ -41,6 +44,9 @@ int main() { RUN_TEST(test_robustness); RUN_TEST(test_stress); RUN_TEST(test_property); + RUN_TEST(test_transport_tcp); + RUN_TEST(test_transport_ssh); + RUN_TEST(test_transport_tls); printf("\n\033[1;36m=== TEST SUMMARY ===\033[0m\n"); printf("Total Tests Run: %d\n", tests_run); diff --git a/tests/test_protocol.c b/tests/test_protocol.c index d0070fe..af1dc00 100644 --- a/tests/test_protocol.c +++ b/tests/test_protocol.c @@ -128,8 +128,9 @@ static void test_send_receive_status() { io_set_fds(p[0], p[1]); io_set_bwlimit(0); - Status statuses[] = {STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, - STATUS_CHUNK, STATUS_CHECK, STATUS_DELTA_SIGNATURE, STATUS_DELTA_DATA}; + Status statuses[] = {STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, + STATUS_CHUNK, STATUS_CHECK, STATUS_DELTA_SIGNATURE, STATUS_DELTA_DATA, + STATUS_KEEPALIVE, STATUS_ABORT, STATUS_CHECK_BATCH}; int count = sizeof(statuses) / sizeof(statuses[0]); for (int i = 0; i < count; i++) { diff --git a/tests/test_transport_ssh.c b/tests/test_transport_ssh.c new file mode 100644 index 0000000..eb08174 --- /dev/null +++ b/tests/test_transport_ssh.c @@ -0,0 +1,18 @@ +#include "test_transport_ssh.h" +#include "test_utils.h" +#include "transport_ssh.h" + +static void test_ssh_connect_invalid_dest_no_colon() { + Client* c = client_connect_ssh("/path/to/dest", 22); + EXPECT_NULL(c); +} + +static void test_ssh_connect_invalid_dest_empty() { + Client* c = client_connect_ssh("", 22); + EXPECT_NULL(c); +} + +void test_transport_ssh() { + test_ssh_connect_invalid_dest_no_colon(); + test_ssh_connect_invalid_dest_empty(); +} diff --git a/tests/test_transport_ssh.h b/tests/test_transport_ssh.h new file mode 100644 index 0000000..0df5a47 --- /dev/null +++ b/tests/test_transport_ssh.h @@ -0,0 +1,6 @@ +#ifndef TEST_TRANSPORT_SSH_H +#define TEST_TRANSPORT_SSH_H + +void test_transport_ssh(); + +#endif diff --git a/tests/test_transport_tcp.c b/tests/test_transport_tcp.c new file mode 100644 index 0000000..516879f --- /dev/null +++ b/tests/test_transport_tcp.c @@ -0,0 +1,43 @@ +#include "test_transport_tcp.h" +#include "test_utils.h" +#include "transport_tcp.h" +#include + +static void test_server_create_ephemeral() { + Server* s = server_create(0); + EXPECT_NOT_NULL(s); + EXPECT_TRUE(s->file_descriptor >= 0); + EXPECT_EQ_INT(s->address.sin_family, AF_INET); + server_delete(&s); + EXPECT_NULL(s); +} + +static void test_server_delete_null() { + Server* s = NULL; + server_delete(&s); + EXPECT_NULL(s); +} + +static void test_client_create() { + Client* c = client_create(); + EXPECT_NOT_NULL(c); + EXPECT_TRUE(c->file_descriptor >= 0); + EXPECT_EQ_INT(c->address.sin_family, AF_INET); + EXPECT_EQ_INT(c->ssh_child_pid, -1); + EXPECT_NULL(c->ssl); + EXPECT_NULL(c->ssl_ctx); + client_disconnect(c); + client_delete(c); +} + +static void test_client_delete_null() { + Client* c = NULL; + client_delete(c); +} + +void test_transport_tcp() { + test_server_create_ephemeral(); + test_server_delete_null(); + test_client_create(); + test_client_delete_null(); +} diff --git a/tests/test_transport_tcp.h b/tests/test_transport_tcp.h new file mode 100644 index 0000000..720209f --- /dev/null +++ b/tests/test_transport_tcp.h @@ -0,0 +1,6 @@ +#ifndef TEST_TRANSPORT_TCP_H +#define TEST_TRANSPORT_TCP_H + +void test_transport_tcp(); + +#endif diff --git a/tests/test_transport_tls.c b/tests/test_transport_tls.c new file mode 100644 index 0000000..657732d --- /dev/null +++ b/tests/test_transport_tls.c @@ -0,0 +1,24 @@ +#include "test_transport_tls.h" +#include "test_utils.h" +#include "transport_tcp.h" +#include "transport_tls.h" + +static void test_tls_global_init() { + bool ok = tls_global_init(); + EXPECT_TRUE(ok); +} + +static void test_server_create_tls_without_certs() { + Server* s = server_create(0); + EXPECT_NOT_NULL(s); + bool ok = server_create_tls(s, NULL, NULL, NULL); + EXPECT_TRUE(ok); + EXPECT_NOT_NULL(s->ssl_ctx); + server_delete(&s); + EXPECT_NULL(s); +} + +void test_transport_tls() { + test_tls_global_init(); + test_server_create_tls_without_certs(); +} diff --git a/tests/test_transport_tls.h b/tests/test_transport_tls.h new file mode 100644 index 0000000..aa13a69 --- /dev/null +++ b/tests/test_transport_tls.h @@ -0,0 +1,6 @@ +#ifndef TEST_TRANSPORT_TLS_H +#define TEST_TRANSPORT_TLS_H + +void test_transport_tls(); + +#endif