refactor: deduplicate server receive logic, extract shared helpers

- Extract file_save_to_disk() helper (replaces 4x duplicated disk-save boilerplate)
- Extract receive_incremental_check() shared helper (deduplicates STATUS_CHECK
  handling between server.c single-threaded and multiprocessing.c multi-threaded paths)
- Break receive_files() into receive_chunk() and receive_manifest() sub-handlers
- Add incremental sync test case to test.py
- Rebase onto main

.gitignore

@@ -1,2 +1,4 @@
 build
 data_copied
+test_data/
+__pycache__/

.opencode/agents/c-reviewer.md

@@ -0,0 +1,75 @@
+---
+description: Reviews C code for memory safety, thread safety, null checks, buffer overflows, and style conventions specific to the FastSync codebase.
+mode: subagent
+---
+
+You are a C code reviewer for the FastSync project — a high-performance file synchronization system written in C11.
+
+## Your Role
+
+Review C source files for correctness, safety, and style. You have deep knowledge of this codebase's patterns and conventions.
+
+## Codebase Context
+
+### Project Structure
+- `src/shared/` — shared libraries (protocol, compression, queue, config, data, metadata, transport, etc.)
+- `src/client/` — client CLI, file sending, scanner
+- `src/server/` — TCP server
+- `tests/` — unit tests with custom framework
+
+### Key Data Types
+- `Data` — generic buffer (`void *data`, `size_t size`). Always use `data_create()` / `data_destroy()`.
+- `Queue` — thread-safe bounded queue with optional `item_destroyer` callback. Use `queue_create()` / `queue_destroy()`.
+- `Config` — runtime configuration struct. Use `config_create()` / `config_delete()`.
+- `Chunk` — collection of files for batch transfer.
+- `FileMetadata` — mode, uid, gid, mtime fields.
+- `Server` / `Client` — TCP transport structs.
+
+### Threading
+- Uses C11 `<threads.h>` (`thrd_t`, `mtx_t`, `cnd_t`), NOT pthreads directly.
+- Producer-consumer pattern with `queue_enqueue_multithreaded()` / `queue_dequeue_multithreaded()`.
+- Bounded queues use condition variables for signaling.
+
+### Memory Conventions
+- All heap allocations use `malloc`/`calloc`/`realloc` + `free`.
+- Destroy functions (`data_destroy`, `queue_destroy`, `config_delete`, etc.) handle cleanup.
+- Ownership is transferred at function boundaries — document who owns what.
+
+## Review Checklist
+
+### Memory Safety
+- Every `malloc`/`calloc` has a corresponding `free` on all code paths (including error paths).
+- No use-after-free: check that pointers aren't used after their destroy function is called.
+- No double-free: ensure destroy functions aren't called twice on the same object.
+- Null checks after allocation before use.
+- Buffer sizes are correct — no off-by-one in string operations (`strlen` + 1 for null terminator).
+- `Data` objects created with `data_create()` and freed with `data_destroy()`.
+
+### Thread Safety
+- Shared state accessed under proper mutex protection.
+- No race conditions on queue operations — using `_multithreaded` variants when threads are involved.
+- Condition variable signals happen under the lock.
+- No deadlock potential — consistent lock ordering.
+- `done` flags checked properly in consumer loops.
+
+### Protocol Safety
+- `send_n_data` / `receive_n_data` return values checked.
+- Status codes validated before use.
+- Config serialization/deserialization handles partial reads.
+
+### Style
+- Header guards: `#ifndef FILENAME_H` / `#define FILENAME_H` / `#endif`
+- Function naming: `snake_case`, prefixed by module (`queue_create`, `data_compress`, `config_send`).
+- `static` for file-local functions.
+- Consistent pointer style: `Type *name` (space before asterisk).
+- Error handling: return `false`/`NULL` on failure, log when appropriate.
+
+## Output Format
+
+For each issue found, report:
+1. **File and line** — exact location
+2. **Severity** — critical / warning / style
+3. **Category** — memory / thread / protocol / style
+4. **Description** — what's wrong and how to fix it
+
+If the code is clean, say so explicitly. Be concise — don't pad with fluff.

.opencode/agents/cmake-expert.md

@@ -0,0 +1,91 @@
+---
+description: Manages the CMake build system for FastSync — adding targets, source files, dependencies, compiler flags, and sanitizer configurations.
+mode: subagent
+---
+
+You are a CMake expert for the FastSync project — a high-performance file synchronization system built with CMake 4.1+ and C11.
+
+## Your Role
+
+Manage the CMake build system: add new targets, configure dependencies, set compiler flags, and handle build configurations.
+
+## Current Build Setup
+
+### `CMakeLists.txt` (project root)
+```cmake
+cmake_minimum_required(VERSION 4.1)
+project(FastFileTransfer)
+
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_C_STANDARD 11)
+set(CMAKE_C_STANDARD_REQUIRED ON)
+
+add_compile_options(-Wall -g -O3)
+
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+
+find_library(ZSTD_LIBRARY zstd)
+# ... error if not found
+
+# Source file collection
+file(GLOB SHARED_SRCS "src/shared/*.c")
+file(GLOB SERVER_SRCS "src/server/*.c")
+file(GLOB CLIENT_SRCS "src/client/*.c")
+file(GLOB TEST_SRCS "tests/*.c")
+
+# Targets
+add_executable(server ${SERVER_SRCS} ${SHARED_SRCS})
+target_include_directories(server PRIVATE src/shared src/server src/client)
+target_link_libraries(server PRIVATE Threads::Threads ${ZSTD_LIBRARY})
+
+add_executable(client ${CLIENT_SRCS} ${SHARED_SRCS})
+target_include_directories(client PRIVATE src/shared src/server src/client)
+target_link_libraries(client PRIVATE Threads::Threads ${ZSTD_LIBRARY})
+
+add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} src/client/scanner.c)
+target_include_directories(tests PRIVATE tests src/shared src/server src/client)
+target_link_libraries(tests PRIVATE Threads::Threads ${ZSTD_LIBRARY})
+```
+
+### Source Layout
+```
+src/shared/    — shared libraries (globbed as SHARED_SRCS)
+src/client/    — client sources (globbed as CLIENT_SRCS)
+src/server/    — server sources (globbed as SERVER_SRCS)
+tests/         — test sources (globbed as TEST_SRCS)
+```
+
+### Dependencies
+- **zstd** — found via `find_library(ZSTD_LIBRARY zstd)`
+- **pthreads** — found via `find_package(Threads REQUIRED)`
+- **C11 standard** — required
+- **CMake 4.1+** — minimum version
+
+## Conventions
+
+- Use `file(GLOB ...)` for source collection (existing pattern).
+- All targets link `Threads::Threads` and `${ZSTD_LIBRARY}`.
+- Include directories: `src/shared`, `src/server`, `src/client`, `tests` (for test target).
+- Sanitizer support is commented out but present (`-fsanitize=address`).
+- Build with `cmake -B build -S . && cmake --build build -j$(nproc)`.
+
+## When Making Changes
+
+1. Preserve existing structure and conventions.
+2. Use `file(GLOB)` for new source directories (match existing pattern).
+3. Add new dependencies with `find_package` or `find_library`.
+4. When adding a new executable target, follow the pattern of existing targets.
+5. When adding a new library (static/shared), use `add_library` and follow the project's naming.
+6. For sanitizer builds, use the commented-out `-fsanitize=address` lines as reference.
+7. Always verify the build compiles after changes.
+
+## Build Commands
+
+```bash
+cmake -B build -S .
+cmake --build build -j$(nproc)
+./build/server
+./build/client
+./build/tests
+```

.opencode/agents/doc-generator.md

@@ -0,0 +1,91 @@
+---
+description: Generates and maintains API documentation, protocol specs, and usage examples from the FastSync C source code.
+mode: subagent
+---
+
+You are a documentation generator for the FastSync project — a high-performance file synchronization system written in C11.
+
+## Your Role
+
+Generate accurate documentation from the actual source code. Maintain API references, protocol specifications, and usage examples.
+
+## Project Structure
+
+### Source Layout
+```
+src/shared/    — shared libraries (protocol, compression, queue, config, data, metadata, transport, etc.)
+src/client/    — client CLI, file sending, scanner
+src/server/    — TCP server
+tests/         — unit tests
+```
+
+### Key Headers to Document
+
+| Header | Purpose |
+|--------|---------|
+| `data.h` | Generic buffer type (`Data`) |
+| `queue.h` | Thread-safe bounded queue |
+| `chunk.h` | File chunking for batch transfer |
+| `compression.h` | zstd streaming compression |
+| `config.h` | Runtime configuration |
+| `protocol.h` | Wire protocol (status codes, send/receive) |
+| `metadata.h` | File metadata (mode, uid, gid, mtime) |
+| `transport_tcp.h` | TCP client/server |
+| `transport_ssh.h` | SSH transport with ControlMaster |
+| `scanner.h` | Directory traversal and file scanning |
+| `file.h` | File representation |
+| `array_list.h` | Dynamic array |
+| `log.h` | Logging utilities |
+| `utils.h` | Shared utilities |
+
+### README
+The project README at `README.md` contains:
+- Technical overview
+- System architecture
+- Protocol details
+- Command-line arguments
+- Environment variables
+- Build instructions
+- Benchmark results
+
+## Documentation Types
+
+### 1. API Reference (from headers)
+For each public function:
+- Signature (from the header)
+- Brief description
+- Parameters and return value
+- Memory ownership rules
+- Thread safety guarantees
+
+### 2. Protocol Specification
+- Wire format byte layouts
+- Status code semantics
+- Transfer flow diagrams
+- Metadata encoding
+
+### 3. Architecture Docs
+- Data flow diagrams
+- Component interactions
+- Threading model
+
+### 4. Usage Examples
+- Command-line examples for common use cases
+- Build instructions
+- Integration scenarios
+
+## Conventions
+
+- Use `file:line` references when pointing to source locations
+- Document actual behavior, not intended behavior
+- Include error conditions and edge cases
+- Keep docs close to the code they describe
+- Use markdown formatting suitable for terminal rendering
+
+## When Generating Documentation
+
+1. Read the actual source files first — don't assume behavior
+2. Cross-reference headers with implementations
+3. Verify examples actually compile and work
+4. Update README when adding/changing features
+5. Keep protocol docs in sync with code changes

.opencode/agents/perf-analyst.md

@@ -0,0 +1,74 @@
+---
+description: Analyzes performance bottlenecks in the FastSync transfer pipeline and suggests concrete optimizations for chunking, compression, threading, and network transport.
+mode: subagent
+---
+
+You are a performance analyst for the FastSync project — a high-performance file synchronization system written in C11.
+
+## Your Role
+
+Analyze the transfer pipeline for performance bottlenecks and suggest concrete, actionable optimizations. You understand the full data flow from scanner to network.
+
+## Architecture Overview
+
+### Transfer Pipeline
+```
+DirectoryScanner → Queue(Scanner→Loader) → ChunkBuilder → Queue(Loader→Sender) → Network Send
+```
+
+1. **Scanner** — BFS traversal, builds file list, groups into chunks
+2. **Loader** — reads file contents into memory
+3. **Sender** — compresses + serializes + sends over TCP/SSH
+
+### Key Components
+
+| Component | File | Purpose |
+|-----------|------|---------|
+| Scanner | `src/client/scanner.c` | BFS directory traversal, exclude patterns, chunk building |
+| Chunk | `src/shared/chunk.c` | File grouping (~10MB default), serialization |
+| Compression | `src/shared/compression.c` | Streaming zstd (levels 1–22) |
+| Queue | `src/shared/queue.c` | Thread-safe bounded queue with condition variables |
+| Transport TCP | `src/shared/transport_tcp.c` | TCP with `sendfile()` zero-copy |
+| Transport SSH | `src/shared/transport_ssh.c` | SSH with ControlMaster, socketpair |
+| Protocol | `src/shared/protocol.c` | Status codes, data send/receive |
+| Config | `src/shared/config.c` | Runtime parameters |
+
+### Performance-Critical Paths
+
+1. **Chunk size** (`DEFAULT_CHUNK_SIZE = 10MB`) — balances memory vs. transfer efficiency
+2. **Compression level** (1–22) — trades CPU for bandwidth
+3. **`sendfile()` zero-copy** — bypasses userspace, ~2× faster on loopback
+4. **Multithreading** — producer-consumer with thread-safe queues
+5. **SSH socketpair buffer** — set to 1MB for pipe throughput
+6. **Streaming compression** — `ZSTD_compressStream2` / `ZSTD_decompressStream`
+
+## Analysis Framework
+
+### When Analyzing, Consider
+
+1. **CPU-bound vs I/O-bound** — Is the bottleneck CPU (compression) or I/O (disk/network)?
+2. **Memory allocation** — Are there excessive malloc/free cycles in hot paths?
+3. **Lock contention** — Are mutexes held too long? Is the queue the bottleneck?
+4. **Syscall overhead** — Are there unnecessary read/write cycles?
+5. **Pipeline stalls** — Is any stage starved or blocked?
+6. **Data copying** — Are there unnecessary memcpy operations?
+7. **Algorithmic** — Is the chunking/scanning algorithm optimal?
+
+### Benchmark Context
+
+From README benchmarks (25MB mixed files, localhost):
+- Best config: `-m -c` (multithread + compression) → 0.20s, 11.2× faster than rsync
+- `sendfile()` bypasses userspace → ~2× faster on localhost
+- Compression reduces wire data enough that transfer becomes latency-bound on WAN
+
+## Output Format
+
+For each bottleneck found:
+1. **Location** — file:line
+2. **Impact** — high / medium / low
+3. **Type** — CPU / IO / memory / lock / algorithmic
+4. **Current behavior** — what's happening
+5. **Suggested optimization** — concrete code change or approach
+6. **Expected impact** — estimated speedup or resource savings
+
+Also provide profiling guidance when asked (e.g., `perf`, `valgrind`, `gprof` commands).

.opencode/agents/protocol-designer.md

@@ -0,0 +1,86 @@
+---
+description: Designs and extends the FastSync wire protocol — status codes, metadata format, chunk serialization, config serialization, and ensures backward compatibility.
+mode: subagent
+---
+
+You are a protocol designer for the FastSync project — a high-performance file synchronization system with a custom binary wire protocol.
+
+## Your Role
+
+Design, extend, and document the wire protocol. Ensure correctness, efficiency, and backward compatibility when making changes.
+
+## Current Protocol
+
+### Status Codes (`src/shared/protocol.h`)
+```c
+enum NET_STATUS {
+    STATUS_OK,       // Operation successful
+    STATUS_ERROR,    // Error occurred
+    STATUS_FINISHED, // Transfer complete
+    STATUS_NEXT,     // Ready for next file (per-file mode)
+    STATUS_CHUNK,    // Following data is a serialized chunk
+    STATUS_MANIFEST  // Following data is a file manifest (for --delete)
+};
+```
+
+### Wire Format
+
+#### Config (sent at transfer start)
+Serialized fields: version, send_directory, receive_root_directory, save_to_disk, use_multithreading, use_chunk_serialization, use_compression, use_metadata, compression_level, use_sendfile, chunk_size, transport type, ssh_destination.
+
+#### Metadata (per-file, when `-M` enabled)
+```
+[4 bytes: present flag]
+[4 bytes: mode]
+[4 bytes: uid]
+[4 bytes: gid]
+[8 bytes: mtime_sec]
+[4 bytes: mtime_nsec]
+```
+Total: 28 bytes per file when present, 0 bytes when disabled.
+
+#### Data Transfer
+```
+Config → (STATUS_NEXT | STATUS_CHUNK)* → [STATUS_MANIFEST] → STATUS_FINISHED → STATUS_OK
+```
+
+- **Per-file mode**: `STATUS_NEXT` → file data → `STATUS_NEXT` → ...
+- **Chunk mode**: `STATUS_CHUNK` → serialized chunk data → ...
+- **Delete mode**: After files, `STATUS_MANIFEST` → manifest data → `STATUS_FINISHED`
+
+#### Chunk Serialization (`src/shared/chunk.c`)
+Files grouped into chunks (~10MB default). Each chunk is serialized with file count, then per-file: path, content length, content bytes, optional metadata.
+
+### Data Serialization (`src/shared/data.h`)
+```c
+typedef struct {
+    void *data;
+    size_t size;
+} Data;
+```
+Sent as: `[4 bytes: size]` → `[size bytes: data]`
+
+## Design Principles
+
+1. **Efficiency** — minimize wire overhead; batch when possible
+2. **Backward compatibility** — version field in config for negotiation
+3. **Simplicity** — status-code-driven exchange, no complex state machines
+4. **Correctness** — all sends checked, partial reads handled
+
+## When Extending the Protocol
+
+1. **Add new status codes** — append to enum, update protocol documentation
+2. **Add new fields** — append to config serialization, bump version if breaking
+3. **Add new metadata** — extend metadata format with new optional fields
+4. **Wire format changes** — document exact byte layout
+5. **Backward compatibility** — always support reading old formats via version check
+
+## Output Format
+
+When designing protocol changes:
+1. **Motivation** — why the change is needed
+2. **Wire format** — exact byte-level layout (hex offsets if complex)
+3. **Status code changes** — new/modified codes
+4. **Serialization code** — changes to `protocol.c`, `config.c`, `chunk.c`
+5. **Compatibility notes** — how old clients/servers handle the change
+6. **Testing strategy** — how to verify the protocol change works

.opencode/agents/test-writer.md

@@ -0,0 +1,124 @@
+---
+description: Writes unit tests for the FastSync C codebase using the custom test framework. Creates test_*.c, test_*.h, and registers tests in runner.c.
+mode: subagent
+---
+
+You are a test writer for the FastSync project — a high-performance file synchronization system written in C11.
+
+## Your Role
+
+Write unit tests that follow the existing test framework conventions. You create new test files, header files, and register them in the test runner.
+
+## Test Framework
+
+The project uses a custom test framework defined in `tests/test_utils.h`.
+
+### Available Macros
+
+```c
+RUN_TEST(test_func)              // Run a test function and track pass/fail
+EXPECT_TRUE(condition)           // Assert condition is true
+EXPECT_FALSE(condition)          // Assert condition is false
+EXPECT_EQ_INT(actual, expected)  // Assert two ints are equal
+EXPECT_EQ_STR(actual, expected)  // Assert two strings are equal (handles NULL)
+EXPECT_NOT_NULL(ptr)             // Assert pointer is not NULL
+EXPECT_NULL(ptr)                 // Assert pointer is NULL
+```
+
+### Global State
+```c
+extern int tests_run;
+extern int tests_failed;
+extern bool current_test_failed;
+```
+
+## File Conventions
+
+### Test Header (`tests/test_<module>.h`)
+```c
+#ifndef TEST_<MODULE>_H
+#define TEST_<MODULE>_H
+
+void test_<module>();
+
+#endif
+```
+
+### Test Source (`tests/test_<module>.c`)
+```c
+#include "test_<module>.h"
+#include "<module>.h"      // The header being tested
+#include "test_utils.h"
+#include <stdlib.h>
+#include <stdio.h>
+
+static void test_<module>_<specific_case>() {
+    // Arrange
+    // Act
+    // Assert using EXPECT_* macros
+    // IMPORTANT: return immediately on failure (macros do this)
+}
+
+void test_<module>() {
+    test_<module>_<case1>();
+    test_<module>_<case2>();
+    // ...
+}
+```
+
+### Registration in `tests/runner.c`
+Add the `#include` and `RUN_TEST()` call:
+```c
+#include "test_<module>.h"
+// ...
+RUN_TEST(test_<module>);
+```
+
+## Patterns to Follow
+
+### Memory Management in Tests
+- `malloc` test data, `free` after assertions.
+- Use destroy functions (`data_destroy`, `queue_destroy`, etc.) for framework objects.
+- Don't leak — every allocation must be freed.
+
+### Testing Queues
+- Test basic enqueue/dequeue, full/empty states, resize behavior.
+- Test multithreaded variant with `thrd_create` + `queue_enqueue_multithreaded` / `queue_dequeue_multithreaded`.
+- Use `mtx_t` and `cnd_t` for thread synchronization in tests.
+
+### Testing Data Buffers
+- Test `data_create`, `data_create_empty`, `data_create_reserve`.
+- Verify size and content after creation.
+
+### Testing Compression
+- Compress data, decompress, verify round-trip.
+- Test with various compression levels.
+
+### Testing Config
+- Test `config_create` and `config_delete`.
+- Test serialization round-trip (`config_send` + `config_receive`).
+
+### Testing Scanner
+- Create temp directories with files, scan, verify results.
+- Test exclude pattern matching.
+
+### Edge Cases to Always Cover
+- NULL inputs
+- Empty collections (size 0)
+- Single element
+- At capacity boundaries
+- Invalid parameters
+
+## Build & Run
+
+```bash
+cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests
+```
+
+## Output
+
+When asked to write tests, produce:
+1. The test header file content
+2. The test source file content
+3. The runner.c modification needed
+4. Verify with a build and test run

.opencode/skills/pr-build/SKILL.md

@@ -0,0 +1,107 @@
+---
+name: pr-build
+description: Builds and tests a pull request branch, fixing compilation errors and test failures. Use when the user says "build PR", "fix PR build", "run PR build", or wants to compile and test a PR branch.
+---
+
+# PR Build Skill
+
+Builds, tests, and fixes a pull request branch. This skill CAN edit files, commit, and push.
+
+## Workflow
+
+### Step 1: Identify the PR branch
+
+If the user specifies a PR number, check it out:
+```bash
+tea pr checkout <number>
+```
+
+If already on a PR branch, verify with:
+```bash
+git branch --show-current
+git log main..HEAD --oneline
+```
+
+### Step 2: Clean build
+
+```bash
+rm -rf build
+cmake -B build -S . 2>&1
+cmake --build build -j$(nproc) 2>&1
+```
+
+Capture both stdout and stderr.
+
+### Step 3: Handle build failures
+
+If the build fails, read the error output carefully. Common issues:
+
+**Missing include / undefined reference:**
+- Check if the new `.c` file is in the right `file(GLOB ...)` directory
+- Check if the new `.h` file is included properly
+- Check if CMakeLists.txt needs updating (new target, new source file, new dependency)
+
+**Type errors / implicit declarations:**
+- Check function signatures match between `.h` and `.c`
+- Check struct field names and types
+
+**Linker errors:**
+- Check if all required libraries are linked in CMakeLists.txt
+- Check if all source files are included in the target
+
+Use the cmake-expert agent to diagnose and fix CMake issues.
+
+### Step 4: Run unit tests
+
+If build succeeds:
+```bash
+./build/tests
+```
+
+### Step 5: Handle test failures
+
+If tests fail:
+- Read the test output carefully
+- Check which test function failed and the assertion line
+- Read the test source file and the module being tested
+- Use the test-writer agent to investigate and fix
+
+### Step 6: Run integration tests (optional)
+
+```bash
+python3 test.py
+```
+
+This runs the integration + benchmark suite. It takes longer — only run if the user asks or if unit tests pass.
+
+### Step 7: Fix and commit
+
+If fixes were needed:
+```bash
+git add -A
+git commit -m "Fix build: <brief description of what was fixed>"
+git push
+```
+
+### Step 8: Report results
+
+Print a summary:
+
+```
+=== PR BUILD SUMMARY ===
+Branch: <branch-name>
+Build: [PASS/FAIL]
+Unit tests: [PASS/FAIL] (<passed>/<total>)
+Integration tests: [PASS/FAIL/SKIPPED]
+
+Fixes applied: <count>
+<list of fixes with commit hashes>
+```
+
+## Rules
+- DO edit source files and CMakeLists.txt to fix issues
+- DO commit and push fixes
+- Always build from clean state (rm -rf build)
+- Read error messages carefully before fixing
+- Don't change functionality — only fix build/test issues
+- Preserve existing code style when making fixes

.opencode/skills/pr-review/SKILL.md

@@ -0,0 +1,119 @@
+---
+name: pr-review
+description: Reviews a pull request for bugs, memory safety, thread safety, and style issues. Use when the user says "review PR", "review this PR", "review pull request", or wants a code review of changes.
+---
+
+# PR Review Skill
+
+Read-only code review of a pull request branch. Produces a report — does NOT edit files.
+
+## Workflow
+
+### Step 1: Identify the PR branch
+
+If the user specifies a PR number, check it out:
+```bash
+tea pr checkout <number>
+```
+
+If already on a PR branch, verify with:
+```bash
+git branch --show-current
+git log main..HEAD --oneline
+```
+
+### Step 2: Get changed files
+
+```bash
+git diff main --name-only -- '*.c' '*.h'
+```
+
+This gives the list of C source and header files changed in the PR.
+
+### Step 3: Read all changed files
+
+Use the Read tool to read every changed `.c` and `.h` file. Read full files — don't skip any.
+
+### Step 4: Review each file
+
+For each changed file, review for:
+
+**Memory Safety**
+- Every `malloc`/`calloc` has a matching `free` on all code paths (including error paths)
+- No use-after-free (pointers used after `*_destroy()` is called)
+- No double-free
+- Null checks after allocation before use
+- Correct buffer sizes (strlen + 1 for null terminators)
+- `Data` objects created/destroyed properly
+
+**Thread Safety**
+- Shared state accessed under mutex
+- No race conditions on queue operations
+- Condition variable signals under lock
+- No deadlock potential (consistent lock ordering)
+- `done` flags checked properly in consumer loops
+
+**Protocol Safety**
+- `send_n_data` / `receive_n_data` return values checked
+- Status codes validated before use
+- Config serialization handles partial reads
+
+**Logic Errors**
+- Off-by-one in loops/buffers
+- Incorrect size calculations
+- Wrong enum values or comparisons
+- Missing break statements in switch
+
+**Error Handling**
+- Resources freed on error paths (no leaks)
+- Functions return appropriate error values
+- Error messages are useful
+
+### Step 5: Categorize findings
+
+For each issue:
+1. **File:line** — exact location
+2. **Severity** — critical / warning / style
+3. **Category** — memory / thread / protocol / logic / error
+4. **Description** — what's wrong and how to fix it
+
+### Step 6: Output report
+
+Print a formatted summary:
+
+```
+=== PR REVIEW SUMMARY ===
+Branch: <branch-name>
+Files reviewed: <count>
+Issues found: <count>
+
+CRITICAL: <count>
+WARNING: <count>
+STYLE: <count>
+
+=== ISSUES ===
+[1] src/shared/compression.c:42 — CRITICAL (memory)
+    Potential leak: data returned from data_compress() not freed on error path
+    Fix: Add data_destroy(compressed) before return false
+
+...
+
+=== VERDICT ===
+[PASS] No critical issues found
+  — or —
+[FAIL] <N> critical issues must be fixed before merge
+```
+
+### Step 7: Optional PR comment
+
+If the user wants to post the review as a PR comment:
+```bash
+tea pr comment <number> --comment "<review report>"
+```
+
+## Rules
+- Do NOT edit any source files
+- Do NOT run builds or tests
+- Do NOT commit or push
+- Report ALL issues — don't filter or minimize
+- Be specific about line numbers and fix suggestions

CMakeLists.txt

@@ -32,7 +32,7 @@ add_executable(client ${CLIENT_SRCS} ${SHARED_SRCS})
 target_include_directories(client PRIVATE src/shared src/server src/client)
 target_link_libraries(client PRIVATE Threads::Threads ${ZSTD_LIBRARY})
 
-add_executable(tests ${TEST_SRCS} ${SHARED_SRCS})
+add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} src/client/scanner.c)
 target_include_directories(tests PRIVATE tests src/shared src/server src/client)
 target_link_libraries(tests PRIVATE Threads::Threads ${ZSTD_LIBRARY})
 

README.md

@@ -1,137 +1,206 @@
-# FastFileTransfer
+# FastSync
 
-A high-performance file synchronization system that implements a custom client-server protocol for efficient file transfer with compression and multithreading support.
+A high-performance file synchronization system with SSH and TCP transport, streaming zstd compression, multithreaded transfer, metadata preservation, and rsync-compatible CLI flags.
 
 ## Technical Overview
 
-FastFileTransfer is a C implementation of a file synchronization system that:
+1. **Dual transport**: custom TCP client-server or SSH subprocess (rsync-style `user@host:/path`)
-
+2. **Chunked file transfer**: files grouped into configurable-size chunks (default ~10 MB)
-1. Uses a custom TCP-based protocol for client-server communication
+3. **Streaming zstd compression** (levels 1–22) using `ZSTD_compressStream2`
-2. Implements chunked file transfer (10MB chunks by default)
+4. **Multithreading**: producer-consumer pipeline with thread-safe queues (scanner → loader → sender)
-3. Supports zstd compression with configurable levels (1-22)
+5. **Metadata preservation**: `mode`, `uid`, `gid`, `mtime` restored on disk when enabled
-4. Utilizes multithreading for parallel file processing
+6. **`sendfile()` zero-copy** on TCP (~2× faster on loopback)
-5. Implements producer-consumer patterns with thread-safe queues
+7. **SSH ControlMaster** for connection reuse across repeated invocations
-6. Provides both in-memory and disk-based storage options
+8. **`--delete`**: receiver removes files not present in sender manifest
+9. **`--exclude`**: glob-pattern filename filtering (`*`, `?`, no `/` crossing)
 
 ## System Architecture
 
-The system consists of two main components:
-
 ### Client
-- Scans source directories recursively
+- Recursively scans source directories (BFS), supports exclude patterns
-- Creates file chunks with configurable size (10MB default)
+- Groups files into chunks (configurable size)
-- Compresses data using zstd algorithm
+- Streaming zstd compression with configurable level
-- Sends files to server using custom protocol
+- Chunk serialization (compact binary format) or per-file transfer
-- Supports both single-threaded and multi-threaded operation
+- Manifests all sent paths when `--delete` is active
+- Sends via TCP `sendfile()` or SSH pipe
+- Optional progress display with throughput
 
 ### Server
-- Listens for client connections on port 8080
+- TCP mode: listens on port 8080; SSH mode: runs via `--stdio`
-- Receives files using the custom protocol
+- Receives and reassembles files
-- Decompresses received data
+- Decompresses (streaming zstd), deserializes, restores metadata
-- Stores files either in memory or on disk
+- Processes `STATUS_MANIFEST` for `--delete`: walks destination tree, removes extras
-- Implements thread pool for parallel processing
+- Thread pool for parallel processing
 
 ## Protocol Details
 
-The client-server communication uses the following status codes:
+### Status Codes
-- `STATUS_OK`: Operation successful
+| Code | Meaning |
-- `STATUS_ERROR`: Error occurred
+|------|---------|
-- `STATUS_FINISHED`: Transfer complete
+| `STATUS_OK` | Operation successful |
-- `STATUS_NEXT`: Ready for next chunk
+| `STATUS_ERROR` | Error occurred |
+| `STATUS_FINISHED` | Transfer complete |
+| `STATUS_NEXT` | Ready for next file (per-file mode) |
+| `STATUS_CHUNK` | Following data is a serialized chunk |
+| `STATUS_MANIFEST` | Following data is a file manifest (for `--delete`) |
 
-## Configuration Options
+### Wire Format — Metadata
+
+When `use_metadata` is enabled (`-M`), each file entry carries a 4-byte `present` flag followed by five fields (`mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`). When disabled globally, no metadata bytes are sent — zero wire overhead.
+
+### Transfer Flow
+```
+Config → (STATUS_NEXT | STATUS_CHUNK)* → [STATUS_MANIFEST] → STATUS_FINISHED → STATUS_OK
+```
+
+## Command-Line Arguments
 
-### Command Line Arguments
 | Argument | Description |
 |----------|-------------|
-| `-m` | Enable multithreading mode |
+| Positional | `<source> <dest>` — automatic SSH detection if dest contains `:` |
-| `-c [level]` | Enable compression with optional level (1-22, default: 5) |
+| `-c [level]` | Compression with optional level (1–22, default 5) |
-| `-s` | Enable chunk serialization |
+| `-z [level]` | Alias for `-c` |
+| `-a, --archive` | Archive mode: enables `-c -m -M` (no `-s`) |
+| `-m` | Multithreading mode |
+| `-s` | Chunk serialization (batch all files per chunk) |
+| `-f` | Sendfile zero-copy. Incompatible with `-c` / `-s`. TCP only. |
+| `-M, --preserve` | Preserve file metadata (mode, uid, gid, mtime) |
+| `-n, --dry-run` | Scan and print what would be transferred |
+| `-p <port>` | SSH port (default: 22) |
+| `--progress` | Show real-time transfer speed |
+| `--delete` | Delete files on receiver not present in source |
+| `--exclude <pattern>` | Exclude files matching glob pattern (repeatable) |
+| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
+| `--source-dir <path>` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) |
+| `--dest-dir <path>` | Server destination directory (overrides `FASTSYNC_DEST_DIR`) |
+| `--save-to-disk` | Write received files to disk |
+| `--server-host <ip>` | Server IP address (default: `127.0.0.1`) |
+| `--server-port <n>` | Server port (default: `8080`) |
+| `-v, --verbose` | Enable debug logging |
 
-### Environment Variables
+## Environment Variables
-| Variable | Description | Default |
+
-|----------|-------------|---------|
+| Variable | Default | Description |
-| `FASTSYNC_SOURCE_DIR` | Source directory for files | Current user's documents directory |
+|----------|---------|-------------|
-| `FASTSYNC_DEST_DIR` | Destination directory | `./data_copied` |
+| `FASTSYNC_SOURCE_DIR` | — | Source directory fallback |
-| `FASTSYNC_SERVER_IP` | Server IP address | `127.0.0.1` |
+| `FASTSYNC_DEST_DIR` | — | Destination directory fallback |
-| `FASTSYNC_SERVER_PORT` | Server port | `8080` |
+| `FASTSYNC_SAVE_TO_DISK` | `false` | Disk persistence fallback |
-| `FASTSYNC_SAVE_TO_DISK` | Save to disk (true/false) | `false` |
 
 ## Implementation Details
 
 ### Data Structures
-
+1. **Chunk** — collection of files (~10 MB total by default)
-1. **Chunk**: Collection of files (default 10MB total size)
+2. **File** — path, content (`Data`), optional `FileMetadata` pointer
-2. **File**: File metadata with path and content
+3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`
-3. **FileReceive**: Received file data structure
+4. **Config** — runtime parameters (transported over wire)
-4. **Config**: Configuration parameters structure
+5. **Queue** — thread-safe bounded queue with condition variables
-5. **Queue**: Thread-safe queue implementation using condition variables
+6. **DirectoryScanner** — recursive BFS traversal with exclude pattern support
 
 ### Key Algorithms
-
+1. **File scanning** — BFS directory traversal; each entry matched against exclude patterns
-1. **File Scanning**: Recursive directory traversal with BFS
+2. **Chunking** — files accumulated until `chunk_size` threshold, then flushed
-2. **Chunking**: Files grouped into chunks with size limit
+3. **Compression** — streaming zstd via `ZSTD_compressStream2` / `ZSTD_decompressStream`
-3. **Compression**: zstd compression with configurable levels
+4. **Network protocol** — status-code-driven exchange with metadata packing
-4. **Network Protocol**: Custom TCP-based protocol with status codes
+5. **Metadata restoration** — `chmod()`, `chown()`, `utimensat()` on the receiving side
-5. **Thread Synchronization**: Condition variables and mutexes for thread coordination
+6. **`--delete`** — sender tracks all sent paths; receiver walks destination tree and removes unlisted files/directories
+7. **SSH transport** — `socketpair()` + `fork()` + `execvp("ssh", ...)` with `ControlMaster` and port support
 
 ## Build Requirements
 
-- C11 compatible compiler
+- C11 compiler
-- CMake 4.1 or later
+- CMake 4.1+
-- zstd library
+- zstd library (≥ 1.4.0 for streaming API)
-- pthread support
+- pthreads
+- SSH client (for SSH transport)
 
 ## Building
 
 ```bash
-mkdir -p build && cd build
+cmake -B build -S . && cmake --build build -j$(nproc)
-cmake ..
-make
 ```
 
 ## Running
 
-### Server
+### Server (TCP mode)
 ```bash
 ./build/server
 ```
 
-### Client
+### Client — SSH (rsync-style)
 ```bash
-# Basic usage
+./build/client /path/to/send user@host:/path/to/receive
-./build/client -m -c 10
 ```
 
+### Client — TCP
+```bash
+./build/client --source-dir /path/to/send --dest-dir /path/to/receive --save-to-disk
+```
+
+### Common Options
+```bash
+# Archive mode (compression + multithreading + metadata)
+./build/client -a /path/to/send user@host:/path
+
+# Dry run
+./build/client -n /path/to/send /path/to/receive
+
+# With progress and custom chunk size
+./build/client --progress --chunk-size 2097152 /src user@host:/dst
+
+# Exclude temporary files + delete extras on receiver
+./build/client --exclude "*.tmp" --exclude "*.o" --delete /src user@host:/dst
+
+# All features
+./build/client -a --progress --chunk-size 5242880 --exclude "*.log" --delete /src /dst
+```
+
+### Server via SSH
+Place the `fastsync-server` binary in the remote `$PATH`. The client runs `ssh user@host fastsync-server --stdio` automatically when an SSH-style destination is given.
+
 ## Testing
 
-The project includes comprehensive unit tests for core functionality:
-
 ```bash
+# Unit tests (7 suites)
 ./build/tests
+
+# Integration + benchmark suite
+python3 test.py
 ```
 
-## Code Organization
+The benchmark prints throughput metrics, best configuration, and speedup vs rsync.
-
-```
-src/
-  client/    # Client implementation
-  server/    # Server implementation
-  shared/    # Shared data structures and utilities
-tests/      # Unit tests
-```
 
 ## Performance Considerations
 
-1. Chunk size (10MB default) affects memory usage and transfer efficiency
+1. Chunk size (~10 MB default) balances memory and transfer efficiency
-2. Compression level (1-22) trades CPU usage for space savings
+2. Compression level trades CPU for bandwidth
-3. Multithreading improves performance on multi-core systems
+3. `sendfile()` bypasses userspace — ~2× faster on localhost for large files
-4. Thread-safe queues minimize contention between producer/consumer threads
+4. Multithreading scales with core count
+5. Metadata transfer adds negligible overhead (~24 bytes per file when enabled)
+6. SSH socketpair buffer set to 1 MB for improved pipe throughput
+7. SSH ControlMaster reuses connections across repeated invocations
 
-## Extensibility
+## Benchmark Results
 
-The system is designed with clear interfaces that allow for:
+25 MB of mixed file sizes over `localhost` with disk I/O throttled (reads ≤ 15 MB/s, writes ≤ 10 MB/s) and network emulation via `tc netem`. Each test was run 3×; the median is reported below.
-1. Additional compression algorithms
+
-2. Different transport protocols
+### LAN (1000 Mbit, 20 ms ±1 ms, 0.1% loss)
-3. Custom storage backends
+
-4. Extended metadata support
+| Configuration | Time | vs rsync (archive) | vs rsync (compress) |
+|---|---|---|---|
+| **Best: `-m -c`** | **0.20 s** | **11.2× faster** | **3.6× faster** |
+| Compression (`-c`) | 0.31 s | 7.3× faster | 2.3× faster |
+| Standard | 1.27 s | 1.8× faster | — |
+| rsync (archive) | 2.27 s | — | — |
+| rsync (archive + compress) | 0.72 s | — | — |
+
+### WAN (100 Mbit, 50 ms ±10 ms, 1% loss)
+
+| Configuration | Time | vs rsync (archive) | vs rsync (compress) |
+|---|---|---|---|
+| **Best: `-m -c`** | **0.39 s** | **44.8× faster** | **3.8× faster** |
+| Compression (`-c`) | 0.64 s | 27.3× faster | 2.3× faster |
+| Standard | 7.12 s | 2.4× faster | — |
+| rsync (archive) | 17.44 s | — | — |
+| rsync (archive + compress) | 1.47 s | — | — |
+
+Compression reduces the data on the wire enough that the transfer becomes latency-bound rather than bandwidth-bound. On WAN, the best configuration runs 10.8× faster than the theoretical limit for uncompressed data, since zstd shrinks the 25 MB payload to a fraction of its original size over the wire.

opencode.json

@@ -1,12 +1,10 @@
 {
-	"$schema": "https://opencode.ai/config.json",
+  "$schema": "https://opencode.ai/config.json",
 	"permission": {
 		"bash": {
 			"*": "allow",
 			"git push origin main": "deny",
-			"git push origin master": "deny",
+			"git push main": "ask"
-			"git push *": "ask",
-			"git commit *": "ask"
 		}
 	}
 }

shell.nix

@@ -1,8 +1,25 @@
-{ pkgs }: {
+{
-  shellHook = ''
+  pkgs ? import <nixpkgs> { },
-    echo "Setting up environment with gh CLI tool"
+}:
-  '';
+
-  buildInputs = [
+pkgs.mkShell {
-    pkgs.gh
+  nativeBuildInputs = with pkgs; [
+    gcc
+    cmake
+    gnumake
+    pkg-config
+    tea
   ];
-}
+
+  buildInputs = with pkgs; [
+    zstd
+  ];
+
+  NIX_ENFORCE_PURITY = 0;
+
+  shellHook = ''
+    export NIX_ENFORCE_PURITY=0
+    cmake -B build
+    export PATH="$PWD/build:$PATH"
+  '';
+}

src/client/client.c

@@ -1,212 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <threads.h>
-#include <unistd.h>
-
-#include "chunk.h"
-#include "config.h"
-#include "data.h"
-#include "file.h"
-#include "log.h"
-#include "multiprocessing.h"
-#include "queue.h"
-#include "scanner.h"
-#include "socket.h"
-#include "utils.h"
-#include <dirent.h>
-
-int send_chunk(Client *client, Chunk *chunk, Config *config) {
-  if (config->use_compression && config->use_chunk_serialization) {
-    Data *data = chunk_compress(chunk, config->compression_level);
-    send_data(client->file_descriptor, data->data, data->size);
-  } else {
-    for (int i = 0; i < chunk->element_count; i++) {
-      send_status(client->file_descriptor, STATUS_NEXT);
-      File *file = chunk->items[i];
-      if (config->use_compression) {
-        Data *compressed_data =
-            data_compress(file->data, config->compression_level);
-        data_destroy(file->data);
-        file->data = compressed_data;
-      }
-      file_send_single_calls(file, client->file_descriptor);
-    }
-  }
-  return 0;
-}
-
-int scan_directory_multithreaded(void *pipeline_context) {
-  PipelineContextSender *context = (PipelineContextSender *)pipeline_context;
-  mtx_lock(&context->mutex_scanner);
-  DirectoryScanner *scanner =
-      directory_scanner_create(context->config->send_directory);
-  mtx_unlock(&context->mutex_scanner);
-
-  Chunk *current_chunk;
-  while ((current_chunk = directory_scanner_next(scanner)) != NULL)
-    queue_enqueue_multithreaded(context->queue_scanner, current_chunk,
-                                &context->mutex_scanner,
-                                &context->condition_not_empty_scanner,
-                                &context->condition_not_full_scanner);
-  mtx_lock(&context->mutex_scanner);
-  context->scanner_done = true;
-  cnd_signal(&context->condition_not_empty_scanner);
-  mtx_unlock(&context->mutex_scanner);
-
-  directory_scanner_destroy(scanner);
-  return thrd_success;
-}
-
-int load_files_multithreaded(void *pipeline_context) {
-  PipelineContextSender *context = (PipelineContextSender *)pipeline_context;
-  while (true) {
-    Chunk *chunk = queue_dequeue_multithreaded(
-        context->queue_scanner, &context->mutex_scanner,
-        &context->condition_not_empty_scanner,
-        &context->condition_not_full_scanner, &context->scanner_done);
-    if (chunk == NULL) {
-      mtx_lock(&context->mutex_loader);
-      context->loader_done = true;
-      cnd_signal(&context->condition_not_empty_loader);
-      mtx_unlock(&context->mutex_loader);
-      return thrd_success;
-    }
-    for (int i = 0; i < chunk->element_count; i++)
-      file_load_data(chunk->items[i]);
-    queue_enqueue_multithreaded(context->queue_loader, chunk,
-                                &context->mutex_loader,
-                                &context->condition_not_empty_loader,
-                                &context->condition_not_full_loader);
-  }
-}
-
-int send_chunks_multithreaded(void *pipeline_context) {
-  PipelineContextSender *context = (PipelineContextSender *)pipeline_context;
-  Client *client = client_create();
-  const char *env_ip = getenv("FASTSYNC_SERVER_IP");
-  const char *ip = env_ip ? env_ip : "127.0.0.1";
-  const char *env_port = getenv("FASTSYNC_SERVER_PORT");
-  int port = env_port ? atoi(env_port) : 8080;
-  client_connect(client, (char *)ip, port);
-  config_send(client->file_descriptor, context->config);
-
-  while (true) {
-    Chunk *current_chunk = queue_dequeue_multithreaded(
-        context->queue_loader, &context->mutex_loader,
-        &context->condition_not_empty_loader,
-        &context->condition_not_full_loader, &context->loader_done);
-    if (current_chunk == NULL) {
-      send_status(client->file_descriptor, STATUS_FINISHED);
-      client_disconnect(client);
-      client_delete(client);
-      return thrd_success;
-    }
-    if (send_chunk(client, current_chunk, context->config) != 0) {
-      perror("Something unexpected happend while sending the chunk");
-      exit(EXIT_FAILURE);
-    }
-    chunk_destroy(current_chunk);
-  }
-}
-
-int send_files(Config *config) {
-  Client *client = client_create();
-  const char *env_ip = getenv("FASTSYNC_SERVER_IP");
-  const char *ip = env_ip ? env_ip : "127.0.0.1";
-  const char *env_port = getenv("FASTSYNC_SERVER_PORT");
-  int port = env_port ? atoi(env_port) : 8080;
-  client_connect(client, (char *)ip, port);
-  config_send(client->file_descriptor, config);
-  DirectoryScanner *scanner = directory_scanner_create(config->send_directory);
-  Chunk *current_chunk;
-  while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
-    for (int i = 0; i < current_chunk->element_count; i++)
-      file_load_data(current_chunk->items[i]);
-    send_chunk(client, current_chunk, config);
-    chunk_destroy(current_chunk);
-  }
-  send_status(client->file_descriptor, STATUS_FINISHED);
-  if (receive_status(client->file_descriptor) != STATUS_OK)
-    return -1;
-  directory_scanner_destroy(scanner);
-  client_disconnect(client);
-  client_delete(client);
-  return 0;
-}
-
-int send_files_multithreaded(Config *config) {
-  PipelineContextSender *context =
-      pipeline_context_sender_create(config, queue_create(100, chunk_destroy),
-                                     queue_create(100, chunk_destroy));
-
-  thrd_t scanner, loader, sender;
-  if (thrd_create(&scanner, scan_directory_multithreaded, context) !=
-          thrd_success ||
-      thrd_create(&loader, load_files_multithreaded, context) != thrd_success ||
-      thrd_create(&sender, send_chunks_multithreaded, context) !=
-          thrd_success) {
-    perror("Error creating threads.\n");
-    return 1;
-  }
-
-  thrd_join(scanner, NULL);
-  thrd_join(loader, NULL);
-  thrd_join(sender, NULL);
-
-  pipeline_context_sender_destroy(context);
-  return 0;
-}
-
-void handle_arg(char *argument_given, char *argument_to_set, bool *result,
-                char *message) {
-  if (strcmp(argument_given, argument_to_set) == 0) {
-    *result = true;
-    log_message(LOG_LEVEL_INFO, message);
-  }
-}
-int main(int argc, char *argv[]) {
-  const char *env_source = getenv("FASTSYNC_SOURCE_DIR");
-  const char *env_dest = getenv("FASTSYNC_DEST_DIR");
-  const char *env_save = getenv("FASTSYNC_SAVE_TO_DISK");
-
-  char *source_dir =
-      env_source ? str_dup((char *)env_source)
-                 : str_dup("/home/taptap/Nextcloud/Uni/moodle/B. Schnor： "
-                           "Konzepte Paralleler Programmierung, SoSe 2026");
-  char *dest_dir =
-      env_dest ? str_dup((char *)env_dest) : str_dup("./data_copied");
-  bool save_to_disk = false;
-  if (env_save &&
-      (strcmp(env_save, "true") == 0 || strcmp(env_save, "1") == 0)) {
-    save_to_disk = true;
-  }
-
-  Config *config = config_create(str_dup("1.0.0"), source_dir, dest_dir,
-                                 save_to_disk, false, false, false, 5, 20);
-  for (int i = 1; i < argc; i++) {
-    if (strcmp(argv[i], "-c") == 0) {
-      config->use_compression = true;
-      log_message(LOG_LEVEL_INFO, "Enabled Compression");
-
-      if (i + 1 < argc) {
-        char *end_ptr;
-        int level = strtol(argv[i + 1], &end_ptr, 10);
-        if (*end_ptr == '\0') {
-          config->compression_level = level;
-          log_message(LOG_LEVEL_INFO, "Set Compression level to %d",
-                      config->compression_level);
-        }
-      }
-    } else {
-      handle_arg(argv[i], "-m", &config->use_multithreading,
-                 "Enabled Multithreading");
-      handle_arg(argv[i], "-s", &config->use_chunk_serialization,
-                 "Enabled Chunk Serialization");
-    }
-  }
-
-  if (config->use_multithreading)
-    return send_files_multithreaded(config);
-  return send_files(config);
-}

src/client/client_cli.c

@@ -0,0 +1,221 @@
+#include "client_send.h"
+#include "config.h"
+#include "log.h"
+#include "protocol.h"
+#include "utils.h"
+#include <errno.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+char *server_host = "127.0.0.1";
+int server_port = 8080;
+
+static void print_usage(void) {
+  printf("Usage:\n");
+  printf("  fastsync [options] <source> <destination>\n");
+  printf("  fastsync [options] --source-dir <src> --dest-dir <dst>\n");
+  printf("\n");
+  printf("Destination formats:\n");
+  printf("  user@host:/path     SSH transport (rsync-style)\n");
+  printf("  host:/path          SSH transport (current user)\n");
+  printf("  /local/path         TCP transport (requires server on localhost:8080)\n");
+  printf("\n");
+  printf("Options:\n");
+  printf("  -c [level]          Enable compression (level 1-22, default 5)\n");
+  printf("  -z [level]          Alias for -c\n");
+  printf("  -a, --archive       Archive mode (-c -m -M)\n");
+  printf("  -n, --dry-run       Show what would be transferred\n");
+  printf("  -p <port>           SSH port (default: 22)\n");
+  printf("  --progress          Show transfer progress\n");
+  printf("  --delete            Delete files on receiver not in source\n");
+  printf("  --exclude <pattern> Exclude files matching pattern\n");
+  printf("  --include <pattern> Only include files matching pattern\n");
+  printf("  --max-size <n>      Skip files larger than n bytes\n");
+  printf("  --min-size <n>      Skip files smaller than n bytes\n");
+  printf("  --incremental       Skip files unchanged since last transfer\n");
+  printf("  -m                  Enable multithreading\n");
+  printf("  -s                  Enable chunk serialization\n");
+  printf("  -f                  Enable sendfile (TCP only, not with -c or -s)\n");
+  printf("  -v, --verbose       Enable debug logging\n");
+  printf("  -M, --preserve      Preserve file metadata\n");
+  printf("  --chunk-size <n>    Chunk size in bytes (default: %d)\n", DEFAULT_CHUNK_SIZE);
+  printf("  --source-dir <path> Source directory\n");
+  printf("  --dest-dir <path>   Destination directory\n");
+  printf("  --save-to-disk      Write received files to disk\n");
+  printf("  --server-host <ip>  Server IP address (default: 127.0.0.1)\n");
+  printf("  --server-port <n>   Server port (default: 8080)\n");
+  printf("  --bwlimit <KB/s>    Bandwidth limit in kilobytes per second\n");
+  printf("  --help              Show this help\n");
+}
+
+int main(int argc, char *argv[]) {
+  const char *env_source = getenv("FASTSYNC_SOURCE_DIR");
+  const char *env_dest = getenv("FASTSYNC_DEST_DIR");
+  const char *env_save = getenv("FASTSYNC_SAVE_TO_DISK");
+
+  bool save_to_disk = false;
+  if (env_save &&
+      (strcmp(env_save, "true") == 0 || strcmp(env_save, "1") == 0)) {
+    save_to_disk = true;
+  }
+
+  Config *config = config_create(str_dup(PROTOCOL_VERSION), NULL, NULL,
+                                 save_to_disk, false, false, false, false, 5, false, 0);
+
+  int positional_args[2];
+  int positional_count = 0;
+
+  for (int i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "--help") == 0) {
+      print_usage();
+      return 0;
+    } else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--archive") == 0) {
+      config->use_compression = true;
+      config->use_multithreading = true;
+      config->use_metadata = true;
+      log_message(LOG_LEVEL_INFO, "Enabled archive mode (-c -m -M)");
+    } else if (strcmp(argv[i], "-n") == 0 || strcmp(argv[i], "--dry-run") == 0) {
+      config->dry_run = true;
+    } else if (strcmp(argv[i], "-p") == 0 && i + 1 < argc) {
+      config->ssh_port = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "--delete") == 0) {
+      config->use_delete = true;
+    } else if (strcmp(argv[i], "--exclude") == 0 && i + 1 < argc) {
+      int idx = config->exclude_count++;
+      config->exclude_patterns = realloc(config->exclude_patterns, config->exclude_count * sizeof(char *));
+      config->exclude_patterns[idx] = str_dup(argv[++i]);
+    } else if (strcmp(argv[i], "--include") == 0 && i + 1 < argc) {
+      int idx = config->include_count++;
+      config->include_patterns = realloc(config->include_patterns, config->include_count * sizeof(char *));
+      config->include_patterns[idx] = str_dup(argv[++i]);
+    } else if (strcmp(argv[i], "--max-size") == 0 && i + 1 < argc) {
+      config->max_size = strtoull(argv[++i], NULL, 10);
+    } else if (strcmp(argv[i], "--min-size") == 0 && i + 1 < argc) {
+      config->min_size = strtoull(argv[++i], NULL, 10);
+    } else if (strcmp(argv[i], "--incremental") == 0) {
+      config->use_incremental = true;
+    } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) {
+      config->use_compression = true;
+      log_message(LOG_LEVEL_INFO, "Enabled Compression");
+      if (i + 1 < argc) {
+        char *end_ptr;
+        int level = strtol(argv[i + 1], &end_ptr, 10);
+        if (*end_ptr == '\0') {
+          config->compression_level = level;
+          log_message(LOG_LEVEL_INFO, "Set Compression level to %d",
+                      config->compression_level);
+          i++;
+        }
+      }
+    } else if (strcmp(argv[i], "--source-dir") == 0 && i + 1 < argc) {
+      free(config->send_directory);
+      config->send_directory = str_dup(argv[++i]);
+    } else if (strcmp(argv[i], "--dest-dir") == 0 && i + 1 < argc) {
+      free(config->receive_root_directory);
+      config->receive_root_directory = str_dup(argv[++i]);
+    } else if (strcmp(argv[i], "--save-to-disk") == 0) {
+      config->save_to_disk = true;
+    } else if (strcmp(argv[i], "-M") == 0 || strcmp(argv[i], "--preserve") == 0) {
+      config->use_metadata = true;
+      log_message(LOG_LEVEL_INFO, "Enabled metadata preservation");
+    } else if (strcmp(argv[i], "-f") == 0 || strcmp(argv[i], "--sendfile") == 0) {
+      config->use_sendfile = true;
+      log_message(LOG_LEVEL_INFO, "Enabled sendfile");
+    } else if (strcmp(argv[i], "-m") == 0) {
+      config->use_multithreading = true;
+      log_message(LOG_LEVEL_INFO, "Enabled Multithreading");
+    } else if (strcmp(argv[i], "-s") == 0) {
+      config->use_chunk_serialization = true;
+      log_message(LOG_LEVEL_INFO, "Enabled Chunk Serialization");
+    } else if (strcmp(argv[i], "--server-host") == 0 && i + 1 < argc) {
+      free(server_host);
+      server_host = str_dup(argv[++i]);
+    } else if (strcmp(argv[i], "--server-port") == 0 && i + 1 < argc) {
+      server_port = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "--bwlimit") == 0 && i + 1 < argc) {
+      char *end;
+      errno = 0;
+      unsigned long long kbps = strtoull(argv[++i], &end, 10);
+      if (errno != 0 || *end != '\0' || kbps == 0) {
+        fprintf(stderr, "Error: --bwlimit must be a positive integer\n");
+        return 1;
+      }
+      if (kbps > ULLONG_MAX / 1024) {
+        fprintf(stderr, "Error: --bwlimit value too large\n");
+        return 1;
+      }
+      io_set_bwlimit(kbps * 1024);
+      log_message(LOG_LEVEL_INFO, "Set bandwidth limit to %llu KB/s", kbps);
+    } else if (strcmp(argv[i], "--progress") == 0) {
+      config->show_progress = true;
+    } else if (strcmp(argv[i], "--chunk-size") == 0 && i + 1 < argc) {
+      unsigned long long val = strtoull(argv[++i], NULL, 10);
+      if (val > 0)
+        config->chunk_size = val;
+    } else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
+      set_log_level(LOG_LEVEL_DEBUG);
+    } else if (argv[i][0] == '-') {
+      fprintf(stderr, "Unknown option: %s\n", argv[i]);
+      print_usage();
+      return 1;
+    } else {
+      if (positional_count < 2)
+        positional_args[positional_count++] = i;
+      else {
+        fprintf(stderr, "Unexpected argument: %s\n", argv[i]);
+        print_usage();
+        return 1;
+      }
+    }
+  }
+
+  if (positional_count == 2) {
+    free(config->send_directory);
+    free(config->receive_root_directory);
+    config->send_directory = str_dup(argv[positional_args[0]]);
+    config->receive_root_directory = str_dup(argv[positional_args[1]]);
+    config->save_to_disk = true;
+
+    config_parse_ssh_dest(config);
+  } else if (positional_count == 1) {
+    fprintf(stderr, "Error: missing destination argument\n");
+    print_usage();
+    return 1;
+  } else {
+    if (!config->send_directory && env_source)
+      config->send_directory = str_dup((char *)env_source);
+    if (!config->receive_root_directory && env_dest)
+      config->receive_root_directory = str_dup((char *)env_dest);
+  }
+
+  if (!config->send_directory || !config->receive_root_directory) {
+    fprintf(stderr, "Error: source and destination directories are required\n");
+    print_usage();
+    return 1;
+  }
+  if (config->use_sendfile && (config->use_chunk_serialization || config->use_compression)) {
+    fprintf(stderr, "Error: -f/--sendfile cannot be combined with -c (compression) or -s (chunk serialization)\n");
+    return 1;
+  }
+
+  if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
+    fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
+    return 1;
+  }
+
+  if (config->use_incremental && config->use_chunk_serialization) {
+    fprintf(stderr, "Error: --incremental is not supported with -s (chunk serialization)\n");
+    return 1;
+  }
+
+  if (config->use_incremental && !config->use_metadata) {
+    log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --incremental");
+    config->use_metadata = true;
+  }
+
+  if (config->use_multithreading)
+    return send_files_multithreaded(config);
+  return send_files(config);
+}

src/client/client_send.c

@@ -0,0 +1,384 @@
+#include "client_send.h"
+#include "array_list.h"
+#include "chunk.h"
+#include "config.h"
+#include "data.h"
+#include "file.h"
+#include "log.h"
+#include "multiprocessing.h"
+#include "protocol.h"
+#include "queue.h"
+#include "scanner.h"
+#include "transport_tcp.h"
+#include "transport_ssh.h"
+#include "utils.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <threads.h>
+#include <time.h>
+
+static int incremental_check(Client *client, File *file) {
+  if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1;
+  if (!send_str(client->file_descriptor, file->path)) return -1;
+  unsigned long long fsize = file->data->size;
+  long long mtime = file->metadata ? file->metadata->mtime_sec : 0;
+  if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1;
+  if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1;
+  Status s;
+  if (!receive_status(client->file_descriptor, &s)) return -1;
+  if (s == STATUS_ERROR) {
+    log_message(LOG_LEVEL_ERROR, "Server reported error for file");
+    return -1;
+  }
+  if (s == STATUS_OK) return 1;
+  if (s != STATUS_NEXT) {
+    log_message(LOG_LEVEL_ERROR, "Unexpected server status");
+    return -1;
+  }
+  return 0;
+}
+
+int send_chunk(Client *client, Chunk *chunk, Config *config) {
+  if (config->use_chunk_serialization) {
+    if (!send_status(client->file_descriptor, STATUS_CHUNK)) return -1;
+    Data *data;
+    if (config->use_compression) {
+      data = chunk_compress(chunk, config->compression_level, config->use_metadata);
+    } else {
+      data = chunk_serialize(chunk, config->use_metadata);
+    }
+    if (data == NULL) return -1;
+    if (!send_data(client->file_descriptor, data)) { data_destroy(data); return -1; }
+    data_destroy(data);
+  } else if (config->use_sendfile && !config->use_compression) {
+    for (int i = 0; i < chunk->element_count; i++) {
+      if (config->use_incremental) {
+        int rc = incremental_check(client, chunk->items[i]);
+        if (rc < 0) return -1;
+        if (rc > 0) continue;
+        if (!file_send_sendfile_no_path(chunk->items[i], client->file_descriptor, config->use_metadata))
+          return -1;
+      } else {
+        if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
+        if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata))
+          return -1;
+      }
+    }
+  } else {
+    for (int i = 0; i < chunk->element_count; i++) {
+      if (config->use_incremental) {
+        int rc = incremental_check(client, chunk->items[i]);
+        if (rc < 0) return -1;
+        if (rc > 0) continue;
+        if (!file_send_single_calls_no_path(chunk->items[i], client->file_descriptor,
+                                       config->use_metadata,
+                                       config->use_compression ? config->compression_level : 0))
+          return -1;
+      } else {
+        if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
+        if (!file_send_single_calls(chunk->items[i], client->file_descriptor,
+                               config->use_metadata,
+                               config->use_compression ? config->compression_level : 0))
+          return -1;
+      }
+    }
+  }
+  return 0;
+}
+
+static int send_chunks_multithreaded(void *pipeline_context) {
+  PipelineContextSender *context = (PipelineContextSender *)pipeline_context;
+  Client *client;
+  if (context->config->transport == TRANSPORT_SSH) {
+    if (context->config->use_sendfile) {
+      fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
+      return 1;
+    }
+    client = client_connect_ssh(context->config->ssh_destination, context->config->ssh_port);
+  } else {
+    client = client_create();
+    if (!client || !client_connect(client, server_host, server_port)) {
+      if (client) client_delete(client);
+      fprintf(stderr, "Error: could not connect to server\n");
+      return thrd_error;
+    }
+  }
+  if (!config_send(client->file_descriptor, context->config)) {
+    client_disconnect(client);
+    client_delete(client);
+    return thrd_error;
+  }
+
+  while (true) {
+    Chunk *current_chunk = queue_dequeue_multithreaded(
+        context->queue_loader, &context->mutex_loader,
+        &context->condition_not_empty_loader,
+        &context->condition_not_full_loader, &context->loader_done);
+    if (current_chunk == NULL) {
+      if (context->config->use_delete) {
+        send_status(client->file_descriptor, STATUS_MANIFEST);
+        send_int(client->file_descriptor, context->manifest->size);
+        for (int i = 0; i < context->manifest->size; i++)
+          send_str(client->file_descriptor,
+                   (char *)context->manifest->items[i]);
+      }
+      send_status(client->file_descriptor, STATUS_FINISHED);
+      Status s;
+      int ok = receive_status(client->file_descriptor, &s) && s == STATUS_OK;
+      client_disconnect(client);
+      client_delete(client);
+      return ok ? thrd_success : thrd_error;
+    }
+    if (send_chunk(client, current_chunk, context->config) != 0) {
+      fprintf(stderr, "Error: unexpected error while sending chunk\n");
+      client_disconnect(client);
+      client_delete(client);
+      return thrd_error;
+    }
+    chunk_destroy(current_chunk);
+  }
+}
+
+static int scan_directory_multithreaded(void *pipeline_context) {
+  PipelineContextSender *context = (PipelineContextSender *)pipeline_context;
+  mtx_lock(&context->mutex_scanner);
+  DirectoryScanner *scanner = directory_scanner_create(
+      context->config->send_directory, context->config->use_metadata,
+      context->config->chunk_size, context->config->exclude_patterns,
+      context->config->exclude_count, context->config->include_patterns,
+      context->config->include_count, context->config->max_size,
+      context->config->min_size);
+  mtx_unlock(&context->mutex_scanner);
+
+  Chunk *current_chunk;
+  while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
+    if (context->config->use_delete) {
+      mtx_lock(&context->mutex_scanner);
+      for (int i = 0; i < current_chunk->element_count; i++) {
+        const char *p = current_chunk->items[i]->path;
+        if (*p == '/') p++;
+        array_list_add(context->manifest, str_dup(p));
+      }
+      mtx_unlock(&context->mutex_scanner);
+    }
+    queue_enqueue_multithreaded(context->queue_scanner, current_chunk,
+                                &context->mutex_scanner,
+                                &context->condition_not_empty_scanner,
+                                &context->condition_not_full_scanner);
+  }
+  mtx_lock(&context->mutex_scanner);
+  context->scanner_done = true;
+  cnd_signal(&context->condition_not_empty_scanner);
+  mtx_unlock(&context->mutex_scanner);
+
+  directory_scanner_destroy(scanner);
+  return thrd_success;
+}
+
+static int load_files_multithreaded(void *pipeline_context) {
+  PipelineContextSender *context = (PipelineContextSender *)pipeline_context;
+  while (true) {
+    Chunk *chunk = queue_dequeue_multithreaded(
+        context->queue_scanner, &context->mutex_scanner,
+        &context->condition_not_empty_scanner,
+        &context->condition_not_full_scanner, &context->scanner_done);
+    if (chunk == NULL) {
+      mtx_lock(&context->mutex_loader);
+      context->loader_done = true;
+      cnd_signal(&context->condition_not_empty_loader);
+      mtx_unlock(&context->mutex_loader);
+      return thrd_success;
+    }
+    if (!context->config->use_sendfile) {
+      for (int i = 0; i < chunk->element_count; i++) {
+        if (!file_load_data(chunk->items[i])) {
+          log_message(LOG_LEVEL_ERROR, "Failed to load file data, skipping");
+          file_destroy(chunk->items[i]);
+          chunk->items[i] = NULL;
+        }
+      }
+    }
+    queue_enqueue_multithreaded(context->queue_loader, chunk,
+                                &context->mutex_loader,
+                                &context->condition_not_empty_loader,
+                                &context->condition_not_full_loader);
+  }
+}
+
+int send_files(Config *config) {
+  if (config->dry_run) {
+    DirectoryScanner *scanner = directory_scanner_create(
+        config->send_directory, config->use_metadata, config->chunk_size,
+        config->exclude_patterns, config->exclude_count,
+        config->include_patterns, config->include_count,
+        config->max_size, config->min_size);
+    Chunk *chunk;
+    int file_count = 0;
+    unsigned long long total_bytes = 0;
+    printf("Dry run: files to be transferred\n");
+    while ((chunk = directory_scanner_next(scanner)) != NULL) {
+      for (int i = 0; i < chunk->element_count; i++) {
+        printf("  %s (%zu bytes)\n", chunk->items[i]->path,
+               chunk->items[i]->data->size);
+        total_bytes += chunk->items[i]->data->size;
+        file_count++;
+      }
+      chunk_destroy(chunk);
+    }
+    directory_scanner_destroy(scanner);
+    printf("Total: %d files, %.1f MB\n", file_count,
+           total_bytes / 1048576.0);
+    return 0;
+  }
+
+  Client *client;
+  if (config->transport == TRANSPORT_SSH) {
+    if (config->use_sendfile) {
+      fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
+      return 1;
+    }
+    client = client_connect_ssh(config->ssh_destination, config->ssh_port);
+    if (!client) return 1;
+  } else {
+    client = client_create();
+    if (!client || !client_connect(client, server_host, server_port)) {
+      if (client) client_delete(client);
+      fprintf(stderr, "Error: could not connect to server\n");
+      return 1;
+    }
+  }
+  if (!config_send(client->file_descriptor, config)) {
+    client_disconnect(client);
+    client_delete(client);
+    return 1;
+  }
+  DirectoryScanner *scanner = directory_scanner_create(
+      config->send_directory, config->use_metadata, config->chunk_size,
+      config->exclude_patterns, config->exclude_count,
+      config->include_patterns, config->include_count,
+      config->max_size, config->min_size);
+  Chunk *current_chunk;
+  unsigned long long total_bytes = 0;
+  time_t last_progress = 0;
+  time_t start = time(NULL);
+  ArrayList *manifest = config->use_delete ? array_list_create(free) : NULL;
+  while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
+    unsigned long long chunk_bytes = 0;
+    for (int i = 0; i < current_chunk->element_count; i++) {
+      chunk_bytes += current_chunk->items[i]->data->size;
+      if (manifest) {
+        const char *p = current_chunk->items[i]->path;
+        if (*p == '/') p++;
+        array_list_add(manifest, str_dup(p));
+      }
+    }
+    if (!config->use_sendfile) {
+      for (int i = 0; i < current_chunk->element_count; i++) {
+        if (!file_load_data(current_chunk->items[i])) {
+          log_message(LOG_LEVEL_ERROR, "Failed to load file data");
+          continue;
+        }
+      }
+    }
+    if (send_chunk(client, current_chunk, config) != 0) {
+      log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
+      chunk_destroy(current_chunk);
+      break;
+    }
+    if (config->show_progress) {
+      total_bytes += chunk_bytes;
+      time_t now = time(NULL);
+      if (now - last_progress >= 1) {
+        last_progress = now;
+        double elapsed = difftime(now, start);
+        double rate = elapsed > 0 ? total_bytes / (1048576.0 * elapsed) : 0;
+        fprintf(stderr, "\rSent %.1f MB  (%.1f MB/s)  ", total_bytes / 1048576.0, rate);
+        fflush(stderr);
+      }
+    }
+    chunk_destroy(current_chunk);
+  }
+  if (config->use_delete) {
+    send_status(client->file_descriptor, STATUS_MANIFEST);
+    send_int(client->file_descriptor, manifest->size);
+    for (int i = 0; i < manifest->size; i++)
+      send_str(client->file_descriptor, (char *)manifest->items[i]);
+    array_list_delete(manifest);
+  }
+  send_status(client->file_descriptor, STATUS_FINISHED);
+  Status s;
+  int ok = receive_status(client->file_descriptor, &s) && s == STATUS_OK;
+  if (config->show_progress) {
+    double elapsed = difftime(time(NULL), start);
+    double rate = elapsed > 0 ? total_bytes / (1048576.0 * elapsed) : 0;
+    fprintf(stderr, "\rSent %.1f MB  (%.1f MB/s)  Done.\n", total_bytes / 1048576.0, rate);
+  }
+  directory_scanner_destroy(scanner);
+  client_disconnect(client);
+  client_delete(client);
+  return ok ? 0 : -1;
+}
+
+int send_files_multithreaded(Config *config) {
+  if (config->dry_run) {
+    DirectoryScanner *scanner = directory_scanner_create(
+        config->send_directory, config->use_metadata, config->chunk_size,
+        config->exclude_patterns, config->exclude_count,
+        config->include_patterns, config->include_count,
+        config->max_size, config->min_size);
+    Chunk *chunk;
+    int file_count = 0;
+    unsigned long long total_bytes = 0;
+    printf("Dry run: files to be transferred\n");
+    while ((chunk = directory_scanner_next(scanner)) != NULL) {
+      for (int i = 0; i < chunk->element_count; i++) {
+        printf("  %s (%zu bytes)\n", chunk->items[i]->path,
+               chunk->items[i]->data->size);
+        total_bytes += chunk->items[i]->data->size;
+        file_count++;
+      }
+      chunk_destroy(chunk);
+    }
+    directory_scanner_destroy(scanner);
+    printf("Total: %d files, %.1f MB\n", file_count,
+           total_bytes / 1048576.0);
+    return 0;
+  }
+
+  Queue *q1 = queue_create(100, chunk_destroy);
+  Queue *q2 = queue_create(100, chunk_destroy);
+  if (!q1 || !q2) {
+    if (q1) queue_destroy(q1);
+    if (q2) queue_destroy(q2);
+    return 1;
+  }
+  PipelineContextSender *context =
+      pipeline_context_sender_create(config, q1, q2);
+  if (!context) {
+    queue_destroy(q1);
+    queue_destroy(q2);
+    return 1;
+  }
+  if (config->use_delete)
+    context->manifest = array_list_create(free);
+
+  thrd_t scanner, loader, sender;
+  if (thrd_create(&scanner, scan_directory_multithreaded, context) !=
+          thrd_success ||
+      thrd_create(&loader, load_files_multithreaded, context) != thrd_success ||
+      thrd_create(&sender, send_chunks_multithreaded, context) !=
+          thrd_success) {
+    perror("Error creating threads.\n");
+    pipeline_context_sender_destroy(context);
+    return 1;
+  }
+
+  int sender_result;
+  thrd_join(scanner, NULL);
+  thrd_join(loader, NULL);
+  thrd_join(sender, &sender_result);
+
+  pipeline_context_sender_destroy(context);
+  return sender_result == thrd_success ? 0 : -1;
+}

src/client/client_send.h

@@ -0,0 +1,15 @@
+#ifndef CLIENT_SEND_H
+#define CLIENT_SEND_H
+
+#include "chunk.h"
+#include "config.h"
+#include "transport_tcp.h"
+
+extern char *server_host;
+extern int server_port;
+
+int send_chunk(Client *client, Chunk *chunk, Config *config);
+int send_files(Config *config);
+int send_files_multithreaded(Config *config);
+
+#endif

src/client/scanner.c

@@ -1,16 +1,29 @@
 #include "scanner.h"
 #include "array_list.h"
 #include "chunk.h"
+#include "file.h"
 #include "queue.h"
 #include "utils.h"
 #include <dirent.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
 
-DirectoryScanner *directory_scanner_create(char *root_directory) {
+DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size, char **exclude_patterns, int exclude_count, char **include_patterns, int include_count, unsigned long long max_size, unsigned long long min_size) {
   DirectoryScanner *scanner = malloc(sizeof(DirectoryScanner));
   scanner->directories = queue_create(100, free);
+  scanner->current_dir = NULL;
+  scanner->current_path = NULL;
+  scanner->use_metadata = use_metadata;
+  scanner->chunk_size = chunk_size > 0 ? chunk_size : DESIRED_CHUNK_SIZE;
+  scanner->exclude_patterns = exclude_patterns;
+  scanner->exclude_count = exclude_count;
+  scanner->include_patterns = include_patterns;
+  scanner->include_count = include_count;
+  scanner->max_size = max_size;
+  scanner->min_size = min_size;
   queue_enqueue(scanner->directories, str_dup(root_directory));
   return scanner;
 }
@@ -18,11 +31,16 @@ DirectoryScanner *directory_scanner_create(char *root_directory) {
 void directory_scanner_destroy(DirectoryScanner *scanner) {
   if (scanner == NULL)
     return;
+  if (scanner->current_dir) {
+    closedir(scanner->current_dir);
+    scanner->current_dir = NULL;
+  }
+  free(scanner->current_path);
   queue_destroy(scanner->directories);
   free(scanner);
 }
 
-Chunk *chunk_data_to_chunk(ArrayList *chunk_data) {
+static Chunk *chunk_data_to_chunk(ArrayList *chunk_data) {
   void **chunk_items = array_list_to_array(chunk_data);
   Chunk *chunk = chunk_create((File **)chunk_items, chunk_data->size);
   free(chunk_items);
@@ -31,40 +49,109 @@ Chunk *chunk_data_to_chunk(ArrayList *chunk_data) {
   return chunk;
 }
 
+static int open_next_directory(DirectoryScanner *scanner) {
+  if (scanner->current_dir) {
+    closedir(scanner->current_dir);
+    scanner->current_dir = NULL;
+  }
+  free(scanner->current_path);
+
+  if (queue_is_empty(scanner->directories))
+    return 0;
+
+  scanner->current_path = (char *)queue_dequeue(scanner->directories);
+  scanner->current_dir = opendir(scanner->current_path);
+  if (scanner->current_dir == NULL) {
+    perror("Could not open directory");
+    free(scanner->current_path);
+    scanner->current_path = NULL;
+    return 0;
+  }
+  return 1;
+}
+
 Chunk *directory_scanner_next(DirectoryScanner *scanner) {
   ArrayList *chunk_data = array_list_create(file_destroy);
   unsigned long long chunk_data_size = 0;
 
-  while (!queue_is_empty(scanner->directories)) {
+  while (1) {
-    char *path = (char *)queue_dequeue(scanner->directories);
+    if (scanner->current_dir == NULL) {
-    DIR *dir;
+      if (!open_next_directory(scanner))
-    struct dirent *entry;
+        break;
-    dir = opendir(path);
-    if (dir == NULL) {
-      perror("Could not open directory!");
-      exit(EXIT_FAILURE);
     }
-    while ((entry = readdir(dir)) != NULL) {
+
-      if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) {
+    struct dirent *entry = readdir(scanner->current_dir);
+    if (entry == NULL) {
+      closedir(scanner->current_dir);
+      scanner->current_dir = NULL;
+      free(scanner->current_path);
+      scanner->current_path = NULL;
+      continue;
+    }
+
+    if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
+      continue;
+
+    char *cur_path = path_cat(scanner->current_path, entry->d_name);
+    struct stat stats;
+    if (stat(cur_path, &stats) != 0) {
+      free(cur_path);
+      continue;
+    }
+
+    if (S_ISDIR(stats.st_mode)) {
+      queue_enqueue(scanner->directories, (void *)cur_path);
+    } else {
+      bool excluded = false;
+      for (int i = 0; i < scanner->exclude_count; i++) {
+        if (glob_match(scanner->exclude_patterns[i], entry->d_name)) {
+          excluded = true;
+          break;
+        }
+      }
+      if (excluded) {
+        free(cur_path);
         continue;
       }
-      char *cur_path = path_cat(path, entry->d_name);
+
-      struct stat stats;
+      if (scanner->include_count > 0) {
-      stat(cur_path, &stats);
+        bool included = false;
-      if (!S_ISREG(stats.st_mode))
+        for (int i = 0; i < scanner->include_count; i++) {
-        queue_enqueue(scanner->directories, (void *)cur_path);
+          if (glob_match(scanner->include_patterns[i], entry->d_name)) {
-      else {
+            included = true;
-        File *file = file_create(cur_path, &stats);
+            break;
-        array_list_add(chunk_data, file);
+          }
-        chunk_data_size += file->stats.st_size;
+        }
-        if (chunk_data_size > DESIRED_CHUNK_SIZE)
+        if (!included) {
-          return chunk_data_to_chunk(chunk_data);
+          free(cur_path);
-        free(cur_path);
+          continue;
+        }
       }
+
+      if ((scanner->max_size > 0 && (unsigned long long)stats.st_size > scanner->max_size) ||
+          (scanner->min_size > 0 && (unsigned long long)stats.st_size < scanner->min_size)) {
+        free(cur_path);
+        continue;
+      }
+
+      File *file = file_create(cur_path);
+      if (file == NULL) {
+        free(cur_path);
+        continue;
+      }
+      file->data->size = stats.st_size;
+      if (scanner->use_metadata)
+        file->metadata = file_metadata_create(&stats);
+      array_list_add(chunk_data, file);
+      chunk_data_size += file->data->size;
+      if (chunk_data_size > scanner->chunk_size) {
+        free(cur_path);
+        return chunk_data_to_chunk(chunk_data);
+      }
+      free(cur_path);
     }
-    closedir(dir);
-    free(path);
   }
+
   if (chunk_data->size > 0)
     return chunk_data_to_chunk(chunk_data);
   return NULL;

src/client/scanner.h

@@ -3,11 +3,24 @@
 
 #include "chunk.h"
 #include "queue.h"
+#include <dirent.h>
+#include <stdbool.h>
+
 typedef struct {
   Queue *directories;
+  DIR *current_dir;
+  char *current_path;
+  bool use_metadata;
+  unsigned long long chunk_size;
+  char **exclude_patterns;
+  int exclude_count;
+  char **include_patterns;
+  int include_count;
+  unsigned long long max_size;
+  unsigned long long min_size;
 } DirectoryScanner;
 
-DirectoryScanner *directory_scanner_create(char *root_directory);
+DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size, char **exclude_patterns, int exclude_count, char **include_patterns, int include_count, unsigned long long max_size, unsigned long long min_size);
 Chunk *directory_scanner_next(DirectoryScanner *scanner);
 void directory_scanner_destroy(DirectoryScanner *scanner);
 

src/server/server.c

@@ -1,113 +1,183 @@
+#include "array_list.h"
+#include "chunk.h"
+#include "compression.h"
 #include "config.h"
 #include "data.h"
 #include "file.h"
 #include "log.h"
 #include "multiprocessing.h"
+#include "protocol.h"
 #include "queue.h"
-#include "socket.h"
+#include "transport_tcp.h"
 #include "unistd.h"
 #include "utils.h"
+#include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <threads.h>
+#include <string.h>
 
-FileReceive *receive_file_receive(Config *config, int file_descriptor) {
+static int receive_chunk(int fd, Config *config) {
-  char *path = (char *)receive_str(file_descriptor);
+  Data *chunk_data = receive_data(fd);
-  Data *file_data = receive_data(file_descriptor);
+  if (chunk_data == NULL) {
-  if (config->use_compression) {
+    log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data");
-    Data *file_data_uncompressed = data_decompress(file_data);
-    free(file_data);
-    file_data = file_data_uncompressed;
-  }
-  FileReceive *file = file_receive_create(path, file_data);
-  return file;
-}
-
-int receive_thread(void *pipeline_context) {
-  PipelineContextReceiver *context =
-      (PipelineContextReceiver *)pipeline_context;
-  mtx_lock(&context->mutex);
-  int file_descriptor = context->file_descriptor;
-  Config *config = context->config;
-  mtx_unlock(&context->mutex);
-
-  while (receive_status(file_descriptor) == STATUS_NEXT) {
-    FileReceive *file = receive_file_receive(config, file_descriptor);
-    queue_enqueue_multithreaded(context->queue, file, &context->mutex,
-                                &context->condition_not_empty,
-                                &context->condition_not_full);
-  }
-  mtx_lock(&context->mutex);
-  context->receiver_done = true;
-  cnd_signal(&context->condition_not_empty);
-  mtx_unlock(&context->mutex);
-  return thrd_success;
-}
-
-int write_thread(void *pipeline_context) {
-  PipelineContextReceiver *context =
-      (PipelineContextReceiver *)pipeline_context;
-  mtx_lock(&context->mutex);
-  bool save_to_disk = context->config->save_to_disk;
-  char *root_directory = str_dup(context->config->receive_root_directory);
-  mtx_unlock(&context->mutex);
-
-  while (true) {
-    FileReceive *file = queue_dequeue_multithreaded(
-        context->queue, &context->mutex, &context->condition_not_empty,
-        &context->condition_not_full, &context->receiver_done);
-    if (file == NULL) {
-      free(root_directory);
-      return thrd_success;
-    }
-    if (save_to_disk)
-      to_disk(path_cat(root_directory, file->path), file->data->data,
-              file->data->size);
-  }
-}
-
-int receive_files(Config *config, int file_descriptor) {
-  Status status = receive_status(file_descriptor);
-  while (status == STATUS_NEXT) {
-    FileReceive *file = receive_file_receive(config, file_descriptor);
-    if (config->save_to_disk)
-      to_disk(path_cat(config->receive_root_directory, file->path),
-              file->data->data, file->data->size);
-    file_receive_destroy(file);
-    // send_status(file_descriptor, STATUS_OK);
-    status = receive_status(file_descriptor);
-  }
-  if (status != STATUS_FINISHED) {
-    log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED or NEXT Status");
-    send_status(file_descriptor, STATUS_ERROR);
     return -1;
   }
-  send_status(file_descriptor, STATUS_OK);
+  Data *data_to_process = chunk_data;
+  if (config->use_compression) {
+    data_to_process = data_decompress(chunk_data);
+    data_destroy(chunk_data);
+    if (data_to_process == NULL) {
+      log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk");
+      return -1;
+    }
+  }
+  Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata);
+  data_destroy(data_to_process);
+  if (chunk == NULL) {
+    log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping");
+    return -1;
+  }
+
+  for (int i = 0; i < chunk->element_count; i++) {
+    if (config->save_to_disk)
+      file_save_to_disk(config->receive_root_directory, chunk->items[i]);
+  }
+  chunk_destroy(chunk);
+  return 0;
+}
+
+static int receive_manifest(int fd, Config *config, Status *next_status) {
+  int count;
+  if (!receive_int(fd, &count)) return -1;
+  ArrayList *manifest = array_list_create(free);
+  if (manifest) {
+    for (int i = 0; i < count; i++) {
+      char *s = receive_str(fd);
+      if (s) array_list_add(manifest, s);
+    }
+    fprintf(stderr, "Deleting files not in manifest...\n");
+    delete_extras(config->receive_root_directory, manifest);
+    array_list_delete(manifest);
+  }
+  if (!receive_status(fd, next_status)) return -1;
+  return 0;
+}
+
+int receive_files(Config *config, int fd) {
+  Status status;
+  if (!receive_status(fd, &status)) return -1;
+
+  while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) {
+    if (status == STATUS_CHECK) {
+      bool skipped;
+      File *file = receive_incremental_check(fd, config, &skipped);
+      if (skipped) goto next;
+      if (file == NULL && !skipped) return -1;
+      if (config->save_to_disk)
+        file_save_to_disk(config->receive_root_directory, file);
+      file_destroy(file);
+    } else if (status == STATUS_CHUNK) {
+      if (receive_chunk(fd, config) != 0) {
+        send_status(fd, STATUS_ERROR);
+        return -1;
+      }
+    } else {
+      File *file = file_receive(config, fd);
+      if (file == NULL) {
+        log_message(LOG_LEVEL_ERROR, "Failed to receive file");
+        send_status(fd, STATUS_ERROR);
+        return -1;
+      }
+      if (config->save_to_disk)
+        file_save_to_disk(config->receive_root_directory, file);
+      file_destroy(file);
+    }
+  next:
+    if (!receive_status(fd, &status)) {
+      send_status(fd, STATUS_ERROR);
+      return -1;
+    }
+  }
+
+  if (status == STATUS_MANIFEST) {
+    if (receive_manifest(fd, config, &status) != 0) return -1;
+  }
+  if (status != STATUS_FINISHED) {
+    log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status");
+    send_status(fd, STATUS_ERROR);
+    return -1;
+  }
+  send_status(fd, STATUS_OK);
   return 0;
 }
 
 void handler(int file_descriptor) {
   Config *config = config_receive(file_descriptor);
+  if (config == NULL) {
+    log_message(LOG_LEVEL_ERROR, "Failed to receive config");
+    close(file_descriptor);
+    return;
+  }
   if (config->use_multithreading) {
+    Queue *q = queue_create(100, file_destroy);
+    if (q == NULL) {
+      config_delete(config);
+      close(file_descriptor);
+      return;
+    }
     PipelineContextReceiver *context = pipeline_context_receiver_create(
-        config, queue_create(100, file_receive_destroy), file_descriptor);
+        config, q, file_descriptor);
+    if (context == NULL) {
+      queue_destroy(q);
+      config_delete(config);
+      close(file_descriptor);
+      return;
+    }
     thrd_t receiver, writer;
     if (thrd_create(&receiver, receive_thread, context) != thrd_success ||
         thrd_create(&writer, write_thread, context) != thrd_success) {
-      perror("Error creating Threads!");
+      perror("Error creating Threads");
-      exit(EXIT_FAILURE);
+      pipeline_context_receiver_destroy(context);
+      close(file_descriptor);
+      return;
     }
     thrd_join(receiver, NULL);
     thrd_join(writer, NULL);
+    send_status(file_descriptor, STATUS_OK);
     pipeline_context_receiver_destroy(context);
   } else
     receive_files(config, file_descriptor);
   close(file_descriptor);
 }
 
-int main() {
+static Server *g_server = NULL;
-  Server *server = server_create(8080);
+
-  server_listen(server, handler);
+static void cleanup(int sig) {
-  server_delete(server);
+  (void)sig;
+  if (g_server) {
+    server_delete(&g_server);
+  }
+  _exit(0);
+}
+
+int main(int argc, char *argv[]) {
+  signal(SIGPIPE, SIG_IGN);
+  for (int i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "--stdio") == 0) {
+      io_set_fds(STDIN_FILENO, STDOUT_FILENO);
+      handler(STDIN_FILENO);
+      return 0;
+    } else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
+      set_log_level(LOG_LEVEL_DEBUG);
+    }
+  }
+  signal(SIGINT, cleanup);
+  signal(SIGTERM, cleanup);
+  g_server = server_create(8080);
+  if (g_server == NULL) {
+    log_message(LOG_LEVEL_ERROR, "Failed to create server");
+    return 1;
+  }
+  server_listen(g_server, handler);
   return 0;
 }

src/shared/array_list.c

@@ -6,15 +6,14 @@
 ArrayList *array_list_create(void (*item_destroyer)(void *item)) {
   ArrayList *list = (ArrayList *)malloc(sizeof(ArrayList));
   if (list == NULL) {
-    perror("FATAL ERROR: Could not allocate memory for array list struct");
+    perror("ERROR: Could not allocate memory for array list struct");
-    exit(EXIT_FAILURE);
+    return NULL;
   }
 
   list->items = malloc(INITIAL_ARRAY_SIZE * sizeof(void *));
   if (list->items == NULL) {
-    perror("FATAL ERROR: Could not allocate memory for list items");
     free(list);
-    exit(EXIT_FAILURE);
+    return NULL;
   }
   list->size = 0;
   list->capacity = INITIAL_ARRAY_SIZE;
@@ -35,37 +34,30 @@ void array_list_delete(ArrayList *array_list) {
   free(array_list);
 }
 
-void array_list_clear(ArrayList *array_list) {
+bool array_list_extend(ArrayList *array_list) {
-  if (array_list == NULL)
+  if (array_list == NULL) return false;
-    return;
-  for (int i = 0; i < array_list->size; i++)
-    array_list->items[i] = NULL;
-  array_list->size = 0;
-}
-
-void array_list_extend(ArrayList *array_list) {
-  if (array_list == NULL)
-    return;
   int new_capacity = array_list->capacity * 2;
   if (new_capacity == 0)
     new_capacity = INITIAL_ARRAY_SIZE;
-  array_list->items = realloc(array_list->items, new_capacity * sizeof(void *));
+  void *new_items = realloc(array_list->items, new_capacity * sizeof(void *));
-  if (array_list->items == NULL) {
+  if (new_items == NULL) {
-    perror("FATAL ERROR: Could not reallocate memory for array list struct");
+    perror("ERROR: Could not reallocate memory for array list items");
-    exit(EXIT_FAILURE);
+    return false;
   }
+  array_list->items = new_items;
   array_list->capacity = new_capacity;
+  return true;
 }
 
-void array_list_add(ArrayList *array_list, void *item) {
+bool array_list_add(ArrayList *array_list, void *item) {
-  if (array_list == NULL) {
+  if (array_list == NULL) return false;
-    return;
-  }
   if (array_list->capacity == array_list->size) {
-    array_list_extend(array_list);
+    if (!array_list_extend(array_list))
+      return false;
   }
   array_list->items[array_list->size] = item;
   array_list->size += 1;
+  return true;
 }
 
 void **array_list_to_array(ArrayList *array_list) {

src/shared/array_list.h

@@ -1,6 +1,8 @@
 #ifndef ARRAY_LIST_H
 #define ARRAY_LIST_H
 
+#include <stdbool.h>
+
 #define INITIAL_ARRAY_SIZE 100
 
 typedef struct ArrayList {
@@ -12,9 +14,8 @@ typedef struct ArrayList {
 
 ArrayList *array_list_create(void (*item_destroyer)(void *item));
 void array_list_delete(ArrayList *array_list);
-void array_list_clear(ArrayList *array_list);
+bool array_list_extend(ArrayList *array_list);
-void array_list_extend(ArrayList *array_list);
+bool array_list_add(ArrayList *array_list, void *item);
-void array_list_add(ArrayList *array_list, void *item);
 void **array_list_to_array(ArrayList *array_list);
 
 #endif

src/shared/chunk.c

@@ -1,30 +1,27 @@
-#include <dirent.h>
-#include <libgen.h>
 #include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <zstd.h>
 
 #include "array_list.h"
 #include "chunk.h"
+#include "compression.h"
 #include "data.h"
 #include "file.h"
 #include "log.h"
+#include "metadata.h"
 
 Chunk *chunk_create(File **items, int element_count) {
   Chunk *chunk = (Chunk *)malloc(sizeof(Chunk));
   if (chunk == NULL) {
-    perror("FATAL ERROR: Could not allocate memory for chunk structure");
+    perror("ERROR: Could not allocate memory for chunk structure");
-    exit(EXIT_FAILURE);
+    return NULL;
   }
 
   chunk->items = (File **)malloc(element_count * sizeof(File *));
   if (chunk->items == NULL) {
-    perror("FATAL ERROR: Could not allocate memory for items of chunk "
-           "structure");
     free(chunk);
-    exit(EXIT_FAILURE);
+    return NULL;
   }
 
   for (int i = 0; i < element_count; i++) {
@@ -48,227 +45,138 @@ void chunk_destroy(void *item) {
   free(chunk);
 }
 
-void chunk_print(void *item) {
+static unsigned long long per_file_serialize_size(File *file, bool use_metadata) {
-  if (item == NULL)
+  return sizeof(size_t) + strlen(file->path) +
-    return;
+         (use_metadata ? sizeof(int) + (file->metadata ? FILE_METADATA_WIRE_SIZE : 0) : 0) +
-  Chunk *chunk = (Chunk *)item;
+         sizeof(size_t) + file->data->size;
-  for (int i = 0; i < chunk->element_count; ++i)
-    if (chunk->items[i] != NULL)
-      file_print(chunk->items[i]);
 }
 
-Data *chunk_format(Chunk *chunk) {
+Data *chunk_serialize(Chunk *chunk, bool use_metadata) {
-  unsigned long long buffer_size = 0;
-  for (int i = 0; i < chunk->element_count; ++i) {
-    buffer_size += sizeof(int);
-    buffer_size += strlen(chunk->items[i]->path);
-    buffer_size += sizeof(unsigned long long);
-    buffer_size += chunk->items[i]->stats.st_size;
-  }
-
-  char *data = malloc(buffer_size);
-  if (data == NULL) {
-    perror("Could not allocate data for ChunkFormated!");
-    exit(EXIT_FAILURE);
-  }
-  char *current_data_pointer = data;
-  // for (int i = 0; i < chunk->element_count; ++i) {
-  //   File *file = chunk->items[i];
-  //   // add path len
-  //   int path_length = (int)strlen(file->path);
-  //   memcpy(current_data_pointer, &path_length, sizeof(int));
-  //   current_data_pointer += sizeof(int);
-  //   // add path
-  //   memcpy(current_data_pointer, file->path, path_length);
-  //   current_data_pointer += path_length;
-  //   // add file data len
-  //   unsigned long long file_length = file->stats.st_size;
-  //   memcpy(current_data_pointer, &file_length, sizeof(unsigned long long));
-  //   current_data_pointer += sizeof(unsigned long long);
-  //   // add file data
-  //   file_content_to_buffer(file, current_data_pointer);
-  //   current_data_pointer += file_length;
-  // }
-  if (current_data_pointer - data != (long)(long)buffer_size) {
-    perror("Buffer of Chunk wasn't filled enough!");
-    exit(EXIT_FAILURE);
-  }
-  return chunk_data_create(data, buffer_size);
-}
-
-Data *chunk_compress(Chunk *chunk, int compression_level) {
-  log_message(LOG_LEVEL_DEBUG, "Starting to gather data for chunk compression");
   unsigned long long data_size = 0;
   for (int i = 0; i < chunk->element_count; i++) {
-    data_size += sizeof(unsigned long long);
+    data_size += per_file_serialize_size(chunk->items[i], use_metadata);
-    data_size += strlen(chunk->items[i]->path);
-    data_size += sizeof(unsigned long long);
-    data_size += chunk->items[i]->stats.st_size;
   }
   Data *data = data_create_empty(data_size);
   if (data == NULL) {
     log_message(LOG_LEVEL_ERROR,
-                "Could not allocate memory for chunk compression");
+                "Could not allocate memory for chunk serialization");
-    exit(EXIT_FAILURE);
+    return NULL;
   }
   char *data_pointer = data->data;
   for (int i = 0; i < chunk->element_count; i++) {
-    // path length
+    File *file = chunk->items[i];
-    size_t path_len = strlen(chunk->items[i]->path);
+    size_t path_len = strlen(file->path);
     memcpy(data_pointer, &path_len, sizeof(size_t));
     data_pointer += sizeof(size_t);
-    memcpy(data_pointer, chunk->items[i]->path, path_len);
+    memcpy(data_pointer, file->path, path_len);
     data_pointer += path_len;
-    // file data
-    unsigned long long data_size = chunk->items[i]->stats.st_size;
-    memcpy(data_pointer, &data_size, sizeof(size_t));
-    data_pointer += sizeof(size_t);
-    memcpy(data_pointer, chunk->items[i]->data, data_size);
-    data_pointer += data_size;
-  }
 
-  log_message(LOG_LEVEL_DEBUG, "Chunk succesfully compressed");
+    if (use_metadata)
-  return data_compress(data, compression_level);
+      metadata_to_buf(&data_pointer, file->metadata);
+
+    size_t file_data_size = file->data->size;
+    memcpy(data_pointer, &file_data_size, sizeof(size_t));
+    data_pointer += sizeof(size_t);
+    memcpy(data_pointer, file->data->data, file_data_size);
+    data_pointer += file_data_size;
+  }
+  return data;
 }
 
-Chunk *chunk_decompress(Data *compressed_data) {
+Chunk *chunk_deserialize(Data *data, bool use_metadata) {
-  log_message(LOG_LEVEL_DEBUG, "Starting to decompress chunk");
-  Data *uncompressed_data = data_decompress(compressed_data);
-  if (uncompressed_data == NULL) {
-    log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk data");
-    return NULL;
-  }
-  
   ArrayList *files = array_list_create(file_destroy);
-  char *data_pointer = uncompressed_data->data;
+  char *data_pointer = data->data;
-  size_t remaining_size = uncompressed_data->size;
+  size_t remaining_size = data->size;
-  
+
   while (remaining_size > 0) {
     if (remaining_size < sizeof(size_t)) {
       log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for path length");
-      array_list_destroy(files);
+      array_list_delete(files);
-      data_delete(uncompressed_data);
       return NULL;
     }
-    
+
     size_t path_len = *(size_t *)data_pointer;
     data_pointer += sizeof(size_t);
     remaining_size -= sizeof(size_t);
-    
+
     if (remaining_size < path_len) {
       log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for path");
-      array_list_destroy(files);
+      array_list_delete(files);
-      data_delete(uncompressed_data);
       return NULL;
     }
-    
+
     char *path = malloc(path_len + 1);
     if (path == NULL) {
       perror("Could not allocate memory for file path");
-      array_list_destroy(files);
+      array_list_delete(files);
-      data_delete(uncompressed_data);
       return NULL;
     }
     memcpy(path, data_pointer, path_len);
     path[path_len] = '\0';
     data_pointer += path_len;
     remaining_size -= path_len;
-    
+
+    File *file = file_create(path);
+    free(path);
+
+    if (use_metadata) {
+      file->metadata = metadata_from_buf(&data_pointer);
+      remaining_size -= sizeof(int);
+      if (file->metadata)
+        remaining_size -= FILE_METADATA_WIRE_SIZE;
+    }
+
     if (remaining_size < sizeof(size_t)) {
       log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for data size");
-      free(path);
+      array_list_delete(files);
-      array_list_destroy(files);
-      data_delete(uncompressed_data);
       return NULL;
     }
-    
+
-    size_t data_size = *(size_t *)data_pointer;
+    size_t file_data_size = *(size_t *)data_pointer;
     data_pointer += sizeof(size_t);
     remaining_size -= sizeof(size_t);
-    
+
-    if (remaining_size < data_size) {
+    if (remaining_size < file_data_size) {
       log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for file content");
-      free(path);
+      array_list_delete(files);
-      array_list_destroy(files);
-      data_delete(uncompressed_data);
       return NULL;
     }
-    
+
-    File *file = file_create(path, data_size);
+    void *file_data = malloc(file_data_size);
-    if (file == NULL) {
+    if (file_data == NULL) {
-      free(path);
+      perror("Could not allocate memory for file data");
-      array_list_destroy(files);
+      array_list_delete(files);
-      data_delete(uncompressed_data);
       return NULL;
     }
-    
+    memcpy(file_data, data_pointer, file_data_size);
-    memcpy(file->data, data_pointer, data_size);
+    data_destroy(file->data);
-    data_pointer += data_size;
+    file->data = data_create(file_data, file_data_size);
-    remaining_size -= data_size;
+    data_pointer += file_data_size;
-    
+    remaining_size -= file_data_size;
+
     array_list_add(files, file);
-    free(path);
   }
-  
+
-  // Create the chunk from the files
   File **file_array = (File **)array_list_to_array(files);
-  Chunk *chunk = chunk_create(file_array, array_list_size(files));
+  Chunk *chunk = chunk_create(file_array, files->size);
-  
+
-  // Clean up
   free(file_array);
-  array_list_destroy(files);
+  files->item_destroyer = NULL;
-  data_delete(uncompressed_data);
+  array_list_delete(files);
-  
+
-  log_message(LOG_LEVEL_DEBUG, "Chunk successfully decompressed");
   return chunk;
 }
 
-Data *chunk_data_create(void *data, unsigned long long data_size) {
+Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata) {
-  Data *chunk_formated = malloc(sizeof(Data));
+  log_message(LOG_LEVEL_DEBUG, "Starting to compress chunk");
-  if (chunk_formated == NULL) {
+  Data *serialized = chunk_serialize(chunk, use_metadata);
-    perror("Could not allocate memory for ChunkFormated");
+  if (serialized == NULL) return NULL;
-    exit(EXIT_FAILURE);
+  Data *compressed = data_compress(serialized, compression_level);
-  }
+  data_destroy(serialized);
-  chunk_formated->data = data;
+  if (compressed == NULL) return NULL;
-  chunk_formated->size = data_size;
+  log_message(LOG_LEVEL_DEBUG, "Chunk successfully compressed");
-  return chunk_formated;
+  return compressed;
 }
 
-void chunk_data_delete(void *chunk) {
-  Data *chunk_data = (Data *)chunk;
-  free(chunk_data->data);
-  free(chunk_data);
-}
 
-// void chunk_data_to_disk(ChunkData *chunk_formated, char *root_directory) {
+
-//   char *current_data_pointer = chunk_formated->data;
-//   while (current_data_pointer - (char *)chunk_formated->data <
-//          chunk_formated->data_size) {
-//     // get path length
-//     int path_length = 0;
-//     memcpy(&path_length, (int *)current_data_pointer, sizeof(int));
-//     current_data_pointer += sizeof(int);
-//     // get path
-//     int path_dir_size =
-//         (strlen(root_directory) + path_length + 1) * sizeof(char);
-//     char *path = (char *)malloc(path_dir_size);
-//     if (path == NULL) {
-//       perror("Could not allocate memory for path!");
-//       exit(EXIT_FAILURE);
-//     }
-//     snprintf(path, path_dir_size, "%s%.*s", root_directory, path_length,
-//              current_data_pointer);
-//     current_data_pointer += sizeof(char) * path_length;
-//     // get data length
-//     unsigned long long data_size = 0;
-//     memcpy(&data_size, (unsigned long long *)current_data_pointer,
-//            sizeof(unsigned long long));
-//     current_data_pointer += sizeof(unsigned long long);
-//     // create File Receive
-//     FileReceive *file =
-//         file_receive_create(path, data_size, current_data_pointer);
-//     file_receive_print(file);
-//     file_receive_to_disk(file);
-//     current_data_pointer += sizeof(char) * data_size;
-//   }
-// }

src/shared/chunk.h

@@ -3,11 +3,10 @@
 
 #include "data.h"
 #include "file.h"
+#include <stdbool.h>
 #include <sys/stat.h>
 
-#define DESIRED_CHUNK_SIZE 10 * 1024 * 1024
+#define DESIRED_CHUNK_SIZE (10 * 1024 * 1024)
-#define FILE_PATH_SEPERATOR "#&&SEPP&&#"
-#define FILE_PATH_DATA_SEPERATOR "#&&SEPD&&#"
 
 typedef struct {
   File **items;
@@ -16,12 +15,8 @@ typedef struct {
 
 Chunk *chunk_create(File **items, int element_count);
 void chunk_destroy(void *chunk);
-void chunk_print(void *chunk);
+Data *chunk_serialize(Chunk *chunk, bool use_metadata);
-Data *chunk_format(Chunk *chunk);
+Chunk *chunk_deserialize(Data *data, bool use_metadata);
-Data *chunk_compress(Chunk *chunk, int compression_level);
+Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata);
-Chunk *chunk_decompress(Data *compressed_data);
 
-Data *chunk_data_create(void *data, unsigned long long data_size);
-void chunk_data_delete(void *chunk);
-void chunk_data_to_disk(Data *chunk, char *root_directory);
 #endif

src/shared/compression.c

@@ -0,0 +1,105 @@
+#include "compression.h"
+#include "data.h"
+#include "log.h"
+#include "stdlib.h"
+#include "zstd.h"
+
+#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
+
+Data *data_compress(Data *data_to_compress, int compression_level) {
+  log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
+  size_t dst_size = ZSTD_compressBound(data_to_compress->size);
+  Data *compressed_data = data_create_empty(dst_size);
+  if (compressed_data == NULL) return NULL;
+
+  ZSTD_CCtx *cctx = ZSTD_createCCtx();
+  if (!cctx) {
+    log_message(LOG_LEVEL_ERROR, "Failed to create ZSTD compression context");
+    data_destroy(compressed_data);
+    return NULL;
+  }
+
+  ZSTD_inBuffer input = {data_to_compress->data, data_to_compress->size, 0};
+  ZSTD_outBuffer output = {compressed_data->data, dst_size, 0};
+
+  size_t ret;
+  do {
+    ret = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
+    if (ZSTD_isError(ret)) {
+      log_message(LOG_LEVEL_ERROR, "Compression failed: %s",
+                  ZSTD_getErrorName(ret));
+      ZSTD_freeCCtx(cctx);
+      data_destroy(compressed_data);
+      return NULL;
+    }
+  } while (ret > 0);
+
+  compressed_data->size = output.pos;
+  ZSTD_freeCCtx(cctx);
+
+  log_message(LOG_LEVEL_DEBUG, "Data succesfully compressed from %zu to %zu",
+              data_to_compress->size, compressed_data->size);
+  return compressed_data;
+}
+
+Data *data_decompress(Data *compressed_data) {
+  log_message(LOG_LEVEL_DEBUG, "Start to decompress data");
+  unsigned long long dst_size = ZSTD_getFrameContentSize(
+      compressed_data->data, compressed_data->size);
+  if (ZSTD_isError(dst_size)) {
+    log_message(LOG_LEVEL_ERROR, "Failed to get decompressed size: %s",
+                ZSTD_getErrorName(dst_size));
+    return NULL;
+  }
+
+  ZSTD_DCtx *dctx = ZSTD_createDCtx();
+  if (!dctx) {
+    log_message(LOG_LEVEL_ERROR,
+                "Failed to create ZSTD decompression context");
+    return NULL;
+  }
+
+  size_t buf_size = (!ZSTD_isError(dst_size) && dst_size > 0)
+                        ? (size_t)dst_size
+                        : INITIAL_DECOMPRESS_BUF_SIZE;
+  Data *uncompressed_data = data_create_empty(buf_size);
+  if (!uncompressed_data) {
+    log_message(LOG_LEVEL_ERROR, "Failed to allocate decompression buffer");
+    ZSTD_freeDCtx(dctx);
+    return NULL;
+  }
+
+  ZSTD_inBuffer input = {compressed_data->data, compressed_data->size, 0};
+  ZSTD_outBuffer output = {uncompressed_data->data, buf_size, 0};
+
+  size_t ret;
+  do {
+    ret = ZSTD_decompressStream(dctx, &output, &input);
+    if (ZSTD_isError(ret)) {
+      log_message(LOG_LEVEL_ERROR, "Decompression failed: %s",
+                  ZSTD_getErrorName(ret));
+      ZSTD_freeDCtx(dctx);
+      data_destroy(uncompressed_data);
+      return NULL;
+    }
+    if (ret > 0 && output.pos == output.size) {
+      buf_size *= 2;
+      void *new_data = realloc(uncompressed_data->data, buf_size);
+      if (!new_data) {
+        log_message(LOG_LEVEL_ERROR, "Failed to grow decompression buffer");
+        ZSTD_freeDCtx(dctx);
+        data_destroy(uncompressed_data);
+        return NULL;
+      }
+      uncompressed_data->data = new_data;
+      output.dst = new_data;
+      output.size = buf_size;
+    }
+  } while (ret > 0);
+
+  uncompressed_data->size = output.pos;
+  ZSTD_freeDCtx(dctx);
+
+  log_message(LOG_LEVEL_DEBUG, "Decompressed data successfully");
+  return uncompressed_data;
+}

src/shared/compression.h

@@ -0,0 +1,9 @@
+#ifndef COMPRESSION_H
+#define COMPRESSION_H
+
+#include "data.h"
+
+Data *data_compress(Data *data_to_compress, int compression_level);
+Data *data_decompress(Data *compressed_data);
+
+#endif

src/shared/config.c

@@ -1,14 +1,18 @@
 #include "config.h"
-#include "socket.h"
+#include "log.h"
+#include "protocol.h"
+#include "utils.h"
 #include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 
 Config *config_create(char *version, char *send_directory,
                       char *receive_directory, bool save_to_disk,
                       bool use_multithreading, bool use_chunk_serialization,
-                      bool use_compression, int compression_level,
+                       bool use_compression, bool use_metadata,
-                      int num_connections) {
+                       int compression_level, bool use_sendfile,
+                       unsigned long long chunk_size) {
 
   Config *config = malloc(sizeof(Config));
   config->version = version;
@@ -18,45 +22,140 @@ Config *config_create(char *version, char *send_directory,
   config->use_multithreading = use_multithreading;
   config->use_chunk_serialization = use_chunk_serialization;
   config->use_compression = use_compression;
+  config->use_metadata = use_metadata;
+  config->show_progress = false;
+  config->dry_run = false;
+  config->use_delete = false;
   config->compression_level = compression_level;
-  config->num_connections = num_connections;
+  config->use_sendfile = use_sendfile;
+  config->chunk_size = chunk_size > 0 ? chunk_size : DEFAULT_CHUNK_SIZE;
+  config->ssh_port = 22;
+  config->transport = TRANSPORT_TCP;
+  config->ssh_destination = NULL;
+  config->exclude_patterns = NULL;
+  config->exclude_count = 0;
+  config->include_patterns = NULL;
+  config->include_count = 0;
+  config->max_size = 0;
+  config->min_size = 0;
+  config->use_incremental = false;
   return config;
 }
 
+bool is_remote_dest(const char *s) {
+  if (s == NULL) return false;
+  const char *colon = strchr(s, ':');
+  if (colon == NULL) return false;
+  if (colon == s) return false;
+  for (const char *p = s; p < colon; p++) {
+    if (*p == '/') return false;
+  }
+  return true;
+}
+
+void config_parse_ssh_dest(Config *config) {
+  if (!is_remote_dest(config->receive_root_directory)) return;
+  config->transport = TRANSPORT_SSH;
+  config->ssh_destination = str_dup(config->receive_root_directory);
+  char *colon = strchr(config->receive_root_directory, ':');
+  char *path = str_dup(colon + 1);
+  free(config->receive_root_directory);
+  config->receive_root_directory = path;
+}
+
 void config_delete(Config *config) {
   free(config->version);
   free(config->send_directory);
   free(config->receive_root_directory);
+  free(config->ssh_destination);
+  for (int i = 0; i < config->exclude_count; i++)
+    free(config->exclude_patterns[i]);
+  free(config->exclude_patterns);
+  for (int i = 0; i < config->include_count; i++)
+    free(config->include_patterns[i]);
+  free(config->include_patterns);
   free(config);
 }
 
-void config_send(int file_descriptor, Config *config) {
+bool config_send(int file_descriptor, Config *config) {
-  send_str(file_descriptor, config->version);
+  if (!send_str(file_descriptor, config->version)) return false;
-  send_str(file_descriptor, config->send_directory);
+  if (!send_str(file_descriptor, config->send_directory)) return false;
-  send_str(file_descriptor, config->receive_root_directory);
+  if (!send_str(file_descriptor, config->receive_root_directory)) return false;
-  send_int(file_descriptor, config->save_to_disk);
+  if (!send_int(file_descriptor, config->save_to_disk)) return false;
-  send_int(file_descriptor, config->use_multithreading);
+  if (!send_int(file_descriptor, config->use_multithreading)) return false;
-  send_int(file_descriptor, config->use_chunk_serialization);
+  if (!send_int(file_descriptor, config->use_chunk_serialization)) return false;
-  send_int(file_descriptor, config->use_compression);
+  if (!send_int(file_descriptor, config->use_compression)) return false;
-  send_int(file_descriptor, config->use_compression);
+  if (!send_int(file_descriptor, config->use_metadata)) return false;
-  send_int(file_descriptor, config->num_connections);
+  if (!send_int(file_descriptor, config->compression_level)) return false;
-  if (receive_status(file_descriptor) != STATUS_OK) {
+  if (!send_int(file_descriptor, (int)config->chunk_size)) return false;
-    perror("Error transmitting config!");
+  if (!send_int(file_descriptor, config->use_sendfile)) return false;
-    exit(EXIT_FAILURE);
+  if (!send_int(file_descriptor, config->use_delete)) return false;
+  if (!send_int(file_descriptor, config->use_incremental)) return false;
+  Status status;
+  if (!receive_status(file_descriptor, &status)) return false;
+  if (status != STATUS_OK) {
+    log_message(LOG_LEVEL_ERROR, "Error transmitting config");
+    return false;
   }
+  return true;
 }
 
 Config *config_receive(int file_descriptor) {
   Config *config = (Config *)malloc(sizeof(Config));
+  if (config == NULL) return NULL;
   config->version = receive_str(file_descriptor);
+  if (!config->version) { free(config); return NULL; }
+  if (strcmp(config->version, PROTOCOL_VERSION) != 0) {
+    fprintf(stderr, "Protocol version mismatch: client=%s, server=%s\n",
+            config->version, PROTOCOL_VERSION);
+    free(config->version);
+    free(config);
+    send_status(file_descriptor, STATUS_ERROR);
+    return NULL;
+  }
   config->send_directory = receive_str(file_descriptor);
+  if (!config->send_directory) { free(config->version); free(config); return NULL; }
   config->receive_root_directory = receive_str(file_descriptor);
-  config->save_to_disk = receive_int(file_descriptor);
+  if (!config->receive_root_directory) { free(config->version); free(config->send_directory); free(config); return NULL; }
-  config->use_multithreading = receive_int(file_descriptor);
+  int tmp;
-  config->use_chunk_serialization = receive_int(file_descriptor);
+  if (!receive_int(file_descriptor, &tmp)) goto error;
-  config->use_compression = receive_int(file_descriptor);
+  config->save_to_disk = tmp;
-  config->compression_level = receive_int(file_descriptor);
+  if (!receive_int(file_descriptor, &tmp)) goto error;
-  config->num_connections = receive_int(file_descriptor);
+  config->use_multithreading = tmp;
-  send_status(file_descriptor, STATUS_OK);
+  if (!receive_int(file_descriptor, &tmp)) goto error;
+  config->use_chunk_serialization = tmp;
+  if (!receive_int(file_descriptor, &tmp)) goto error;
+  config->use_compression = tmp;
+  if (!receive_int(file_descriptor, &tmp)) goto error;
+  config->use_metadata = tmp;
+  if (!receive_int(file_descriptor, &tmp)) goto error;
+  config->compression_level = tmp;
+  if (!receive_int(file_descriptor, &tmp)) goto error;
+  config->chunk_size = (unsigned long long)tmp;
+  if (!receive_int(file_descriptor, &tmp)) goto error;
+  config->use_sendfile = tmp;
+  if (!receive_int(file_descriptor, &tmp)) goto error;
+  config->use_delete = tmp;
+  if (!receive_int(file_descriptor, &tmp)) goto error;
+  config->use_incremental = tmp;
+  config->show_progress = false;
+  config->dry_run = false;
+  config->ssh_port = 22;
+  config->transport = TRANSPORT_TCP;
+  config->ssh_destination = NULL;
+  config->exclude_patterns = NULL;
+  config->exclude_count = 0;
+  config->include_patterns = NULL;
+  config->include_count = 0;
+  config->max_size = 0;
+  config->min_size = 0;
+  if (!send_status(file_descriptor, STATUS_OK)) goto error;
   return config;
+
+error:
+  free(config->version);
+  free(config->send_directory);
+  free(config->receive_root_directory);
+  free(config);
+  return NULL;
 }

src/shared/config.h

@@ -3,6 +3,11 @@
 
 #include <stdbool.h>
 
+typedef enum {
+  TRANSPORT_TCP,
+  TRANSPORT_SSH
+} TransportType;
+
 typedef struct Config {
   char *version;
   char *send_directory;
@@ -11,18 +16,38 @@ typedef struct Config {
   bool use_multithreading;
   bool use_chunk_serialization;
   bool use_compression;
-  bool use_single_send_per_file;
+  bool use_sendfile;
+  bool use_metadata;
+  bool show_progress;
+  bool dry_run;
+  bool use_delete;
   int compression_level;
-  int num_connections;
+  unsigned long long chunk_size;
+  int ssh_port;
+  TransportType transport;
+  char *ssh_destination;
+  char **exclude_patterns;
+  int exclude_count;
+  char **include_patterns;
+  int include_count;
+  unsigned long long max_size;
+  unsigned long long min_size;
+  bool use_incremental;
 } Config;
 
+#define PROTOCOL_VERSION "1.1.0"
+#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
+
 Config *config_create(char *version, char *send_directory,
                       char *receive_directory, bool save_to_disk,
                       bool use_multithreading, bool use_chunk_serialization,
-                      bool use_compression, int compression_level,
+                       bool use_compression, bool use_metadata,
-                      int num_connections);
+                       int compression_level, bool use_sendfile,
+                       unsigned long long chunk_size);
 void config_delete(Config *config);
-void config_send(int file_descriptor, Config *config);
+bool config_send(int file_descriptor, Config *config);
 Config *config_receive(int file_descriptor);
+bool is_remote_dest(const char *s);
+void config_parse_ssh_dest(Config *config);
 
 #endif

src/shared/data.c

@@ -1,23 +1,33 @@
-
 #include "data.h"
 #include "log.h"
 #include "stdlib.h"
-#include "zstd.h"
 
 Data *data_create_empty(size_t data_size) {
   void *data = malloc(data_size);
   if (data == NULL) {
     log_message(LOG_LEVEL_ERROR, "Could not allocate memory for empty data");
-    exit(EXIT_FAILURE);
+    return NULL;
   }
   return data_create(data, data_size);
 }
 
+Data *data_create_reserve(size_t size) {
+  Data *d = malloc(sizeof(Data));
+  if (d == NULL) {
+    log_message(LOG_LEVEL_ERROR, "Could not allocate memory for data");
+    return NULL;
+  }
+  d->data = NULL;
+  d->size = size;
+  return d;
+}
+
 Data *data_create(void *data, size_t data_size) {
   Data *new_data = malloc(sizeof(Data));
   if (new_data == NULL) {
     log_message(LOG_LEVEL_ERROR, "Could not allocate memory for data");
-    exit(EXIT_FAILURE);
+    free(data);
+    return NULL;
   }
   new_data->data = data;
   new_data->size = data_size;
@@ -25,47 +35,7 @@ Data *data_create(void *data, size_t data_size) {
 }
 
 void data_destroy(Data *data) {
+  if (data == NULL) return;
   free(data->data);
   free(data);
 }
-
-Data *data_compress(Data *data_to_compress, int compression_level) {
-  log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
-  Data *compressed_data =
-      data_create_empty(ZSTD_compressBound(data_to_compress->size));
-
-  compressed_data->size = ZSTD_compress(
-      compressed_data->data, compressed_data->size, data_to_compress->data,
-      data_to_compress->size, compression_level);
-  if (ZSTD_isError(compressed_data->size)) {
-    log_message(LOG_LEVEL_ERROR, "Compression failed: %s",
-                ZSTD_getErrorName(compressed_data->size));
-    exit(EXIT_FAILURE);
-  }
-
-  log_message(LOG_LEVEL_DEBUG, "Data succesfully compressed from %zu to %zu",
-              data_to_compress->size, compressed_data->size);
-  return compressed_data;
-}
-
-Data *data_decompress(Data *compressed_data) {
-  log_message(LOG_LEVEL_DEBUG, "Start to decompress data");
-  Data *uncompressed_data = data_create_empty(
-      ZSTD_getFrameContentSize(compressed_data->data, compressed_data->size));
-  if (ZSTD_isError(uncompressed_data->size)) {
-    log_message(LOG_LEVEL_ERROR, "Decompression failed: %s",
-                ZSTD_getErrorName(uncompressed_data->size));
-    exit(EXIT_FAILURE);
-  }
-
-  uncompressed_data->size =
-      ZSTD_decompress(uncompressed_data->data, uncompressed_data->size,
-                      compressed_data->data, compressed_data->size);
-  if (ZSTD_isError(uncompressed_data->size)) {
-    log_message(LOG_LEVEL_ERROR, "Decompression failed: %s",
-                ZSTD_getErrorName(uncompressed_data->size));
-    exit(EXIT_FAILURE);
-  }
-  log_message(LOG_LEVEL_DEBUG, "Decompressed data successfully");
-  return uncompressed_data;
-}

src/shared/data.h

@@ -9,9 +9,8 @@ typedef struct {
 } Data;
 
 Data *data_create_empty(size_t data_size);
+Data *data_create_reserve(size_t size);
 Data *data_create(void *data, size_t data_size);
 void data_destroy(Data *data);
-Data *data_compress(Data *data_to_compress, int compression_level);
-Data *data_decompress(Data *compressed_data);
 
 #endif

src/shared/file.c

@@ -1,35 +1,46 @@
 #include <dirent.h>
+#include <fcntl.h>
 #include <libgen.h>
 #include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <zstd.h>
+#include <sys/sendfile.h>
+#include <sys/stat.h>
+#include <unistd.h>
 
+#include "compression.h"
+#include "log.h"
+#include "config.h"
 #include "data.h"
 #include "file.h"
 #include "log.h"
-#include "socket.h"
+#include "metadata.h"
+#include "protocol.h"
+#include "utils.h"
 
-File *file_create(const char *path, struct stat *stats) {
+File *file_create(const char *path) {
   File *file = (File *)malloc(sizeof(File));
   if (file == NULL) {
-    perror("FATAL ERROR: Could not allocate memory for file struct");
+    perror("ERROR: Could not allocate memory for file struct");
-    exit(EXIT_FAILURE);
+    return NULL;
   }
 
-  file->stats = *stats;
-
   int path_len = strlen(path);
   file->path = (char *)malloc(path_len + 1);
   if (file->path == NULL) {
-    perror("FATAL ERROR: Could not allocate memory for path file string");
     free(file);
-    exit(EXIT_FAILURE);
+    return NULL;
   }
 
   strcpy(file->path, path);
-  file->data = NULL;
+  file->data = data_create_reserve(0);
+  if (file->data == NULL) {
+    free(file->path);
+    free(file);
+    return NULL;
+  }
+  file->metadata = NULL;
   return file;
 }
 
@@ -37,35 +48,287 @@ void file_destroy(void *item) {
   if (item == NULL)
     return;
   File *file = (File *)item;
-  free(file->data);
+  data_destroy(file->data);
   file->data = NULL;
+  file_metadata_destroy(file->metadata);
+  file->metadata = NULL;
   free(file->path);
   file->path = NULL;
   free(file);
 }
 
-void file_load_data(File *file) {
+FileMetadata *file_metadata_create(struct stat *stats) {
-  if (file == NULL)
+  FileMetadata *m = malloc(sizeof(FileMetadata));
-    return;
+  if (m == NULL) {
-  file->data = data_create_empty(file->stats.st_size);
+    perror("ERROR: Could not allocate memory for file metadata");
-  printf("%ld is file big", file->data->size);
+    return NULL;
+  }
+  m->mode = stats->st_mode;
+  m->uid = stats->st_uid;
+  m->gid = stats->st_gid;
+  m->mtime_sec = stats->st_mtime;
+#ifdef __linux__
+  m->mtime_nsec = stats->st_mtim.tv_nsec;
+#else
+  m->mtime_nsec = 0;
+#endif
+  return m;
+}
+
+void file_metadata_destroy(void *metadata) {
+  free(metadata);
+}
+
+bool file_load_data(File *file) {
+  if (file == NULL) return false;
+  if (file->data->data == NULL) {
+    file->data->data = malloc(file->data->size);
+    if (file->data->data == NULL) {
+      perror("Could not allocate memory for file data");
+      return false;
+    }
+  }
   size_t bytes_read = file_content_to_buffer(file);
   if (bytes_read != file->data->size) {
-    log_message(STATUS_ERROR, "Didnt read expected amount of bytes from file");
+    log_message(LOG_LEVEL_ERROR, "Did not read expected amount of bytes from file");
-    exit(EXIT_FAILURE);
+    return false;
   }
+  return true;
 }
 
-void file_print(void *item) {
+bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level) {
-  if (item == NULL)
+  Data *data_to_send = file->data;
-    return;
+  Data *compressed_data = NULL;
-  printf("%s\n", ((File *)item)->path);
+  if (compression_level > 0) {
+    compressed_data = data_compress(file->data, compression_level);
+    if (compressed_data == NULL) {
+      log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
+      return false;
+    }
+    data_to_send = compressed_data;
+  }
+  if (use_metadata && !metadata_send(file_descriptor, file->metadata)) {
+    data_destroy(compressed_data);
+    return false;
+  }
+  if (!send_data(file_descriptor, data_to_send)) {
+    data_destroy(compressed_data);
+    return false;
+  }
+  data_destroy(compressed_data);
+  return true;
 }
 
-void file_send_single_calls(File *file, int file_descriptor) {
+bool file_send_sendfile_no_path(File *file, int file_descriptor, bool use_metadata) {
-  send_str(file_descriptor, file->path);
+  if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false;
-  printf("Sending File: %ld", file->data->size);
+
-  send_data(file_descriptor, file->data->data, file->data->size);
+  int fd = open(file->path, O_RDONLY);
+  if (fd == -1) {
+    perror("Could not open file for sendfile");
+    return false;
+  }
+
+  unsigned long long file_size = file->data->size;
+  if (!send_n_data(file_descriptor, &file_size, sizeof(unsigned long long))) {
+    close(fd);
+    return false;
+  }
+
+  off_t offset = 0;
+  while (offset < file_size) {
+    ssize_t sent = sendfile(file_descriptor, fd, &offset, file_size - offset);
+    if (sent == -1) {
+      perror("sendfile failed");
+      close(fd);
+      return false;
+    }
+  }
+
+  close(fd);
+  return true;
+}
+
+bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) {
+  Data *data_to_send = file->data;
+  Data *compressed_data = NULL;
+  if (compression_level > 0) {
+    compressed_data = data_compress(file->data, compression_level);
+    if (compressed_data == NULL) {
+      log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
+      return false;
+    }
+    data_to_send = compressed_data;
+  }
+  if (!send_str(file_descriptor, file->path)) {
+    data_destroy(compressed_data);
+    return false;
+  }
+  if (use_metadata && !metadata_send(file_descriptor, file->metadata)) {
+    data_destroy(compressed_data);
+    return false;
+  }
+  if (!send_data(file_descriptor, data_to_send)) {
+    data_destroy(compressed_data);
+    return false;
+  }
+  data_destroy(compressed_data);
+  return true;
+}
+
+bool file_save_to_disk(const char *root_directory, File *file) {
+  char *disk_path = path_cat((char *)root_directory, file->path);
+  if (disk_path == NULL) return false;
+  bool ok = to_disk(disk_path, file->data->data, file->data->size);
+  if (ok) file_restore_metadata(disk_path, file->metadata);
+  free(disk_path);
+  return ok;
+}
+
+File *receive_incremental_check(int fd, Config *config, bool *skipped) {
+  *skipped = false;
+  char *check_path = receive_str(fd);
+  if (check_path == NULL) { send_status(fd, STATUS_ERROR); return NULL; }
+
+  unsigned long long check_size;
+  long long check_mtime;
+  if (!receive_n_data(fd, &check_size, sizeof(check_size)) ||
+      !receive_n_data(fd, &check_mtime, sizeof(check_mtime))) {
+    free(check_path);
+    send_status(fd, STATUS_ERROR);
+    return NULL;
+  }
+
+  char *full_path = path_cat(config->receive_root_directory, check_path);
+  struct stat st;
+  bool match = false;
+  if (full_path && stat(full_path, &st) == 0 &&
+      (unsigned long long)st.st_size == check_size &&
+      (long long)st.st_mtime == check_mtime) {
+    match = true;
+  }
+  free(full_path);
+
+  if (match) {
+    if (!send_status(fd, STATUS_OK)) { free(check_path); return NULL; }
+    free(check_path);
+    *skipped = true;
+    return NULL;
+  }
+
+  if (!send_status(fd, STATUS_NEXT)) { free(check_path); return NULL; }
+
+  File *file = file_create(check_path);
+  free(check_path);
+  if (file == NULL) { send_status(fd, STATUS_ERROR); return NULL; }
+
+  if (config->use_metadata) {
+    int meta_ok = 1;
+    file->metadata = metadata_receive(fd, &meta_ok);
+    if (!meta_ok) { file_destroy(file); send_status(fd, STATUS_ERROR); return NULL; }
+  }
+
+  Data *file_data = receive_data(fd);
+  if (file_data == NULL) {
+    file_destroy(file);
+    send_status(fd, STATUS_ERROR);
+    return NULL;
+  }
+
+  if (config->use_compression) {
+    Data *uncompressed = data_decompress(file_data);
+    data_destroy(file_data);
+    if (uncompressed == NULL) { file_destroy(file); send_status(fd, STATUS_ERROR); return NULL; }
+    file_data = uncompressed;
+  }
+
+  data_destroy(file->data);
+  file->data = file_data;
+  return file;
+}
+
+bool to_disk(const char *path, const void *data, unsigned long long data_size) {
+  char *directory = str_dup(path);
+  char *dir_to_free = directory;
+  directory = dirname(directory);
+  if (!mkdir_r(directory)) {
+    free(dir_to_free);
+    return false;
+  }
+  FILE *file_pointer = fopen(path, "wb");
+  if (file_pointer == NULL) {
+    perror("Could not open File");
+    free(dir_to_free);
+    return false;
+  }
+  if (fwrite(data, 1, data_size, file_pointer) != data_size) {
+    perror("Failed to write all data to disk");
+    fclose(file_pointer);
+    free(dir_to_free);
+    return false;
+  }
+  fclose(file_pointer);
+  free(dir_to_free);
+  return true;
+}
+
+bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata) {
+  if (!send_str(file_descriptor, file->path)) return false;
+  if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false;
+
+  int fd = open(file->path, O_RDONLY);
+  if (fd == -1) {
+    perror("Could not open file for sendfile");
+    return false;
+  }
+
+  unsigned long long file_size = file->data->size;
+  if (!send_n_data(file_descriptor, &file_size, sizeof(unsigned long long))) {
+    close(fd);
+    return false;
+  }
+
+  off_t offset = 0;
+  while (offset < file_size) {
+    ssize_t sent = sendfile(file_descriptor, fd, &offset, file_size - offset);
+    if (sent == -1) {
+      perror("sendfile failed");
+      close(fd);
+      return false;
+    }
+  }
+
+  close(fd);
+  return true;
+}
+
+File *file_receive(Config *config, int file_descriptor) {
+  char *path = receive_str(file_descriptor);
+  if (path == NULL) return NULL;
+  File *file = file_create(path);
+  free(path);
+  if (file == NULL) return NULL;
+  if (config->use_metadata) {
+    int meta_ok = 1;
+    file->metadata = metadata_receive(file_descriptor, &meta_ok);
+    if (!meta_ok) { file_destroy(file); return NULL; }
+  }
+  Data *file_data = receive_data(file_descriptor);
+  if (file_data == NULL) {
+    file_destroy(file);
+    return NULL;
+  }
+  if (config->use_compression) {
+    Data *file_data_uncompressed = data_decompress(file_data);
+    data_destroy(file_data);
+    if (file_data_uncompressed == NULL) {
+      file_destroy(file);
+      return NULL;
+    }
+    file_data = file_data_uncompressed;
+  }
+  data_destroy(file->data);
+  file->data = file_data;
+  return file;
 }
 
 size_t file_content_to_buffer(File *file) {
@@ -75,29 +338,14 @@ size_t file_content_to_buffer(File *file) {
     return 0;
   }
   size_t bytes_read =
-      fread(file->data->data, 1, file->stats.st_size, file_pointer);
+      fread(file->data->data, 1, file->data->size, file_pointer);
-  if (bytes_read != (size_t)file->stats.st_size) {
+  if (bytes_read != (size_t)file->data->size) {
-    perror("Read to many or to less bytes from File!");
+    fclose(file_pointer);
+    perror("Read unexpected number of bytes from File!");
     return 0;
   }
   fclose(file_pointer);
   return bytes_read;
 }
 
-FileReceive *file_receive_create(char *path, Data *data) {
-  FileReceive *file = malloc(sizeof(FileReceive));
-  file->path = path;
-  file->data = data;
-  return file;
-}
 
-void file_receive_destroy(void *file_receive) {
-  if (file_receive == NULL)
-    return;
-  FileReceive *file = (FileReceive *)file_receive;
-  data_destroy(file->data);
-  free(file->path);
-  free(file);
-}
-
-FileReceive *file_receive_from_buffer(void *buffer) {}

src/shared/file.h

@@ -1,31 +1,38 @@
 #ifndef FILE_H
 #define FILE_H
 
+#include "config.h"
 #include "data.h"
+#include <stdbool.h>
 #include <sys/stat.h>
 
 typedef struct {
-  char *path;
+  mode_t mode;
-  struct stat stats;
+  uid_t uid;
-  Data *data;
+  gid_t gid;
-} File;
+  time_t mtime_sec;
+  long mtime_nsec;
+} FileMetadata;
 
 typedef struct {
   char *path;
   Data *data;
-} FileReceive;
+  FileMetadata *metadata;
+} File;
 
-File *file_create(const char *path, struct stat *stats);
+File *file_create(const char *path);
 void file_destroy(void *item);
-void file_load_data(File *file);
+bool file_load_data(File *file);
-void file_print(void *item);
+File *file_receive(Config *config, int file_descriptor);
-void file_send_single_calls(File *file, int file_descriptor);
+bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level);
+bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level);
+bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata);
+bool file_send_sendfile_no_path(File *file, int file_descriptor, bool use_metadata);
 size_t file_content_to_buffer(File *file);
-Data *file_compress(File *file);
+FileMetadata *file_metadata_create(struct stat *stats);
-
+void file_metadata_destroy(void *metadata);
-FileReceive *file_receive_create(char *path, Data *data);
+bool to_disk(const char *path, const void *data, unsigned long long data_size);
-void file_receive_destroy(void *file_receive);
+bool file_save_to_disk(const char *root_directory, File *file);
-FileReceive *file_receive_from_buffer(void *buffer);
+File *receive_incremental_check(int fd, Config *config, bool *skipped);
-FileReceive *file_receive_decompress(void *FileReceive);
 
 #endif

src/shared/log.c

@@ -4,7 +4,11 @@
 #include <time.h>
 
 static const char *log_level_strings[] = {"DEBUG", "INFO", "WARN", "ERROR"};
-static LogLevel current_log_level = LOG_LEVEL_DEBUG;
+static LogLevel current_log_level = LOG_LEVEL_WARNING;
+
+void set_log_level(LogLevel level) {
+  current_log_level = level;
+}
 
 void log_message(LogLevel log_level, char *format, ...) {
   if (log_level < current_log_level)
@@ -12,14 +16,13 @@ void log_message(LogLevel log_level, char *format, ...) {
   time_t now = time(NULL);
   struct tm *t = localtime(&now);
 
-  // Print timestamp and log level to the file
+  fprintf(stderr, "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t->tm_year + 1900,
-  printf("%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t->tm_year + 1900,
+          t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min, t->tm_sec,
-         t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min, t->tm_sec,
+          log_level_strings[log_level]);
-         log_level_strings[log_level]);
 
   va_list args;
   va_start(args, format);
-  vprintf(format, args);
+  vfprintf(stderr, format, args);
   va_end(args);
-  printf("\n");
+  fprintf(stderr, "\n");
 }

src/shared/log.h

@@ -9,5 +9,6 @@ typedef enum {
 } LogLevel;
 
 void log_message(LogLevel log_level, char *message, ...);
+void set_log_level(LogLevel level);
 
 #endif

src/shared/metadata.c

@@ -0,0 +1,90 @@
+#include "metadata.h"
+#include "file.h"
+#include "protocol.h"
+#include <fcntl.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <unistd.h>
+
+void metadata_to_buf(char **buf, FileMetadata *m) {
+  int present = (m != NULL) ? 1 : 0;
+  memcpy(*buf, &present, sizeof(int));
+  *buf += sizeof(int);
+  if (m == NULL)
+    return;
+  memcpy(*buf, &m->mode, sizeof(mode_t));        *buf += sizeof(mode_t);
+  memcpy(*buf, &m->uid, sizeof(uid_t));          *buf += sizeof(uid_t);
+  memcpy(*buf, &m->gid, sizeof(gid_t));          *buf += sizeof(gid_t);
+  memcpy(*buf, &m->mtime_sec, sizeof(time_t));   *buf += sizeof(time_t);
+  memcpy(*buf, &m->mtime_nsec, sizeof(long));    *buf += sizeof(long);
+}
+
+FileMetadata *metadata_from_buf(char **buf) {
+  int present;
+  memcpy(&present, *buf, sizeof(int));
+  *buf += sizeof(int);
+  if (!present)
+    return NULL;
+  FileMetadata *m = malloc(sizeof(FileMetadata));
+  memcpy(&m->mode, *buf, sizeof(mode_t));        *buf += sizeof(mode_t);
+  memcpy(&m->uid, *buf, sizeof(uid_t));          *buf += sizeof(uid_t);
+  memcpy(&m->gid, *buf, sizeof(gid_t));          *buf += sizeof(gid_t);
+  memcpy(&m->mtime_sec, *buf, sizeof(time_t));   *buf += sizeof(time_t);
+  memcpy(&m->mtime_nsec, *buf, sizeof(long));    *buf += sizeof(long);
+  return m;
+}
+
+bool metadata_send(int file_descriptor, FileMetadata *m) {
+  if (m == NULL) {
+    int zero = 0;
+    return send_n_data(file_descriptor, &zero, sizeof(int));
+  }
+  int present = 1;
+  return send_n_data(file_descriptor, &present, sizeof(int)) &&
+         send_n_data(file_descriptor, &m->mode, sizeof(mode_t)) &&
+         send_n_data(file_descriptor, &m->uid, sizeof(uid_t)) &&
+         send_n_data(file_descriptor, &m->gid, sizeof(gid_t)) &&
+         send_n_data(file_descriptor, &m->mtime_sec, sizeof(time_t)) &&
+         send_n_data(file_descriptor, &m->mtime_nsec, sizeof(long));
+}
+
+FileMetadata *metadata_receive(int file_descriptor, int *ok) {
+  int present;
+  if (!receive_n_data(file_descriptor, &present, sizeof(int))) {
+    if (ok) *ok = 0;
+    return NULL;
+  }
+  if (!present) {
+    if (ok) *ok = 1;
+    return NULL;
+  }
+  FileMetadata *m = malloc(sizeof(FileMetadata));
+  if (m == NULL) { if (ok) *ok = 0; return NULL; }
+  if (!receive_n_data(file_descriptor, &m->mode, sizeof(mode_t)) ||
+      !receive_n_data(file_descriptor, &m->uid, sizeof(uid_t)) ||
+      !receive_n_data(file_descriptor, &m->gid, sizeof(gid_t)) ||
+      !receive_n_data(file_descriptor, &m->mtime_sec, sizeof(time_t)) ||
+      !receive_n_data(file_descriptor, &m->mtime_nsec, sizeof(long))) {
+    free(m);
+    if (ok) *ok = 0;
+    return NULL;
+  }
+  if (ok) *ok = 1;
+  return m;
+}
+
+void file_restore_metadata(const char *path, FileMetadata *metadata) {
+  if (metadata == NULL)
+    return;
+  chmod(path, metadata->mode & 07777);
+  int chown_ret = chown(path, metadata->uid, metadata->gid);
+  (void)chown_ret;
+  struct timespec times[2];
+  times[0].tv_sec = 0;
+  times[0].tv_nsec = UTIME_OMIT;
+  times[1].tv_sec = metadata->mtime_sec;
+  times[1].tv_nsec = metadata->mtime_nsec;
+  utimensat(AT_FDCWD, path, times, 0);
+}

src/shared/metadata.h

@@ -0,0 +1,16 @@
+#ifndef METADATA_H
+#define METADATA_H
+
+#include "file.h"
+#include <stdbool.h>
+#include <sys/stat.h>
+
+#define FILE_METADATA_WIRE_SIZE (sizeof(mode_t) + sizeof(uid_t) + sizeof(gid_t) + sizeof(time_t) + sizeof(long))
+
+void metadata_to_buf(char **buf, FileMetadata *m);
+FileMetadata *metadata_from_buf(char **buf);
+bool metadata_send(int file_descriptor, FileMetadata *m);
+FileMetadata *metadata_receive(int file_descriptor, int *ok);
+void file_restore_metadata(const char *path, FileMetadata *metadata);
+
+#endif

src/shared/multiprocessing.c

@@ -1,32 +1,47 @@
 #include "multiprocessing.h"
+#include "array_list.h"
+#include "chunk.h"
+#include "compression.h"
 #include "config.h"
+#include "data.h"
+#include "file.h"
+#include "log.h"
+#include "protocol.h"
 #include "queue.h"
+#include "utils.h"
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <threads.h>
 
 PipelineContextSender *pipeline_context_sender_create(Config *config,
                                                       Queue *queue_scanner,
                                                       Queue *queue_loader) {
   PipelineContextSender *context = malloc(sizeof(PipelineContextSender));
+  if (context == NULL) return NULL;
   context->config = config;
   context->queue_scanner = queue_scanner;
   context->queue_loader = queue_loader;
   context->scanner_done = false;
   context->loader_done = false;
+  context->manifest = NULL;
   if (mtx_init(&context->mutex_scanner, mtx_plain) != thrd_success ||
       cnd_init(&context->condition_not_full_scanner) != thrd_success ||
       cnd_init(&context->condition_not_empty_scanner) != thrd_success ||
       mtx_init(&context->mutex_loader, mtx_plain) != thrd_success ||
       cnd_init(&context->condition_not_full_loader) != thrd_success ||
       cnd_init(&context->condition_not_empty_loader) != thrd_success) {
-    perror("Error initializing synchronization objects!");
+    perror("Error initializing synchronization objects");
-    exit(EXIT_FAILURE);
+    free(context);
+    return NULL;
   }
   return context;
 }
 
 void pipeline_context_sender_destroy(PipelineContextSender *context) {
+  if (context->manifest) {
+    array_list_delete(context->manifest);
+  }
   config_delete(context->config);
   queue_destroy(context->queue_scanner);
   queue_destroy(context->queue_loader);
@@ -43,6 +58,7 @@ PipelineContextReceiver *pipeline_context_receiver_create(Config *config,
                                                           Queue *queue,
                                                           int file_descriptor) {
   PipelineContextReceiver *context = malloc(sizeof(PipelineContextReceiver));
+  if (context == NULL) return NULL;
   context->config = config;
   context->queue = queue;
   context->file_descriptor = file_descriptor;
@@ -50,8 +66,9 @@ PipelineContextReceiver *pipeline_context_receiver_create(Config *config,
   if (mtx_init(&context->mutex, mtx_plain) != thrd_success ||
       cnd_init(&context->condition_not_full) != thrd_success ||
       cnd_init(&context->condition_not_empty) != thrd_success) {
-    perror("Error initializing synchronization objects!");
+    perror("Error initializing synchronization objects");
-    exit(EXIT_FAILURE);
+    free(context);
+    return NULL;
   }
   return context;
 }
@@ -64,3 +81,117 @@ void pipeline_context_receiver_destroy(PipelineContextReceiver *context) {
   cnd_destroy(&context->condition_not_empty);
   free(context);
 }
+
+static void receive_chunk_enqueue(int file_descriptor,
+                                  PipelineContextReceiver *context) {
+  Data *chunk_data = receive_data(file_descriptor);
+  if (chunk_data == NULL) {
+    log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data");
+    return;
+  }
+  Data *data_to_process = chunk_data;
+  if (context->config->use_compression) {
+    data_to_process = data_decompress(chunk_data);
+    data_destroy(chunk_data);
+    if (data_to_process == NULL) {
+      log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk");
+      return;
+    }
+  }
+  Chunk *chunk = chunk_deserialize(data_to_process, context->config->use_metadata);
+  data_destroy(data_to_process);
+  if (chunk == NULL) {
+    log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping");
+    return;
+  }
+
+  for (int i = 0; i < chunk->element_count; i++) {
+    File *file = chunk->items[i];
+    chunk->items[i] = NULL;
+    queue_enqueue_multithreaded(context->queue, file, &context->mutex,
+                                &context->condition_not_empty,
+                                &context->condition_not_full);
+  }
+  chunk_destroy(chunk);
+}
+
+int receive_thread(void *pipeline_context) {
+  PipelineContextReceiver *context =
+      (PipelineContextReceiver *)pipeline_context;
+  mtx_lock(&context->mutex);
+  int file_descriptor = context->file_descriptor;
+  Config *config = context->config;
+  mtx_unlock(&context->mutex);
+
+  Status status;
+  if (!receive_status(file_descriptor, &status)) return thrd_error;
+  while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) {
+    if (status == STATUS_CHECK) {
+      bool skipped;
+      File *file = receive_incremental_check(file_descriptor, config, &skipped);
+      if (!skipped) {
+        if (file == NULL) return thrd_error;
+        queue_enqueue_multithreaded(context->queue, file, &context->mutex,
+                                    &context->condition_not_empty,
+                                    &context->condition_not_full);
+      }
+    } else if (status == STATUS_CHUNK) {
+      receive_chunk_enqueue(file_descriptor, context);
+    } else {
+      File *file = file_receive(config, file_descriptor);
+      if (file) {
+        queue_enqueue_multithreaded(context->queue, file, &context->mutex,
+                                    &context->condition_not_empty,
+                                    &context->condition_not_full);
+      } else {
+        log_message(LOG_LEVEL_ERROR, "Failed to receive file");
+      }
+    }
+    if (!receive_status(file_descriptor, &status)) return thrd_error;
+  }
+  if (status == STATUS_MANIFEST) {
+    int count;
+    if (!receive_int(file_descriptor, &count)) return thrd_error;
+    ArrayList *manifest = array_list_create(free);
+    if (manifest) {
+      for (int i = 0; i < count; i++) {
+        char *s = receive_str(file_descriptor);
+        if (s) {
+          array_list_add(manifest, s);
+        }
+      }
+      delete_extras(context->config->receive_root_directory, manifest);
+      for (int i = 0; i < manifest->size; i++)
+        free(manifest->items[i]);
+      array_list_delete(manifest);
+    }
+    if (!receive_status(file_descriptor, &status)) return thrd_error;
+  }
+  mtx_lock(&context->mutex);
+  context->receiver_done = true;
+  cnd_signal(&context->condition_not_empty);
+  mtx_unlock(&context->mutex);
+  return thrd_success;
+}
+
+int write_thread(void *pipeline_context) {
+  PipelineContextReceiver *context =
+      (PipelineContextReceiver *)pipeline_context;
+  mtx_lock(&context->mutex);
+  bool save_to_disk = context->config->save_to_disk;
+  char *root_directory = str_dup(context->config->receive_root_directory);
+  mtx_unlock(&context->mutex);
+
+  while (true) {
+    File *file = queue_dequeue_multithreaded(
+        context->queue, &context->mutex, &context->condition_not_empty,
+        &context->condition_not_full, &context->receiver_done);
+    if (file == NULL) {
+      free(root_directory);
+      return thrd_success;
+    }
+    if (save_to_disk)
+      file_save_to_disk(root_directory, file);
+    file_destroy(file);
+  }
+}

src/shared/multiprocessing.h

@@ -3,7 +3,9 @@
 
 #include <threads.h>
 
+#include "array_list.h"
 #include "config.h"
+#include "file.h"
 #include "queue.h"
 
 typedef struct {
@@ -18,6 +20,7 @@ typedef struct {
   cnd_t condition_not_full_loader;
   cnd_t condition_not_empty_loader;
   bool loader_done;
+  ArrayList *manifest;
 } PipelineContextSender;
 
 typedef struct PipelineContextReceiver {
@@ -38,4 +41,6 @@ PipelineContextReceiver *pipeline_context_receiver_create(Config *config,
                                                           Queue *queue_receiver,
                                                           int file_descriptor);
 void pipeline_context_receiver_destroy(PipelineContextReceiver *context);
+int receive_thread(void *pipeline_context);
+int write_thread(void *pipeline_context);
 #endif

src/shared/protocol.c

@@ -0,0 +1,189 @@
+#include "protocol.h"
+#include "log.h"
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+static __thread int io_read_fd = -1;
+static __thread int io_write_fd = -1;
+
+static unsigned long long io_bwlimit = 0;
+static long long bw_tokens = 0;
+static struct timespec bw_last_refill = {0, 0};
+
+void io_set_fds(int read_fd, int write_fd) {
+  io_read_fd = read_fd;
+  io_write_fd = write_fd;
+}
+
+void io_set_bwlimit(unsigned long long bytes_per_sec) {
+  io_bwlimit = bytes_per_sec;
+  bw_tokens = (long long)io_bwlimit;
+  clock_gettime(CLOCK_MONOTONIC, &bw_last_refill);
+}
+
+static void bw_throttle(size_t bytes_written) {
+  if (io_bwlimit == 0) return;
+
+  struct timespec now;
+  clock_gettime(CLOCK_MONOTONIC, &now);
+
+  long long elapsed_ns = (now.tv_sec - bw_last_refill.tv_sec) * 1000000000LL +
+                         (now.tv_nsec - bw_last_refill.tv_nsec);
+  bw_last_refill = now;
+
+  long long tokens_to_add = (long long)((double)io_bwlimit * elapsed_ns / 1000000000.0);
+  bw_tokens += tokens_to_add;
+  if (bw_tokens > (long long)io_bwlimit)
+    bw_tokens = (long long)io_bwlimit;
+
+  bw_tokens -= (long long)bytes_written;
+
+  if (bw_tokens < 0) {
+    long long deficit_ns = (long long)((double)(-bw_tokens) / io_bwlimit * 1000000000.0);
+    struct timespec sleep_time, remaining;
+    sleep_time.tv_sec = deficit_ns / 1000000000LL;
+    sleep_time.tv_nsec = deficit_ns % 1000000000LL;
+    while (nanosleep(&sleep_time, &remaining) < 0 && errno == EINTR)
+      sleep_time = remaining;
+    bw_tokens = 0;
+    clock_gettime(CLOCK_MONOTONIC, &bw_last_refill);
+  }
+}
+
+static int io_fd(int dir_fd, int file_descriptor) {
+  return (dir_fd != -1) ? dir_fd : file_descriptor;
+}
+
+bool send_n_data(int file_descriptor, void *data, size_t data_size) {
+  log_message(LOG_LEVEL_DEBUG, "    Sending n Data: %zu", data_size);
+  int fd = io_fd(io_write_fd, file_descriptor);
+  ssize_t total_bytes_send = 0;
+  while (total_bytes_send < data_size) {
+    size_t chunk = data_size - total_bytes_send;
+    if (io_bwlimit > 0 && chunk > 65536)
+      chunk = 65536;
+    ssize_t bytes_send =
+        write(fd, (char *)data + total_bytes_send, chunk);
+    if (bytes_send <= 0) {
+      log_message(LOG_LEVEL_ERROR, "Could not send data");
+      return false;
+    }
+    bw_throttle((size_t)bytes_send);
+    total_bytes_send += bytes_send;
+  }
+  log_message(LOG_LEVEL_DEBUG, "    Send n Data: %zu", total_bytes_send);
+  return true;
+}
+
+bool receive_n_data(int file_descriptor, void *data, size_t data_size) {
+  log_message(LOG_LEVEL_DEBUG, "    Receiving n Data: %zu", data_size);
+  int fd = io_fd(io_read_fd, file_descriptor);
+  size_t total_bytes_received = 0;
+  while (total_bytes_received < data_size) {
+    ssize_t bytes_received =
+        read(fd, (char *)data + total_bytes_received, data_size - total_bytes_received);
+    if (bytes_received <= 0) {
+      if (bytes_received == 0)
+        log_message(LOG_LEVEL_ERROR, "Connection closed while receiving data");
+      else
+        log_message(LOG_LEVEL_ERROR, "Could not receive bytes");
+      return false;
+    }
+    total_bytes_received += bytes_received;
+  }
+  log_message(LOG_LEVEL_DEBUG, "    Received n Data: %zu", total_bytes_received);
+  return true;
+}
+
+static const char *status_to_string(Status status) {
+  switch (status) {
+  case STATUS_OK:
+    return "OK";
+  case STATUS_ERROR:
+    return "ERROR";
+  case STATUS_FINISHED:
+    return "FINISHED";
+  case STATUS_NEXT:
+    return "NEXT";
+  case STATUS_CHUNK:
+    return "CHUNK";
+  case STATUS_CHECK:
+    return "CHECK";
+  default:
+    return "UNKNOWN";
+  }
+}
+
+bool send_str(int file_descriptor, char *data) {
+  size_t size = strlen(data);
+  if (!send_n_data(file_descriptor, &size, sizeof(size_t))) return false;
+  if (!send_n_data(file_descriptor, data, size)) return false;
+  log_message(LOG_LEVEL_DEBUG, "Send String: %s", data);
+  return true;
+}
+
+char *receive_str(int file_descriptor) {
+  size_t size;
+  if (!receive_n_data(file_descriptor, &size, sizeof(size_t))) return NULL;
+  char *data = (char *)malloc(size + 1);
+  if (data == NULL) return NULL;
+  if (!receive_n_data(file_descriptor, data, size)) {
+    free(data);
+    return NULL;
+  }
+  data[size] = '\0';
+  log_message(LOG_LEVEL_DEBUG, "Received String: %s", data);
+  return data;
+}
+
+bool send_data(int file_descriptor, Data *data) {
+  unsigned long long data_size = data->size;
+  if (!send_n_data(file_descriptor, &data_size, sizeof(unsigned long long)))
+    return false;
+  if (!send_n_data(file_descriptor, data->data, data_size))
+    return false;
+  log_message(LOG_LEVEL_DEBUG, "Send %lld data", data_size);
+  return true;
+}
+
+Data *receive_data(int file_descriptor) {
+  unsigned long long size = 0;
+  if (!receive_n_data(file_descriptor, &size, sizeof(unsigned long long)))
+    return NULL;
+  void *data = malloc((size_t)size);
+  if (data == NULL) return NULL;
+  if (!receive_n_data(file_descriptor, data, (size_t)size)) {
+    free(data);
+    return NULL;
+  }
+  log_message(LOG_LEVEL_DEBUG, "Received %lld data", size);
+  return data_create(data, (size_t)size);
+}
+
+bool send_int(int file_descriptor, int data) {
+  if (!send_n_data(file_descriptor, &data, sizeof(int))) return false;
+  log_message(LOG_LEVEL_DEBUG, "Send Int: %d", data);
+  return true;
+}
+
+bool receive_int(int file_descriptor, int *data) {
+  if (!receive_n_data(file_descriptor, data, sizeof(int))) return false;
+  log_message(LOG_LEVEL_DEBUG, "Received Int: %d", *data);
+  return true;
+}
+
+bool send_status(int file_descriptor, Status status) {
+  if (!send_n_data(file_descriptor, &status, sizeof(Status))) return false;
+  log_message(LOG_LEVEL_DEBUG, "Send Status: %s", status_to_string(status));
+  return true;
+}
+
+bool receive_status(int file_descriptor, Status *status) {
+  if (!receive_n_data(file_descriptor, status, sizeof(Status))) return false;
+  log_message(LOG_LEVEL_DEBUG, "Received Status: %s", status_to_string(*status));
+  return true;
+}

src/shared/protocol.h

@@ -0,0 +1,25 @@
+#ifndef PROTOCOL_H
+#define PROTOCOL_H
+
+#include "data.h"
+#include <stdbool.h>
+#include <stddef.h>
+
+typedef int Status;
+enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST, STATUS_CHECK };
+
+void io_set_fds(int read_fd, int write_fd);
+void io_set_bwlimit(unsigned long long bytes_per_sec);
+bool send_n_data(int file_descriptor, void *data, size_t data_size);
+bool receive_n_data(int file_descriptor, void *data, size_t data_size);
+
+bool send_str(int file_descriptor, char *data);
+char *receive_str(int file_descriptor);
+bool send_data(int file_descriptor, Data *data);
+Data *receive_data(int file_descriptor);
+bool send_int(int file_descriptor, int data);
+bool receive_int(int file_descriptor, int *data);
+bool send_status(int file_descriptor, Status status);
+bool receive_status(int file_descriptor, Status *status);
+
+#endif

src/shared/queue.c

@@ -9,15 +9,14 @@
 Queue *queue_create(int capacity, void (*destroyer)(void *item)) {
   Queue *queue = (Queue *)malloc(sizeof(Queue));
   if (queue == NULL) {
-    perror("FATAL ERROR: Could not allocate memory for queue structure");
+    perror("ERROR: Could not allocate memory for queue structure");
-    exit(EXIT_FAILURE);
+    return NULL;
   }
 
   queue->items = malloc(capacity * sizeof(void *));
   if (queue->items == NULL) {
-    perror("FATAL ERROR: Could not allocate memory for queue items");
     free(queue);
-    exit(EXIT_FAILURE);
+    return NULL;
   }
 
   for (int i = 0; i < capacity; ++i) {
@@ -59,17 +58,15 @@ bool queue_is_full(Queue *queue) {
   return queue->size == queue->capacity;
 }
 
-void queue_double_capacity(Queue *queue) {
+static bool queue_double_capacity(Queue *queue) {
-  if (queue == NULL)
+  if (queue == NULL) return false;
-    return;
   unsigned int new_capacity = queue->capacity * 2;
   if (new_capacity <= 1)
     new_capacity = 100;
   void **new_items = malloc(new_capacity * sizeof(void *));
   if (new_items == NULL) {
-    perror("FATAL ERROR: Could not allocate memory for doubling capacity of "
+    perror("ERROR: Could not allocate memory for doubling capacity of queue.");
-           "queue.");
+    return false;
-    exit(EXIT_FAILURE);
   }
   for (int i = 0; i < queue->size; i++)
     new_items[i] = queue->items[(i + queue->front) % queue->capacity];
@@ -78,29 +75,30 @@ void queue_double_capacity(Queue *queue) {
   queue->front = 0;
   queue->rear = queue->size;
   queue->capacity = new_capacity;
+  return true;
 }
 
-void queue_enqueue(Queue *queue, void *item) {
+bool queue_enqueue(Queue *queue, void *item) {
-  if (queue == NULL || item == NULL) {
+  if (queue == NULL || item == NULL) return false;
-    perror("ERROR: Cannot enqueue with a null queue or item.\n");
+  if (queue_is_full(queue)) {
-    exit(EXIT_FAILURE);
+    if (!queue_double_capacity(queue)) return false;
   }
-  if (queue_is_full(queue))
-    queue_double_capacity(queue);
   queue->items[queue->rear] = item;
   queue->rear = (queue->rear + 1) % queue->capacity;
   queue->size++;
+  return true;
 }
 
-void queue_enqueue_multithreaded(Queue *queue, void *item, mtx_t *mutex,
+bool queue_enqueue_multithreaded(Queue *queue, void *item, mtx_t *mutex,
                                  cnd_t *condition_not_empty,
                                  cnd_t *condition_not_full) {
   mtx_lock(mutex);
   while (queue_is_full(queue))
     cnd_wait(condition_not_full, mutex);
-  queue_enqueue(queue, item);
+  bool ok = queue_enqueue(queue, item);
   cnd_signal(condition_not_empty);
   mtx_unlock(mutex);
+  return ok;
 }
 
 void *queue_dequeue(Queue *queue) {

src/shared/queue.h

@@ -17,9 +17,8 @@ Queue *queue_create(int capacity, void (*destroyer)(void *item));
 void queue_destroy(Queue *queue);
 bool queue_is_empty(Queue *queue);
 bool queue_is_full(Queue *queue);
-void queue_double_capacity(Queue *queue);
+bool queue_enqueue(Queue *queue, void *item);
-void queue_enqueue(Queue *queue, void *item);
+bool queue_enqueue_multithreaded(Queue *queue, void *item, mtx_t *mutex,
-void queue_enqueue_multithreaded(Queue *queue, void *item, mtx_t *mutex,
                                  cnd_t *condition_not_empty,
                                  cnd_t *condition_not_full);
 void *queue_dequeue(Queue *queue);

src/shared/socket.c

@@ -1,213 +0,0 @@
-#include "socket.h"
-#include "log.h"
-#include <arpa/inet.h>
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-Server *server_create(int port) {
-  Server *server = (Server *)malloc(sizeof(Server));
-  if (server == NULL) {
-    perror("Could not allocate space for Server");
-    exit(EXIT_FAILURE);
-  }
-
-  int file_descriptor = socket(AF_INET, SOCK_STREAM, 0);
-  if (file_descriptor < 0) {
-    perror("Could not create Socket!");
-    exit(EXIT_FAILURE);
-  }
-  server->file_descriptor = file_descriptor;
-  int opt = 1;
-  if (setsockopt(server->file_descriptor, SOL_SOCKET, SO_REUSEADDR, &opt,
-                 sizeof(opt))) {
-    perror("Error setting a socket option!");
-    close(server->file_descriptor);
-    free(server);
-    exit(EXIT_FAILURE);
-  }
-
-  server->address.sin_family = AF_INET;
-  server->address.sin_addr.s_addr = INADDR_ANY;
-  server->address.sin_port = htons(port);
-  server->address_length = sizeof(server->address);
-
-  if (bind(server->file_descriptor, (struct sockaddr *)&server->address,
-           server->address_length) < 0) {
-    perror("Could not bind server");
-    close(server->file_descriptor);
-    free(server);
-    exit(EXIT_FAILURE);
-  }
-
-  return server;
-}
-
-void server_delete(Server *server) {
-  free(server);
-  server = NULL;
-};
-
-void server_listen(Server *server, void (*handler)(int file_descriptor)) {
-  log_message(LOG_LEVEL_INFO, "Start Listening on Port: %d",
-              server->address.sin_port);
-  if (listen(server->file_descriptor, 3) < 0) {
-    perror("Could not listen on port!");
-    exit(EXIT_FAILURE);
-  }
-
-  int file_descriptor =
-      accept(server->file_descriptor, (struct sockaddr *)&server->address,
-             &server->address_length);
-  if (file_descriptor < 0) {
-    perror("Could not accept the connection");
-    exit(EXIT_FAILURE);
-  }
-  log_message(LOG_LEVEL_INFO, "Received Connection");
-  handler(file_descriptor);
-  close(server->file_descriptor);
-  close(file_descriptor);
-}
-
-Client *client_create() {
-  int file_descriptor = socket(AF_INET, SOCK_STREAM, 0);
-  if (file_descriptor < 0) {
-    perror("Could not create Socket!");
-    exit(EXIT_FAILURE);
-  };
-
-  Client *client = (Client *)malloc(sizeof(Client));
-  client->file_descriptor = file_descriptor;
-  client->address.sin_family = AF_INET;
-  client->address_length = sizeof(client->address);
-  return client;
-}
-
-void client_connect(Client *client, char *host, int port) {
-  client->address.sin_port = htons(port);
-
-  if (inet_pton(AF_INET, host, &client->address.sin_addr) <= 0) {
-    perror("Could not convert host address!");
-    exit(EXIT_FAILURE);
-  }
-
-  if (connect(client->file_descriptor, (struct sockaddr *)&client->address,
-              client->address_length) < 0) {
-    perror("Could not connect to Server!");
-    exit(EXIT_FAILURE);
-  }
-}
-
-void client_disconnect(Client *client) { close(client->file_descriptor); }
-
-void client_delete(Client *client) {
-  if (client == NULL)
-    return;
-  free(client);
-}
-
-void send_n_data(int file_descriptor, void *data, size_t data_size) {
-  log_message(LOG_LEVEL_DEBUG, "    Sending n Data: %d", data_size);
-  ssize_t total_bytes_send = 0;
-  while (total_bytes_send < data_size) {
-    printf("Trying: %zu\n", data_size - total_bytes_send);
-    ssize_t bytes_send = send(file_descriptor, (char *)data + total_bytes_send,
-                              data_size - total_bytes_send, 0);
-    printf("Bytes send: %zd\n", bytes_send);
-    if (bytes_send <= 0) {
-      perror("Could not send data!");
-      exit(EXIT_FAILURE);
-    }
-    total_bytes_send += bytes_send;
-  }
-  log_message(LOG_LEVEL_DEBUG, "    Send n Data: %zu", total_bytes_send);
-}
-
-void receive_n_data(int file_descriptor, void *data, size_t data_size) {
-  log_message(LOG_LEVEL_DEBUG, "    Receiving n Data: %d", data_size);
-  size_t total_bytes_received = 0;
-  while (total_bytes_received < data_size) {
-    long long bytes_received =
-        recv(file_descriptor, data + total_bytes_received,
-             data_size - total_bytes_received, 0);
-    if (bytes_received == -1 || bytes_received == 0) {
-      perror("Could not receive bytes!");
-      exit(EXIT_FAILURE);
-    }
-    total_bytes_received += bytes_received;
-  }
-  log_message(LOG_LEVEL_DEBUG, "    Received n Data: %d", total_bytes_received);
-}
-
-void send_str(int file_descriptor, char *data) {
-  size_t size = strlen(data);
-  send_n_data(file_descriptor, &size, sizeof(size_t));
-  send_n_data(file_descriptor, data, size);
-  log_message(LOG_LEVEL_DEBUG, "Send String: %s", data);
-}
-
-char *receive_str(int file_descriptor) {
-  size_t size;
-  receive_n_data(file_descriptor, &size, sizeof(size_t));
-  char *data = (char *)malloc(size + 1);
-  receive_n_data(file_descriptor, data, size);
-  data[size] = '\0';
-  log_message(LOG_LEVEL_DEBUG, "Received String: %s", data);
-  return data;
-}
-
-void send_data(int file_descriptor, void *data, unsigned long long data_size) {
-  send_n_data(file_descriptor, &data_size, sizeof(unsigned long long));
-  send_n_data(file_descriptor, data, data_size);
-  log_message(LOG_LEVEL_DEBUG, "Send %lld data", data_size);
-}
-
-Data *receive_data(int file_descriptor) {
-  size_t size = 0;
-  receive_n_data(file_descriptor, &size, sizeof(unsigned long long));
-  void *data = malloc(size);
-  receive_n_data(file_descriptor, data, size);
-  log_message(LOG_LEVEL_DEBUG, "Received %lld data", size);
-  return data_create(data, size);
-}
-
-void send_int(int file_descriptor, int data) {
-  send_n_data(file_descriptor, &data, sizeof(int));
-  log_message(LOG_LEVEL_DEBUG, "Send Int: %d", data);
-}
-
-int receive_int(int file_descriptor) {
-  int data;
-  receive_n_data(file_descriptor, &data, sizeof(int));
-  log_message(LOG_LEVEL_DEBUG, "Received Int: %d", data);
-  return data;
-}
-
-const char *status_to_string(Status status) {
-  switch (status) {
-  case STATUS_OK:
-    return "OK";
-  case STATUS_ERROR:
-    return "ERROR";
-  case STATUS_FINISHED:
-    return "FINISHED";
-  case STATUS_NEXT:
-    return "NEXT";
-  default:
-    return "UNKNOWN";
-  }
-}
-
-void send_status(int file_descriptor, Status status) {
-  send_n_data(file_descriptor, &status, sizeof(Status));
-  log_message(LOG_LEVEL_DEBUG, "Send Status: %s", status_to_string(status));
-}
-
-Status receive_status(int file_descriptor) {
-  Status data;
-  receive_n_data(file_descriptor, &data, sizeof(Status));
-  log_message(LOG_LEVEL_DEBUG, "Received Status: %s", status_to_string(data));
-  return data;
-}

src/shared/socket.h

@@ -1,42 +0,0 @@
-#ifndef SOCKET_H
-#define SOCKET_H
-
-#include "data.h"
-#include <netinet/in.h>
-
-typedef int Status;
-enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT };
-
-typedef struct Server {
-  struct sockaddr_in address;
-  unsigned int address_length;
-  int file_descriptor;
-} Server;
-
-Server *server_create(int port);
-void server_listen(Server *server, void (*handler)(int file_descriptor));
-void server_delete(Server *server);
-
-typedef struct Client {
-  struct sockaddr_in address;
-  unsigned int address_length;
-  int file_descriptor;
-} Client;
-
-Client *client_create();
-void client_disconnect(Client *client);
-void client_delete(Client *client);
-void client_connect(Client *client, char *host, int port);
-
-void send_n_data(int file_descriptor, void *data, size_t data_size);
-void receive_n_data(int file_descriptor, void *data, size_t data_size);
-void send_str(int file_descriptor, char *data);
-char *receive_str(int file_descriptor);
-void send_data(int file_descriptor, void *data, unsigned long long data_size);
-Data *receive_data(int file_descriptor);
-void send_int(int file_descriptor, int data);
-int receive_int(int file_descriptor);
-void send_status(int file_descriptor, Status status);
-Status receive_status(int file_descriptor);
-
-#endif

src/shared/transport_ssh.c

@@ -0,0 +1,147 @@
+#include "transport_ssh.h"
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+typedef struct {
+  char user[256];
+  char host[256];
+  char remote_path[4096];
+} RemoteDest;
+
+static int parse_remote_dest(const char *dest, RemoteDest *r) {
+  const char *colon = strchr(dest, ':');
+  if (!colon) return -1;
+
+  size_t remote_path_len = strlen(colon + 1);
+  if (remote_path_len >= sizeof(r->remote_path)) return -1;
+  memcpy(r->remote_path, colon + 1, remote_path_len + 1);
+
+  const char *at = memchr(dest, '@', colon - dest);
+  if (at) {
+    size_t user_len = at - dest;
+    if (user_len >= sizeof(r->user)) return -1;
+    memcpy(r->user, dest, user_len);
+    r->user[user_len] = '\0';
+
+    size_t host_len = colon - at - 1;
+    if (host_len >= sizeof(r->host)) return -1;
+    memcpy(r->host, at + 1, host_len);
+    r->host[host_len] = '\0';
+  } else {
+    r->user[0] = '\0';
+    size_t host_len = colon - dest;
+    if (host_len >= sizeof(r->host)) return -1;
+    memcpy(r->host, dest, host_len);
+    r->host[host_len] = '\0';
+  }
+  return 0;
+}
+
+Client *client_connect_ssh(char *destination, int port) {
+  RemoteDest r;
+  if (parse_remote_dest(destination, &r) != 0) {
+    fprintf(stderr, "Invalid remote destination: %s\n", destination);
+    return NULL;
+  }
+
+  int sv[2];
+  if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) < 0) {
+    perror("socketpair failed");
+    return NULL;
+  }
+
+  int buf_size = 1024 * 1024;
+  setsockopt(sv[0], SOL_SOCKET, SO_SNDBUF, &buf_size, sizeof(buf_size));
+  setsockopt(sv[0], SOL_SOCKET, SO_RCVBUF, &buf_size, sizeof(buf_size));
+  setsockopt(sv[1], SOL_SOCKET, SO_SNDBUF, &buf_size, sizeof(buf_size));
+  setsockopt(sv[1], SOL_SOCKET, SO_RCVBUF, &buf_size, sizeof(buf_size));
+
+  int exec_pipe[2];
+  if (pipe(exec_pipe) < 0) {
+    perror("pipe failed");
+    close(sv[0]); close(sv[1]);
+    return NULL;
+  }
+
+  pid_t pid = fork();
+  if (pid < 0) {
+    perror("fork failed");
+    close(sv[0]); close(sv[1]);
+    close(exec_pipe[0]); close(exec_pipe[1]);
+    return NULL;
+  }
+
+  if (pid == 0) {
+    close(sv[0]);
+    close(exec_pipe[0]);
+    fcntl(exec_pipe[1], F_SETFD, FD_CLOEXEC);
+
+    if (sv[1] != STDIN_FILENO)
+      dup2(sv[1], STDIN_FILENO);
+    if (sv[1] != STDOUT_FILENO)
+      dup2(sv[1], STDOUT_FILENO);
+    if (sv[1] > 1) close(sv[1]);
+
+    char ssh_user[512];
+    if (r.user[0] != '\0')
+      snprintf(ssh_user, sizeof(ssh_user), "%s@%s", r.user, r.host);
+    else
+      snprintf(ssh_user, sizeof(ssh_user), "%s", r.host);
+
+    char *ssh_argv[16];
+    int ac = 0;
+    char port_str[16];
+    ssh_argv[ac++] = "ssh";
+    ssh_argv[ac++] = "-o";
+    ssh_argv[ac++] = "Compression=no";
+    ssh_argv[ac++] = "-o";
+    ssh_argv[ac++] = "ControlMaster=auto";
+    ssh_argv[ac++] = "-o";
+    ssh_argv[ac++] = "ControlPath=~/.cache/fastsync-%r@%h:%p";
+    if (port > 0 && port != 22) {
+      ssh_argv[ac++] = "-p";
+      snprintf(port_str, sizeof(port_str), "%d", port);
+      ssh_argv[ac++] = port_str;
+    }
+    ssh_argv[ac++] = ssh_user;
+    ssh_argv[ac++] = "fastsync-server";
+    ssh_argv[ac++] = "--stdio";
+    ssh_argv[ac] = NULL;
+    execvp("ssh", ssh_argv);
+    perror("exec of ssh failed");
+    ssize_t wret = write(exec_pipe[1], "x", 1);
+    (void)wret;
+    _exit(1);
+  }
+
+  close(sv[1]);
+  close(exec_pipe[1]);
+
+  char exec_status;
+  ssize_t n = read(exec_pipe[0], &exec_status, 1);
+  close(exec_pipe[0]);
+
+  if (n > 0) {
+    close(sv[0]);
+    waitpid(pid, NULL, 0);
+    fprintf(stderr, "Error: could not launch 'fastsync-server --stdio' on remote\n");
+    return NULL;
+  }
+
+  Client *client = malloc(sizeof(Client));
+  if (client == NULL) {
+    close(sv[0]);
+    waitpid(pid, NULL, 0);
+    return NULL;
+  }
+  client->file_descriptor = sv[0];
+  client->address.sin_family = AF_UNIX;
+  client->address_length = 0;
+  client->ssh_child_pid = pid;
+  return client;
+}

src/shared/transport_ssh.h

@@ -0,0 +1,8 @@
+#ifndef TRANSPORT_SSH_H
+#define TRANSPORT_SSH_H
+
+#include "transport_tcp.h"
+
+Client *client_connect_ssh(char *destination, int port);
+
+#endif

src/shared/transport_tcp.c

@@ -0,0 +1,139 @@
+#include "transport_tcp.h"
+#include "log.h"
+#include <arpa/inet.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+Server *server_create(int port) {
+  Server *server = (Server *)malloc(sizeof(Server));
+  if (server == NULL) {
+    perror("Could not allocate space for Server");
+    return NULL;
+  }
+
+  int file_descriptor = socket(AF_INET, SOCK_STREAM, 0);
+  if (file_descriptor < 0) {
+    perror("Could not create Socket!");
+    free(server);
+    return NULL;
+  }
+  server->file_descriptor = file_descriptor;
+  int opt = 1;
+  if (setsockopt(server->file_descriptor, SOL_SOCKET, SO_REUSEADDR, &opt,
+                 sizeof(opt))) {
+    perror("Error setting a socket option!");
+    close(server->file_descriptor);
+    free(server);
+    return NULL;
+  }
+
+  server->address.sin_family = AF_INET;
+  server->address.sin_addr.s_addr = INADDR_ANY;
+  server->address.sin_port = htons(port);
+  server->address_length = sizeof(server->address);
+
+  if (bind(server->file_descriptor, (struct sockaddr *)&server->address,
+           server->address_length) < 0) {
+    perror("Could not bind server");
+    close(server->file_descriptor);
+    free(server);
+    return NULL;
+  }
+
+  return server;
+}
+
+void server_delete(Server **server) {
+  if (server == NULL || *server == NULL) return;
+  close((*server)->file_descriptor);
+  free(*server);
+  *server = NULL;
+}
+
+bool server_listen(Server *server, void (*handler)(int file_descriptor)) {
+  log_message(LOG_LEVEL_INFO, "Start Listening on Port: %d",
+              server->address.sin_port);
+  if (listen(server->file_descriptor, SOMAXCONN) < 0) {
+    perror("Could not listen on port!");
+    return false;
+  }
+
+  signal(SIGCHLD, SIG_IGN);
+
+  while (1) {
+    struct sockaddr_in client_addr;
+    socklen_t client_len = sizeof(client_addr);
+    int file_descriptor =
+        accept(server->file_descriptor, (struct sockaddr *)&client_addr,
+               &client_len);
+    if (file_descriptor < 0) {
+      perror("Could not accept the connection");
+      continue;
+    }
+    log_message(LOG_LEVEL_INFO, "Received Connection");
+    pid_t pid = fork();
+    if (pid == 0) {
+      close(server->file_descriptor);
+      handler(file_descriptor);
+      close(file_descriptor);
+      _exit(0);
+    }
+    close(file_descriptor);
+  }
+  return true;
+}
+
+Client *client_create() {
+  int file_descriptor = socket(AF_INET, SOCK_STREAM, 0);
+  if (file_descriptor < 0) {
+    perror("Could not create Socket!");
+    return NULL;
+  }
+
+  Client *client = (Client *)malloc(sizeof(Client));
+  if (client == NULL) {
+    close(file_descriptor);
+    return NULL;
+  }
+  client->file_descriptor = file_descriptor;
+  client->address.sin_family = AF_INET;
+  client->address_length = sizeof(client->address);
+  client->ssh_child_pid = -1;
+  return client;
+}
+
+bool client_connect(Client *client, char *host, int port) {
+  client->address.sin_port = htons(port);
+
+  if (inet_pton(AF_INET, host, &client->address.sin_addr) <= 0) {
+    perror("Could not convert host address!");
+    return false;
+  }
+
+  if (connect(client->file_descriptor, (struct sockaddr *)&client->address,
+              client->address_length) < 0) {
+    perror("Could not connect to Server!");
+    return false;
+  }
+  return true;
+}
+
+void client_disconnect(Client *client) {
+  close(client->file_descriptor);
+  if (client->ssh_child_pid > 0) {
+    int status;
+    waitpid(client->ssh_child_pid, &status, 0);
+    client->ssh_child_pid = -1;
+  }
+}
+
+void client_delete(Client *client) {
+  if (client == NULL)
+    return;
+  free(client);
+}

src/shared/transport_tcp.h

@@ -0,0 +1,29 @@
+#ifndef TRANSPORT_TCP_H
+#define TRANSPORT_TCP_H
+
+#include <netinet/in.h>
+#include <stdbool.h>
+#include <sys/types.h>
+
+typedef struct Server {
+  struct sockaddr_in address;
+  unsigned int address_length;
+  int file_descriptor;
+} Server;
+
+typedef struct Client {
+  struct sockaddr_in address;
+  unsigned int address_length;
+  int file_descriptor;
+  pid_t ssh_child_pid;
+} Client;
+
+Server *server_create(int port);
+bool server_listen(Server *server, void (*handler)(int file_descriptor));
+void server_delete(Server **server);
+Client *client_create();
+bool client_connect(Client *client, char *host, int port);
+void client_disconnect(Client *client);
+void client_delete(Client *client);
+
+#endif

src/shared/utils.c

@@ -1,14 +1,19 @@
 #include "utils.h"
+#include "array_list.h"
 #include "libgen.h"
-#include "sys/stat.h"
+#include <dirent.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
 
-void mkdir_r(char *path) {
+bool mkdir_r(char *path) {
   char *path_duplicate = malloc(strlen(path) + 1);
+  if (!path_duplicate) return false;
   strcpy(path_duplicate, path);
   char *path_current = (char *)malloc((strlen(path) + 2) * sizeof(char));
+  if (!path_current) { free(path_duplicate); return false; }
   char *path_current_position = path_current;
   if (path[0] == '/') {
     strcpy(path_current, "/");
@@ -18,6 +23,7 @@ void mkdir_r(char *path) {
   }
   const char *delimiter = "/";
   char *part = strtok(path_duplicate, delimiter);
+  bool ok = true;
   while (part != NULL) {
     strcpy(path_current_position, part);
     path_current_position += strlen(part) * sizeof(char);
@@ -27,16 +33,18 @@ void mkdir_r(char *path) {
     if (stat(path_current, &st) != 0) {
       if (mkdir(path_current, 0755) != 0) {
         perror("Could not create directory");
-        exit(EXIT_FAILURE);
+        ok = false;
+        break;
       }
     }
     part = strtok(NULL, delimiter);
   }
   free(path_duplicate);
   free(path_current);
+  return ok;
 }
 
-char *str_dup(char *string) {
+char *str_dup(const char *string) {
   if (string == NULL)
     return NULL;
   char *new_string = (char *)malloc(strlen(string) + 1);
@@ -44,19 +52,73 @@ char *str_dup(char *string) {
   return new_string;
 }
 
-void to_disk(char *path, void *data, unsigned long long data_size) {
+bool glob_match(const char *pattern, const char *str) {
-  char *directory = str_dup(path);
+  while (*pattern) {
-  char *dir_to_free = directory;
+    if (*pattern == '*') {
-  directory = dirname(directory);
+      pattern++;
-  mkdir_r(directory);
+      while (*str && *str != '/') {
-  FILE *file_pointer = fopen(path, "wb");
+        if (glob_match(pattern, str))
-  if (file_pointer == NULL) {
+          return true;
-    perror("Could not open File");
+        str++;
-    exit(EXIT_FAILURE);
+      }
+      return glob_match(pattern, str);
+    } else if (*pattern == '?') {
+      if (!*str || *str == '/')
+        return false;
+      pattern++;
+      str++;
+    } else {
+      if (*pattern != *str)
+        return false;
+      pattern++;
+      str++;
+    }
   }
-  fwrite(data, 1, data_size, file_pointer);
+  return *str == '\0';
-  fclose(file_pointer);
+}
-  free(dir_to_free);
+
+static void delete_extras_walk(const char *abs_path, const char *rel_path,
+                               ArrayList *manifest) {
+  DIR *dir = opendir(abs_path);
+  if (!dir)
+    return;
+  struct dirent *entry;
+  while ((entry = readdir(dir)) != NULL) {
+    if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
+      continue;
+    char *child_abs = path_cat((char *)abs_path, entry->d_name);
+    char *child_rel = path_cat((char *)rel_path, entry->d_name);
+    struct stat st;
+    if (stat(child_abs, &st) != 0) {
+      free(child_abs);
+      free(child_rel);
+      continue;
+    }
+    if (S_ISDIR(st.st_mode)) {
+      delete_extras_walk(child_abs, child_rel, manifest);
+    } else {
+      // Check if relative path is in manifest
+      bool found = false;
+      for (int i = 0; i < manifest->size; i++) {
+        if (strcmp((char *)manifest->items[i], child_rel) == 0) {
+          found = true;
+          break;
+        }
+      }
+      if (!found) {
+        unlink(child_abs);
+        fprintf(stderr, "  Deleted: %s\n", child_rel);
+      }
+    }
+    free(child_abs);
+    free(child_rel);
+  }
+  closedir(dir);
+  rmdir(abs_path);
+}
+
+void delete_extras(const char *dest_root, ArrayList *manifest) {
+  delete_extras_walk(dest_root, "", manifest);
 }
 
 char *path_cat(char *path1, char *path2) {
@@ -74,6 +136,7 @@ char *path_cat(char *path1, char *path2) {
     path2_len -= 1;
   }
   char *new_path = malloc(path1_len + path2_len + 2);
+  if (new_path == NULL) return NULL;
   memcpy(new_path, path1, path1_len);
   new_path[path1_len] = '/';
   memcpy(new_path + path1_len + 1, path2_pointer, path2_len);

src/shared/utils.h

@@ -1,9 +1,13 @@
 #ifndef UTILS_H
 #define UTILS_H
 
-void mkdir_r(char *path);
+#include "array_list.h"
-char *str_dup(char *string);
+#include <stdbool.h>
-void to_disk(char *path, void *data, unsigned long long data_size);
+
+bool mkdir_r(char *path);
+char *str_dup(const char *string);
 char *path_cat(char *path1, char *path2);
+bool glob_match(const char *pattern, const char *str);
+void delete_extras(const char *dest_root, ArrayList *manifest);
 
 #endif

test.py

@@ -1,27 +1,43 @@
+import argparse
+import filecmp
 import os
+import random
+import re
+import shutil
 import subprocess
+import sys
+import tempfile
 import time
+import socket
+
+TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_data")
+DEFAULT_SOURCE_DIR = os.path.join(TEST_DIR, "source")
+DEFAULT_DEST_DIR = os.path.join(TEST_DIR, "dest")
 
-# --- Configuration ---
 SERVER_CMD = ["./build/server"]
-base_client_cmd = ["./build/client"]
+BASE_CLIENT_CMD = ["./build/client"]
 
-# --- Resource Limit Configuration ---
+DISK_DEVICE = "/dev/nvme0n1p5"
-# 💾 Disk throttling settings
+READ_BPS_MAX = "15M"
-DISK_DEVICE = (
+WRITE_BPS_MAX = "10M"
-    "/dev/nvme0n1p5"  # IMPORTANT: Change this to your disk (e.g., /dev/nvme0n1)
-)
-READ_BPS_MAX = "15M"  # Max read speed (M for megabytes)
-WRITE_BPS_MAX = "10M"  # Max write speed
-
-# 🐢 Network throttling settings (Linux tc)
-NET_LIMIT = "100mbit"
-NET_DELAY = "100ms"
 NETWORK_INTERFACE = "lo"
-NET_LIMIT_CMD = f"sudo tc qdisc add dev {NETWORK_INTERFACE} root netem rate {NET_LIMIT} delay {NET_DELAY}".split()
-NET_RESET_CMD = f"sudo tc qdisc del dev {NETWORK_INTERFACE} root".split()
 
-# --- Build the client command prefix with throttling ---
+NETWORK_PROFILES = {
+    "Unlimited": {},
+    "LAN": {
+        "rate": "1000mbit",
+        "delay": "20ms",
+        "jitter": "1ms",
+        "loss": "0.1%",
+    },
+    "WAN": {
+        "rate": "100mbit",
+        "delay": "50ms",
+        "jitter": "10ms",
+        "loss": "1%",
+    },
+}
+
 CLIENT_CMD_PREFIX = [
     "sudo",
     "systemd-run",
@@ -32,198 +48,653 @@ CLIENT_CMD_PREFIX = [
     f"IOWriteBandwidthMax={DISK_DEVICE} {WRITE_BPS_MAX}",
 ]
 
-# --- Test Cases ---
+BASE_CLIENT_FLAGS = ["--save-to-disk"]
+
 TEST_CASES = [
-    {"name": "Standard (Single-threaded)", "flags": []},
+    {"name": "Standard", "flags": []},
+    {"name": "Posix Args (no flags)", "flags": [], "posix": True},
+    {"name": "Standard (no metadata)", "flags": [], "use_metadata": False},
     {"name": "Multithreading (-m)", "flags": ["-m"]},
-    {"name": "Compression (-c -5)", "flags": ["-c -5"]},
+    {"name": "Compression (-c)", "flags": ["-c"]},
-    {"name": "Compression (-c 0)", "flags": ["-c 0"]},
+    {"name": "Chunk Serialization (-s)", "flags": ["-s"]},
-    {"name": "Compression (-c 10)", "flags": ["-c 10"]},
+    {"name": "Compression + Chunk Serialization (-c -s)", "flags": ["-c", "-s"]},
-    {"name": "Compression (-c 20)", "flags": ["-c 20"]},
-    # {"name": "Chunk Serialization (-s)", "flags": ["-s"]},
     {"name": "Multithreading + Compression (-m -c)", "flags": ["-m", "-c"]},
-    # {"name": "Multithreading + Chunk Serialization (-m -s)", "flags": ["-m", "-s"]},
+    {"name": "Multithreading + Chunk Serialization (-m -s)", "flags": ["-m", "-s"]},
-    # {"name": "Compression + Chunk Serialization (-c -s)", "flags": ["-c", "-s"]},
+    {
-    # {
+        "name": "Multithreading + Compression + Chunk Serialization (-m -c -s)",
-    #     "name": "Multithreading + Compression + Chunk Serialization (-m -c -s)",
+        "flags": ["-m", "-c", "-s"],
-    #     "flags": ["-m", "-c", "-s"],
+    },
-    # },
+    {"name": "Sendfile (-f)", "flags": ["-f"]},
+    {"name": "Sendfile + Multithreading (-f -m)", "flags": ["-f", "-m"]},
+]
+
+SSH_CASES = [
+    {"name": "SSH (localhost)", "flags": []},
+    {"name": "SSH Multithreading (-m)", "flags": ["-m"]},
+    {"name": "SSH Compression (-c)", "flags": ["-c"]},
+    {"name": "SSH Chunk Serialization (-s)", "flags": ["-s"]},
+    {"name": "SSH Compression + Chunk Serialization (-c -s)", "flags": ["-c", "-s"]},
+    {"name": "SSH Multithreading + Compression (-m -c)", "flags": ["-m", "-c"]},
+    {"name": "SSH Multithreading + Chunk Serialization (-m -s)", "flags": ["-m", "-s"]},
+    {"name": "SSH Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]},
+]
+
+RSYNC_CASES = [
+    {"name": "rsync (archive)", "args": ["-aH"]},
+    {"name": "rsync (archive + compress)", "args": ["-aHz"]},
 ]
 
 
-def run_suite(env_name, apply_limits):
+def netem_apply(profile):
-    results = []
+    params = NETWORK_PROFILES[profile]
-    print(f"\n{'=' * 60}")
+    if not params:
-    print(f"🚀 Starting Suite: {env_name}")
+        netem_reset()
-    print(f"{'=' * 60}")
+        return
+    netem_reset()
+    cmd = ["sudo", "tc", "qdisc", "add", "dev", NETWORK_INTERFACE, "root", "netem"]
+    cmd += ["rate", params["rate"]]
+    cmd += ["delay", params["delay"], params["jitter"]]
+    cmd += ["loss", params["loss"]]
+    subprocess.run(cmd, check=True, capture_output=True)
 
-    if apply_limits:
+
-        print(
+def netem_reset():
-            f"Applying Disk I/O Limits: Reads <= {READ_BPS_MAX}, Writes <= {WRITE_BPS_MAX}"
+    subprocess.run(
-        )
+        f"sudo tc qdisc del dev {NETWORK_INTERFACE} root".split(),
-        print(f"Applying Network Limits: {NET_LIMIT}, {NET_DELAY} delay")
+        capture_output=True,
-        client_prefix = CLIENT_CMD_PREFIX
+    )
+
+
+def wait_proc(proc, timeout=5):
+    try:
+        proc.wait(timeout=timeout)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+        proc.wait()
+
+
+def find_free_port():
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(('', 0))
+        return s.getsockname()[1]
+
+
+def generate_test_files(source_dir):
+    if os.path.exists(source_dir):
+        shutil.rmtree(source_dir)
+    os.makedirs(source_dir)
+
+    target_total = 25 * 1024 * 1024
+    written = 0
+
+    files = {
+        "small.txt": b"hello world\n",
+        "medium.txt": b"the quick brown fox jumps over the lazy dog\n" * 5000,
+        "binary.bin": bytes(range(256)) * 1000,
+        "nested/subdir/deep.txt": b"deeply nested file\n",
+        "nested/another.txt": b"another nested file\n" * 50,
+    }
+    for rel_path, content in files.items():
+        full_path = os.path.join(source_dir, rel_path)
+        os.makedirs(os.path.dirname(full_path), exist_ok=True)
+        with open(full_path, "wb") as f:
+            f.write(content)
+        written += len(content)
+
+    os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True)
+    i = 0
+    while written < target_total:
+        chunk_size = min(5 * 1024 * 1024, target_total - written)
+        with open(os.path.join(source_dir, f"bulk/file_{i}.dat"), "wb") as f:
+            f.write(random.randbytes(chunk_size))
+        written += chunk_size
+        i += 1
+
+    total_mb = written / (1024 * 1024)
+    small_bytes = sum(len(c) for c in files.values())
+    print(f"  Generated {total_mb:.1f} MB of test data in {source_dir} "
+          f"({(written - small_bytes)/(1024*1024):.1f} MB random, "
+          f"{small_bytes} B structured)")
+    return written
+
+
+def verify_transfer(source_dir, received_dir):
+    source_dir = os.path.abspath(source_dir)
+    received_dir = os.path.abspath(received_dir)
+    if not os.path.exists(received_dir):
+        return [], ["no received files found"]
+    mismatches, missing = [], []
+    for root, dirs, files in os.walk(source_dir):
+        for f in files:
+            src_path = os.path.join(root, f)
+            rel = os.path.relpath(src_path, source_dir)
+            dst_path = os.path.join(received_dir, rel)
+            if not os.path.exists(dst_path):
+                missing.append(rel)
+            elif not filecmp.cmp(src_path, dst_path, shallow=False):
+                mismatches.append(rel)
+    return mismatches, missing
+
+
+def start_rsync_daemon(source_dir):
+    port = find_free_port()
+    conf = os.path.join(tempfile.gettempdir(), f"rsyncd-{port}.conf")
+    with open(conf, "w") as f:
+        f.write(f"port = {port}\nread only = yes\n\n[source]\n    path = {source_dir}\n")
+    daemon = subprocess.Popen(
+        ["rsync", "--daemon", "--no-detach", f"--config={conf}"],
+        stdout=subprocess.DEVNULL, stderr=None,
+    )
+    for _ in range(50):
+        time.sleep(0.1)
+        if daemon.poll() is not None:
+            raise RuntimeError(f"rsync daemon exited (rc={daemon.returncode})")
+        try:
+            with socket.create_connection(("127.0.0.1", port), timeout=0.3):
+                break
+        except (ConnectionRefusedError, OSError):
+            continue
     else:
-        print("Running Baseline (No limits applied)")
+        raise RuntimeError("rsync daemon did not start")
-        client_prefix = []  # Run normally without systemd-run/limits
+    return port, conf, daemon
+
+
+def run_single_test(cmd, name, source_dir, dest_dir, *, source_prefix=None, no_server=False, expected_missing=None):
+    if os.path.exists(dest_dir):
+        shutil.rmtree(dest_dir)
+    if no_server:
+        server = None
+    else:
+        server = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
+        time.sleep(0.5)
+    try:
+        start = time.monotonic()
+        result = subprocess.run(cmd, text=True, capture_output=True)
+        duration = time.monotonic() - start
+    finally:
+        if server:
+            wait_proc(server)
+
+    mismatches, missing = [], []
+    if result.returncode == 0:
+        received = os.path.join(dest_dir, source_prefix if source_prefix is not None
+                                else os.path.abspath(source_dir).lstrip(os.sep))
+        mismatches, missing = verify_transfer(source_dir, received)
+        if expected_missing:
+            missing = [m for m in missing if m not in expected_missing]
+
+    first_line = lambda s: (s or "").strip().split("\n")[0]
+    entry = {
+        "name": name,
+        "time": f"{duration:.4f}s" if result.returncode == 0 else "N/A",
+    }
+    if result.returncode == 0 and not mismatches and not missing:
+        entry["status"] = "Success"
+        entry["error"] = ""
+    else:
+        entry["status"] = "Failed"
+        errors = []
+        if result.returncode != 0:
+            errors.append(f"Exit code {result.returncode}: {first_line(result.stderr) or first_line(result.stdout) or 'No output'[:80]}")
+        if missing:
+            errors.append(f"Missing ({len(missing)}): {', '.join(missing[:5])}")
+        if mismatches:
+            errors.append(f"Mismatch ({len(mismatches)}): {', '.join(mismatches[:3])}")
+        entry["error"] = " | ".join(errors)
+    return entry
+
+
+def print_profile_header(profile_name):
+    params = NETWORK_PROFILES[profile_name]
+    print(f"\n{'=' * 60}\nProfile: {profile_name}\n{'=' * 60}")
+    if params:
+        print(f"  Network: rate={params['rate']}, delay={params['delay']} ±{params['jitter']}, loss={params['loss']}")
+        print(f"  Disk I/O: Reads <= {READ_BPS_MAX}, Writes <= {WRITE_BPS_MAX}")
+    else:
+        print("  No limits applied")
+
+
+def run_profile(profile_name, source_dir, dest_dir):
+    print_profile_header(profile_name)
+    is_limited = profile_name != "Unlimited"
+    client_prefix = CLIENT_CMD_PREFIX if is_limited else []
 
     try:
-        # SETUP: Apply or ensure clean network limits
+        if is_limited:
-        if apply_limits:
+            netem_apply(profile_name)
-            subprocess.run(NET_LIMIT_CMD, check=True)
         else:
-            # Silently attempt to clear any leftover rules just to ensure a clean baseline
+            netem_reset()
-            subprocess.run(NET_RESET_CMD, capture_output=True)
 
+        results = []
         for case in TEST_CASES:
-            name = case["name"]
+            flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"]
-            flags = case["flags"]
+            if case.get("posix"):
-
+                cmd = client_prefix + BASE_CLIENT_CMD + [source_dir, dest_dir] + flags
-            print(f"\n--- Running: {name} ---")
+            else:
-
+                cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
-            server_process = None
+            print(f"\n  --- {case['name']} ---\n    Running: {' '.join(cmd)}")
             try:
-                # 1. Start the server
+                r = run_single_test(cmd, case["name"], source_dir, dest_dir)
-                print("  Starting server...")
+                r["suite"] = profile_name
-                server_process = subprocess.Popen(
+                results.append(r)
-                    SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None
+            except Exception as e:
-                )
+                results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
-                time.sleep(0.5)  # Allow server to bind to port
 
-                # 2. Build and run the client
+        if SSH_AVAILABLE:
-                client_cmd = client_prefix + base_client_cmd + flags
+            for case in SSH_CASES:
-                print(f"  Running client: {' '.join(client_cmd)}")
+                flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"]
-
+                ssh_dest = f"localhost:{dest_dir}_ssh"
-                start_time = time.monotonic()
+                cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags
-                client_result = subprocess.run(
+                print(f"\n  --- {case['name']} ---\n    Running: {' '.join(cmd)}")
-                    client_cmd, text=True, capture_output=True
+                try:
-                )
+                    r = run_single_test(cmd, case["name"], source_dir, f"{dest_dir}_ssh", no_server=True)
-                end_time = time.monotonic()
+                    r["suite"] = profile_name
-
+                    results.append(r)
-                duration = end_time - start_time
+                except Exception as e:
-
+                    results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
-                if client_result.returncode == 0:
+    
-                    results.append(
+        port, conf, daemon = start_rsync_daemon(source_dir)
-                        {
+        try:
-                            "environment": env_name,
+            for case in RSYNC_CASES:
-                            "name": name,
+                cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"]
-                            "status": "Success",
+                print(f"\n  --- {case['name']} ---\n    Running: {' '.join(cmd)}")
-                            "time": f"{duration:.4f}s",
+                try:
-                            "error": "",
+                    r = run_single_test(cmd, case["name"], source_dir, dest_dir, source_prefix="")
-                        }
+                    r["suite"] = profile_name
-                    )
+                except subprocess.TimeoutExpired:
+                    r = {"name": case["name"], "suite": profile_name, "status": "Timeout", "time": "N/A", "error": "Exceeded 120s"}
+                except Exception as e:
+                    r = {"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}
                 else:
-                    print(f"  ⚠️ Failed (code: {client_result.returncode})")
+                    if r["status"] != "Success" and client_prefix:
-                    err_msg = (
+                        tmp = tempfile.mkdtemp()
-                        client_result.stderr.strip().split("\n")[0]
+                        try:
-                        if client_result.stderr
+                            plain = subprocess.run(["rsync", "-aH", f"rsync://localhost:{port}/source/", f"{tmp}/"], capture_output=True, text=True, timeout=30)
-                        else (
+                            if plain.returncode != 0:
-                            client_result.stdout.strip().split("\n")[0]
+                                errs = [l for l in (plain.stderr or "").split("\n") if l.strip()]
-                            if client_result.stdout
+                                if errs:
-                            else "No output"
+                                    r["error"] += f" | raw: {errs[-1][:150]}"
-                        )
+                        finally:
-                    )
+                            shutil.rmtree(tmp, ignore_errors=True)
-                    results.append(
+                results.append(r)
-                        {
+        finally:
-                            "environment": env_name,
+            wait_proc(daemon)
-                            "name": name,
-                            "status": "Failed",
-                            "time": "N/A",
-                            "error": f"Exit code {client_result.returncode}: {err_msg[:40]}",
-                        }
-                    )
-
-            except subprocess.TimeoutExpired:
-                print("  ⚠️ Timeout (exceeded 15s)")
-                results.append(
-                    {
-                        "environment": env_name,
-                        "name": name,
-                        "status": "Timeout",
-                        "time": "N/A",
-                        "error": "Exceeded 15 seconds",
-                    }
-                )
-            except Exception as e:
-                print(f"  ❌ Error: {e}")
-                results.append(
-                    {
-                        "environment": env_name,
-                        "name": name,
-                        "status": "Error",
-                        "time": "N/A",
-                        "error": str(e),
-                    }
-                )
-            finally:
-                # Clean up the server for this test case
-                if server_process:
-                    print("  Stopping server...")
-                    try:
-                        server_process.terminate()
-                        server_process.wait(timeout=5)
-                    except subprocess.TimeoutExpired:
-                        server_process.kill()
-                        server_process.wait()
-
-    except subprocess.CalledProcessError as e:
-        print(f"❌ Error running system limit command: {' '.join(e.cmd)}")
-        print("Are you running this script with 'sudo' privileges?")
-
-    finally:
-        # TEARDOWN: Remove network limits if they were applied
-        if apply_limits:
-            print("\nCleaning up limits for this suite...")
             try:
-                subprocess.run(NET_RESET_CMD, check=True, capture_output=True)
+                os.unlink(conf)
-                print("Network limits removed.")
+            except Exception:
-            except Exception as e:
+                pass
-                print(f"⚠️ Could not reset network settings: {e}")
+
+        # Feature-specific tests for rsync-compatible flags
+        print("\n  " + "─" * 56 + "\n  Feature Tests\n  " + "─" * 56)
+
+        # Dry run (-n) — no server needed
+        print("\n  --- Dry run (-n) ---")
+        flags = BASE_CLIENT_FLAGS + ["-n"]
+        cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
+        print(f"    Running: {' '.join(cmd)}")
+        try:
+            start = time.monotonic()
+            result = subprocess.run(cmd, text=True, capture_output=True)
+            duration = time.monotonic() - start
+            r = {"name": "Dry run (-n)", "suite": profile_name}
+            if result.returncode == 0 and "Dry run:" in result.stdout:
+                r["status"] = "Success"
+                r["time"] = f"{duration:.4f}s"
+                r["error"] = ""
+            else:
+                r["status"] = "Failed"
+                r["time"] = "N/A"
+                r["error"] = f"Exit {result.returncode}: {(result.stderr or result.stdout)[:100]}"
+            results.append(r)
+        except Exception as e:
+            results.append({"name": "Dry run (-n)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
+
+        # Archive mode (-a)
+        feature_flags = BASE_CLIENT_FLAGS + ["-a"]
+        cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
+        print(f"\n  --- Archive mode (-a) ---\n    Running: {' '.join(cmd)}")
+        try:
+            r = run_single_test(cmd, "Archive mode (-a)", source_dir, dest_dir)
+            r["suite"] = profile_name
+            results.append(r)
+        except Exception as e:
+            results.append({"name": "Archive mode (-a)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
+
+        # Exclude (--exclude small.txt)
+        feature_flags = BASE_CLIENT_FLAGS + ["--exclude", "small.txt"]
+        cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
+        print(f"\n  --- Exclude (--exclude small.txt) ---\n    Running: {' '.join(cmd)}")
+        try:
+            r = run_single_test(cmd, "Exclude (--exclude small.txt)", source_dir, dest_dir,
+                                expected_missing=["small.txt"])
+            r["suite"] = profile_name
+            results.append(r)
+        except Exception as e:
+            results.append({"name": "Exclude (--exclude small.txt)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
+
+        # Progress (--progress)
+        feature_flags = BASE_CLIENT_FLAGS + ["--progress"]
+        cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
+        print(f"\n  --- Progress (--progress) ---\n    Running: {' '.join(cmd)}")
+        try:
+            r = run_single_test(cmd, "Progress (--progress)", source_dir, dest_dir)
+            r["suite"] = profile_name
+            results.append(r)
+        except Exception as e:
+            results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
+
+        # Incremental sync (--incremental) — first sync, then second sync should skip all
+        print(f"\n  --- Incremental (--incremental) ---")
+        try:
+            flags = BASE_CLIENT_FLAGS + ["-M"]
+            srv = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
+            time.sleep(0.5)
+            first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
+            r1 = subprocess.run(first_cmd, text=True, capture_output=True)
+            wait_proc(srv)
+            if r1.returncode != 0:
+                raise RuntimeError(f"First sync failed: {r1.stderr[:100]}")
+            srv2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
+            time.sleep(0.5)
+            second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--incremental"]
+            start = time.monotonic()
+            r2 = subprocess.run(second_cmd, text=True, capture_output=True, timeout=30)
+            duration = time.monotonic() - start
+            wait_proc(srv2)
+            r = {"name": "Incremental (--incremental)", "suite": profile_name,
+                 "status": "Success" if r2.returncode == 0 else "Failed",
+                 "time": f"{duration:.4f}s" if r2.returncode == 0 else "N/A",
+                 "error": "" if r2.returncode == 0 else f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}"}
+            results.append(r)
+        except Exception as e:
+            results.append({"name": "Incremental (--incremental)", "suite": profile_name,
+                            "status": "Error", "time": "N/A", "error": str(e)})
+
+        # Chunk size (--chunk-size 5242880)
+        feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"]
+        cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
+        print(f"\n  --- Chunk size (--chunk-size 5242880) ---\n    Running: {' '.join(cmd)}")
+        try:
+            r = run_single_test(cmd, "Chunk size (--chunk-size 5242880)", source_dir, dest_dir)
+            r["suite"] = profile_name
+            results.append(r)
+        except Exception as e:
+            results.append({"name": "Chunk size (--chunk-size 5242880)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
+
+        # Delete (--delete) — pre-populate dest, add extra files, then sync with --delete
+        # Note: server handles one client per launch, so we restart between syncs
+        print(f"\n  --- Delete (--delete) ---")
+        try:
+            flags = BASE_CLIENT_FLAGS + ["-M"]
+            # First sync (no delete) to populate dest
+            s1 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
+            time.sleep(0.5)
+            first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
+            r1 = subprocess.run(first_cmd, text=True, capture_output=True)
+            wait_proc(s1)
+            if r1.returncode != 0:
+                raise RuntimeError(f"First sync failed: {r1.stderr[:100]}")
+            # Add extra files to received dir
+            received = os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep))
+            extra_path = os.path.join(received, "extra_file.txt")
+            with open(extra_path, "w") as f:
+                f.write("should be deleted")
+            extra_dir = os.path.join(received, "extra_dir")
+            os.makedirs(extra_dir, exist_ok=True)
+            with open(os.path.join(extra_dir, "nested.txt"), "w") as f:
+                f.write("nested extra")
+            # Second sync with --delete (fresh server)
+            s2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
+            time.sleep(0.5)
+            second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--delete"]
+            start = time.monotonic()
+            r2 = subprocess.run(second_cmd, text=True, capture_output=True)
+            duration = time.monotonic() - start
+            wait_proc(s2)
+            r = {"name": "Delete (--delete)", "suite": profile_name}
+            if r2.returncode == 0 and not os.path.exists(extra_path) and not os.path.exists(extra_dir):
+                mismatches, missing = verify_transfer(source_dir, received)
+                if not mismatches and not missing:
+                    r["status"] = "Success"
+                    r["time"] = f"{duration:.4f}s"
+                    r["error"] = ""
+                else:
+                    r["status"] = "Failed"
+                    r["time"] = "N/A"
+                    r["error"] = f"post-delete verify: mismatches={len(mismatches)}, missing={len(missing)}"
+            else:
+                r["status"] = "Failed"
+                r["time"] = "N/A"
+                errs = []
+                if r2.returncode != 0:
+                    errs.append(f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}")
+                if os.path.exists(extra_path):
+                    errs.append("extra_file.txt remains")
+                if os.path.exists(extra_dir):
+                    errs.append("extra_dir remains")
+                r["error"] = " | ".join(errs)
+            results.append(r)
+        except Exception as e:
+            results.append({"name": "Delete (--delete)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
+
+        # SSH feature tests
+        if SSH_AVAILABLE:
+            ssh_dest = f"localhost:{dest_dir}_ssh"
+            ssh_feature_cases = [
+                {"name": "SSH Archive (-a)", "flags": ["-a"]},
+                {"name": "SSH Exclude (--exclude small.txt)", "flags": ["--exclude", "small.txt"], "expected_missing": ["small.txt"]},
+            ]
+            for case in ssh_feature_cases:
+                flags = BASE_CLIENT_FLAGS + case["flags"]
+                cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags
+                print(f"\n  --- {case['name']} ---\n    Running: {' '.join(cmd)}")
+                try:
+                    r = run_single_test(cmd, case["name"], source_dir, f"{dest_dir}_ssh",
+                                        no_server=True,
+                                        expected_missing=case.get("expected_missing"))
+                    r["suite"] = profile_name
+                    results.append(r)
+                except Exception as e:
+                    results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
+
+    except (subprocess.CalledProcessError, RuntimeError) as e:
+        print(f"  Error: {e}")
+        results = []
+    finally:
+        if is_limited:
+            try:
+                netem_reset()
+            except Exception:
+                pass
 
     return results
 
 
-os.system("cmake -B build -S .")
+def print_metrics(profile_name, results, total_bytes):
-os.system("cd build && make")
+    params = NETWORK_PROFILES.get(profile_name)
+    if not params or "rate" not in params:
+        return
 
-# --- Main Execution ---
+    client_times, rsync_times = [], {}
-all_results = []
+    for r in results:
+        if r["status"] != "Success" or r["time"] == "N/A":
+            continue
+        t = float(r["time"].rstrip("s"))
+        if r["name"].startswith("rsync"):
+            rsync_times[r["name"]] = t
+        elif "Dry run" not in r["name"]:
+            client_times.append((t, r["name"]))
+    if not client_times or len(rsync_times) < 2:
+        return
 
-# 1. Run Baseline (No Limits)
+    m = re.match(r'(\d+)\s*(mbit|gbit|kbit|bit)', params["rate"])
-all_results.extend(run_suite("Unlimited", apply_limits=False))
+    rate_val = int(m.group(1)) * {'mbit': 1_000_000, 'gbit': 1_000_000_000, 'kbit': 1000, 'bit': 1}[m.group(2)] / 8 if m else None
 
-# # 2. Run Throttled (With Limits)
+    best_time, best_name = min(client_times, key=lambda x: x[0])
-all_results.extend(run_suite("Throttled", apply_limits=True))
+    theoretical_max = total_bytes / rate_val if rate_val else None
 
-# --- Print Comparison Table ---
+    print(f"\n  {'─' * 90}\n  Profile: {profile_name}\n  {'─' * 90}")
-print("\n" + "=" * 105)
+    print(f"  Total data size:               {total_bytes / (1024*1024):.1f} MB")
-print(f"{'fastSync BENCHMARK RESULTS (COMPARISON)':^105}")
+    if rate_val:
-print("=" * 105)
+        print(f"  Network rate:                  {params['rate']} ({format_throughput(rate_val)})")
-print(
+    print(f"  Best client configuration:     {best_name}")
-    f"{'Configuration':<45} | {'Environment':<12} | {'Status':<10} | {'Time':<10} | {'Details/Error':<20}"
+    print(f"  Best client time:              {best_time:.4f}s")
-)
+    if theoretical_max:
-print("-" * 105)
+        print(f"  Theoretical max (uncompressed): {theoretical_max:.4f}s")
+        print(f"  Speedup vs theoretical max:    {theoretical_max / best_time:.2f}x")
+    if (a := rsync_times.get("rsync (archive)")):
+        print(f"  Speedup vs rsync (archive):    {a / best_time:.2f}x")
+    if (c := rsync_times.get("rsync (archive + compress)")):
+        print(f"  Speedup vs rsync (compress):   {c / best_time:.2f}x")
 
-# Sort results by test case name first, then environment to easily compare
-# This groups the baseline and throttled results for the same test next to each other
-# sorted_results = sorted(
-#     all_results,
-#     key=lambda x: (
-#         TEST_CASES.index(
-#             next(item for item in TEST_CASES if item["name"] == x["name"])
-#         ),
-#         x["environment"],
-#     ),
-# )
 
-for res in all_results:
+def format_throughput(bps):
-    status_symbol = (
+    for unit, threshold in [("GB/s", 1_000_000_000), ("MB/s", 1_000_000), ("KB/s", 1000)]:
-        "✅"
+        if bps >= threshold:
-        if res["status"] == "Success"
+            return f"{bps/threshold:.1f} {unit}"
-        else ("⏳" if res["status"] == "Timeout" else "❌")
+    return f"{bps:.0f} B/s"
-    )
+
-    status_str = f"{status_symbol} {res['status']}"
+
-    print(
+SSH_AVAILABLE = False
-        f"{res['name']:<45} | {res['environment']:<12} | {status_str:<10} | {res['time']:<10} | {res['error']:<20}"
+
-    )
+def check_ssh_localhost():
-print("=" * 105)
+    global SSH_AVAILABLE
+    build_dir = os.path.abspath("build")
+    server_path = os.path.join(build_dir, "server")
+
+    r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5",
+                        "localhost", "which", "fastsync-server"],
+                       capture_output=True, timeout=10)
+    if r.returncode == 0:
+        SSH_AVAILABLE = True
+        return
+
+    SSH_AVAILABLE = False
+    # Try each PATH dir: create symlink, then verify with which
+    r = subprocess.run(
+        ["ssh", "-o", "BatchMode=yes", "localhost",
+         'echo "$PATH"'],
+        capture_output=True, timeout=10, text=True)
+    if r.returncode != 0:
+        return
+    for d in r.stdout.strip().split(":"):
+        d = d.strip()
+        if not d:
+            continue
+        if "wrappers" in d:
+            continue
+        test = subprocess.run(
+            ["ssh", "-o", "BatchMode=yes", "localhost",
+             f'test -w "{d}" && ln -sf {server_path} "{d}/fastsync-server" && which fastsync-server'],
+            capture_output=True, timeout=10)
+        if test.returncode == 0:
+            SSH_AVAILABLE = True
+            return
+
+
+def preflight_checks():
+    global SSH_AVAILABLE
+    errors = []
+    print("Pre-flight checks:")
+    print("  [1] --help flag...", end=" ")
+    r = subprocess.run(BASE_CLIENT_CMD + ["--help"], capture_output=True, text=True)
+    if r.returncode == 0 and "Usage:" in r.stdout and "SSH transport" in r.stdout:
+        print("OK")
+    else:
+        print("FAIL")
+        errors.append("--help failed")
+
+    print("  [2] Remote SSH dest detection...", end=" ")
+    r = subprocess.run(BASE_CLIENT_CMD + ["/x", "somehost:/y"], capture_output=True, text=True, timeout=5)
+    if r.returncode != 0 and ("ssh" in r.stderr or "Could not receive" in r.stderr or "could not launch" in r.stderr or "Error" in r.stderr):
+        print("OK (detected as SSH)")
+    else:
+        print("FAIL (not detected as SSH dest)")
+        errors.append("SSH detection failed")
+
+    print("  [3] Server --stdio flag...", end=" ")
+    try:
+        r = subprocess.run(["./build/server", "--stdio"], capture_output=True, text=True, timeout=3)
+        if r.returncode != 0 and ("receiving" in r.stderr or "receiving" in r.stdout or "Receiving" in r.stderr):
+            print("OK (started in stdio mode)")
+        else:
+            print("WARN (stdio exited: rc=%d)" % r.returncode)
+    except subprocess.TimeoutExpired:
+        print("OK (waiting for stdin)")
+
+    print("  [4] Posix arg syntax (no server, expect failure)...", end=" ")
+    r = subprocess.run(BASE_CLIENT_CMD + ["/tmp/x", "/tmp/y"], capture_output=True, text=True, timeout=5)
+    if r.returncode != 0 and "connect" in r.stderr:
+        print("OK (TCP fallback)")
+    else:
+        print("FAIL")
+        errors.append("Posix arg syntax failed")
+
+    check_ssh_localhost()
+    print("  [5] SSH to localhost...", end=" ")
+    if SSH_AVAILABLE:
+        print("OK")
+    else:
+        print("SKIP (install fastsync-server in PATH on remote)")
+
+    if errors:
+        print(f"\n  {len(errors)} pre-flight check(s) failed: {', '.join(errors)}")
+        sys.exit(1)
+    print("  All pre-flight checks passed.\n")
+
+
+def main():
+    os.system("cmake -B build -S . > /dev/null 2>&1")
+    if os.system("cd build && make -j$(nproc) 2>&1 | tail -3") != 0:
+        print("Build failed")
+        sys.exit(1)
+    preflight_checks()
+    parser = argparse.ArgumentParser(description="FastSync integration test / benchmark")
+    parser.add_argument("--source-dir", default=DEFAULT_SOURCE_DIR)
+    parser.add_argument("--dest-dir", default=DEFAULT_DEST_DIR)
+    parser.add_argument("--keep-data", action="store_true")
+    parser.add_argument("--unlimited", action="store_true")
+    parser.add_argument("--wan", action="store_true")
+    args = parser.parse_args()
+
+    total_bytes = generate_test_files(args.source_dir)
+    if os.path.exists(args.dest_dir):
+        shutil.rmtree(args.dest_dir)
+    os.makedirs(args.dest_dir, exist_ok=True)
+
+    profiles = []
+    if args.unlimited:
+        profiles.append("Unlimited")
+    elif args.wan:
+        profiles.append("WAN")
+    else:
+        profiles.append("LAN")
+
+    try:
+        all_results = []
+        for p in profiles:
+            all_results.extend(run_profile(p, args.source_dir, args.dest_dir))
+
+        print("\n" + "=" * 130)
+        print(f"{'RESULTS':^130}")
+        print("=" * 130)
+        print(f"{'Configuration':<45} | {'Profile':<12} | {'Status':<8} | {'Time':<10} | {'Details'}")
+        print("-" * 130)
+        for r in all_results:
+            print(f"{r['name']:<45} | {r['suite']:<12} | {r['status']:<8} | {r['time']:<10} | {r['error']}")
+
+        for p in profiles:
+            print_metrics(p, [r for r in all_results if r["suite"] == p], total_bytes)
+
+        failed = [r for r in all_results if r["status"] != "Success"]
+        if failed:
+            print(f"\n  {len(failed)} test(s) FAILED")
+            sys.exit(1)
+        print(f"\n  ALL {len(all_results)} TESTS PASSED")
+    finally:
+        if not args.keep_data:
+            shutil.rmtree(TEST_DIR, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    main()

tests/runner.c

@@ -1,7 +1,9 @@
 #include "test_array_list.h"
 #include "test_chunk.h"
+#include "test_compression.h"
 #include "test_config.h"
 #include "test_queue.h"
+#include "test_scanner.h"
 #include "test_shared_utils.h"
 #include "test_utils.h"
 #include <stdio.h>
@@ -19,6 +21,8 @@ int main() {
   RUN_TEST(test_shared_utils);
   RUN_TEST(test_chunk);
   RUN_TEST(test_config);
+  RUN_TEST(test_compression);
+  RUN_TEST(test_scanner);
 
   printf("\n\033[1;36m=== TEST SUMMARY ===\033[0m\n");
   printf("Total Tests Run: %d\n", tests_run);

tests/test_array_list.c

@@ -49,16 +49,4 @@ void test_array_list() {
   list->item_destroyer = test_destroyer;
   array_list_delete(list);
   EXPECT_EQ_INT(destroyer_calls, 106);
-
-  // Test clear with NULL destroyer
-  list = array_list_create(NULL);
-  int a = 1, b = 2;
-  array_list_add(list, &a);
-  array_list_add(list, &b);
-  EXPECT_EQ_INT(list->size, 2);
-  array_list_clear(list);
-  EXPECT_EQ_INT(list->size, 0);
-  EXPECT_NULL(list->items[0]);
-  EXPECT_NULL(list->items[1]);
-  array_list_delete(list);
 }

tests/test_chunk.c

@@ -13,43 +13,27 @@ static void test_file_operations() {
 
   to_disk(test_path, test_content, test_len);
 
-  struct stat st;
+  File *f = file_create(test_path);
-  int stat_res = stat(test_path, &st);
-  EXPECT_EQ_INT(stat_res, 0);
-  EXPECT_EQ_INT((int)st.st_size, (int)test_len);
-
-  File *f = file_create(test_path, &st);
   EXPECT_NOT_NULL(f);
   EXPECT_EQ_STR(f->path, test_path);
-  EXPECT_NULL(f->data);
+  EXPECT_NOT_NULL(f->data);
+  EXPECT_NULL(f->data->data);
+  EXPECT_EQ_INT((int)f->data->size, 0);
+
+  struct stat st;
+  stat(test_path, &st);
+  f->data->size = st.st_size;
 
   file_load_data(f);
   EXPECT_NOT_NULL(f->data);
-  EXPECT_EQ_INT(memcmp(f->data, test_content, test_len), 0);
+  EXPECT_NOT_NULL(f->data->data);
+  EXPECT_EQ_INT((int)f->data->size, (int)test_len);
+  EXPECT_EQ_INT(memcmp(f->data->data, test_content, test_len), 0);
 
   file_destroy(f);
   unlink(test_path);
 }
 
-static void test_file_receive_operations() {
-  char *path = str_dup("temp_receive.txt");
-  char *data = str_dup("receive data content");
-  unsigned long long size = strlen(data);
-
-  Data *df = data_create(data, size);
-  EXPECT_NOT_NULL(df);
-  EXPECT_EQ_INT((int)df->size, (int)size);
-  EXPECT_EQ_STR(df->data, "receive data content");
-
-  FileReceive *fr = file_receive_create(path, df);
-  EXPECT_NOT_NULL(fr);
-  EXPECT_EQ_STR(fr->path, "temp_receive.txt");
-  EXPECT_NOT_NULL(fr->data);
-  EXPECT_EQ_STR(fr->data->data, "receive data content");
-
-  file_receive_destroy(fr);
-}
-
 static void test_chunk_operations() {
   char *path1 = "temp_chunk_1.txt";
   char *content1 = "chunk item 1";
@@ -66,8 +50,10 @@ static void test_chunk_operations() {
   stat(path1, &st1);
   stat(path2, &st2);
 
-  File *f1 = file_create(path1, &st1);
+  File *f1 = file_create(path1);
-  File *f2 = file_create(path2, &st2);
+  f1->data->size = st1.st_size;
+  File *f2 = file_create(path2);
+  f2->data->size = st2.st_size;
 
   File *files[2] = {f1, f2};
   Chunk *chunk = chunk_create(files, 2);
@@ -76,63 +62,27 @@ static void test_chunk_operations() {
   EXPECT_NOT_NULL(chunk->items[0]);
   EXPECT_NOT_NULL(chunk->items[1]);
 
-  Data *formatted = chunk_format(chunk);
+  // load data before serializing
-  EXPECT_NOT_NULL(formatted);
+  file_load_data(f1);
+  file_load_data(f2);
 
-  unsigned long long expected_size =
+  // Test chunk_serialize / chunk_deserialize round-trip
-      (sizeof(int) + strlen(path1) + sizeof(unsigned long long) + len1) +
+  Data *serialized = chunk_serialize(chunk, false);
-      (sizeof(int) + strlen(path2) + sizeof(unsigned long long) + len2);
+  EXPECT_NOT_NULL(serialized);
-  EXPECT_EQ_INT((int)formatted->size, (int)expected_size);
 
-  char *ptr = (char *)formatted->data;
+  Chunk *deserialized = chunk_deserialize(serialized, false);
+  EXPECT_NOT_NULL(deserialized);
+  EXPECT_EQ_INT(deserialized->element_count, 2);
+  EXPECT_EQ_STR(deserialized->items[0]->path, path1);
+  EXPECT_EQ_STR(deserialized->items[1]->path, path2);
+  EXPECT_EQ_INT((int)deserialized->items[0]->data->size, (int)len1);
+  EXPECT_EQ_INT((int)deserialized->items[1]->data->size, (int)len2);
+  EXPECT_EQ_INT(memcmp(deserialized->items[0]->data->data, content1, len1), 0);
+  EXPECT_EQ_INT(memcmp(deserialized->items[1]->data->data, content2, len2), 0);
 
-  // File 1
+  data_destroy(serialized);
-  int p_len1;
+  chunk_destroy(deserialized);
-  memcpy(&p_len1, ptr, sizeof(int));
-  ptr += sizeof(int);
-  EXPECT_EQ_INT(p_len1, (int)strlen(path1));
 
-  char read_path1[256];
-  memcpy(read_path1, ptr, p_len1);
-  read_path1[p_len1] = '\0';
-  ptr += p_len1;
-  EXPECT_EQ_STR(read_path1, path1);
-
-  unsigned long long d_len1;
-  memcpy(&d_len1, ptr, sizeof(unsigned long long));
-  ptr += sizeof(unsigned long long);
-  EXPECT_EQ_INT((int)d_len1, (int)len1);
-
-  char read_content1[256];
-  memcpy(read_content1, ptr, d_len1);
-  read_content1[d_len1] = '\0';
-  ptr += d_len1;
-  EXPECT_EQ_STR(read_content1, content1);
-
-  // File 2
-  int p_len2;
-  memcpy(&p_len2, ptr, sizeof(int));
-  ptr += sizeof(int);
-  EXPECT_EQ_INT(p_len2, (int)strlen(path2));
-
-  char read_path2[256];
-  memcpy(read_path2, ptr, p_len2);
-  read_path2[p_len2] = '\0';
-  ptr += p_len2;
-  EXPECT_EQ_STR(read_path2, path2);
-
-  unsigned long long d_len2;
-  memcpy(&d_len2, ptr, sizeof(unsigned long long));
-  ptr += sizeof(unsigned long long);
-  EXPECT_EQ_INT((int)d_len2, (int)len2);
-
-  char read_content2[256];
-  memcpy(read_content2, ptr, d_len2);
-  read_content2[d_len2] = '\0';
-  ptr += d_len2;
-  EXPECT_EQ_STR(read_content2, content2);
-
-  chunk_data_delete(formatted);
   chunk_destroy(chunk);
 
   unlink(path1);
@@ -141,6 +91,5 @@ static void test_chunk_operations() {
 
 void test_chunk() {
   test_file_operations();
-  test_file_receive_operations();
   test_chunk_operations();
 }

tests/test_compression.c

@@ -0,0 +1,117 @@
+#include "test_utils.h"
+#include "chunk.h"
+#include "compression.h"
+#include "data.h"
+#include "file.h"
+#include "utils.h"
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+static void test_data_compress_decompress_roundtrip() {
+  char original[] = "Hello, World! This is test data for compression round-trip!";
+  size_t len = strlen(original);
+
+  char *buf = malloc(len);
+  memcpy(buf, original, len);
+  Data *original_data = data_create(buf, len);
+  EXPECT_NOT_NULL(original_data);
+
+  Data *compressed = data_compress(original_data, 3);
+  EXPECT_NOT_NULL(compressed);
+
+  Data *decompressed = data_decompress(compressed);
+  EXPECT_NOT_NULL(decompressed);
+  EXPECT_EQ_INT((int)decompressed->size, (int)len);
+  EXPECT_EQ_INT(memcmp(decompressed->data, original, len), 0);
+
+  data_destroy(original_data);
+  data_destroy(compressed);
+  data_destroy(decompressed);
+}
+
+static void test_data_compress_decompress_large() {
+  size_t size = 1024 * 10;
+  char *original = malloc(size);
+  EXPECT_NOT_NULL(original);
+  for (size_t i = 0; i < size; i++)
+    original[i] = (char)(i % 256);
+
+  Data *original_data = data_create(original, size);
+  EXPECT_NOT_NULL(original_data);
+
+  Data *compressed = data_compress(original_data, 1);
+  EXPECT_NOT_NULL(compressed);
+
+  Data *decompressed = data_decompress(compressed);
+  EXPECT_NOT_NULL(decompressed);
+  EXPECT_EQ_INT((int)decompressed->size, (int)size);
+  EXPECT_EQ_INT(memcmp(decompressed->data, original, size), 0);
+
+  data_destroy(original_data);
+  data_destroy(compressed);
+  data_destroy(decompressed);
+}
+
+static void test_chunk_compress_decompress_roundtrip() {
+  char *path1 = "temp_comp_test_1.txt";
+  char *content1 = "chunk compression test file 1";
+  unsigned long long len1 = strlen(content1);
+
+  char *path2 = "temp_comp_test_2.txt";
+  char *content2 = "chunk compression test file 2 with more data";
+  unsigned long long len2 = strlen(content2);
+
+  to_disk(path1, content1, len1);
+  to_disk(path2, content2, len2);
+
+  struct stat st1, st2;
+  EXPECT_EQ_INT(stat(path1, &st1), 0);
+  EXPECT_EQ_INT(stat(path2, &st2), 0);
+
+  File *f1 = file_create(path1);
+  f1->data->size = st1.st_size;
+  File *f2 = file_create(path2);
+  f2->data->size = st2.st_size;
+  EXPECT_NOT_NULL(f1);
+  EXPECT_NOT_NULL(f2);
+
+  file_load_data(f1);
+  file_load_data(f2);
+
+  File *files[2] = {f1, f2};
+  Chunk *chunk = chunk_create(files, 2);
+  EXPECT_NOT_NULL(chunk);
+
+  Data *compressed = chunk_compress(chunk, 3, false);
+  EXPECT_NOT_NULL(compressed);
+
+  Data *decompressed_data = data_decompress(compressed);
+  EXPECT_NOT_NULL(decompressed_data);
+
+  Chunk *decompressed_chunk = chunk_deserialize(decompressed_data, false);
+  EXPECT_NOT_NULL(decompressed_chunk);
+  EXPECT_EQ_INT(decompressed_chunk->element_count, 2);
+
+  EXPECT_EQ_STR(decompressed_chunk->items[0]->path, path1);
+  EXPECT_EQ_INT((int)decompressed_chunk->items[0]->data->size, (int)len1);
+  EXPECT_EQ_INT(memcmp(decompressed_chunk->items[0]->data->data, content1, len1), 0);
+
+  EXPECT_EQ_STR(decompressed_chunk->items[1]->path, path2);
+  EXPECT_EQ_INT((int)decompressed_chunk->items[1]->data->size, (int)len2);
+  EXPECT_EQ_INT(memcmp(decompressed_chunk->items[1]->data->data, content2, len2), 0);
+
+  chunk_destroy(chunk);
+  data_destroy(compressed);
+  data_destroy(decompressed_data);
+  chunk_destroy(decompressed_chunk);
+
+  unlink(path1);
+  unlink(path2);
+}
+
+void test_compression() {
+  test_data_compress_decompress_roundtrip();
+  test_data_compress_decompress_large();
+  test_chunk_compress_decompress_roundtrip();
+}

tests/test_compression.h

@@ -0,0 +1,6 @@
+#ifndef TEST_COMPRESSION_H
+#define TEST_COMPRESSION_H
+
+void test_compression();
+
+#endif

tests/test_config.c

@@ -8,7 +8,7 @@
 
 static void test_config_lifecycle() {
   Config *cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("/dst"),
-                              true, true, false, false, 1, 4);
+                              true, true, false, false, false, 1, false, 0);
   EXPECT_NOT_NULL(cfg);
   EXPECT_EQ_STR(cfg->version, "1.0");
   EXPECT_EQ_STR(cfg->send_directory, "/src");
@@ -17,13 +17,49 @@ static void test_config_lifecycle() {
   EXPECT_TRUE(cfg->use_multithreading);
   EXPECT_FALSE(cfg->use_chunk_serialization);
   EXPECT_FALSE(cfg->use_compression);
-  EXPECT_EQ_INT(cfg->num_connections, 4);
+  EXPECT_EQ_INT(cfg->transport, TRANSPORT_TCP);
+  EXPECT_NULL(cfg->ssh_destination);
+  config_delete(cfg);
+}
+
+static void test_config_ssh_dest() {
+  Config *cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("user@host:/dst"),
+                              true, false, false, false, false, 1, false, 0);
+  EXPECT_NOT_NULL(cfg);
+  EXPECT_EQ_INT(cfg->transport, TRANSPORT_TCP);
+  EXPECT_NULL(cfg->ssh_destination);
+  EXPECT_EQ_STR(cfg->receive_root_directory, "user@host:/dst");
+
+  config_parse_ssh_dest(cfg);
+  EXPECT_EQ_INT(cfg->transport, TRANSPORT_SSH);
+  EXPECT_EQ_STR(cfg->ssh_destination, "user@host:/dst");
+  EXPECT_EQ_STR(cfg->receive_root_directory, "/dst");
+  config_delete(cfg);
+}
+
+static void test_config_ssh_dest_local_path() {
+  Config *cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("/local/path"),
+                              true, false, false, false, false, 1, false, 0);
+  config_parse_ssh_dest(cfg);
+  EXPECT_EQ_INT(cfg->transport, TRANSPORT_TCP);
+  EXPECT_NULL(cfg->ssh_destination);
+  EXPECT_EQ_STR(cfg->receive_root_directory, "/local/path");
+  config_delete(cfg);
+}
+
+static void test_config_ssh_dest_no_user() {
+  Config *cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("host:/remote"),
+                              true, false, false, false, false, 1, false, 0);
+  config_parse_ssh_dest(cfg);
+  EXPECT_EQ_INT(cfg->transport, TRANSPORT_SSH);
+  EXPECT_EQ_STR(cfg->ssh_destination, "host:/remote");
+  EXPECT_EQ_STR(cfg->receive_root_directory, "/remote");
   config_delete(cfg);
 }
 
 static void test_pipeline_sender_lifecycle() {
   Config *cfg = config_create(str_dup("2.0"), str_dup("/src2"),
-                              str_dup("/dst2"), false, false, true, true, 1, 8);
+                              str_dup("/dst2"), false, false, true, true, false, 1, false, 0);
   Queue *q1 = queue_create(5, NULL);
   Queue *q2 = queue_create(15, NULL);
 
@@ -40,7 +76,7 @@ static void test_pipeline_sender_lifecycle() {
 
 static void test_pipeline_receiver_lifecycle() {
   Config *cfg = config_create(str_dup("3.0"), str_dup("/src3"),
-                              str_dup("/dst3"), true, true, true, true, 1, 2);
+                              str_dup("/dst3"), true, true, true, true, false, 1, false, 0);
   Queue *q = queue_create(20, NULL);
 
   PipelineContextReceiver *pcr = pipeline_context_receiver_create(cfg, q, 42);
@@ -55,6 +91,9 @@ static void test_pipeline_receiver_lifecycle() {
 
 void test_config() {
   test_config_lifecycle();
+  test_config_ssh_dest();
+  test_config_ssh_dest_local_path();
+  test_config_ssh_dest_no_user();
   test_pipeline_sender_lifecycle();
   test_pipeline_receiver_lifecycle();
 }

tests/test_scanner.c

@@ -0,0 +1,124 @@
+#include "test_utils.h"
+#include "scanner.h"
+#include "file.h"
+#include "utils.h"
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+static void create_test_file(const char *path, const char *content) {
+  (void)to_disk(path, content, strlen(content));
+}
+
+static void test_scanner_single_file() {
+  const char *dir = "test_scan_dir_single";
+  const char *file1 = "test_scan_dir_single/file1.txt";
+  const char *content1 = "hello scanner";
+
+  mkdir(dir, 0755);
+  create_test_file(file1, content1);
+
+  DirectoryScanner *scanner = directory_scanner_create((char *)dir, false, 0, NULL, 0, NULL, 0, 0, 0);
+  EXPECT_NOT_NULL(scanner);
+
+  Chunk *chunk = directory_scanner_next(scanner);
+  EXPECT_NOT_NULL(chunk);
+  EXPECT_EQ_INT(chunk->element_count, 1);
+  EXPECT_EQ_STR(chunk->items[0]->path, file1);
+
+  Chunk *next = directory_scanner_next(scanner);
+  EXPECT_NULL(next);
+
+  chunk_destroy(chunk);
+  directory_scanner_destroy(scanner);
+  unlink(file1);
+  rmdir(dir);
+}
+
+static void test_scanner_multiple_files() {
+  const char *dir = "test_scan_dir_multi";
+  const char *file1 = "test_scan_dir_multi/a.txt";
+  const char *file2 = "test_scan_dir_multi/b.txt";
+  const char *content1 = "alpha";
+  const char *content2 = "beta";
+
+  mkdir(dir, 0755);
+  create_test_file(file1, content1);
+  create_test_file(file2, content2);
+
+  DirectoryScanner *scanner = directory_scanner_create((char *)dir, false, 0, NULL, 0, NULL, 0, 0, 0);
+  EXPECT_NOT_NULL(scanner);
+
+  Chunk *chunk = directory_scanner_next(scanner);
+  EXPECT_NOT_NULL(chunk);
+  EXPECT_EQ_INT(chunk->element_count, 2);
+
+  int found1 = 0, found2 = 0;
+  for (int i = 0; i < chunk->element_count; i++) {
+    if (strcmp(chunk->items[i]->path, file1) == 0) found1 = 1;
+    if (strcmp(chunk->items[i]->path, file2) == 0) found2 = 1;
+  }
+  EXPECT_TRUE(found1);
+  EXPECT_TRUE(found2);
+
+  Chunk *next = directory_scanner_next(scanner);
+  EXPECT_NULL(next);
+
+  chunk_destroy(chunk);
+  directory_scanner_destroy(scanner);
+  unlink(file1);
+  unlink(file2);
+  rmdir(dir);
+}
+
+static void test_scanner_subdirectory() {
+  const char *root = "test_scan_sub";
+  const char *sub = "test_scan_sub/sub";
+  const char *root_file = "test_scan_sub/root.txt";
+  const char *sub_file = "test_scan_sub/sub/sub_file.txt";
+  const char *content = "nested content";
+
+  mkdir(root, 0755);
+  mkdir(sub, 0755);
+  create_test_file(root_file, content);
+  create_test_file(sub_file, content);
+
+  DirectoryScanner *scanner = directory_scanner_create((char *)root, false, 0, NULL, 0, NULL, 0, 0, 0);
+  EXPECT_NOT_NULL(scanner);
+
+  int total_files = 0;
+  Chunk *chunk;
+  while ((chunk = directory_scanner_next(scanner)) != NULL) {
+    total_files += chunk->element_count;
+    chunk_destroy(chunk);
+  }
+  EXPECT_EQ_INT(total_files, 2);
+
+  directory_scanner_destroy(scanner);
+  unlink(root_file);
+  unlink(sub_file);
+  rmdir(sub);
+  rmdir(root);
+}
+
+static void test_scanner_empty_directory() {
+  const char *dir = "test_scan_empty";
+
+  mkdir(dir, 0755);
+
+  DirectoryScanner *scanner = directory_scanner_create((char *)dir, false, 0, NULL, 0, NULL, 0, 0, 0);
+  EXPECT_NOT_NULL(scanner);
+
+  Chunk *chunk = directory_scanner_next(scanner);
+  EXPECT_NULL(chunk);
+
+  directory_scanner_destroy(scanner);
+  rmdir(dir);
+}
+
+void test_scanner() {
+  test_scanner_single_file();
+  test_scanner_multiple_files();
+  test_scanner_subdirectory();
+  test_scanner_empty_directory();
+}

tests/test_scanner.h

@@ -0,0 +1,6 @@
+#ifndef TEST_SCANNER_H
+#define TEST_SCANNER_H
+
+void test_scanner();
+
+#endif