From 0c3da958e25ccefc9d1aa46d6ec14ba79379d7b6 Mon Sep 17 00:00:00 2001 From: TapTap Date: Thu, 16 Jul 2026 14:33:51 +0200 Subject: [PATCH] Initial commit --- .gitignore | 4 + CMakeLists.txt | 38 ++ README.md | 206 +++++++++++ compile_commands.json | 211 +++++++++++ opencode.json | 10 + shell.nix | 25 ++ src/client/client_cli.c | 179 ++++++++++ src/client/client_send.c | 288 +++++++++++++++ src/client/client_send.h | 15 + src/client/scanner.c | 127 +++++++ src/client/scanner.h | 23 ++ src/server/server.c | 129 +++++++ src/shared/array_list.c | 74 ++++ src/shared/array_list.h | 19 + src/shared/chunk.c | 182 ++++++++++ src/shared/chunk.h | 22 ++ src/shared/compression.c | 103 ++++++ src/shared/compression.h | 9 + src/shared/config.c | 121 +++++++ src/shared/config.h | 48 +++ src/shared/data.c | 40 +++ src/shared/data.h | 16 + src/shared/file.c | 191 ++++++++++ src/shared/file.h | 34 ++ src/shared/log.c | 28 ++ src/shared/log.h | 14 + src/shared/metadata.c | 80 +++++ src/shared/metadata.h | 15 + src/shared/multiprocessing.c | 172 +++++++++ src/shared/multiprocessing.h | 46 +++ src/shared/protocol.c | 124 +++++++ src/shared/protocol.h | 23 ++ src/shared/queue.c | 134 +++++++ src/shared/queue.h | 30 ++ src/shared/transport_ssh.c | 139 ++++++++ src/shared/transport_ssh.h | 8 + src/shared/transport_tcp.c | 132 +++++++ src/shared/transport_tcp.h | 28 ++ src/shared/utils.c | 139 ++++++++ src/shared/utils.h | 13 + test.py | 673 +++++++++++++++++++++++++++++++++++ tests/runner.c | 36 ++ tests/test_array_list.c | 52 +++ tests/test_array_list.h | 6 + tests/test_chunk.c | 95 +++++ tests/test_chunk.h | 6 + tests/test_compression.c | 117 ++++++ tests/test_compression.h | 6 + tests/test_config.c | 99 ++++++ tests/test_config.h | 6 + tests/test_queue.c | 206 +++++++++++ tests/test_queue.h | 6 + tests/test_scanner.c | 124 +++++++ tests/test_scanner.h | 6 + tests/test_shared_utils.c | 62 ++++ tests/test_shared_utils.h | 6 + tests/test_utils.h | 94 +++++ tmux.sh | 15 + to_one_file.py | 14 + 59 files changed, 4868 insertions(+) create mode 100644 .gitignore create mode 100644 CMakeLists.txt create mode 100644 README.md create mode 100644 compile_commands.json create mode 100644 opencode.json create mode 100644 shell.nix create mode 100644 src/client/client_cli.c create mode 100644 src/client/client_send.c create mode 100644 src/client/client_send.h create mode 100644 src/client/scanner.c create mode 100644 src/client/scanner.h create mode 100644 src/server/server.c create mode 100644 src/shared/array_list.c create mode 100644 src/shared/array_list.h create mode 100644 src/shared/chunk.c create mode 100644 src/shared/chunk.h create mode 100644 src/shared/compression.c create mode 100644 src/shared/compression.h create mode 100644 src/shared/config.c create mode 100644 src/shared/config.h create mode 100644 src/shared/data.c create mode 100644 src/shared/data.h create mode 100644 src/shared/file.c create mode 100644 src/shared/file.h create mode 100644 src/shared/log.c create mode 100644 src/shared/log.h create mode 100644 src/shared/metadata.c create mode 100644 src/shared/metadata.h create mode 100644 src/shared/multiprocessing.c create mode 100644 src/shared/multiprocessing.h create mode 100644 src/shared/protocol.c create mode 100644 src/shared/protocol.h create mode 100644 src/shared/queue.c create mode 100644 src/shared/queue.h create mode 100644 src/shared/transport_ssh.c create mode 100644 src/shared/transport_ssh.h create mode 100644 src/shared/transport_tcp.c create mode 100644 src/shared/transport_tcp.h create mode 100644 src/shared/utils.c create mode 100644 src/shared/utils.h create mode 100644 test.py create mode 100644 tests/runner.c create mode 100644 tests/test_array_list.c create mode 100644 tests/test_array_list.h create mode 100644 tests/test_chunk.c create mode 100644 tests/test_chunk.h create mode 100644 tests/test_compression.c create mode 100644 tests/test_compression.h create mode 100644 tests/test_config.c create mode 100644 tests/test_config.h create mode 100644 tests/test_queue.c create mode 100644 tests/test_queue.h create mode 100644 tests/test_scanner.c create mode 100644 tests/test_scanner.h create mode 100644 tests/test_shared_utils.c create mode 100644 tests/test_shared_utils.h create mode 100644 tests/test_utils.h create mode 100644 tmux.sh create mode 100644 to_one_file.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ac92afb --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +build +data_copied +test_data/ +__pycache__/ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..6555105 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,38 @@ +cmake_minimum_required(VERSION 4.1) + +project(FastFileTransfer) + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) + +add_compile_options(-Wall -g -O3) +# add_compile_options(-Wall -g -O1 -fsanitize=address) + +# add_link_options(-fsanitize=address) + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + +find_library(ZSTD_LIBRARY zstd) +if(NOT ZSTD_LIBRARY) + message(FATAL_ERROR "zstd library not found. Ensure it is in your nix-shell!") +endif() + +file(GLOB SHARED_SRCS "src/shared/*.c") +file(GLOB SERVER_SRCS "src/server/*.c") +file(GLOB CLIENT_SRCS "src/client/*.c") +file(GLOB TEST_SRCS "tests/*.c") + +add_executable(server ${SERVER_SRCS} ${SHARED_SRCS}) +target_include_directories(server PRIVATE src/shared src/server src/client) +target_link_libraries(server PRIVATE Threads::Threads ${ZSTD_LIBRARY}) + +add_executable(client ${CLIENT_SRCS} ${SHARED_SRCS}) +target_include_directories(client PRIVATE src/shared src/server src/client) +target_link_libraries(client PRIVATE Threads::Threads ${ZSTD_LIBRARY}) + +add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} src/client/scanner.c) +target_include_directories(tests PRIVATE tests src/shared src/server src/client) +target_link_libraries(tests PRIVATE Threads::Threads ${ZSTD_LIBRARY}) + diff --git a/README.md b/README.md new file mode 100644 index 0000000..9c78a91 --- /dev/null +++ b/README.md @@ -0,0 +1,206 @@ +# FastSync + +A high-performance file synchronization system with SSH and TCP transport, streaming zstd compression, multithreaded transfer, metadata preservation, and rsync-compatible CLI flags. + +## Technical Overview + +1. **Dual transport**: custom TCP client-server or SSH subprocess (rsync-style `user@host:/path`) +2. **Chunked file transfer**: files grouped into configurable-size chunks (default ~10 MB) +3. **Streaming zstd compression** (levels 1–22) using `ZSTD_compressStream2` +4. **Multithreading**: producer-consumer pipeline with thread-safe queues (scanner → loader → sender) +5. **Metadata preservation**: `mode`, `uid`, `gid`, `mtime` restored on disk when enabled +6. **`sendfile()` zero-copy** on TCP (~2× faster on loopback) +7. **SSH ControlMaster** for connection reuse across repeated invocations +8. **`--delete`**: receiver removes files not present in sender manifest +9. **`--exclude`**: glob-pattern filename filtering (`*`, `?`, no `/` crossing) + +## System Architecture + +### Client +- Recursively scans source directories (BFS), supports exclude patterns +- Groups files into chunks (configurable size) +- Streaming zstd compression with configurable level +- Chunk serialization (compact binary format) or per-file transfer +- Manifests all sent paths when `--delete` is active +- Sends via TCP `sendfile()` or SSH pipe +- Optional progress display with throughput + +### Server +- TCP mode: listens on port 8080; SSH mode: runs via `--stdio` +- Receives and reassembles files +- Decompresses (streaming zstd), deserializes, restores metadata +- Processes `STATUS_MANIFEST` for `--delete`: walks destination tree, removes extras +- Thread pool for parallel processing + +## Protocol Details + +### Status Codes +| Code | Meaning | +|------|---------| +| `STATUS_OK` | Operation successful | +| `STATUS_ERROR` | Error occurred | +| `STATUS_FINISHED` | Transfer complete | +| `STATUS_NEXT` | Ready for next file (per-file mode) | +| `STATUS_CHUNK` | Following data is a serialized chunk | +| `STATUS_MANIFEST` | Following data is a file manifest (for `--delete`) | + +### Wire Format — Metadata + +When `use_metadata` is enabled (`-M`), each file entry carries a 4-byte `present` flag followed by five fields (`mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`). When disabled globally, no metadata bytes are sent — zero wire overhead. + +### Transfer Flow +``` +Config → (STATUS_NEXT | STATUS_CHUNK)* → [STATUS_MANIFEST] → STATUS_FINISHED → STATUS_OK +``` + +## Command-Line Arguments + +| Argument | Description | +|----------|-------------| +| Positional | ` ` — automatic SSH detection if dest contains `:` | +| `-c [level]` | Compression with optional level (1–22, default 5) | +| `-z [level]` | Alias for `-c` | +| `-a, --archive` | Archive mode: enables `-c -m -M` (no `-s`) | +| `-m` | Multithreading mode | +| `-s` | Chunk serialization (batch all files per chunk) | +| `-f` | Sendfile zero-copy. Incompatible with `-c` / `-s`. TCP only. | +| `-M, --preserve` | Preserve file metadata (mode, uid, gid, mtime) | +| `-n, --dry-run` | Scan and print what would be transferred | +| `-p ` | SSH port (default: 22) | +| `--progress` | Show real-time transfer speed | +| `--delete` | Delete files on receiver not present in source | +| `--exclude ` | Exclude files matching glob pattern (repeatable) | +| `--chunk-size ` | Chunk size in bytes (default: 10485760) | +| `--source-dir ` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) | +| `--dest-dir ` | Server destination directory (overrides `FASTSYNC_DEST_DIR`) | +| `--save-to-disk` | Write received files to disk | +| `--server-host ` | Server IP address (default: `127.0.0.1`) | +| `--server-port ` | Server port (default: `8080`) | +| `-v, --verbose` | Enable debug logging | + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `FASTSYNC_SOURCE_DIR` | — | Source directory fallback | +| `FASTSYNC_DEST_DIR` | — | Destination directory fallback | +| `FASTSYNC_SAVE_TO_DISK` | `false` | Disk persistence fallback | + +## Implementation Details + +### Data Structures +1. **Chunk** — collection of files (~10 MB total by default) +2. **File** — path, content (`Data`), optional `FileMetadata` pointer +3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec` +4. **Config** — runtime parameters (transported over wire) +5. **Queue** — thread-safe bounded queue with condition variables +6. **DirectoryScanner** — recursive BFS traversal with exclude pattern support + +### Key Algorithms +1. **File scanning** — BFS directory traversal; each entry matched against exclude patterns +2. **Chunking** — files accumulated until `chunk_size` threshold, then flushed +3. **Compression** — streaming zstd via `ZSTD_compressStream2` / `ZSTD_decompressStream` +4. **Network protocol** — status-code-driven exchange with metadata packing +5. **Metadata restoration** — `chmod()`, `chown()`, `utimensat()` on the receiving side +6. **`--delete`** — sender tracks all sent paths; receiver walks destination tree and removes unlisted files/directories +7. **SSH transport** — `socketpair()` + `fork()` + `execvp("ssh", ...)` with `ControlMaster` and port support + +## Build Requirements + +- C11 compiler +- CMake 4.1+ +- zstd library (≥ 1.4.0 for streaming API) +- pthreads +- SSH client (for SSH transport) + +## Building + +```bash +cmake -B build -S . && cmake --build build -j$(nproc) +``` + +## Running + +### Server (TCP mode) +```bash +./build/server +``` + +### Client — SSH (rsync-style) +```bash +./build/client /path/to/send user@host:/path/to/receive +``` + +### Client — TCP +```bash +./build/client --source-dir /path/to/send --dest-dir /path/to/receive --save-to-disk +``` + +### Common Options +```bash +# Archive mode (compression + multithreading + metadata) +./build/client -a /path/to/send user@host:/path + +# Dry run +./build/client -n /path/to/send /path/to/receive + +# With progress and custom chunk size +./build/client --progress --chunk-size 2097152 /src user@host:/dst + +# Exclude temporary files + delete extras on receiver +./build/client --exclude "*.tmp" --exclude "*.o" --delete /src user@host:/dst + +# All features +./build/client -a --progress --chunk-size 5242880 --exclude "*.log" --delete /src /dst +``` + +### Server via SSH +Place the `fastsync-server` binary in the remote `$PATH`. The client runs `ssh user@host fastsync-server --stdio` automatically when an SSH-style destination is given. + +## Testing + +```bash +# Unit tests (7 suites) +./build/tests + +# Integration + benchmark suite +python3 test.py +``` + +The benchmark prints throughput metrics, best configuration, and speedup vs rsync. + +## Performance Considerations + +1. Chunk size (~10 MB default) balances memory and transfer efficiency +2. Compression level trades CPU for bandwidth +3. `sendfile()` bypasses userspace — ~2× faster on localhost for large files +4. Multithreading scales with core count +5. Metadata transfer adds negligible overhead (~24 bytes per file when enabled) +6. SSH socketpair buffer set to 1 MB for improved pipe throughput +7. SSH ControlMaster reuses connections across repeated invocations + +## Benchmark Results + +25 MB of mixed file sizes over `localhost` with disk I/O throttled (reads ≤ 15 MB/s, writes ≤ 10 MB/s) and network emulation via `tc netem`. Each test was run 3×; the median is reported below. + +### LAN (1000 Mbit, 20 ms ±1 ms, 0.1% loss) + +| Configuration | Time | vs rsync (archive) | vs rsync (compress) | +|---|---|---|---| +| **Best: `-m -c`** | **0.20 s** | **11.2× faster** | **3.6× faster** | +| Compression (`-c`) | 0.31 s | 7.3× faster | 2.3× faster | +| Standard | 1.27 s | 1.8× faster | — | +| rsync (archive) | 2.27 s | — | — | +| rsync (archive + compress) | 0.72 s | — | — | + +### WAN (100 Mbit, 50 ms ±10 ms, 1% loss) + +| Configuration | Time | vs rsync (archive) | vs rsync (compress) | +|---|---|---|---| +| **Best: `-m -c`** | **0.39 s** | **44.8× faster** | **3.8× faster** | +| Compression (`-c`) | 0.64 s | 27.3× faster | 2.3× faster | +| Standard | 7.12 s | 2.4× faster | — | +| rsync (archive) | 17.44 s | — | — | +| rsync (archive + compress) | 1.47 s | — | — | + +Compression reduces the data on the wire enough that the transfer becomes latency-bound rather than bandwidth-bound. On WAN, the best configuration runs 10.8× faster than the theoretical limit for uncompressed data, since zstd shrinks the 25 MB payload to a fraction of its original size over the wire. diff --git a/compile_commands.json b/compile_commands.json new file mode 100644 index 0000000..fdc7e46 --- /dev/null +++ b/compile_commands.json @@ -0,0 +1,211 @@ +[ + { + "arguments": [ + "/nix/store/vr15iyyykg9zai6fpgvhcgyw7gckl78w-gcc-wrapper-14.3.0/bin/gcc", + "-Wall", + "-g", + "-pthread", + "-std=c11", + "-Isrc/shared", + "-Isrc/server", + "-Isrc/client", + "-c", + "-o", + "obj/shared/array_list.o", + "src/shared/array_list.c" + ], + "directory": "/home/taptap/Nextcloud/Programming/C/fastSync", + "file": "/home/taptap/Nextcloud/Programming/C/fastSync/src/shared/array_list.c", + "output": "/home/taptap/Nextcloud/Programming/C/fastSync/obj/shared/array_list.o" + }, + { + "arguments": [ + "/nix/store/vr15iyyykg9zai6fpgvhcgyw7gckl78w-gcc-wrapper-14.3.0/bin/gcc", + "-Wall", + "-g", + "-pthread", + "-std=c11", + "-Isrc/shared", + "-Isrc/server", + "-Isrc/client", + "-c", + "-o", + "obj/shared/chunk.o", + "src/shared/chunk.c" + ], + "directory": "/home/taptap/Nextcloud/Programming/C/fastSync", + "file": "/home/taptap/Nextcloud/Programming/C/fastSync/src/shared/chunk.c", + "output": "/home/taptap/Nextcloud/Programming/C/fastSync/obj/shared/chunk.o" + }, + { + "arguments": [ + "/nix/store/vr15iyyykg9zai6fpgvhcgyw7gckl78w-gcc-wrapper-14.3.0/bin/gcc", + "-Wall", + "-g", + "-pthread", + "-std=c11", + "-Isrc/shared", + "-Isrc/server", + "-Isrc/client", + "-c", + "-o", + "obj/shared/config.o", + "src/shared/config.c" + ], + "directory": "/home/taptap/Nextcloud/Programming/C/fastSync", + "file": "/home/taptap/Nextcloud/Programming/C/fastSync/src/shared/config.c", + "output": "/home/taptap/Nextcloud/Programming/C/fastSync/obj/shared/config.o" + }, + { + "arguments": [ + "/nix/store/vr15iyyykg9zai6fpgvhcgyw7gckl78w-gcc-wrapper-14.3.0/bin/gcc", + "-Wall", + "-g", + "-pthread", + "-std=c11", + "-Isrc/shared", + "-Isrc/server", + "-Isrc/client", + "-c", + "-o", + "obj/shared/log.o", + "src/shared/log.c" + ], + "directory": "/home/taptap/Nextcloud/Programming/C/fastSync", + "file": "/home/taptap/Nextcloud/Programming/C/fastSync/src/shared/log.c", + "output": "/home/taptap/Nextcloud/Programming/C/fastSync/obj/shared/log.o" + }, + { + "arguments": [ + "/nix/store/vr15iyyykg9zai6fpgvhcgyw7gckl78w-gcc-wrapper-14.3.0/bin/gcc", + "-Wall", + "-g", + "-pthread", + "-std=c11", + "-Isrc/shared", + "-Isrc/server", + "-Isrc/client", + "-c", + "-o", + "obj/shared/multiprocessing.o", + "src/shared/multiprocessing.c" + ], + "directory": "/home/taptap/Nextcloud/Programming/C/fastSync", + "file": "/home/taptap/Nextcloud/Programming/C/fastSync/src/shared/multiprocessing.c", + "output": "/home/taptap/Nextcloud/Programming/C/fastSync/obj/shared/multiprocessing.o" + }, + { + "arguments": [ + "/nix/store/vr15iyyykg9zai6fpgvhcgyw7gckl78w-gcc-wrapper-14.3.0/bin/gcc", + "-Wall", + "-g", + "-pthread", + "-std=c11", + "-Isrc/shared", + "-Isrc/server", + "-Isrc/client", + "-c", + "-o", + "obj/shared/queue.o", + "src/shared/queue.c" + ], + "directory": "/home/taptap/Nextcloud/Programming/C/fastSync", + "file": "/home/taptap/Nextcloud/Programming/C/fastSync/src/shared/queue.c", + "output": "/home/taptap/Nextcloud/Programming/C/fastSync/obj/shared/queue.o" + }, + { + "arguments": [ + "/nix/store/vr15iyyykg9zai6fpgvhcgyw7gckl78w-gcc-wrapper-14.3.0/bin/gcc", + "-Wall", + "-g", + "-pthread", + "-std=c11", + "-Isrc/shared", + "-Isrc/server", + "-Isrc/client", + "-c", + "-o", + "obj/shared/socket.o", + "src/shared/socket.c" + ], + "directory": "/home/taptap/Nextcloud/Programming/C/fastSync", + "file": "/home/taptap/Nextcloud/Programming/C/fastSync/src/shared/socket.c", + "output": "/home/taptap/Nextcloud/Programming/C/fastSync/obj/shared/socket.o" + }, + { + "arguments": [ + "/nix/store/vr15iyyykg9zai6fpgvhcgyw7gckl78w-gcc-wrapper-14.3.0/bin/gcc", + "-Wall", + "-g", + "-pthread", + "-std=c11", + "-Isrc/shared", + "-Isrc/server", + "-Isrc/client", + "-c", + "-o", + "obj/shared/utils.o", + "src/shared/utils.c" + ], + "directory": "/home/taptap/Nextcloud/Programming/C/fastSync", + "file": "/home/taptap/Nextcloud/Programming/C/fastSync/src/shared/utils.c", + "output": "/home/taptap/Nextcloud/Programming/C/fastSync/obj/shared/utils.o" + }, + { + "arguments": [ + "/nix/store/vr15iyyykg9zai6fpgvhcgyw7gckl78w-gcc-wrapper-14.3.0/bin/gcc", + "-Wall", + "-g", + "-pthread", + "-std=c11", + "-Isrc/shared", + "-Isrc/server", + "-Isrc/client", + "-c", + "-o", + "obj/server/server.o", + "src/server/server.c" + ], + "directory": "/home/taptap/Nextcloud/Programming/C/fastSync", + "file": "/home/taptap/Nextcloud/Programming/C/fastSync/src/server/server.c", + "output": "/home/taptap/Nextcloud/Programming/C/fastSync/obj/server/server.o" + }, + { + "arguments": [ + "/nix/store/vr15iyyykg9zai6fpgvhcgyw7gckl78w-gcc-wrapper-14.3.0/bin/gcc", + "-Wall", + "-g", + "-pthread", + "-std=c11", + "-Isrc/shared", + "-Isrc/server", + "-Isrc/client", + "-c", + "-o", + "obj/client/client.o", + "src/client/client.c" + ], + "directory": "/home/taptap/Nextcloud/Programming/C/fastSync", + "file": "/home/taptap/Nextcloud/Programming/C/fastSync/src/client/client.c", + "output": "/home/taptap/Nextcloud/Programming/C/fastSync/obj/client/client.o" + }, + { + "arguments": [ + "/nix/store/vr15iyyykg9zai6fpgvhcgyw7gckl78w-gcc-wrapper-14.3.0/bin/gcc", + "-Wall", + "-g", + "-pthread", + "-std=c11", + "-Isrc/shared", + "-Isrc/server", + "-Isrc/client", + "-c", + "-o", + "obj/client/scanner.o", + "src/client/scanner.c" + ], + "directory": "/home/taptap/Nextcloud/Programming/C/fastSync", + "file": "/home/taptap/Nextcloud/Programming/C/fastSync/src/client/scanner.c", + "output": "/home/taptap/Nextcloud/Programming/C/fastSync/obj/client/scanner.o" + } +] diff --git a/opencode.json b/opencode.json new file mode 100644 index 0000000..4efe427 --- /dev/null +++ b/opencode.json @@ -0,0 +1,10 @@ +{ + "$schema": "https://opencode.ai/config.json", + "permission": { + "bash": { + "*": "allow", + "git push origin main": "deny", + "git push main": "ask" + } + } +} diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..1259d7b --- /dev/null +++ b/shell.nix @@ -0,0 +1,25 @@ +{ + pkgs ? import { }, +}: + +pkgs.mkShell { + nativeBuildInputs = with pkgs; [ + gcc + cmake + gnumake + pkg-config + tea + ]; + + buildInputs = with pkgs; [ + zstd + ]; + + NIX_ENFORCE_PURITY = 0; + + shellHook = '' + export NIX_ENFORCE_PURITY=0 + cmake -B build + export PATH="$PWD/build:$PATH" + ''; +} diff --git a/src/client/client_cli.c b/src/client/client_cli.c new file mode 100644 index 0000000..c3232b9 --- /dev/null +++ b/src/client/client_cli.c @@ -0,0 +1,179 @@ +#include "client_send.h" +#include "config.h" +#include "log.h" +#include "utils.h" +#include +#include +#include + +char *server_host = "127.0.0.1"; +int server_port = 8080; + +static void print_usage(void) { + printf("Usage:\n"); + printf(" fastsync [options] \n"); + printf(" fastsync [options] --source-dir --dest-dir \n"); + printf("\n"); + printf("Destination formats:\n"); + printf(" user@host:/path SSH transport (rsync-style)\n"); + printf(" host:/path SSH transport (current user)\n"); + printf(" /local/path TCP transport (requires server on localhost:8080)\n"); + printf("\n"); + printf("Options:\n"); + printf(" -c [level] Enable compression (level 1-22, default 5)\n"); + printf(" -z [level] Alias for -c\n"); + printf(" -a, --archive Archive mode (-c -m -M)\n"); + printf(" -n, --dry-run Show what would be transferred\n"); + printf(" -p SSH port (default: 22)\n"); + printf(" --progress Show transfer progress\n"); + printf(" --delete Delete files on receiver not in source\n"); + printf(" --exclude Exclude files matching pattern\n"); + printf(" -m Enable multithreading\n"); + printf(" -s Enable chunk serialization\n"); + printf(" -f Enable sendfile (TCP only, not with -c or -s)\n"); + printf(" -v, --verbose Enable debug logging\n"); + printf(" -M, --preserve Preserve file metadata\n"); + printf(" --chunk-size Chunk size in bytes (default: %d)\n", DEFAULT_CHUNK_SIZE); + printf(" --source-dir Source directory\n"); + printf(" --dest-dir Destination directory\n"); + printf(" --save-to-disk Write received files to disk\n"); + printf(" --server-host Server IP address (default: 127.0.0.1)\n"); + printf(" --server-port Server port (default: 8080)\n"); + printf(" --help Show this help\n"); +} + +int main(int argc, char *argv[]) { + const char *env_source = getenv("FASTSYNC_SOURCE_DIR"); + const char *env_dest = getenv("FASTSYNC_DEST_DIR"); + const char *env_save = getenv("FASTSYNC_SAVE_TO_DISK"); + + bool save_to_disk = false; + if (env_save && + (strcmp(env_save, "true") == 0 || strcmp(env_save, "1") == 0)) { + save_to_disk = true; + } + + Config *config = config_create(str_dup(PROTOCOL_VERSION), NULL, NULL, + save_to_disk, false, false, false, false, 5, false, 0); + + int positional_args[2]; + int positional_count = 0; + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--help") == 0) { + print_usage(); + return 0; + } else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--archive") == 0) { + config->use_compression = true; + config->use_multithreading = true; + config->use_metadata = true; + log_message(LOG_LEVEL_INFO, "Enabled archive mode (-c -m -M)"); + } else if (strcmp(argv[i], "-n") == 0 || strcmp(argv[i], "--dry-run") == 0) { + config->dry_run = true; + } else if (strcmp(argv[i], "-p") == 0 && i + 1 < argc) { + config->ssh_port = atoi(argv[++i]); + } else if (strcmp(argv[i], "--delete") == 0) { + config->use_delete = true; + } else if (strcmp(argv[i], "--exclude") == 0 && i + 1 < argc) { + int idx = config->exclude_count++; + config->exclude_patterns = realloc(config->exclude_patterns, config->exclude_count * sizeof(char *)); + config->exclude_patterns[idx] = str_dup(argv[++i]); + } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) { + config->use_compression = true; + log_message(LOG_LEVEL_INFO, "Enabled Compression"); + if (i + 1 < argc) { + char *end_ptr; + int level = strtol(argv[i + 1], &end_ptr, 10); + if (*end_ptr == '\0') { + config->compression_level = level; + log_message(LOG_LEVEL_INFO, "Set Compression level to %d", + config->compression_level); + i++; + } + } + } else if (strcmp(argv[i], "--source-dir") == 0 && i + 1 < argc) { + free(config->send_directory); + config->send_directory = str_dup(argv[++i]); + } else if (strcmp(argv[i], "--dest-dir") == 0 && i + 1 < argc) { + free(config->receive_root_directory); + config->receive_root_directory = str_dup(argv[++i]); + } else if (strcmp(argv[i], "--save-to-disk") == 0) { + config->save_to_disk = true; + } else if (strcmp(argv[i], "-M") == 0 || strcmp(argv[i], "--preserve") == 0) { + config->use_metadata = true; + log_message(LOG_LEVEL_INFO, "Enabled metadata preservation"); + } else if (strcmp(argv[i], "-f") == 0 || strcmp(argv[i], "--sendfile") == 0) { + config->use_sendfile = true; + log_message(LOG_LEVEL_INFO, "Enabled sendfile"); + } else if (strcmp(argv[i], "-m") == 0) { + config->use_multithreading = true; + log_message(LOG_LEVEL_INFO, "Enabled Multithreading"); + } else if (strcmp(argv[i], "-s") == 0) { + config->use_chunk_serialization = true; + log_message(LOG_LEVEL_INFO, "Enabled Chunk Serialization"); + } else if (strcmp(argv[i], "--server-host") == 0 && i + 1 < argc) { + free(server_host); + server_host = str_dup(argv[++i]); + } else if (strcmp(argv[i], "--server-port") == 0 && i + 1 < argc) { + server_port = atoi(argv[++i]); + } else if (strcmp(argv[i], "--progress") == 0) { + config->show_progress = true; + } else if (strcmp(argv[i], "--chunk-size") == 0 && i + 1 < argc) { + unsigned long long val = strtoull(argv[++i], NULL, 10); + if (val > 0) + config->chunk_size = val; + } else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) { + set_log_level(LOG_LEVEL_DEBUG); + } else if (argv[i][0] == '-') { + fprintf(stderr, "Unknown option: %s\n", argv[i]); + print_usage(); + return 1; + } else { + if (positional_count < 2) + positional_args[positional_count++] = i; + else { + fprintf(stderr, "Unexpected argument: %s\n", argv[i]); + print_usage(); + return 1; + } + } + } + + if (positional_count == 2) { + free(config->send_directory); + free(config->receive_root_directory); + config->send_directory = str_dup(argv[positional_args[0]]); + config->receive_root_directory = str_dup(argv[positional_args[1]]); + config->save_to_disk = true; + + config_parse_ssh_dest(config); + } else if (positional_count == 1) { + fprintf(stderr, "Error: missing destination argument\n"); + print_usage(); + return 1; + } else { + if (!config->send_directory && env_source) + config->send_directory = str_dup((char *)env_source); + if (!config->receive_root_directory && env_dest) + config->receive_root_directory = str_dup((char *)env_dest); + } + + if (!config->send_directory || !config->receive_root_directory) { + fprintf(stderr, "Error: source and destination directories are required\n"); + print_usage(); + return 1; + } + if (config->use_sendfile && (config->use_chunk_serialization || config->use_compression)) { + fprintf(stderr, "Error: -f/--sendfile cannot be combined with -c (compression) or -s (chunk serialization)\n"); + return 1; + } + + if (config->transport == TRANSPORT_SSH && config->use_sendfile) { + fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n"); + return 1; + } + + if (config->use_multithreading) + return send_files_multithreaded(config); + return send_files(config); +} diff --git a/src/client/client_send.c b/src/client/client_send.c new file mode 100644 index 0000000..4797d31 --- /dev/null +++ b/src/client/client_send.c @@ -0,0 +1,288 @@ +#include "client_send.h" +#include "array_list.h" +#include "chunk.h" +#include "config.h" +#include "data.h" +#include "file.h" +#include "log.h" +#include "multiprocessing.h" +#include "protocol.h" +#include "queue.h" +#include "scanner.h" +#include "transport_tcp.h" +#include "transport_ssh.h" +#include "utils.h" +#include +#include +#include +#include +#include + +int send_chunk(Client *client, Chunk *chunk, Config *config) { + if (config->use_chunk_serialization) { + send_status(client->file_descriptor, STATUS_CHUNK); + Data *data; + if (config->use_compression) { + data = chunk_compress(chunk, config->compression_level, config->use_metadata); + } else { + data = chunk_serialize(chunk, config->use_metadata); + } + send_data(client->file_descriptor, data); + data_destroy(data); + } else if (config->use_sendfile && !config->use_compression) { + for (int i = 0; i < chunk->element_count; i++) { + send_status(client->file_descriptor, STATUS_NEXT); + file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata); + } + } else { + for (int i = 0; i < chunk->element_count; i++) { + send_status(client->file_descriptor, STATUS_NEXT); + file_send_single_calls(chunk->items[i], client->file_descriptor, + config->use_metadata, + config->use_compression ? config->compression_level : 0); + } + } + return 0; +} + +static int send_chunks_multithreaded(void *pipeline_context) { + PipelineContextSender *context = (PipelineContextSender *)pipeline_context; + Client *client; + if (context->config->transport == TRANSPORT_SSH) { + if (context->config->use_sendfile) { + fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n"); + return 1; + } + client = client_connect_ssh(context->config->ssh_destination, context->config->ssh_port); + } else { + client = client_create(); + client_connect(client, server_host, server_port); + } + config_send(client->file_descriptor, context->config); + + while (true) { + Chunk *current_chunk = queue_dequeue_multithreaded( + context->queue_loader, &context->mutex_loader, + &context->condition_not_empty_loader, + &context->condition_not_full_loader, &context->loader_done); + if (current_chunk == NULL) { + if (context->config->use_delete) { + send_status(client->file_descriptor, STATUS_MANIFEST); + send_int(client->file_descriptor, context->manifest->size); + for (int i = 0; i < context->manifest->size; i++) + send_str(client->file_descriptor, + (char *)context->manifest->items[i]); + } + send_status(client->file_descriptor, STATUS_FINISHED); + int ok = receive_status(client->file_descriptor) == STATUS_OK; + client_disconnect(client); + client_delete(client); + return ok ? thrd_success : thrd_error; + } + if (send_chunk(client, current_chunk, context->config) != 0) { + perror("Something unexpected happend while sending the chunk"); + exit(EXIT_FAILURE); + } + chunk_destroy(current_chunk); + } +} + +static int scan_directory_multithreaded(void *pipeline_context) { + PipelineContextSender *context = (PipelineContextSender *)pipeline_context; + mtx_lock(&context->mutex_scanner); + DirectoryScanner *scanner = directory_scanner_create( + context->config->send_directory, context->config->use_metadata, + context->config->chunk_size, context->config->exclude_patterns, + context->config->exclude_count); + mtx_unlock(&context->mutex_scanner); + + Chunk *current_chunk; + while ((current_chunk = directory_scanner_next(scanner)) != NULL) { + if (context->config->use_delete) { + mtx_lock(&context->mutex_scanner); + for (int i = 0; i < current_chunk->element_count; i++) { + const char *p = current_chunk->items[i]->path; + if (*p == '/') p++; + array_list_add(context->manifest, str_dup(p)); + } + mtx_unlock(&context->mutex_scanner); + } + queue_enqueue_multithreaded(context->queue_scanner, current_chunk, + &context->mutex_scanner, + &context->condition_not_empty_scanner, + &context->condition_not_full_scanner); + } + mtx_lock(&context->mutex_scanner); + context->scanner_done = true; + cnd_signal(&context->condition_not_empty_scanner); + mtx_unlock(&context->mutex_scanner); + + directory_scanner_destroy(scanner); + return thrd_success; +} + +static int load_files_multithreaded(void *pipeline_context) { + PipelineContextSender *context = (PipelineContextSender *)pipeline_context; + while (true) { + Chunk *chunk = queue_dequeue_multithreaded( + context->queue_scanner, &context->mutex_scanner, + &context->condition_not_empty_scanner, + &context->condition_not_full_scanner, &context->scanner_done); + if (chunk == NULL) { + mtx_lock(&context->mutex_loader); + context->loader_done = true; + cnd_signal(&context->condition_not_empty_loader); + mtx_unlock(&context->mutex_loader); + return thrd_success; + } + if (!context->config->use_sendfile) { + for (int i = 0; i < chunk->element_count; i++) + file_load_data(chunk->items[i]); + } + queue_enqueue_multithreaded(context->queue_loader, chunk, + &context->mutex_loader, + &context->condition_not_empty_loader, + &context->condition_not_full_loader); + } +} + +int send_files(Config *config) { + if (config->dry_run) { + DirectoryScanner *scanner = directory_scanner_create( + config->send_directory, config->use_metadata, config->chunk_size, + config->exclude_patterns, config->exclude_count); + Chunk *chunk; + int file_count = 0; + unsigned long long total_bytes = 0; + printf("Dry run: files to be transferred\n"); + while ((chunk = directory_scanner_next(scanner)) != NULL) { + for (int i = 0; i < chunk->element_count; i++) { + printf(" %s (%zu bytes)\n", chunk->items[i]->path, + chunk->items[i]->data->size); + total_bytes += chunk->items[i]->data->size; + file_count++; + } + chunk_destroy(chunk); + } + directory_scanner_destroy(scanner); + printf("Total: %d files, %.1f MB\n", file_count, + total_bytes / 1048576.0); + return 0; + } + + Client *client; + if (config->transport == TRANSPORT_SSH) { + if (config->use_sendfile) { + fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n"); + return 1; + } + client = client_connect_ssh(config->ssh_destination, config->ssh_port); + } else { + client = client_create(); + client_connect(client, server_host, server_port); + } + config_send(client->file_descriptor, config); + DirectoryScanner *scanner = directory_scanner_create( + config->send_directory, config->use_metadata, config->chunk_size, + config->exclude_patterns, config->exclude_count); + Chunk *current_chunk; + unsigned long long total_bytes = 0; + time_t last_progress = 0; + time_t start = time(NULL); + ArrayList *manifest = config->use_delete ? array_list_create(free) : NULL; + while ((current_chunk = directory_scanner_next(scanner)) != NULL) { + unsigned long long chunk_bytes = 0; + for (int i = 0; i < current_chunk->element_count; i++) { + chunk_bytes += current_chunk->items[i]->data->size; + if (manifest) { + const char *p = current_chunk->items[i]->path; + if (*p == '/') p++; + array_list_add(manifest, str_dup(p)); + } + } + if (!config->use_sendfile) { + for (int i = 0; i < current_chunk->element_count; i++) + file_load_data(current_chunk->items[i]); + } + send_chunk(client, current_chunk, config); + if (config->show_progress) { + total_bytes += chunk_bytes; + time_t now = time(NULL); + if (now - last_progress >= 1) { + last_progress = now; + double elapsed = difftime(now, start); + double rate = elapsed > 0 ? total_bytes / (1048576.0 * elapsed) : 0; + fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) ", total_bytes / 1048576.0, rate); + fflush(stderr); + } + } + chunk_destroy(current_chunk); + } + if (config->use_delete) { + send_status(client->file_descriptor, STATUS_MANIFEST); + send_int(client->file_descriptor, manifest->size); + for (int i = 0; i < manifest->size; i++) + send_str(client->file_descriptor, (char *)manifest->items[i]); + array_list_delete(manifest); + } + send_status(client->file_descriptor, STATUS_FINISHED); + int ok = receive_status(client->file_descriptor) == STATUS_OK; + if (config->show_progress) { + double elapsed = difftime(time(NULL), start); + double rate = elapsed > 0 ? total_bytes / (1048576.0 * elapsed) : 0; + fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) Done.\n", total_bytes / 1048576.0, rate); + } + directory_scanner_destroy(scanner); + client_disconnect(client); + client_delete(client); + return ok ? 0 : -1; +} + +int send_files_multithreaded(Config *config) { + if (config->dry_run) { + DirectoryScanner *scanner = directory_scanner_create( + config->send_directory, config->use_metadata, config->chunk_size, + config->exclude_patterns, config->exclude_count); + Chunk *chunk; + int file_count = 0; + unsigned long long total_bytes = 0; + printf("Dry run: files to be transferred\n"); + while ((chunk = directory_scanner_next(scanner)) != NULL) { + for (int i = 0; i < chunk->element_count; i++) { + printf(" %s (%zu bytes)\n", chunk->items[i]->path, + chunk->items[i]->data->size); + total_bytes += chunk->items[i]->data->size; + file_count++; + } + chunk_destroy(chunk); + } + directory_scanner_destroy(scanner); + printf("Total: %d files, %.1f MB\n", file_count, + total_bytes / 1048576.0); + return 0; + } + + PipelineContextSender *context = + pipeline_context_sender_create(config, queue_create(100, chunk_destroy), + queue_create(100, chunk_destroy)); + if (config->use_delete) + context->manifest = array_list_create(free); + + thrd_t scanner, loader, sender; + if (thrd_create(&scanner, scan_directory_multithreaded, context) != + thrd_success || + thrd_create(&loader, load_files_multithreaded, context) != thrd_success || + thrd_create(&sender, send_chunks_multithreaded, context) != + thrd_success) { + perror("Error creating threads.\n"); + return 1; + } + + int sender_result; + thrd_join(scanner, NULL); + thrd_join(loader, NULL); + thrd_join(sender, &sender_result); + + pipeline_context_sender_destroy(context); + return sender_result == thrd_success ? 0 : -1; +} diff --git a/src/client/client_send.h b/src/client/client_send.h new file mode 100644 index 0000000..132cb15 --- /dev/null +++ b/src/client/client_send.h @@ -0,0 +1,15 @@ +#ifndef CLIENT_SEND_H +#define CLIENT_SEND_H + +#include "chunk.h" +#include "config.h" +#include "transport_tcp.h" + +extern char *server_host; +extern int server_port; + +int send_chunk(Client *client, Chunk *chunk, Config *config); +int send_files(Config *config); +int send_files_multithreaded(Config *config); + +#endif diff --git a/src/client/scanner.c b/src/client/scanner.c new file mode 100644 index 0000000..175e238 --- /dev/null +++ b/src/client/scanner.c @@ -0,0 +1,127 @@ +#include "scanner.h" +#include "array_list.h" +#include "chunk.h" +#include "file.h" +#include "queue.h" +#include "utils.h" +#include +#include +#include +#include +#include +#include + +DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size, char **exclude_patterns, int exclude_count) { + DirectoryScanner *scanner = malloc(sizeof(DirectoryScanner)); + scanner->directories = queue_create(100, free); + scanner->current_dir = NULL; + scanner->current_path = NULL; + scanner->use_metadata = use_metadata; + scanner->chunk_size = chunk_size > 0 ? chunk_size : DESIRED_CHUNK_SIZE; + scanner->exclude_patterns = exclude_patterns; + scanner->exclude_count = exclude_count; + queue_enqueue(scanner->directories, str_dup(root_directory)); + return scanner; +} + +void directory_scanner_destroy(DirectoryScanner *scanner) { + if (scanner == NULL) + return; + if (scanner->current_dir) { + closedir(scanner->current_dir); + scanner->current_dir = NULL; + } + free(scanner->current_path); + queue_destroy(scanner->directories); + free(scanner); +} + +static Chunk *chunk_data_to_chunk(ArrayList *chunk_data) { + void **chunk_items = array_list_to_array(chunk_data); + Chunk *chunk = chunk_create((File **)chunk_items, chunk_data->size); + free(chunk_items); + chunk_data->item_destroyer = NULL; + array_list_delete(chunk_data); + return chunk; +} + +static int open_next_directory(DirectoryScanner *scanner) { + if (scanner->current_dir) { + closedir(scanner->current_dir); + scanner->current_dir = NULL; + } + free(scanner->current_path); + + if (queue_is_empty(scanner->directories)) + return 0; + + scanner->current_path = (char *)queue_dequeue(scanner->directories); + scanner->current_dir = opendir(scanner->current_path); + if (scanner->current_dir == NULL) { + perror("Could not open directory!"); + exit(EXIT_FAILURE); + } + return 1; +} + +Chunk *directory_scanner_next(DirectoryScanner *scanner) { + ArrayList *chunk_data = array_list_create(file_destroy); + unsigned long long chunk_data_size = 0; + + while (1) { + if (scanner->current_dir == NULL) { + if (!open_next_directory(scanner)) + break; + } + + struct dirent *entry = readdir(scanner->current_dir); + if (entry == NULL) { + closedir(scanner->current_dir); + scanner->current_dir = NULL; + free(scanner->current_path); + scanner->current_path = NULL; + continue; + } + + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) + continue; + + char *cur_path = path_cat(scanner->current_path, entry->d_name); + struct stat stats; + if (stat(cur_path, &stats) != 0) { + free(cur_path); + continue; + } + + if (S_ISDIR(stats.st_mode)) { + queue_enqueue(scanner->directories, (void *)cur_path); + } else { + bool excluded = false; + for (int i = 0; i < scanner->exclude_count; i++) { + if (glob_match(scanner->exclude_patterns[i], entry->d_name)) { + excluded = true; + break; + } + } + if (excluded) { + free(cur_path); + continue; + } + File *file = file_create(cur_path); + file->data->size = stats.st_size; + if (scanner->use_metadata) + file->metadata = file_metadata_create(&stats); + array_list_add(chunk_data, file); + chunk_data_size += file->data->size; + if (chunk_data_size > scanner->chunk_size) { + free(cur_path); + return chunk_data_to_chunk(chunk_data); + } + free(cur_path); + } + } + + if (chunk_data->size > 0) + return chunk_data_to_chunk(chunk_data); + return NULL; +} diff --git a/src/client/scanner.h b/src/client/scanner.h new file mode 100644 index 0000000..76cf437 --- /dev/null +++ b/src/client/scanner.h @@ -0,0 +1,23 @@ +#ifndef SCANNER_H +#define SCANNER_H + +#include "chunk.h" +#include "queue.h" +#include +#include + +typedef struct { + Queue *directories; + DIR *current_dir; + char *current_path; + bool use_metadata; + unsigned long long chunk_size; + char **exclude_patterns; + int exclude_count; +} DirectoryScanner; + +DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size, char **exclude_patterns, int exclude_count); +Chunk *directory_scanner_next(DirectoryScanner *scanner); +void directory_scanner_destroy(DirectoryScanner *scanner); + +#endif diff --git a/src/server/server.c b/src/server/server.c new file mode 100644 index 0000000..25ac13f --- /dev/null +++ b/src/server/server.c @@ -0,0 +1,129 @@ +#include "array_list.h" +#include "chunk.h" +#include "compression.h" +#include "config.h" +#include "data.h" +#include "file.h" +#include "log.h" +#include "metadata.h" +#include "multiprocessing.h" +#include "protocol.h" +#include "queue.h" +#include "transport_tcp.h" +#include "unistd.h" +#include "utils.h" +#include +#include +#include +#include + +int receive_files(Config *config, int file_descriptor) { + Status status = receive_status(file_descriptor); + while (status == STATUS_NEXT || status == STATUS_CHUNK) { + if (status == STATUS_CHUNK) { + Data *chunk_data = receive_data(file_descriptor); + Data *data_to_process = chunk_data; + if (config->use_compression) { + data_to_process = data_decompress(chunk_data); + data_destroy(chunk_data); + if (data_to_process == NULL) { + log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk"); + send_status(file_descriptor, STATUS_ERROR); + return -1; + } + } + Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata); + data_destroy(data_to_process); + if (chunk == NULL) { + log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping"); + send_status(file_descriptor, STATUS_ERROR); + return -1; + } + + for (int i = 0; i < chunk->element_count; i++) { + if (config->save_to_disk) { + char *disk_path = path_cat(config->receive_root_directory, chunk->items[i]->path); + to_disk(disk_path, chunk->items[i]->data->data, chunk->items[i]->data->size); + file_restore_metadata(disk_path, chunk->items[i]->metadata); + free(disk_path); + } + } + chunk_destroy(chunk); + } else { + File *file = file_receive(config, file_descriptor); + if (config->save_to_disk) { + char *disk_path = path_cat(config->receive_root_directory, file->path); + to_disk(disk_path, file->data->data, file->data->size); + file_restore_metadata(disk_path, file->metadata); + free(disk_path); + } + file_destroy(file); + } + status = receive_status(file_descriptor); + } + if (status == STATUS_MANIFEST) { + int count = receive_int(file_descriptor); + ArrayList *manifest = array_list_create(free); + for (int i = 0; i < count; i++) + array_list_add(manifest, receive_str(file_descriptor)); + fprintf(stderr, "Deleting files not in manifest...\n"); + delete_extras(config->receive_root_directory, manifest); + array_list_delete(manifest); + status = receive_status(file_descriptor); + } + if (status != STATUS_FINISHED) { + log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status"); + send_status(file_descriptor, STATUS_ERROR); + return -1; + } + send_status(file_descriptor, STATUS_OK); + return 0; +} + +void handler(int file_descriptor) { + Config *config = config_receive(file_descriptor); + if (config->use_multithreading) { + PipelineContextReceiver *context = pipeline_context_receiver_create( + config, queue_create(100, file_destroy), file_descriptor); + thrd_t receiver, writer; + if (thrd_create(&receiver, receive_thread, context) != thrd_success || + thrd_create(&writer, write_thread, context) != thrd_success) { + perror("Error creating Threads!"); + exit(EXIT_FAILURE); + } + thrd_join(receiver, NULL); + thrd_join(writer, NULL); + pipeline_context_receiver_destroy(context); + send_status(file_descriptor, STATUS_OK); + } else + receive_files(config, file_descriptor); + close(file_descriptor); +} + +static Server *g_server = NULL; + +static void cleanup(int sig) { + (void)sig; + if (g_server) { + server_delete(&g_server); + } + _exit(0); +} + +int main(int argc, char *argv[]) { + signal(SIGPIPE, SIG_IGN); + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--stdio") == 0) { + io_set_fds(STDIN_FILENO, STDOUT_FILENO); + handler(STDIN_FILENO); + return 0; + } else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) { + set_log_level(LOG_LEVEL_DEBUG); + } + } + signal(SIGINT, cleanup); + signal(SIGTERM, cleanup); + g_server = server_create(8080); + server_listen(g_server, handler); + return 0; +} diff --git a/src/shared/array_list.c b/src/shared/array_list.c new file mode 100644 index 0000000..1c33c06 --- /dev/null +++ b/src/shared/array_list.c @@ -0,0 +1,74 @@ +#include "array_list.h" +#include +#include +#include + +ArrayList *array_list_create(void (*item_destroyer)(void *item)) { + ArrayList *list = (ArrayList *)malloc(sizeof(ArrayList)); + if (list == NULL) { + perror("FATAL ERROR: Could not allocate memory for array list struct"); + exit(EXIT_FAILURE); + } + + list->items = malloc(INITIAL_ARRAY_SIZE * sizeof(void *)); + if (list->items == NULL) { + perror("FATAL ERROR: Could not allocate memory for list items"); + free(list); + exit(EXIT_FAILURE); + } + list->size = 0; + list->capacity = INITIAL_ARRAY_SIZE; + list->item_destroyer = item_destroyer; + return list; +} + +void array_list_delete(ArrayList *array_list) { + if (array_list == NULL) + return; + if (array_list->item_destroyer != NULL) { + for (int i = 0; i < array_list->size; i++) { + array_list->item_destroyer(array_list->items[i]); + array_list->items[i] = NULL; + } + } + free(array_list->items); + free(array_list); +} + +void array_list_extend(ArrayList *array_list) { + if (array_list == NULL) + return; + int new_capacity = array_list->capacity * 2; + if (new_capacity == 0) + new_capacity = INITIAL_ARRAY_SIZE; + array_list->items = realloc(array_list->items, new_capacity * sizeof(void *)); + if (array_list->items == NULL) { + perror("FATAL ERROR: Could not reallocate memory for array list struct"); + exit(EXIT_FAILURE); + } + array_list->capacity = new_capacity; +} + +void array_list_add(ArrayList *array_list, void *item) { + if (array_list == NULL) { + return; + } + if (array_list->capacity == array_list->size) { + array_list_extend(array_list); + } + array_list->items[array_list->size] = item; + array_list->size += 1; +} + +void **array_list_to_array(ArrayList *array_list) { + if (array_list == NULL) { + return NULL; + } + void **array = malloc(array_list->size * sizeof(void *)); + if (array == NULL) { + perror("Could not malloc space for array from array list!"); + return NULL; + } + memcpy(array, array_list->items, array_list->size * sizeof(void *)); + return array; +} diff --git a/src/shared/array_list.h b/src/shared/array_list.h new file mode 100644 index 0000000..e0e4afd --- /dev/null +++ b/src/shared/array_list.h @@ -0,0 +1,19 @@ +#ifndef ARRAY_LIST_H +#define ARRAY_LIST_H + +#define INITIAL_ARRAY_SIZE 100 + +typedef struct ArrayList { + void **items; + int size; + int capacity; + void (*item_destroyer)(void *item); +} ArrayList; + +ArrayList *array_list_create(void (*item_destroyer)(void *item)); +void array_list_delete(ArrayList *array_list); +void array_list_extend(ArrayList *array_list); +void array_list_add(ArrayList *array_list, void *item); +void **array_list_to_array(ArrayList *array_list); + +#endif diff --git a/src/shared/chunk.c b/src/shared/chunk.c new file mode 100644 index 0000000..cd26510 --- /dev/null +++ b/src/shared/chunk.c @@ -0,0 +1,182 @@ +#include +#include +#include +#include + +#include "array_list.h" +#include "chunk.h" +#include "compression.h" +#include "data.h" +#include "file.h" +#include "log.h" +#include "metadata.h" + +Chunk *chunk_create(File **items, int element_count) { + Chunk *chunk = (Chunk *)malloc(sizeof(Chunk)); + if (chunk == NULL) { + perror("FATAL ERROR: Could not allocate memory for chunk structure"); + exit(EXIT_FAILURE); + } + + chunk->items = (File **)malloc(element_count * sizeof(File *)); + if (chunk->items == NULL) { + perror("FATAL ERROR: Could not allocate memory for items of chunk " + "structure"); + free(chunk); + exit(EXIT_FAILURE); + } + + for (int i = 0; i < element_count; i++) { + chunk->items[i] = items[i]; + } + chunk->element_count = element_count; + return chunk; +} + +void chunk_destroy(void *item) { + if (item == NULL) { + return; + } + Chunk *chunk = (Chunk *)item; + for (int i = 0; i < chunk->element_count; ++i) { + if (chunk->items[i] != NULL) { + file_destroy(chunk->items[i]); + } + } + free(chunk->items); + free(chunk); +} + +static unsigned long long per_file_serialize_size(File *file, bool use_metadata) { + return sizeof(size_t) + strlen(file->path) + + (use_metadata ? sizeof(int) + (file->metadata ? FILE_METADATA_WIRE_SIZE : 0) : 0) + + sizeof(size_t) + file->data->size; +} + +Data *chunk_serialize(Chunk *chunk, bool use_metadata) { + unsigned long long data_size = 0; + for (int i = 0; i < chunk->element_count; i++) { + data_size += per_file_serialize_size(chunk->items[i], use_metadata); + } + Data *data = data_create_empty(data_size); + if (data == NULL) { + log_message(LOG_LEVEL_ERROR, + "Could not allocate memory for chunk serialization"); + exit(EXIT_FAILURE); + } + char *data_pointer = data->data; + for (int i = 0; i < chunk->element_count; i++) { + File *file = chunk->items[i]; + size_t path_len = strlen(file->path); + memcpy(data_pointer, &path_len, sizeof(size_t)); + data_pointer += sizeof(size_t); + memcpy(data_pointer, file->path, path_len); + data_pointer += path_len; + + if (use_metadata) + metadata_to_buf(&data_pointer, file->metadata); + + size_t file_data_size = file->data->size; + memcpy(data_pointer, &file_data_size, sizeof(size_t)); + data_pointer += sizeof(size_t); + memcpy(data_pointer, file->data->data, file_data_size); + data_pointer += file_data_size; + } + return data; +} + +Chunk *chunk_deserialize(Data *data, bool use_metadata) { + ArrayList *files = array_list_create(file_destroy); + char *data_pointer = data->data; + size_t remaining_size = data->size; + + while (remaining_size > 0) { + if (remaining_size < sizeof(size_t)) { + log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for path length"); + array_list_delete(files); + return NULL; + } + + size_t path_len = *(size_t *)data_pointer; + data_pointer += sizeof(size_t); + remaining_size -= sizeof(size_t); + + if (remaining_size < path_len) { + log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for path"); + array_list_delete(files); + return NULL; + } + + char *path = malloc(path_len + 1); + if (path == NULL) { + perror("Could not allocate memory for file path"); + array_list_delete(files); + return NULL; + } + memcpy(path, data_pointer, path_len); + path[path_len] = '\0'; + data_pointer += path_len; + remaining_size -= path_len; + + File *file = file_create(path); + free(path); + + if (use_metadata) { + file->metadata = metadata_from_buf(&data_pointer); + remaining_size -= sizeof(int); + if (file->metadata) + remaining_size -= FILE_METADATA_WIRE_SIZE; + } + + if (remaining_size < sizeof(size_t)) { + log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for data size"); + array_list_delete(files); + return NULL; + } + + size_t file_data_size = *(size_t *)data_pointer; + data_pointer += sizeof(size_t); + remaining_size -= sizeof(size_t); + + if (remaining_size < file_data_size) { + log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for file content"); + array_list_delete(files); + return NULL; + } + + void *file_data = malloc(file_data_size); + if (file_data == NULL) { + perror("Could not allocate memory for file data"); + array_list_delete(files); + return NULL; + } + memcpy(file_data, data_pointer, file_data_size); + data_destroy(file->data); + file->data = data_create(file_data, file_data_size); + data_pointer += file_data_size; + remaining_size -= file_data_size; + + array_list_add(files, file); + } + + File **file_array = (File **)array_list_to_array(files); + Chunk *chunk = chunk_create(file_array, files->size); + + free(file_array); + files->item_destroyer = NULL; + array_list_delete(files); + + return chunk; +} + +Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata) { + log_message(LOG_LEVEL_DEBUG, "Starting to compress chunk"); + Data *serialized = chunk_serialize(chunk, use_metadata); + Data *compressed = data_compress(serialized, compression_level); + data_destroy(serialized); + log_message(LOG_LEVEL_DEBUG, "Chunk successfully compressed"); + return compressed; +} + + + diff --git a/src/shared/chunk.h b/src/shared/chunk.h new file mode 100644 index 0000000..7837936 --- /dev/null +++ b/src/shared/chunk.h @@ -0,0 +1,22 @@ +#ifndef CHUNK_H +#define CHUNK_H + +#include "data.h" +#include "file.h" +#include +#include + +#define DESIRED_CHUNK_SIZE (10 * 1024 * 1024) + +typedef struct { + File **items; + int element_count; +} Chunk; + +Chunk *chunk_create(File **items, int element_count); +void chunk_destroy(void *chunk); +Data *chunk_serialize(Chunk *chunk, bool use_metadata); +Chunk *chunk_deserialize(Data *data, bool use_metadata); +Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata); + +#endif diff --git a/src/shared/compression.c b/src/shared/compression.c new file mode 100644 index 0000000..8ff5de2 --- /dev/null +++ b/src/shared/compression.c @@ -0,0 +1,103 @@ +#include "compression.h" +#include "data.h" +#include "log.h" +#include "stdlib.h" +#include "zstd.h" + +#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024) + +Data *data_compress(Data *data_to_compress, int compression_level) { + log_message(LOG_LEVEL_DEBUG, "Starting to compress data"); + size_t dst_size = ZSTD_compressBound(data_to_compress->size); + Data *compressed_data = data_create_empty(dst_size); + if (!compressed_data) { + log_message(LOG_LEVEL_ERROR, "Failed to allocate compression buffer"); + return NULL; + } + + ZSTD_CCtx *cctx = ZSTD_createCCtx(); + if (!cctx) { + log_message(LOG_LEVEL_ERROR, "Failed to create ZSTD compression context"); + data_destroy(compressed_data); + return NULL; + } + + ZSTD_inBuffer input = {data_to_compress->data, data_to_compress->size, 0}; + ZSTD_outBuffer output = {compressed_data->data, dst_size, 0}; + + size_t ret; + do { + ret = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end); + if (ZSTD_isError(ret)) { + log_message(LOG_LEVEL_ERROR, "Compression failed: %s", + ZSTD_getErrorName(ret)); + ZSTD_freeCCtx(cctx); + data_destroy(compressed_data); + return NULL; + } + } while (ret > 0); + + compressed_data->size = output.pos; + ZSTD_freeCCtx(cctx); + + log_message(LOG_LEVEL_DEBUG, "Data succesfully compressed from %zu to %zu", + data_to_compress->size, compressed_data->size); + return compressed_data; +} + +Data *data_decompress(Data *compressed_data) { + log_message(LOG_LEVEL_DEBUG, "Start to decompress data"); + unsigned long long dst_size = ZSTD_getFrameContentSize( + compressed_data->data, compressed_data->size); + + ZSTD_DCtx *dctx = ZSTD_createDCtx(); + if (!dctx) { + log_message(LOG_LEVEL_ERROR, + "Failed to create ZSTD decompression context"); + return NULL; + } + + size_t buf_size = (!ZSTD_isError(dst_size) && dst_size > 0) + ? (size_t)dst_size + : INITIAL_DECOMPRESS_BUF_SIZE; + Data *uncompressed_data = data_create_empty(buf_size); + if (!uncompressed_data) { + log_message(LOG_LEVEL_ERROR, "Failed to allocate decompression buffer"); + ZSTD_freeDCtx(dctx); + return NULL; + } + + ZSTD_inBuffer input = {compressed_data->data, compressed_data->size, 0}; + ZSTD_outBuffer output = {uncompressed_data->data, buf_size, 0}; + + size_t ret; + do { + ret = ZSTD_decompressStream(dctx, &output, &input); + if (ZSTD_isError(ret)) { + log_message(LOG_LEVEL_ERROR, "Decompression failed: %s", + ZSTD_getErrorName(ret)); + ZSTD_freeDCtx(dctx); + data_destroy(uncompressed_data); + return NULL; + } + if (ret > 0 && output.pos == output.size) { + buf_size *= 2; + void *new_data = realloc(uncompressed_data->data, buf_size); + if (!new_data) { + log_message(LOG_LEVEL_ERROR, "Failed to grow decompression buffer"); + ZSTD_freeDCtx(dctx); + data_destroy(uncompressed_data); + return NULL; + } + uncompressed_data->data = new_data; + output.dst = new_data; + output.size = buf_size; + } + } while (ret > 0); + + uncompressed_data->size = output.pos; + ZSTD_freeDCtx(dctx); + + log_message(LOG_LEVEL_DEBUG, "Decompressed data successfully"); + return uncompressed_data; +} diff --git a/src/shared/compression.h b/src/shared/compression.h new file mode 100644 index 0000000..d902cfa --- /dev/null +++ b/src/shared/compression.h @@ -0,0 +1,9 @@ +#ifndef COMPRESSION_H +#define COMPRESSION_H + +#include "data.h" + +Data *data_compress(Data *data_to_compress, int compression_level); +Data *data_decompress(Data *compressed_data); + +#endif diff --git a/src/shared/config.c b/src/shared/config.c new file mode 100644 index 0000000..f4bbbfe --- /dev/null +++ b/src/shared/config.c @@ -0,0 +1,121 @@ +#include "config.h" +#include "protocol.h" +#include "utils.h" +#include +#include +#include +#include + +Config *config_create(char *version, char *send_directory, + char *receive_directory, bool save_to_disk, + bool use_multithreading, bool use_chunk_serialization, + bool use_compression, bool use_metadata, + int compression_level, bool use_sendfile, + unsigned long long chunk_size) { + + Config *config = malloc(sizeof(Config)); + config->version = version; + config->send_directory = send_directory; + config->receive_root_directory = receive_directory; + config->save_to_disk = save_to_disk; + config->use_multithreading = use_multithreading; + config->use_chunk_serialization = use_chunk_serialization; + config->use_compression = use_compression; + config->use_metadata = use_metadata; + config->show_progress = false; + config->dry_run = false; + config->use_delete = false; + config->compression_level = compression_level; + config->use_sendfile = use_sendfile; + config->chunk_size = chunk_size > 0 ? chunk_size : DEFAULT_CHUNK_SIZE; + config->ssh_port = 22; + config->transport = TRANSPORT_TCP; + config->ssh_destination = NULL; + config->exclude_patterns = NULL; + config->exclude_count = 0; + return config; +} + +bool is_remote_dest(const char *s) { + if (s == NULL) return false; + const char *colon = strchr(s, ':'); + if (colon == NULL) return false; + if (colon == s) return false; + for (const char *p = s; p < colon; p++) { + if (*p == '/') return false; + } + return true; +} + +void config_parse_ssh_dest(Config *config) { + if (!is_remote_dest(config->receive_root_directory)) return; + config->transport = TRANSPORT_SSH; + config->ssh_destination = str_dup(config->receive_root_directory); + char *colon = strchr(config->receive_root_directory, ':'); + char *path = str_dup(colon + 1); + free(config->receive_root_directory); + config->receive_root_directory = path; +} + +void config_delete(Config *config) { + free(config->version); + free(config->send_directory); + free(config->receive_root_directory); + free(config->ssh_destination); + for (int i = 0; i < config->exclude_count; i++) + free(config->exclude_patterns[i]); + free(config->exclude_patterns); + free(config); +} + +void config_send(int file_descriptor, Config *config) { + send_str(file_descriptor, config->version); + send_str(file_descriptor, config->send_directory); + send_str(file_descriptor, config->receive_root_directory); + send_int(file_descriptor, config->save_to_disk); + send_int(file_descriptor, config->use_multithreading); + send_int(file_descriptor, config->use_chunk_serialization); + send_int(file_descriptor, config->use_compression); + send_int(file_descriptor, config->use_metadata); + send_int(file_descriptor, config->compression_level); + send_int(file_descriptor, (int)config->chunk_size); + send_int(file_descriptor, config->use_sendfile); + send_int(file_descriptor, config->use_delete); + if (receive_status(file_descriptor) != STATUS_OK) { + perror("Error transmitting config!"); + exit(EXIT_FAILURE); + } +} + +Config *config_receive(int file_descriptor) { + Config *config = (Config *)malloc(sizeof(Config)); + config->version = receive_str(file_descriptor); + if (strcmp(config->version, PROTOCOL_VERSION) != 0) { + fprintf(stderr, "Protocol version mismatch: client=%s, server=%s\n", + config->version, PROTOCOL_VERSION); + free(config->version); + free(config); + send_status(file_descriptor, STATUS_ERROR); + exit(EXIT_FAILURE); + } + config->send_directory = receive_str(file_descriptor); + config->receive_root_directory = receive_str(file_descriptor); + config->save_to_disk = receive_int(file_descriptor); + config->use_multithreading = receive_int(file_descriptor); + config->use_chunk_serialization = receive_int(file_descriptor); + config->use_compression = receive_int(file_descriptor); + config->use_metadata = receive_int(file_descriptor); + config->compression_level = receive_int(file_descriptor); + config->chunk_size = (unsigned long long)receive_int(file_descriptor); + config->use_sendfile = receive_int(file_descriptor); + config->use_delete = receive_int(file_descriptor); + config->show_progress = false; + config->dry_run = false; + config->ssh_port = 22; + config->transport = TRANSPORT_TCP; + config->ssh_destination = NULL; + config->exclude_patterns = NULL; + config->exclude_count = 0; + send_status(file_descriptor, STATUS_OK); + return config; +} diff --git a/src/shared/config.h b/src/shared/config.h new file mode 100644 index 0000000..58f452e --- /dev/null +++ b/src/shared/config.h @@ -0,0 +1,48 @@ +#ifndef CONFIG_H +#define CONFIG_H + +#include + +typedef enum { + TRANSPORT_TCP, + TRANSPORT_SSH +} TransportType; + +typedef struct Config { + char *version; + char *send_directory; + char *receive_root_directory; + bool save_to_disk; + bool use_multithreading; + bool use_chunk_serialization; + bool use_compression; + bool use_sendfile; + bool use_metadata; + bool show_progress; + bool dry_run; + bool use_delete; + int compression_level; + unsigned long long chunk_size; + int ssh_port; + TransportType transport; + char *ssh_destination; + char **exclude_patterns; + int exclude_count; +} Config; + +#define PROTOCOL_VERSION "1.0.0" +#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) + +Config *config_create(char *version, char *send_directory, + char *receive_directory, bool save_to_disk, + bool use_multithreading, bool use_chunk_serialization, + bool use_compression, bool use_metadata, + int compression_level, bool use_sendfile, + unsigned long long chunk_size); +void config_delete(Config *config); +void config_send(int file_descriptor, Config *config); +Config *config_receive(int file_descriptor); +bool is_remote_dest(const char *s); +void config_parse_ssh_dest(Config *config); + +#endif diff --git a/src/shared/data.c b/src/shared/data.c new file mode 100644 index 0000000..648eb1d --- /dev/null +++ b/src/shared/data.c @@ -0,0 +1,40 @@ +#include "data.h" +#include "log.h" +#include "stdlib.h" + +Data *data_create_empty(size_t data_size) { + void *data = malloc(data_size); + if (data == NULL) { + log_message(LOG_LEVEL_ERROR, "Could not allocate memory for empty data"); + exit(EXIT_FAILURE); + } + return data_create(data, data_size); +} + +Data *data_create_reserve(size_t size) { + Data *d = malloc(sizeof(Data)); + if (d == NULL) { + log_message(LOG_LEVEL_ERROR, "Could not allocate memory for data"); + exit(EXIT_FAILURE); + } + d->data = NULL; + d->size = size; + return d; +} + +Data *data_create(void *data, size_t data_size) { + Data *new_data = malloc(sizeof(Data)); + if (new_data == NULL) { + log_message(LOG_LEVEL_ERROR, "Could not allocate memory for data"); + exit(EXIT_FAILURE); + } + new_data->data = data; + new_data->size = data_size; + return new_data; +} + +void data_destroy(Data *data) { + if (data == NULL) return; + free(data->data); + free(data); +} diff --git a/src/shared/data.h b/src/shared/data.h new file mode 100644 index 0000000..53ed762 --- /dev/null +++ b/src/shared/data.h @@ -0,0 +1,16 @@ +#ifndef DATA_H +#define DATA_H + +#include "stdlib.h" + +typedef struct { + void *data; + size_t size; +} Data; + +Data *data_create_empty(size_t data_size); +Data *data_create_reserve(size_t size); +Data *data_create(void *data, size_t data_size); +void data_destroy(Data *data); + +#endif diff --git a/src/shared/file.c b/src/shared/file.c new file mode 100644 index 0000000..6b4c87f --- /dev/null +++ b/src/shared/file.c @@ -0,0 +1,191 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "compression.h" +#include "log.h" +#include "config.h" +#include "data.h" +#include "file.h" +#include "log.h" +#include "metadata.h" +#include "protocol.h" +#include "utils.h" + +File *file_create(const char *path) { + File *file = (File *)malloc(sizeof(File)); + if (file == NULL) { + perror("FATAL ERROR: Could not allocate memory for file struct"); + exit(EXIT_FAILURE); + } + + int path_len = strlen(path); + file->path = (char *)malloc(path_len + 1); + if (file->path == NULL) { + perror("FATAL ERROR: Could not allocate memory for path file string"); + free(file); + exit(EXIT_FAILURE); + } + + strcpy(file->path, path); + file->data = data_create_reserve(0); + file->metadata = NULL; + return file; +} + +void file_destroy(void *item) { + if (item == NULL) + return; + File *file = (File *)item; + data_destroy(file->data); + file->data = NULL; + file_metadata_destroy(file->metadata); + file->metadata = NULL; + free(file->path); + file->path = NULL; + free(file); +} + +FileMetadata *file_metadata_create(struct stat *stats) { + FileMetadata *m = malloc(sizeof(FileMetadata)); + if (m == NULL) { + perror("FATAL ERROR: Could not allocate memory for file metadata"); + exit(EXIT_FAILURE); + } + m->mode = stats->st_mode; + m->uid = stats->st_uid; + m->gid = stats->st_gid; + m->mtime_sec = stats->st_mtime; +#ifdef __linux__ + m->mtime_nsec = stats->st_mtim.tv_nsec; +#else + m->mtime_nsec = 0; +#endif + return m; +} + +void file_metadata_destroy(void *metadata) { + free(metadata); +} + +void file_load_data(File *file) { + if (file == NULL) + return; + if (file->data->data == NULL) { + file->data->data = malloc(file->data->size); + if (file->data->data == NULL) { + perror("Could not allocate memory for file data"); + exit(EXIT_FAILURE); + } + } + size_t bytes_read = file_content_to_buffer(file); + if (bytes_read != file->data->size) { + log_message(STATUS_ERROR, "Didnt read expected amount of bytes from file"); + exit(EXIT_FAILURE); + } +} + +void file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) { + if (compression_level > 0) { + Data *compressed_data = data_compress(file->data, compression_level); + data_destroy(file->data); + if (compressed_data == NULL) { + log_message(LOG_LEVEL_ERROR, "Compression failed in file_send_single_calls"); + exit(EXIT_FAILURE); + } + file->data = compressed_data; + } + send_str(file_descriptor, file->path); + if (use_metadata) + metadata_send(file_descriptor, file->metadata); + send_data(file_descriptor, file->data); +} + +void to_disk(const char *path, const void *data, unsigned long long data_size) { + char *directory = str_dup(path); + char *dir_to_free = directory; + directory = dirname(directory); + mkdir_r(directory); + FILE *file_pointer = fopen(path, "wb"); + if (file_pointer == NULL) { + perror("Could not open File"); + exit(EXIT_FAILURE); + } + if (fwrite(data, 1, data_size, file_pointer) != data_size) { + perror("Failed to write all data to disk"); + fclose(file_pointer); + exit(EXIT_FAILURE); + } + fclose(file_pointer); + free(dir_to_free); +} + +void file_send_sendfile(File *file, int file_descriptor, bool use_metadata) { + send_str(file_descriptor, file->path); + if (use_metadata) + metadata_send(file_descriptor, file->metadata); + + int fd = open(file->path, O_RDONLY); + if (fd == -1) { + perror("Could not open file for sendfile"); + exit(EXIT_FAILURE); + } + + unsigned long long file_size = file->data->size; + send_n_data(file_descriptor, &file_size, sizeof(unsigned long long)); + + off_t offset = 0; + while (offset < file_size) { + ssize_t sent = sendfile(file_descriptor, fd, &offset, file_size - offset); + if (sent == -1) { + perror("sendfile failed"); + close(fd); + exit(EXIT_FAILURE); + } + } + + close(fd); +} + +File *file_receive(Config *config, int file_descriptor) { + char *path = (char *)receive_str(file_descriptor); + File *file = file_create(path); + free(path); + if (config->use_metadata) + file->metadata = metadata_receive(file_descriptor); + Data *file_data = receive_data(file_descriptor); + if (config->use_compression) { + Data *file_data_uncompressed = data_decompress(file_data); + data_destroy(file_data); + file_data = file_data_uncompressed; + } + data_destroy(file->data); + file->data = file_data; + return file; +} + +size_t file_content_to_buffer(File *file) { + FILE *file_pointer = fopen(file->path, "rb"); + if (file_pointer == NULL) { + perror("Could not open the file!"); + return 0; + } + size_t bytes_read = + fread(file->data->data, 1, file->data->size, file_pointer); + if (bytes_read != (size_t)file->data->size) { + fclose(file_pointer); + perror("Read unexpected number of bytes from File!"); + return 0; + } + fclose(file_pointer); + return bytes_read; +} + + diff --git a/src/shared/file.h b/src/shared/file.h new file mode 100644 index 0000000..cf2bd52 --- /dev/null +++ b/src/shared/file.h @@ -0,0 +1,34 @@ +#ifndef FILE_H +#define FILE_H + +#include "config.h" +#include "data.h" +#include +#include + +typedef struct { + mode_t mode; + uid_t uid; + gid_t gid; + time_t mtime_sec; + long mtime_nsec; +} FileMetadata; + +typedef struct { + char *path; + Data *data; + FileMetadata *metadata; +} File; + +File *file_create(const char *path); +void file_destroy(void *item); +void file_load_data(File *file); +File *file_receive(Config *config, int file_descriptor); +void file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level); +void file_send_sendfile(File *file, int file_descriptor, bool use_metadata); +size_t file_content_to_buffer(File *file); +FileMetadata *file_metadata_create(struct stat *stats); +void file_metadata_destroy(void *metadata); +void to_disk(const char *path, const void *data, unsigned long long data_size); + +#endif diff --git a/src/shared/log.c b/src/shared/log.c new file mode 100644 index 0000000..953a117 --- /dev/null +++ b/src/shared/log.c @@ -0,0 +1,28 @@ +#include "log.h" +#include +#include +#include + +static const char *log_level_strings[] = {"DEBUG", "INFO", "WARN", "ERROR"}; +static LogLevel current_log_level = LOG_LEVEL_WARNING; + +void set_log_level(LogLevel level) { + current_log_level = level; +} + +void log_message(LogLevel log_level, char *format, ...) { + if (log_level < current_log_level) + return; + time_t now = time(NULL); + struct tm *t = localtime(&now); + + fprintf(stderr, "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t->tm_year + 1900, + t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min, t->tm_sec, + log_level_strings[log_level]); + + va_list args; + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + fprintf(stderr, "\n"); +} diff --git a/src/shared/log.h b/src/shared/log.h new file mode 100644 index 0000000..1510b6b --- /dev/null +++ b/src/shared/log.h @@ -0,0 +1,14 @@ +#ifndef LOG_H +#define LOG_H + +typedef enum { + LOG_LEVEL_DEBUG, + LOG_LEVEL_INFO, + LOG_LEVEL_WARNING, + LOG_LEVEL_ERROR +} LogLevel; + +void log_message(LogLevel log_level, char *message, ...); +void set_log_level(LogLevel level); + +#endif diff --git a/src/shared/metadata.c b/src/shared/metadata.c new file mode 100644 index 0000000..3fe822d --- /dev/null +++ b/src/shared/metadata.c @@ -0,0 +1,80 @@ +#include "metadata.h" +#include "file.h" +#include "protocol.h" +#include +#include +#include +#include +#include +#include + +void metadata_to_buf(char **buf, FileMetadata *m) { + int present = (m != NULL) ? 1 : 0; + memcpy(*buf, &present, sizeof(int)); + *buf += sizeof(int); + if (m == NULL) + return; + memcpy(*buf, &m->mode, sizeof(mode_t)); *buf += sizeof(mode_t); + memcpy(*buf, &m->uid, sizeof(uid_t)); *buf += sizeof(uid_t); + memcpy(*buf, &m->gid, sizeof(gid_t)); *buf += sizeof(gid_t); + memcpy(*buf, &m->mtime_sec, sizeof(time_t)); *buf += sizeof(time_t); + memcpy(*buf, &m->mtime_nsec, sizeof(long)); *buf += sizeof(long); +} + +FileMetadata *metadata_from_buf(char **buf) { + int present; + memcpy(&present, *buf, sizeof(int)); + *buf += sizeof(int); + if (!present) + return NULL; + FileMetadata *m = malloc(sizeof(FileMetadata)); + memcpy(&m->mode, *buf, sizeof(mode_t)); *buf += sizeof(mode_t); + memcpy(&m->uid, *buf, sizeof(uid_t)); *buf += sizeof(uid_t); + memcpy(&m->gid, *buf, sizeof(gid_t)); *buf += sizeof(gid_t); + memcpy(&m->mtime_sec, *buf, sizeof(time_t)); *buf += sizeof(time_t); + memcpy(&m->mtime_nsec, *buf, sizeof(long)); *buf += sizeof(long); + return m; +} + +void metadata_send(int file_descriptor, FileMetadata *m) { + if (m == NULL) { + int zero = 0; + send_n_data(file_descriptor, &zero, sizeof(int)); + return; + } + int present = 1; + send_n_data(file_descriptor, &present, sizeof(int)); + send_n_data(file_descriptor, &m->mode, sizeof(mode_t)); + send_n_data(file_descriptor, &m->uid, sizeof(uid_t)); + send_n_data(file_descriptor, &m->gid, sizeof(gid_t)); + send_n_data(file_descriptor, &m->mtime_sec, sizeof(time_t)); + send_n_data(file_descriptor, &m->mtime_nsec, sizeof(long)); +} + +FileMetadata *metadata_receive(int file_descriptor) { + int present; + receive_n_data(file_descriptor, &present, sizeof(int)); + if (!present) + return NULL; + FileMetadata *m = malloc(sizeof(FileMetadata)); + receive_n_data(file_descriptor, &m->mode, sizeof(mode_t)); + receive_n_data(file_descriptor, &m->uid, sizeof(uid_t)); + receive_n_data(file_descriptor, &m->gid, sizeof(gid_t)); + receive_n_data(file_descriptor, &m->mtime_sec, sizeof(time_t)); + receive_n_data(file_descriptor, &m->mtime_nsec, sizeof(long)); + return m; +} + +void file_restore_metadata(const char *path, FileMetadata *metadata) { + if (metadata == NULL) + return; + chmod(path, metadata->mode & 07777); + int chown_ret = chown(path, metadata->uid, metadata->gid); + (void)chown_ret; + struct timespec times[2]; + times[0].tv_sec = 0; + times[0].tv_nsec = UTIME_OMIT; + times[1].tv_sec = metadata->mtime_sec; + times[1].tv_nsec = metadata->mtime_nsec; + utimensat(AT_FDCWD, path, times, 0); +} diff --git a/src/shared/metadata.h b/src/shared/metadata.h new file mode 100644 index 0000000..0edcdcf --- /dev/null +++ b/src/shared/metadata.h @@ -0,0 +1,15 @@ +#ifndef METADATA_H +#define METADATA_H + +#include "file.h" +#include + +#define FILE_METADATA_WIRE_SIZE (sizeof(mode_t) + sizeof(uid_t) + sizeof(gid_t) + sizeof(time_t) + sizeof(long)) + +void metadata_to_buf(char **buf, FileMetadata *m); +FileMetadata *metadata_from_buf(char **buf); +void metadata_send(int file_descriptor, FileMetadata *m); +FileMetadata *metadata_receive(int file_descriptor); +void file_restore_metadata(const char *path, FileMetadata *metadata); + +#endif diff --git a/src/shared/multiprocessing.c b/src/shared/multiprocessing.c new file mode 100644 index 0000000..912be13 --- /dev/null +++ b/src/shared/multiprocessing.c @@ -0,0 +1,172 @@ +#include "multiprocessing.h" +#include "array_list.h" +#include "chunk.h" +#include "compression.h" +#include "config.h" +#include "data.h" +#include "file.h" +#include "log.h" +#include "metadata.h" +#include "protocol.h" +#include "queue.h" +#include "utils.h" +#include +#include +#include +#include + +PipelineContextSender *pipeline_context_sender_create(Config *config, + Queue *queue_scanner, + Queue *queue_loader) { + PipelineContextSender *context = malloc(sizeof(PipelineContextSender)); + context->config = config; + context->queue_scanner = queue_scanner; + context->queue_loader = queue_loader; + context->scanner_done = false; + context->loader_done = false; + if (mtx_init(&context->mutex_scanner, mtx_plain) != thrd_success || + cnd_init(&context->condition_not_full_scanner) != thrd_success || + cnd_init(&context->condition_not_empty_scanner) != thrd_success || + mtx_init(&context->mutex_loader, mtx_plain) != thrd_success || + cnd_init(&context->condition_not_full_loader) != thrd_success || + cnd_init(&context->condition_not_empty_loader) != thrd_success) { + perror("Error initializing synchronization objects!"); + exit(EXIT_FAILURE); + } + return context; +} + +void pipeline_context_sender_destroy(PipelineContextSender *context) { + if (context->manifest) { + array_list_delete(context->manifest); + } + config_delete(context->config); + queue_destroy(context->queue_scanner); + queue_destroy(context->queue_loader); + mtx_destroy(&context->mutex_scanner); + cnd_destroy(&context->condition_not_full_scanner); + cnd_destroy(&context->condition_not_empty_scanner); + mtx_destroy(&context->mutex_loader); + cnd_destroy(&context->condition_not_full_loader); + cnd_destroy(&context->condition_not_empty_loader); + free(context); +} + +PipelineContextReceiver *pipeline_context_receiver_create(Config *config, + Queue *queue, + int file_descriptor) { + PipelineContextReceiver *context = malloc(sizeof(PipelineContextReceiver)); + context->config = config; + context->queue = queue; + context->file_descriptor = file_descriptor; + context->receiver_done = false; + if (mtx_init(&context->mutex, mtx_plain) != thrd_success || + cnd_init(&context->condition_not_full) != thrd_success || + cnd_init(&context->condition_not_empty) != thrd_success) { + perror("Error initializing synchronization objects!"); + exit(EXIT_FAILURE); + } + return context; +} + +void pipeline_context_receiver_destroy(PipelineContextReceiver *context) { + config_delete(context->config); + queue_destroy(context->queue); + mtx_destroy(&context->mutex); + cnd_destroy(&context->condition_not_full); + cnd_destroy(&context->condition_not_empty); + free(context); +} + +static void receive_chunk_enqueue(int file_descriptor, + PipelineContextReceiver *context) { + Data *chunk_data = receive_data(file_descriptor); + Data *data_to_process = chunk_data; + if (context->config->use_compression) { + data_to_process = data_decompress(chunk_data); + data_destroy(chunk_data); + if (data_to_process == NULL) { + log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk, skipping"); + return; + } + } + Chunk *chunk = chunk_deserialize(data_to_process, context->config->use_metadata); + data_destroy(data_to_process); + if (chunk == NULL) { + log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping"); + return; + } + + for (int i = 0; i < chunk->element_count; i++) { + File *file = chunk->items[i]; + chunk->items[i] = NULL; + queue_enqueue_multithreaded(context->queue, file, &context->mutex, + &context->condition_not_empty, + &context->condition_not_full); + } + chunk_destroy(chunk); +} + +int receive_thread(void *pipeline_context) { + PipelineContextReceiver *context = + (PipelineContextReceiver *)pipeline_context; + mtx_lock(&context->mutex); + int file_descriptor = context->file_descriptor; + Config *config = context->config; + mtx_unlock(&context->mutex); + + Status status = receive_status(file_descriptor); + while (status == STATUS_NEXT || status == STATUS_CHUNK) { + if (status == STATUS_CHUNK) { + receive_chunk_enqueue(file_descriptor, context); + } else { + File *file = file_receive(config, file_descriptor); + queue_enqueue_multithreaded(context->queue, file, &context->mutex, + &context->condition_not_empty, + &context->condition_not_full); + } + status = receive_status(file_descriptor); + } + if (status == STATUS_MANIFEST) { + int count = receive_int(file_descriptor); + ArrayList *manifest = array_list_create(free); + for (int i = 0; i < count; i++) + array_list_add(manifest, receive_str(file_descriptor)); + delete_extras(context->config->receive_root_directory, manifest); + for (int i = 0; i < manifest->size; i++) + free(manifest->items[i]); + array_list_delete(manifest); + status = receive_status(file_descriptor); + } + mtx_lock(&context->mutex); + context->receiver_done = true; + cnd_signal(&context->condition_not_empty); + mtx_unlock(&context->mutex); + return thrd_success; +} + +int write_thread(void *pipeline_context) { + PipelineContextReceiver *context = + (PipelineContextReceiver *)pipeline_context; + mtx_lock(&context->mutex); + bool save_to_disk = context->config->save_to_disk; + char *root_directory = str_dup(context->config->receive_root_directory); + mtx_unlock(&context->mutex); + + while (true) { + File *file = queue_dequeue_multithreaded( + context->queue, &context->mutex, &context->condition_not_empty, + &context->condition_not_full, &context->receiver_done); + if (file == NULL) { + free(root_directory); + return thrd_success; + } + if (save_to_disk) { + char *disk_path = path_cat(root_directory, file->path); + to_disk(disk_path, file->data->data, file->data->size); + file_restore_metadata(disk_path, file->metadata); + free(disk_path); + } + file_destroy(file); + } +} diff --git a/src/shared/multiprocessing.h b/src/shared/multiprocessing.h new file mode 100644 index 0000000..605c42a --- /dev/null +++ b/src/shared/multiprocessing.h @@ -0,0 +1,46 @@ +#ifndef MULTIPROCESSING_H +#define MULTIPROCESSING_H + +#include + +#include "array_list.h" +#include "config.h" +#include "file.h" +#include "queue.h" + +typedef struct { + Config *config; + Queue *queue_scanner; + mtx_t mutex_scanner; + cnd_t condition_not_full_scanner; + cnd_t condition_not_empty_scanner; + bool scanner_done; + Queue *queue_loader; + mtx_t mutex_loader; + cnd_t condition_not_full_loader; + cnd_t condition_not_empty_loader; + bool loader_done; + ArrayList *manifest; +} PipelineContextSender; + +typedef struct PipelineContextReceiver { + Queue *queue; + Config *config; + int file_descriptor; + mtx_t mutex; + cnd_t condition_not_full; + cnd_t condition_not_empty; + bool receiver_done; +} PipelineContextReceiver; + +PipelineContextSender *pipeline_context_sender_create(Config *config, + Queue *queue_scanner, + Queue *queue_loader); +void pipeline_context_sender_destroy(PipelineContextSender *context); +PipelineContextReceiver *pipeline_context_receiver_create(Config *config, + Queue *queue_receiver, + int file_descriptor); +void pipeline_context_receiver_destroy(PipelineContextReceiver *context); +int receive_thread(void *pipeline_context); +int write_thread(void *pipeline_context); +#endif diff --git a/src/shared/protocol.c b/src/shared/protocol.c new file mode 100644 index 0000000..5802912 --- /dev/null +++ b/src/shared/protocol.c @@ -0,0 +1,124 @@ +#include "protocol.h" +#include "log.h" +#include +#include +#include +#include + +static __thread int io_read_fd = -1; +static __thread int io_write_fd = -1; + +void io_set_fds(int read_fd, int write_fd) { + io_read_fd = read_fd; + io_write_fd = write_fd; +} + +static int io_fd(int dir_fd, int file_descriptor) { + return (dir_fd != -1) ? dir_fd : file_descriptor; +} + +void send_n_data(int file_descriptor, void *data, size_t data_size) { + log_message(LOG_LEVEL_DEBUG, " Sending n Data: %zu", data_size); + int fd = io_fd(io_write_fd, file_descriptor); + ssize_t total_bytes_send = 0; + while (total_bytes_send < data_size) { + ssize_t bytes_send = + write(fd, (char *)data + total_bytes_send, data_size - total_bytes_send); + if (bytes_send <= 0) { + perror("Could not send data!"); + exit(EXIT_FAILURE); + } + total_bytes_send += bytes_send; + } + log_message(LOG_LEVEL_DEBUG, " Send n Data: %zu", total_bytes_send); +} + +void receive_n_data(int file_descriptor, void *data, size_t data_size) { + log_message(LOG_LEVEL_DEBUG, " Receiving n Data: %zu", data_size); + int fd = io_fd(io_read_fd, file_descriptor); + size_t total_bytes_received = 0; + while (total_bytes_received < data_size) { + ssize_t bytes_received = + read(fd, (char *)data + total_bytes_received, data_size - total_bytes_received); + if (bytes_received == -1 || bytes_received == 0) { + perror("Could not receive bytes!"); + exit(EXIT_FAILURE); + } + total_bytes_received += bytes_received; + } + log_message(LOG_LEVEL_DEBUG, " Received n Data: %zu", total_bytes_received); +} + +static const char *status_to_string(Status status) { + switch (status) { + case STATUS_OK: + return "OK"; + case STATUS_ERROR: + return "ERROR"; + case STATUS_FINISHED: + return "FINISHED"; + case STATUS_NEXT: + return "NEXT"; + case STATUS_CHUNK: + return "CHUNK"; + default: + return "UNKNOWN"; + } +} + +void send_str(int file_descriptor, char *data) { + size_t size = strlen(data); + send_n_data(file_descriptor, &size, sizeof(size_t)); + send_n_data(file_descriptor, data, size); + log_message(LOG_LEVEL_DEBUG, "Send String: %s", data); +} + +char *receive_str(int file_descriptor) { + size_t size; + receive_n_data(file_descriptor, &size, sizeof(size_t)); + char *data = (char *)malloc(size + 1); + receive_n_data(file_descriptor, data, size); + data[size] = '\0'; + log_message(LOG_LEVEL_DEBUG, "Received String: %s", data); + return data; +} + +void send_data(int file_descriptor, Data *data) { + unsigned long long data_size = data->size; + send_n_data(file_descriptor, &data_size, sizeof(unsigned long long)); + send_n_data(file_descriptor, data->data, data_size); + log_message(LOG_LEVEL_DEBUG, "Send %lld data", data_size); +} + +Data *receive_data(int file_descriptor) { + unsigned long long size = 0; + receive_n_data(file_descriptor, &size, sizeof(unsigned long long)); + void *data = malloc((size_t)size); + receive_n_data(file_descriptor, data, (size_t)size); + log_message(LOG_LEVEL_DEBUG, "Received %lld data", size); + return data_create(data, (size_t)size); +} + +void send_int(int file_descriptor, int data) { + send_n_data(file_descriptor, &data, sizeof(int)); + log_message(LOG_LEVEL_DEBUG, "Send Int: %d", data); +} + +int receive_int(int file_descriptor) { + int data; + receive_n_data(file_descriptor, &data, sizeof(int)); + log_message(LOG_LEVEL_DEBUG, "Received Int: %d", data); + return data; +} + +void send_status(int file_descriptor, Status status) { + send_n_data(file_descriptor, &status, sizeof(Status)); + log_message(LOG_LEVEL_DEBUG, "Send Status: %s", status_to_string(status)); +} + +Status receive_status(int file_descriptor) { + Status data; + receive_n_data(file_descriptor, &data, sizeof(Status)); + log_message(LOG_LEVEL_DEBUG, "Received Status: %s", status_to_string(data)); + return data; +} diff --git a/src/shared/protocol.h b/src/shared/protocol.h new file mode 100644 index 0000000..d981097 --- /dev/null +++ b/src/shared/protocol.h @@ -0,0 +1,23 @@ +#ifndef PROTOCOL_H +#define PROTOCOL_H + +#include "data.h" +#include + +typedef int Status; +enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST }; + +void io_set_fds(int read_fd, int write_fd); +void send_n_data(int file_descriptor, void *data, size_t data_size); +void receive_n_data(int file_descriptor, void *data, size_t data_size); + +void send_str(int file_descriptor, char *data); +char *receive_str(int file_descriptor); +void send_data(int file_descriptor, Data *data); +Data *receive_data(int file_descriptor); +void send_int(int file_descriptor, int data); +int receive_int(int file_descriptor); +void send_status(int file_descriptor, Status status); +Status receive_status(int file_descriptor); + +#endif diff --git a/src/shared/queue.c b/src/shared/queue.c new file mode 100644 index 0000000..6267e51 --- /dev/null +++ b/src/shared/queue.c @@ -0,0 +1,134 @@ +#include +#include +#include +#include +#include + +#include "queue.h" + +Queue *queue_create(int capacity, void (*destroyer)(void *item)) { + Queue *queue = (Queue *)malloc(sizeof(Queue)); + if (queue == NULL) { + perror("FATAL ERROR: Could not allocate memory for queue structure"); + exit(EXIT_FAILURE); + } + + queue->items = malloc(capacity * sizeof(void *)); + if (queue->items == NULL) { + perror("FATAL ERROR: Could not allocate memory for queue items"); + free(queue); + exit(EXIT_FAILURE); + } + + for (int i = 0; i < capacity; ++i) { + queue->items[i] = NULL; + } + + queue->capacity = capacity; + queue->front = 0; + queue->rear = 0; + queue->size = 0; + queue->item_destroyer = destroyer; + + return queue; +} + +void queue_destroy(Queue *queue) { + if (queue == NULL) + return; + + if (queue->item_destroyer != NULL) { + for (int i = 0; i < queue->size; ++i) { + int index = (queue->front + i) % queue->capacity; + queue->item_destroyer(queue->items[index]); + } + } + free(queue->items); + free(queue); +} + +bool queue_is_empty(Queue *queue) { + if (queue == NULL) + return true; + return queue->size == 0; +} + +bool queue_is_full(Queue *queue) { + if (queue == NULL) + return false; + return queue->size == queue->capacity; +} + +static void queue_double_capacity(Queue *queue) { + if (queue == NULL) + return; + unsigned int new_capacity = queue->capacity * 2; + if (new_capacity <= 1) + new_capacity = 100; + void **new_items = malloc(new_capacity * sizeof(void *)); + if (new_items == NULL) { + perror("FATAL ERROR: Could not allocate memory for doubling capacity of " + "queue."); + exit(EXIT_FAILURE); + } + for (int i = 0; i < queue->size; i++) + new_items[i] = queue->items[(i + queue->front) % queue->capacity]; + free(queue->items); + queue->items = new_items; + queue->front = 0; + queue->rear = queue->size; + queue->capacity = new_capacity; +} + +void queue_enqueue(Queue *queue, void *item) { + if (queue == NULL || item == NULL) { + perror("ERROR: Cannot enqueue with a null queue or item.\n"); + exit(EXIT_FAILURE); + } + if (queue_is_full(queue)) + queue_double_capacity(queue); + queue->items[queue->rear] = item; + queue->rear = (queue->rear + 1) % queue->capacity; + queue->size++; +} + +void queue_enqueue_multithreaded(Queue *queue, void *item, mtx_t *mutex, + cnd_t *condition_not_empty, + cnd_t *condition_not_full) { + mtx_lock(mutex); + while (queue_is_full(queue)) + cnd_wait(condition_not_full, mutex); + queue_enqueue(queue, item); + cnd_signal(condition_not_empty); + mtx_unlock(mutex); +} + +void *queue_dequeue(Queue *queue) { + if (queue == NULL || queue_is_empty(queue)) { + perror("ERROR: Could not dequeue from null or empty queue."); + return NULL; + } + + void *item = queue->items[queue->front]; + queue->items[queue->front] = NULL; + queue->front = (queue->front + 1) % queue->capacity; + queue->size--; + return item; +} + +void *queue_dequeue_multithreaded(Queue *queue, mtx_t *mutex, + cnd_t *condition_not_empty, + cnd_t *condition_not_full, + bool *other_thread_done) { + mtx_lock(mutex); + while (queue_is_empty(queue) && !*other_thread_done) + cnd_wait(condition_not_empty, mutex); + if (queue_is_empty(queue) && *other_thread_done) { + mtx_unlock(mutex); + return NULL; + } + void *item = queue_dequeue(queue); + cnd_signal(condition_not_full); + mtx_unlock(mutex); + return item; +} diff --git a/src/shared/queue.h b/src/shared/queue.h new file mode 100644 index 0000000..9666b5a --- /dev/null +++ b/src/shared/queue.h @@ -0,0 +1,30 @@ +#ifndef QUEUE_H +#define QUEUE_H + +#include +#include + +typedef struct Queue { + void **items; + int front; + int rear; + int size; + int capacity; + void (*item_destroyer)(void *item); +} Queue; + +Queue *queue_create(int capacity, void (*destroyer)(void *item)); +void queue_destroy(Queue *queue); +bool queue_is_empty(Queue *queue); +bool queue_is_full(Queue *queue); +void queue_enqueue(Queue *queue, void *item); +void queue_enqueue_multithreaded(Queue *queue, void *item, mtx_t *mutex, + cnd_t *condition_not_empty, + cnd_t *condition_not_full); +void *queue_dequeue(Queue *queue); +void *queue_dequeue_multithreaded(Queue *queue, mtx_t *mutex, + cnd_t *condition_not_empty, + cnd_t *condition_not_full, + bool *other_thread_done); + +#endif diff --git a/src/shared/transport_ssh.c b/src/shared/transport_ssh.c new file mode 100644 index 0000000..4fc54cb --- /dev/null +++ b/src/shared/transport_ssh.c @@ -0,0 +1,139 @@ +#include "transport_ssh.h" +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + char user[256]; + char host[256]; + char remote_path[4096]; +} RemoteDest; + +static int parse_remote_dest(const char *dest, RemoteDest *r) { + const char *colon = strchr(dest, ':'); + if (!colon) return -1; + + size_t remote_path_len = strlen(colon + 1); + if (remote_path_len >= sizeof(r->remote_path)) return -1; + memcpy(r->remote_path, colon + 1, remote_path_len + 1); + + const char *at = memchr(dest, '@', colon - dest); + if (at) { + size_t user_len = at - dest; + if (user_len >= sizeof(r->user)) return -1; + memcpy(r->user, dest, user_len); + r->user[user_len] = '\0'; + + size_t host_len = colon - at - 1; + if (host_len >= sizeof(r->host)) return -1; + memcpy(r->host, at + 1, host_len); + r->host[host_len] = '\0'; + } else { + r->user[0] = '\0'; + size_t host_len = colon - dest; + if (host_len >= sizeof(r->host)) return -1; + memcpy(r->host, dest, host_len); + r->host[host_len] = '\0'; + } + return 0; +} + +Client *client_connect_ssh(char *destination, int port) { + RemoteDest r; + if (parse_remote_dest(destination, &r) != 0) { + fprintf(stderr, "Invalid remote destination: %s\n", destination); + exit(EXIT_FAILURE); + } + + int sv[2]; + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) < 0) { + perror("socketpair failed"); + exit(EXIT_FAILURE); + } + + int buf_size = 1024 * 1024; + setsockopt(sv[0], SOL_SOCKET, SO_SNDBUF, &buf_size, sizeof(buf_size)); + setsockopt(sv[0], SOL_SOCKET, SO_RCVBUF, &buf_size, sizeof(buf_size)); + setsockopt(sv[1], SOL_SOCKET, SO_SNDBUF, &buf_size, sizeof(buf_size)); + setsockopt(sv[1], SOL_SOCKET, SO_RCVBUF, &buf_size, sizeof(buf_size)); + + int exec_pipe[2]; + if (pipe(exec_pipe) < 0) { + perror("pipe failed"); + exit(EXIT_FAILURE); + } + + pid_t pid = fork(); + if (pid < 0) { + perror("fork failed"); + exit(EXIT_FAILURE); + } + + if (pid == 0) { + close(sv[0]); + close(exec_pipe[0]); + fcntl(exec_pipe[1], F_SETFD, FD_CLOEXEC); + + if (sv[1] != STDIN_FILENO) + dup2(sv[1], STDIN_FILENO); + if (sv[1] != STDOUT_FILENO) + dup2(sv[1], STDOUT_FILENO); + if (sv[1] > 1) close(sv[1]); + + char ssh_user[512]; + if (r.user[0] != '\0') + snprintf(ssh_user, sizeof(ssh_user), "%s@%s", r.user, r.host); + else + snprintf(ssh_user, sizeof(ssh_user), "%s", r.host); + + char *ssh_argv[16]; + int ac = 0; + char port_str[16]; + ssh_argv[ac++] = "ssh"; + ssh_argv[ac++] = "-o"; + ssh_argv[ac++] = "Compression=no"; + ssh_argv[ac++] = "-o"; + ssh_argv[ac++] = "ControlMaster=auto"; + ssh_argv[ac++] = "-o"; + ssh_argv[ac++] = "ControlPath=~/.cache/fastsync-%r@%h:%p"; + if (port > 0 && port != 22) { + ssh_argv[ac++] = "-p"; + snprintf(port_str, sizeof(port_str), "%d", port); + ssh_argv[ac++] = port_str; + } + ssh_argv[ac++] = ssh_user; + ssh_argv[ac++] = "fastsync-server"; + ssh_argv[ac++] = "--stdio"; + ssh_argv[ac] = NULL; + execvp("ssh", ssh_argv); + perror("exec of ssh failed"); + ssize_t wret = write(exec_pipe[1], "x", 1); + (void)wret; + _exit(1); + } + + close(sv[1]); + close(exec_pipe[1]); + + char exec_status; + ssize_t n = read(exec_pipe[0], &exec_status, 1); + close(exec_pipe[0]); + + if (n > 0) { + close(sv[0]); + waitpid(pid, NULL, 0); + fprintf(stderr, "Error: could not launch 'fastsync-server --stdio' on remote\n"); + exit(EXIT_FAILURE); + } + + Client *client = malloc(sizeof(Client)); + client->file_descriptor = sv[0]; + client->address.sin_family = AF_UNIX; + client->address_length = 0; + client->ssh_child_pid = pid; + return client; +} diff --git a/src/shared/transport_ssh.h b/src/shared/transport_ssh.h new file mode 100644 index 0000000..f542870 --- /dev/null +++ b/src/shared/transport_ssh.h @@ -0,0 +1,8 @@ +#ifndef TRANSPORT_SSH_H +#define TRANSPORT_SSH_H + +#include "transport_tcp.h" + +Client *client_connect_ssh(char *destination, int port); + +#endif diff --git a/src/shared/transport_tcp.c b/src/shared/transport_tcp.c new file mode 100644 index 0000000..9e8dd92 --- /dev/null +++ b/src/shared/transport_tcp.c @@ -0,0 +1,132 @@ +#include "transport_tcp.h" +#include "log.h" +#include +#include +#include +#include +#include +#include +#include +#include + +Server *server_create(int port) { + Server *server = (Server *)malloc(sizeof(Server)); + if (server == NULL) { + perror("Could not allocate space for Server"); + exit(EXIT_FAILURE); + } + + int file_descriptor = socket(AF_INET, SOCK_STREAM, 0); + if (file_descriptor < 0) { + perror("Could not create Socket!"); + exit(EXIT_FAILURE); + } + server->file_descriptor = file_descriptor; + int opt = 1; + if (setsockopt(server->file_descriptor, SOL_SOCKET, SO_REUSEADDR, &opt, + sizeof(opt))) { + perror("Error setting a socket option!"); + close(server->file_descriptor); + free(server); + exit(EXIT_FAILURE); + } + + server->address.sin_family = AF_INET; + server->address.sin_addr.s_addr = INADDR_ANY; + server->address.sin_port = htons(port); + server->address_length = sizeof(server->address); + + if (bind(server->file_descriptor, (struct sockaddr *)&server->address, + server->address_length) < 0) { + perror("Could not bind server"); + close(server->file_descriptor); + free(server); + exit(EXIT_FAILURE); + } + + return server; +} + +void server_delete(Server **server) { + if (server == NULL || *server == NULL) return; + close((*server)->file_descriptor); + free(*server); + *server = NULL; +} + +void server_listen(Server *server, void (*handler)(int file_descriptor)) { + log_message(LOG_LEVEL_INFO, "Start Listening on Port: %d", + server->address.sin_port); + if (listen(server->file_descriptor, SOMAXCONN) < 0) { + perror("Could not listen on port!"); + exit(EXIT_FAILURE); + } + + signal(SIGCHLD, SIG_IGN); + + while (1) { + struct sockaddr_in client_addr; + socklen_t client_len = sizeof(client_addr); + int file_descriptor = + accept(server->file_descriptor, (struct sockaddr *)&client_addr, + &client_len); + if (file_descriptor < 0) { + perror("Could not accept the connection"); + continue; + } + log_message(LOG_LEVEL_INFO, "Received Connection"); + pid_t pid = fork(); + if (pid == 0) { + close(server->file_descriptor); + handler(file_descriptor); + close(file_descriptor); + _exit(0); + } + close(file_descriptor); + } +} + +Client *client_create() { + int file_descriptor = socket(AF_INET, SOCK_STREAM, 0); + if (file_descriptor < 0) { + perror("Could not create Socket!"); + exit(EXIT_FAILURE); + } + + Client *client = (Client *)malloc(sizeof(Client)); + client->file_descriptor = file_descriptor; + client->address.sin_family = AF_INET; + client->address_length = sizeof(client->address); + client->ssh_child_pid = -1; + return client; +} + +void client_connect(Client *client, char *host, int port) { + client->address.sin_port = htons(port); + + if (inet_pton(AF_INET, host, &client->address.sin_addr) <= 0) { + perror("Could not convert host address!"); + exit(EXIT_FAILURE); + } + + if (connect(client->file_descriptor, (struct sockaddr *)&client->address, + client->address_length) < 0) { + perror("Could not connect to Server!"); + exit(EXIT_FAILURE); + } +} + +void client_disconnect(Client *client) { + close(client->file_descriptor); + if (client->ssh_child_pid > 0) { + int status; + waitpid(client->ssh_child_pid, &status, 0); + client->ssh_child_pid = -1; + } +} + +void client_delete(Client *client) { + if (client == NULL) + return; + free(client); +} diff --git a/src/shared/transport_tcp.h b/src/shared/transport_tcp.h new file mode 100644 index 0000000..7c394ac --- /dev/null +++ b/src/shared/transport_tcp.h @@ -0,0 +1,28 @@ +#ifndef TRANSPORT_TCP_H +#define TRANSPORT_TCP_H + +#include +#include + +typedef struct Server { + struct sockaddr_in address; + unsigned int address_length; + int file_descriptor; +} Server; + +typedef struct Client { + struct sockaddr_in address; + unsigned int address_length; + int file_descriptor; + pid_t ssh_child_pid; +} Client; + +Server *server_create(int port); +void server_listen(Server *server, void (*handler)(int file_descriptor)); +void server_delete(Server **server); +Client *client_create(); +void client_connect(Client *client, char *host, int port); +void client_disconnect(Client *client); +void client_delete(Client *client); + +#endif diff --git a/src/shared/utils.c b/src/shared/utils.c new file mode 100644 index 0000000..b7c2c7a --- /dev/null +++ b/src/shared/utils.c @@ -0,0 +1,139 @@ +#include "utils.h" +#include "array_list.h" +#include "libgen.h" +#include +#include +#include +#include +#include +#include + +void mkdir_r(char *path) { + char *path_duplicate = malloc(strlen(path) + 1); + strcpy(path_duplicate, path); + char *path_current = (char *)malloc((strlen(path) + 2) * sizeof(char)); + char *path_current_position = path_current; + if (path[0] == '/') { + strcpy(path_current, "/"); + path_current_position += 1; + } else { + path_current[0] = '\0'; + } + const char *delimiter = "/"; + char *part = strtok(path_duplicate, delimiter); + while (part != NULL) { + strcpy(path_current_position, part); + path_current_position += strlen(part) * sizeof(char); + strcpy(path_current_position, "/"); + path_current_position += sizeof(char); + struct stat st; + if (stat(path_current, &st) != 0) { + if (mkdir(path_current, 0755) != 0) { + perror("Could not create directory"); + exit(EXIT_FAILURE); + } + } + part = strtok(NULL, delimiter); + } + free(path_duplicate); + free(path_current); +} + +char *str_dup(const char *string) { + if (string == NULL) + return NULL; + char *new_string = (char *)malloc(strlen(string) + 1); + strcpy(new_string, string); + return new_string; +} + +bool glob_match(const char *pattern, const char *str) { + while (*pattern) { + if (*pattern == '*') { + pattern++; + while (*str && *str != '/') { + if (glob_match(pattern, str)) + return true; + str++; + } + return glob_match(pattern, str); + } else if (*pattern == '?') { + if (!*str || *str == '/') + return false; + pattern++; + str++; + } else { + if (*pattern != *str) + return false; + pattern++; + str++; + } + } + return *str == '\0'; +} + +static void delete_extras_walk(const char *abs_path, const char *rel_path, + ArrayList *manifest) { + DIR *dir = opendir(abs_path); + if (!dir) + return; + struct dirent *entry; + while ((entry = readdir(dir)) != NULL) { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) + continue; + char *child_abs = path_cat((char *)abs_path, entry->d_name); + char *child_rel = path_cat((char *)rel_path, entry->d_name); + struct stat st; + if (stat(child_abs, &st) != 0) { + free(child_abs); + free(child_rel); + continue; + } + if (S_ISDIR(st.st_mode)) { + delete_extras_walk(child_abs, child_rel, manifest); + } else { + // Check if relative path is in manifest + bool found = false; + for (int i = 0; i < manifest->size; i++) { + if (strcmp((char *)manifest->items[i], child_rel) == 0) { + found = true; + break; + } + } + if (!found) { + unlink(child_abs); + fprintf(stderr, " Deleted: %s\n", child_rel); + } + } + free(child_abs); + free(child_rel); + } + closedir(dir); + rmdir(abs_path); +} + +void delete_extras(const char *dest_root, ArrayList *manifest) { + delete_extras_walk(dest_root, "", manifest); +} + +char *path_cat(char *path1, char *path2) { + if (path1 == NULL || *path1 == '\0') + return str_dup(path2); + if (path2 == NULL || *path2 == '\0') + return str_dup(path1); + int path1_len = strlen(path1); + int path2_len = strlen(path2); + char *path2_pointer = path2; + if (path1[path1_len - 1] == '/') + path1_len -= 1; + if (path2[0] == '/') { + path2_pointer += 1; + path2_len -= 1; + } + char *new_path = malloc(path1_len + path2_len + 2); + memcpy(new_path, path1, path1_len); + new_path[path1_len] = '/'; + memcpy(new_path + path1_len + 1, path2_pointer, path2_len); + new_path[path1_len + path2_len + 1] = '\0'; + return new_path; +} diff --git a/src/shared/utils.h b/src/shared/utils.h new file mode 100644 index 0000000..67030ad --- /dev/null +++ b/src/shared/utils.h @@ -0,0 +1,13 @@ +#ifndef UTILS_H +#define UTILS_H + +#include "array_list.h" +#include + +void mkdir_r(char *path); +char *str_dup(const char *string); +char *path_cat(char *path1, char *path2); +bool glob_match(const char *pattern, const char *str); +void delete_extras(const char *dest_root, ArrayList *manifest); + +#endif diff --git a/test.py b/test.py new file mode 100644 index 0000000..f15cd76 --- /dev/null +++ b/test.py @@ -0,0 +1,673 @@ +import argparse +import filecmp +import os +import random +import re +import shutil +import subprocess +import sys +import tempfile +import time +import socket + +TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_data") +DEFAULT_SOURCE_DIR = os.path.join(TEST_DIR, "source") +DEFAULT_DEST_DIR = os.path.join(TEST_DIR, "dest") + +SERVER_CMD = ["./build/server"] +BASE_CLIENT_CMD = ["./build/client"] + +DISK_DEVICE = "/dev/nvme0n1p5" +READ_BPS_MAX = "15M" +WRITE_BPS_MAX = "10M" +NETWORK_INTERFACE = "lo" + +NETWORK_PROFILES = { + "Unlimited": {}, + "LAN": { + "rate": "1000mbit", + "delay": "20ms", + "jitter": "1ms", + "loss": "0.1%", + }, + "WAN": { + "rate": "100mbit", + "delay": "50ms", + "jitter": "10ms", + "loss": "1%", + }, +} + +CLIENT_CMD_PREFIX = [ + "sudo", + "systemd-run", + "--scope", + "-p", + f"IOReadBandwidthMax={DISK_DEVICE} {READ_BPS_MAX}", + "-p", + f"IOWriteBandwidthMax={DISK_DEVICE} {WRITE_BPS_MAX}", +] + +BASE_CLIENT_FLAGS = ["--save-to-disk"] + +TEST_CASES = [ + {"name": "Standard", "flags": []}, + {"name": "Posix Args (no flags)", "flags": [], "posix": True}, + {"name": "Standard (no metadata)", "flags": [], "use_metadata": False}, + {"name": "Multithreading (-m)", "flags": ["-m"]}, + {"name": "Compression (-c)", "flags": ["-c"]}, + {"name": "Chunk Serialization (-s)", "flags": ["-s"]}, + {"name": "Compression + Chunk Serialization (-c -s)", "flags": ["-c", "-s"]}, + {"name": "Multithreading + Compression (-m -c)", "flags": ["-m", "-c"]}, + {"name": "Multithreading + Chunk Serialization (-m -s)", "flags": ["-m", "-s"]}, + { + "name": "Multithreading + Compression + Chunk Serialization (-m -c -s)", + "flags": ["-m", "-c", "-s"], + }, + {"name": "Sendfile (-f)", "flags": ["-f"]}, + {"name": "Sendfile + Multithreading (-f -m)", "flags": ["-f", "-m"]}, +] + +SSH_CASES = [ + {"name": "SSH (localhost)", "flags": []}, + {"name": "SSH Multithreading (-m)", "flags": ["-m"]}, + {"name": "SSH Compression (-c)", "flags": ["-c"]}, + {"name": "SSH Chunk Serialization (-s)", "flags": ["-s"]}, + {"name": "SSH Compression + Chunk Serialization (-c -s)", "flags": ["-c", "-s"]}, + {"name": "SSH Multithreading + Compression (-m -c)", "flags": ["-m", "-c"]}, + {"name": "SSH Multithreading + Chunk Serialization (-m -s)", "flags": ["-m", "-s"]}, + {"name": "SSH Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]}, +] + +RSYNC_CASES = [ + {"name": "rsync (archive)", "args": ["-aH"]}, + {"name": "rsync (archive + compress)", "args": ["-aHz"]}, +] + + +def netem_apply(profile): + params = NETWORK_PROFILES[profile] + if not params: + netem_reset() + return + netem_reset() + cmd = ["sudo", "tc", "qdisc", "add", "dev", NETWORK_INTERFACE, "root", "netem"] + cmd += ["rate", params["rate"]] + cmd += ["delay", params["delay"], params["jitter"]] + cmd += ["loss", params["loss"]] + subprocess.run(cmd, check=True, capture_output=True) + + +def netem_reset(): + subprocess.run( + f"sudo tc qdisc del dev {NETWORK_INTERFACE} root".split(), + capture_output=True, + ) + + +def wait_proc(proc, timeout=5): + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + + +def find_free_port(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('', 0)) + return s.getsockname()[1] + + +def generate_test_files(source_dir): + if os.path.exists(source_dir): + shutil.rmtree(source_dir) + os.makedirs(source_dir) + + target_total = 25 * 1024 * 1024 + written = 0 + + files = { + "small.txt": b"hello world\n", + "medium.txt": b"the quick brown fox jumps over the lazy dog\n" * 5000, + "binary.bin": bytes(range(256)) * 1000, + "nested/subdir/deep.txt": b"deeply nested file\n", + "nested/another.txt": b"another nested file\n" * 50, + } + for rel_path, content in files.items(): + full_path = os.path.join(source_dir, rel_path) + os.makedirs(os.path.dirname(full_path), exist_ok=True) + with open(full_path, "wb") as f: + f.write(content) + written += len(content) + + os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True) + i = 0 + while written < target_total: + chunk_size = min(5 * 1024 * 1024, target_total - written) + with open(os.path.join(source_dir, f"bulk/file_{i}.dat"), "wb") as f: + f.write(random.randbytes(chunk_size)) + written += chunk_size + i += 1 + + total_mb = written / (1024 * 1024) + small_bytes = sum(len(c) for c in files.values()) + print(f" Generated {total_mb:.1f} MB of test data in {source_dir} " + f"({(written - small_bytes)/(1024*1024):.1f} MB random, " + f"{small_bytes} B structured)") + return written + + +def verify_transfer(source_dir, received_dir): + source_dir = os.path.abspath(source_dir) + received_dir = os.path.abspath(received_dir) + if not os.path.exists(received_dir): + return [], ["no received files found"] + mismatches, missing = [], [] + for root, dirs, files in os.walk(source_dir): + for f in files: + src_path = os.path.join(root, f) + rel = os.path.relpath(src_path, source_dir) + dst_path = os.path.join(received_dir, rel) + if not os.path.exists(dst_path): + missing.append(rel) + elif not filecmp.cmp(src_path, dst_path, shallow=False): + mismatches.append(rel) + return mismatches, missing + + +def start_rsync_daemon(source_dir): + port = find_free_port() + conf = os.path.join(tempfile.gettempdir(), f"rsyncd-{port}.conf") + with open(conf, "w") as f: + f.write(f"port = {port}\nread only = yes\n\n[source]\n path = {source_dir}\n") + daemon = subprocess.Popen( + ["rsync", "--daemon", "--no-detach", f"--config={conf}"], + stdout=subprocess.DEVNULL, stderr=None, + ) + for _ in range(50): + time.sleep(0.1) + if daemon.poll() is not None: + raise RuntimeError(f"rsync daemon exited (rc={daemon.returncode})") + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.3): + break + except (ConnectionRefusedError, OSError): + continue + else: + raise RuntimeError("rsync daemon did not start") + return port, conf, daemon + + +def run_single_test(cmd, name, source_dir, dest_dir, *, source_prefix=None, no_server=False, expected_missing=None): + if os.path.exists(dest_dir): + shutil.rmtree(dest_dir) + if no_server: + server = None + else: + server = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + try: + start = time.monotonic() + result = subprocess.run(cmd, text=True, capture_output=True) + duration = time.monotonic() - start + finally: + if server: + wait_proc(server) + + mismatches, missing = [], [] + if result.returncode == 0: + received = os.path.join(dest_dir, source_prefix if source_prefix is not None + else os.path.abspath(source_dir).lstrip(os.sep)) + mismatches, missing = verify_transfer(source_dir, received) + if expected_missing: + missing = [m for m in missing if m not in expected_missing] + + first_line = lambda s: (s or "").strip().split("\n")[0] + entry = { + "name": name, + "time": f"{duration:.4f}s" if result.returncode == 0 else "N/A", + } + if result.returncode == 0 and not mismatches and not missing: + entry["status"] = "Success" + entry["error"] = "" + else: + entry["status"] = "Failed" + errors = [] + if result.returncode != 0: + errors.append(f"Exit code {result.returncode}: {first_line(result.stderr) or first_line(result.stdout) or 'No output'[:80]}") + if missing: + errors.append(f"Missing ({len(missing)}): {', '.join(missing[:5])}") + if mismatches: + errors.append(f"Mismatch ({len(mismatches)}): {', '.join(mismatches[:3])}") + entry["error"] = " | ".join(errors) + return entry + + +def print_profile_header(profile_name): + params = NETWORK_PROFILES[profile_name] + print(f"\n{'=' * 60}\nProfile: {profile_name}\n{'=' * 60}") + if params: + print(f" Network: rate={params['rate']}, delay={params['delay']} ±{params['jitter']}, loss={params['loss']}") + print(f" Disk I/O: Reads <= {READ_BPS_MAX}, Writes <= {WRITE_BPS_MAX}") + else: + print(" No limits applied") + + +def run_profile(profile_name, source_dir, dest_dir): + print_profile_header(profile_name) + is_limited = profile_name != "Unlimited" + client_prefix = CLIENT_CMD_PREFIX if is_limited else [] + + try: + if is_limited: + netem_apply(profile_name) + else: + netem_reset() + + results = [] + for case in TEST_CASES: + flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"] + if case.get("posix"): + cmd = client_prefix + BASE_CLIENT_CMD + [source_dir, dest_dir] + flags + else: + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, case["name"], source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + if SSH_AVAILABLE: + for case in SSH_CASES: + flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"] + ssh_dest = f"localhost:{dest_dir}_ssh" + cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags + print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, case["name"], source_dir, f"{dest_dir}_ssh", no_server=True) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + port, conf, daemon = start_rsync_daemon(source_dir) + try: + for case in RSYNC_CASES: + cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"] + print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, case["name"], source_dir, dest_dir, source_prefix="") + r["suite"] = profile_name + except subprocess.TimeoutExpired: + r = {"name": case["name"], "suite": profile_name, "status": "Timeout", "time": "N/A", "error": "Exceeded 120s"} + except Exception as e: + r = {"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)} + else: + if r["status"] != "Success" and client_prefix: + tmp = tempfile.mkdtemp() + try: + plain = subprocess.run(["rsync", "-aH", f"rsync://localhost:{port}/source/", f"{tmp}/"], capture_output=True, text=True, timeout=30) + if plain.returncode != 0: + errs = [l for l in (plain.stderr or "").split("\n") if l.strip()] + if errs: + r["error"] += f" | raw: {errs[-1][:150]}" + finally: + shutil.rmtree(tmp, ignore_errors=True) + results.append(r) + finally: + wait_proc(daemon) + try: + os.unlink(conf) + except Exception: + pass + + # Feature-specific tests for rsync-compatible flags + print("\n " + "─" * 56 + "\n Feature Tests\n " + "─" * 56) + + # Dry run (-n) — no server needed + print("\n --- Dry run (-n) ---") + flags = BASE_CLIENT_FLAGS + ["-n"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + print(f" Running: {' '.join(cmd)}") + try: + start = time.monotonic() + result = subprocess.run(cmd, text=True, capture_output=True) + duration = time.monotonic() - start + r = {"name": "Dry run (-n)", "suite": profile_name} + if result.returncode == 0 and "Dry run:" in result.stdout: + r["status"] = "Success" + r["time"] = f"{duration:.4f}s" + r["error"] = "" + else: + r["status"] = "Failed" + r["time"] = "N/A" + r["error"] = f"Exit {result.returncode}: {(result.stderr or result.stdout)[:100]}" + results.append(r) + except Exception as e: + results.append({"name": "Dry run (-n)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Archive mode (-a) + feature_flags = BASE_CLIENT_FLAGS + ["-a"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Archive mode (-a) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Archive mode (-a)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Archive mode (-a)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Exclude (--exclude small.txt) + feature_flags = BASE_CLIENT_FLAGS + ["--exclude", "small.txt"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Exclude (--exclude small.txt) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Exclude (--exclude small.txt)", source_dir, dest_dir, + expected_missing=["small.txt"]) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Exclude (--exclude small.txt)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Progress (--progress) + feature_flags = BASE_CLIENT_FLAGS + ["--progress"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Progress (--progress) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Progress (--progress)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Chunk size (--chunk-size 5242880) + feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"] + cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags + print(f"\n --- Chunk size (--chunk-size 5242880) ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, "Chunk size (--chunk-size 5242880)", source_dir, dest_dir) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": "Chunk size (--chunk-size 5242880)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # Delete (--delete) — pre-populate dest, add extra files, then sync with --delete + # Note: server handles one client per launch, so we restart between syncs + print(f"\n --- Delete (--delete) ---") + try: + flags = BASE_CLIENT_FLAGS + ["-M"] + # First sync (no delete) to populate dest + s1 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + r1 = subprocess.run(first_cmd, text=True, capture_output=True) + wait_proc(s1) + if r1.returncode != 0: + raise RuntimeError(f"First sync failed: {r1.stderr[:100]}") + # Add extra files to received dir + received = os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep)) + extra_path = os.path.join(received, "extra_file.txt") + with open(extra_path, "w") as f: + f.write("should be deleted") + extra_dir = os.path.join(received, "extra_dir") + os.makedirs(extra_dir, exist_ok=True) + with open(os.path.join(extra_dir, "nested.txt"), "w") as f: + f.write("nested extra") + # Second sync with --delete (fresh server) + s2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None) + time.sleep(0.5) + second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--delete"] + start = time.monotonic() + r2 = subprocess.run(second_cmd, text=True, capture_output=True) + duration = time.monotonic() - start + wait_proc(s2) + r = {"name": "Delete (--delete)", "suite": profile_name} + if r2.returncode == 0 and not os.path.exists(extra_path) and not os.path.exists(extra_dir): + mismatches, missing = verify_transfer(source_dir, received) + if not mismatches and not missing: + r["status"] = "Success" + r["time"] = f"{duration:.4f}s" + r["error"] = "" + else: + r["status"] = "Failed" + r["time"] = "N/A" + r["error"] = f"post-delete verify: mismatches={len(mismatches)}, missing={len(missing)}" + else: + r["status"] = "Failed" + r["time"] = "N/A" + errs = [] + if r2.returncode != 0: + errs.append(f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}") + if os.path.exists(extra_path): + errs.append("extra_file.txt remains") + if os.path.exists(extra_dir): + errs.append("extra_dir remains") + r["error"] = " | ".join(errs) + results.append(r) + except Exception as e: + results.append({"name": "Delete (--delete)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + # SSH feature tests + if SSH_AVAILABLE: + ssh_dest = f"localhost:{dest_dir}_ssh" + ssh_feature_cases = [ + {"name": "SSH Archive (-a)", "flags": ["-a"]}, + {"name": "SSH Exclude (--exclude small.txt)", "flags": ["--exclude", "small.txt"], "expected_missing": ["small.txt"]}, + ] + for case in ssh_feature_cases: + flags = BASE_CLIENT_FLAGS + case["flags"] + cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags + print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}") + try: + r = run_single_test(cmd, case["name"], source_dir, f"{dest_dir}_ssh", + no_server=True, + expected_missing=case.get("expected_missing")) + r["suite"] = profile_name + results.append(r) + except Exception as e: + results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) + + except (subprocess.CalledProcessError, RuntimeError) as e: + print(f" Error: {e}") + results = [] + finally: + if is_limited: + try: + netem_reset() + except Exception: + pass + + return results + + +def print_metrics(profile_name, results, total_bytes): + params = NETWORK_PROFILES.get(profile_name) + if not params or "rate" not in params: + return + + client_times, rsync_times = [], {} + for r in results: + if r["status"] != "Success" or r["time"] == "N/A": + continue + t = float(r["time"].rstrip("s")) + if r["name"].startswith("rsync"): + rsync_times[r["name"]] = t + elif "Dry run" not in r["name"]: + client_times.append((t, r["name"])) + if not client_times or len(rsync_times) < 2: + return + + m = re.match(r'(\d+)\s*(mbit|gbit|kbit|bit)', params["rate"]) + rate_val = int(m.group(1)) * {'mbit': 1_000_000, 'gbit': 1_000_000_000, 'kbit': 1000, 'bit': 1}[m.group(2)] / 8 if m else None + + best_time, best_name = min(client_times, key=lambda x: x[0]) + theoretical_max = total_bytes / rate_val if rate_val else None + + print(f"\n {'─' * 90}\n Profile: {profile_name}\n {'─' * 90}") + print(f" Total data size: {total_bytes / (1024*1024):.1f} MB") + if rate_val: + print(f" Network rate: {params['rate']} ({format_throughput(rate_val)})") + print(f" Best client configuration: {best_name}") + print(f" Best client time: {best_time:.4f}s") + if theoretical_max: + print(f" Theoretical max (uncompressed): {theoretical_max:.4f}s") + print(f" Speedup vs theoretical max: {theoretical_max / best_time:.2f}x") + if (a := rsync_times.get("rsync (archive)")): + print(f" Speedup vs rsync (archive): {a / best_time:.2f}x") + if (c := rsync_times.get("rsync (archive + compress)")): + print(f" Speedup vs rsync (compress): {c / best_time:.2f}x") + + +def format_throughput(bps): + for unit, threshold in [("GB/s", 1_000_000_000), ("MB/s", 1_000_000), ("KB/s", 1000)]: + if bps >= threshold: + return f"{bps/threshold:.1f} {unit}" + return f"{bps:.0f} B/s" + + +SSH_AVAILABLE = False + +def check_ssh_localhost(): + global SSH_AVAILABLE + build_dir = os.path.abspath("build") + server_path = os.path.join(build_dir, "server") + + r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", + "localhost", "which", "fastsync-server"], + capture_output=True, timeout=10) + if r.returncode == 0: + SSH_AVAILABLE = True + return + + SSH_AVAILABLE = False + # Try each PATH dir: create symlink, then verify with which + r = subprocess.run( + ["ssh", "-o", "BatchMode=yes", "localhost", + 'echo "$PATH"'], + capture_output=True, timeout=10, text=True) + if r.returncode != 0: + return + for d in r.stdout.strip().split(":"): + d = d.strip() + if not d: + continue + if "wrappers" in d: + continue + test = subprocess.run( + ["ssh", "-o", "BatchMode=yes", "localhost", + f'test -w "{d}" && ln -sf {server_path} "{d}/fastsync-server" && which fastsync-server'], + capture_output=True, timeout=10) + if test.returncode == 0: + SSH_AVAILABLE = True + return + + +def preflight_checks(): + global SSH_AVAILABLE + errors = [] + print("Pre-flight checks:") + print(" [1] --help flag...", end=" ") + r = subprocess.run(BASE_CLIENT_CMD + ["--help"], capture_output=True, text=True) + if r.returncode == 0 and "Usage:" in r.stdout and "SSH transport" in r.stdout: + print("OK") + else: + print("FAIL") + errors.append("--help failed") + + print(" [2] Remote SSH dest detection...", end=" ") + r = subprocess.run(BASE_CLIENT_CMD + ["/x", "somehost:/y"], capture_output=True, text=True, timeout=5) + if r.returncode != 0 and ("ssh" in r.stderr or "Could not receive" in r.stderr or "could not launch" in r.stderr or "Error" in r.stderr): + print("OK (detected as SSH)") + else: + print("FAIL (not detected as SSH dest)") + errors.append("SSH detection failed") + + print(" [3] Server --stdio flag...", end=" ") + try: + r = subprocess.run(["./build/server", "--stdio"], capture_output=True, text=True, timeout=3) + if r.returncode != 0 and ("receiving" in r.stderr or "receiving" in r.stdout or "Receiving" in r.stderr): + print("OK (started in stdio mode)") + else: + print("WARN (stdio exited: rc=%d)" % r.returncode) + except subprocess.TimeoutExpired: + print("OK (waiting for stdin)") + + print(" [4] Posix arg syntax (no server, expect failure)...", end=" ") + r = subprocess.run(BASE_CLIENT_CMD + ["/tmp/x", "/tmp/y"], capture_output=True, text=True, timeout=5) + if r.returncode != 0 and "connect" in r.stderr: + print("OK (TCP fallback)") + else: + print("FAIL") + errors.append("Posix arg syntax failed") + + check_ssh_localhost() + print(" [5] SSH to localhost...", end=" ") + if SSH_AVAILABLE: + print("OK") + else: + print("SKIP (install fastsync-server in PATH on remote)") + + if errors: + print(f"\n {len(errors)} pre-flight check(s) failed: {', '.join(errors)}") + sys.exit(1) + print(" All pre-flight checks passed.\n") + + +def main(): + os.system("cmake -B build -S . > /dev/null 2>&1") + if os.system("cd build && make -j$(nproc) 2>&1 | tail -3") != 0: + print("Build failed") + sys.exit(1) + preflight_checks() + parser = argparse.ArgumentParser(description="FastSync integration test / benchmark") + parser.add_argument("--source-dir", default=DEFAULT_SOURCE_DIR) + parser.add_argument("--dest-dir", default=DEFAULT_DEST_DIR) + parser.add_argument("--keep-data", action="store_true") + parser.add_argument("--unlimited", action="store_true") + parser.add_argument("--wan", action="store_true") + args = parser.parse_args() + + total_bytes = generate_test_files(args.source_dir) + if os.path.exists(args.dest_dir): + shutil.rmtree(args.dest_dir) + os.makedirs(args.dest_dir, exist_ok=True) + + profiles = [] + if args.unlimited: + profiles.append("Unlimited") + elif args.wan: + profiles.append("WAN") + else: + profiles.append("LAN") + + try: + all_results = [] + for p in profiles: + all_results.extend(run_profile(p, args.source_dir, args.dest_dir)) + + print("\n" + "=" * 130) + print(f"{'RESULTS':^130}") + print("=" * 130) + print(f"{'Configuration':<45} | {'Profile':<12} | {'Status':<8} | {'Time':<10} | {'Details'}") + print("-" * 130) + for r in all_results: + print(f"{r['name']:<45} | {r['suite']:<12} | {r['status']:<8} | {r['time']:<10} | {r['error']}") + + for p in profiles: + print_metrics(p, [r for r in all_results if r["suite"] == p], total_bytes) + + failed = [r for r in all_results if r["status"] != "Success"] + if failed: + print(f"\n {len(failed)} test(s) FAILED") + sys.exit(1) + print(f"\n ALL {len(all_results)} TESTS PASSED") + finally: + if not args.keep_data: + shutil.rmtree(TEST_DIR, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/tests/runner.c b/tests/runner.c new file mode 100644 index 0000000..b05f007 --- /dev/null +++ b/tests/runner.c @@ -0,0 +1,36 @@ +#include "test_array_list.h" +#include "test_chunk.h" +#include "test_compression.h" +#include "test_config.h" +#include "test_queue.h" +#include "test_scanner.h" +#include "test_shared_utils.h" +#include "test_utils.h" +#include + +// Define global test state variables +int tests_run = 0; +int tests_failed = 0; +bool current_test_failed = false; + +int main() { + printf("\033[1;36m=== RUNNING UNIT TESTS ===\033[0m\n\n"); + + RUN_TEST(test_queue); + RUN_TEST(test_array_list); + RUN_TEST(test_shared_utils); + RUN_TEST(test_chunk); + RUN_TEST(test_config); + RUN_TEST(test_compression); + RUN_TEST(test_scanner); + + printf("\n\033[1;36m=== TEST SUMMARY ===\033[0m\n"); + printf("Total Tests Run: %d\n", tests_run); + if (tests_failed > 0) { + printf("Status: \033[1;31m%d FAILED\033[0m\n", tests_failed); + return 1; + } else { + printf("Status: \033[1;32mALL PASSED\033[0m\n"); + return 0; + } +} diff --git a/tests/test_array_list.c b/tests/test_array_list.c new file mode 100644 index 0000000..d7bc9ea --- /dev/null +++ b/tests/test_array_list.c @@ -0,0 +1,52 @@ +#include "test_array_list.h" +#include "array_list.h" +#include "test_utils.h" +#include + +static int destroyer_calls = 0; +static void test_destroyer(void *item) { + destroyer_calls++; + free(item); +} + +void test_array_list() { + ArrayList *list = array_list_create(free); + EXPECT_NOT_NULL(list); + EXPECT_EQ_INT(list->size, 0); + EXPECT_EQ_INT(list->capacity, 100); + + // Test adding + int *val1 = malloc(sizeof(int)); + *val1 = 42; + array_list_add(list, val1); + EXPECT_EQ_INT(list->size, 1); + EXPECT_EQ_INT(*(int *)list->items[0], 42); + + // Test extending capacity + // Initial capacity is 100. Let's add 105 elements. + for (int i = 0; i < 105; i++) { + int *val = malloc(sizeof(int)); + *val = i; + array_list_add(list, val); + } + EXPECT_EQ_INT(list->size, 106); + EXPECT_EQ_INT(list->capacity, 200); // 100 * 2 + + // Verify contents + EXPECT_EQ_INT(*(int *)list->items[0], 42); + EXPECT_EQ_INT(*(int *)list->items[1], 0); + EXPECT_EQ_INT(*(int *)list->items[105], 104); + + // Test array conversion + void **arr = array_list_to_array(list); + EXPECT_NOT_NULL(arr); + EXPECT_EQ_INT(*(int *)arr[0], 42); + EXPECT_EQ_INT(*(int *)arr[105], 104); + free(arr); + + // Delete list, verifying the destroyer is called 106 times + destroyer_calls = 0; + list->item_destroyer = test_destroyer; + array_list_delete(list); + EXPECT_EQ_INT(destroyer_calls, 106); +} diff --git a/tests/test_array_list.h b/tests/test_array_list.h new file mode 100644 index 0000000..cb952b3 --- /dev/null +++ b/tests/test_array_list.h @@ -0,0 +1,6 @@ +#ifndef TEST_ARRAY_LIST_H +#define TEST_ARRAY_LIST_H + +void test_array_list(); + +#endif diff --git a/tests/test_chunk.c b/tests/test_chunk.c new file mode 100644 index 0000000..024e729 --- /dev/null +++ b/tests/test_chunk.c @@ -0,0 +1,95 @@ + +#include "chunk.h" +#include "test_utils.h" +#include "utils.h" +#include +#include +#include + +static void test_file_operations() { + char *test_path = "temp_file_test.txt"; + char *test_content = "Hello, Chunk System!"; + unsigned long long test_len = strlen(test_content); + + to_disk(test_path, test_content, test_len); + + File *f = file_create(test_path); + EXPECT_NOT_NULL(f); + EXPECT_EQ_STR(f->path, test_path); + EXPECT_NOT_NULL(f->data); + EXPECT_NULL(f->data->data); + EXPECT_EQ_INT((int)f->data->size, 0); + + struct stat st; + stat(test_path, &st); + f->data->size = st.st_size; + + file_load_data(f); + EXPECT_NOT_NULL(f->data); + EXPECT_NOT_NULL(f->data->data); + EXPECT_EQ_INT((int)f->data->size, (int)test_len); + EXPECT_EQ_INT(memcmp(f->data->data, test_content, test_len), 0); + + file_destroy(f); + unlink(test_path); +} + +static void test_chunk_operations() { + char *path1 = "temp_chunk_1.txt"; + char *content1 = "chunk item 1"; + unsigned long long len1 = strlen(content1); + + char *path2 = "temp_chunk_2.txt"; + char *content2 = "chunk item number 2"; + unsigned long long len2 = strlen(content2); + + to_disk(path1, content1, len1); + to_disk(path2, content2, len2); + + struct stat st1, st2; + stat(path1, &st1); + stat(path2, &st2); + + File *f1 = file_create(path1); + f1->data->size = st1.st_size; + File *f2 = file_create(path2); + f2->data->size = st2.st_size; + + File *files[2] = {f1, f2}; + Chunk *chunk = chunk_create(files, 2); + EXPECT_NOT_NULL(chunk); + EXPECT_EQ_INT(chunk->element_count, 2); + EXPECT_NOT_NULL(chunk->items[0]); + EXPECT_NOT_NULL(chunk->items[1]); + + // load data before serializing + file_load_data(f1); + file_load_data(f2); + + // Test chunk_serialize / chunk_deserialize round-trip + Data *serialized = chunk_serialize(chunk, false); + EXPECT_NOT_NULL(serialized); + + Chunk *deserialized = chunk_deserialize(serialized, false); + EXPECT_NOT_NULL(deserialized); + EXPECT_EQ_INT(deserialized->element_count, 2); + EXPECT_EQ_STR(deserialized->items[0]->path, path1); + EXPECT_EQ_STR(deserialized->items[1]->path, path2); + EXPECT_EQ_INT((int)deserialized->items[0]->data->size, (int)len1); + EXPECT_EQ_INT((int)deserialized->items[1]->data->size, (int)len2); + EXPECT_EQ_INT(memcmp(deserialized->items[0]->data->data, content1, len1), 0); + EXPECT_EQ_INT(memcmp(deserialized->items[1]->data->data, content2, len2), 0); + + data_destroy(serialized); + chunk_destroy(deserialized); + + chunk_destroy(chunk); + + unlink(path1); + unlink(path2); +} + +void test_chunk() { + test_file_operations(); + test_chunk_operations(); +} diff --git a/tests/test_chunk.h b/tests/test_chunk.h new file mode 100644 index 0000000..5084d22 --- /dev/null +++ b/tests/test_chunk.h @@ -0,0 +1,6 @@ +#ifndef TEST_CHUNK_H +#define TEST_CHUNK_H + +void test_chunk(); + +#endif diff --git a/tests/test_compression.c b/tests/test_compression.c new file mode 100644 index 0000000..46ab006 --- /dev/null +++ b/tests/test_compression.c @@ -0,0 +1,117 @@ +#include "test_utils.h" +#include "chunk.h" +#include "compression.h" +#include "data.h" +#include "file.h" +#include "utils.h" +#include +#include +#include + +static void test_data_compress_decompress_roundtrip() { + char original[] = "Hello, World! This is test data for compression round-trip!"; + size_t len = strlen(original); + + char *buf = malloc(len); + memcpy(buf, original, len); + Data *original_data = data_create(buf, len); + EXPECT_NOT_NULL(original_data); + + Data *compressed = data_compress(original_data, 3); + EXPECT_NOT_NULL(compressed); + + Data *decompressed = data_decompress(compressed); + EXPECT_NOT_NULL(decompressed); + EXPECT_EQ_INT((int)decompressed->size, (int)len); + EXPECT_EQ_INT(memcmp(decompressed->data, original, len), 0); + + data_destroy(original_data); + data_destroy(compressed); + data_destroy(decompressed); +} + +static void test_data_compress_decompress_large() { + size_t size = 1024 * 10; + char *original = malloc(size); + EXPECT_NOT_NULL(original); + for (size_t i = 0; i < size; i++) + original[i] = (char)(i % 256); + + Data *original_data = data_create(original, size); + EXPECT_NOT_NULL(original_data); + + Data *compressed = data_compress(original_data, 1); + EXPECT_NOT_NULL(compressed); + + Data *decompressed = data_decompress(compressed); + EXPECT_NOT_NULL(decompressed); + EXPECT_EQ_INT((int)decompressed->size, (int)size); + EXPECT_EQ_INT(memcmp(decompressed->data, original, size), 0); + + data_destroy(original_data); + data_destroy(compressed); + data_destroy(decompressed); +} + +static void test_chunk_compress_decompress_roundtrip() { + char *path1 = "temp_comp_test_1.txt"; + char *content1 = "chunk compression test file 1"; + unsigned long long len1 = strlen(content1); + + char *path2 = "temp_comp_test_2.txt"; + char *content2 = "chunk compression test file 2 with more data"; + unsigned long long len2 = strlen(content2); + + to_disk(path1, content1, len1); + to_disk(path2, content2, len2); + + struct stat st1, st2; + EXPECT_EQ_INT(stat(path1, &st1), 0); + EXPECT_EQ_INT(stat(path2, &st2), 0); + + File *f1 = file_create(path1); + f1->data->size = st1.st_size; + File *f2 = file_create(path2); + f2->data->size = st2.st_size; + EXPECT_NOT_NULL(f1); + EXPECT_NOT_NULL(f2); + + file_load_data(f1); + file_load_data(f2); + + File *files[2] = {f1, f2}; + Chunk *chunk = chunk_create(files, 2); + EXPECT_NOT_NULL(chunk); + + Data *compressed = chunk_compress(chunk, 3, false); + EXPECT_NOT_NULL(compressed); + + Data *decompressed_data = data_decompress(compressed); + EXPECT_NOT_NULL(decompressed_data); + + Chunk *decompressed_chunk = chunk_deserialize(decompressed_data, false); + EXPECT_NOT_NULL(decompressed_chunk); + EXPECT_EQ_INT(decompressed_chunk->element_count, 2); + + EXPECT_EQ_STR(decompressed_chunk->items[0]->path, path1); + EXPECT_EQ_INT((int)decompressed_chunk->items[0]->data->size, (int)len1); + EXPECT_EQ_INT(memcmp(decompressed_chunk->items[0]->data->data, content1, len1), 0); + + EXPECT_EQ_STR(decompressed_chunk->items[1]->path, path2); + EXPECT_EQ_INT((int)decompressed_chunk->items[1]->data->size, (int)len2); + EXPECT_EQ_INT(memcmp(decompressed_chunk->items[1]->data->data, content2, len2), 0); + + chunk_destroy(chunk); + data_destroy(compressed); + data_destroy(decompressed_data); + chunk_destroy(decompressed_chunk); + + unlink(path1); + unlink(path2); +} + +void test_compression() { + test_data_compress_decompress_roundtrip(); + test_data_compress_decompress_large(); + test_chunk_compress_decompress_roundtrip(); +} diff --git a/tests/test_compression.h b/tests/test_compression.h new file mode 100644 index 0000000..d86ff93 --- /dev/null +++ b/tests/test_compression.h @@ -0,0 +1,6 @@ +#ifndef TEST_COMPRESSION_H +#define TEST_COMPRESSION_H + +void test_compression(); + +#endif diff --git a/tests/test_config.c b/tests/test_config.c new file mode 100644 index 0000000..df71e3a --- /dev/null +++ b/tests/test_config.c @@ -0,0 +1,99 @@ +#include "test_config.h" +#include "config.h" +#include "multiprocessing.h" +#include "queue.h" +#include "test_utils.h" +#include "utils.h" +#include + +static void test_config_lifecycle() { + Config *cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("/dst"), + true, true, false, false, false, 1, false, 0); + EXPECT_NOT_NULL(cfg); + EXPECT_EQ_STR(cfg->version, "1.0"); + EXPECT_EQ_STR(cfg->send_directory, "/src"); + EXPECT_EQ_STR(cfg->receive_root_directory, "/dst"); + EXPECT_TRUE(cfg->save_to_disk); + EXPECT_TRUE(cfg->use_multithreading); + EXPECT_FALSE(cfg->use_chunk_serialization); + EXPECT_FALSE(cfg->use_compression); + EXPECT_EQ_INT(cfg->transport, TRANSPORT_TCP); + EXPECT_NULL(cfg->ssh_destination); + config_delete(cfg); +} + +static void test_config_ssh_dest() { + Config *cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("user@host:/dst"), + true, false, false, false, false, 1, false, 0); + EXPECT_NOT_NULL(cfg); + EXPECT_EQ_INT(cfg->transport, TRANSPORT_TCP); + EXPECT_NULL(cfg->ssh_destination); + EXPECT_EQ_STR(cfg->receive_root_directory, "user@host:/dst"); + + config_parse_ssh_dest(cfg); + EXPECT_EQ_INT(cfg->transport, TRANSPORT_SSH); + EXPECT_EQ_STR(cfg->ssh_destination, "user@host:/dst"); + EXPECT_EQ_STR(cfg->receive_root_directory, "/dst"); + config_delete(cfg); +} + +static void test_config_ssh_dest_local_path() { + Config *cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("/local/path"), + true, false, false, false, false, 1, false, 0); + config_parse_ssh_dest(cfg); + EXPECT_EQ_INT(cfg->transport, TRANSPORT_TCP); + EXPECT_NULL(cfg->ssh_destination); + EXPECT_EQ_STR(cfg->receive_root_directory, "/local/path"); + config_delete(cfg); +} + +static void test_config_ssh_dest_no_user() { + Config *cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("host:/remote"), + true, false, false, false, false, 1, false, 0); + config_parse_ssh_dest(cfg); + EXPECT_EQ_INT(cfg->transport, TRANSPORT_SSH); + EXPECT_EQ_STR(cfg->ssh_destination, "host:/remote"); + EXPECT_EQ_STR(cfg->receive_root_directory, "/remote"); + config_delete(cfg); +} + +static void test_pipeline_sender_lifecycle() { + Config *cfg = config_create(str_dup("2.0"), str_dup("/src2"), + str_dup("/dst2"), false, false, true, true, false, 1, false, 0); + Queue *q1 = queue_create(5, NULL); + Queue *q2 = queue_create(15, NULL); + + PipelineContextSender *pcs = pipeline_context_sender_create(cfg, q1, q2); + EXPECT_NOT_NULL(pcs); + EXPECT_EQ_STR(pcs->config->version, "2.0"); + EXPECT_EQ_INT(pcs->queue_scanner->capacity, 5); + EXPECT_EQ_INT(pcs->queue_loader->capacity, 15); + EXPECT_FALSE(pcs->scanner_done); + EXPECT_FALSE(pcs->loader_done); + + pipeline_context_sender_destroy(pcs); +} + +static void test_pipeline_receiver_lifecycle() { + Config *cfg = config_create(str_dup("3.0"), str_dup("/src3"), + str_dup("/dst3"), true, true, true, true, false, 1, false, 0); + Queue *q = queue_create(20, NULL); + + PipelineContextReceiver *pcr = pipeline_context_receiver_create(cfg, q, 42); + EXPECT_NOT_NULL(pcr); + EXPECT_EQ_STR(pcr->config->version, "3.0"); + EXPECT_EQ_INT(pcr->queue->capacity, 20); + EXPECT_EQ_INT(pcr->file_descriptor, 42); + EXPECT_FALSE(pcr->receiver_done); + + pipeline_context_receiver_destroy(pcr); +} + +void test_config() { + test_config_lifecycle(); + test_config_ssh_dest(); + test_config_ssh_dest_local_path(); + test_config_ssh_dest_no_user(); + test_pipeline_sender_lifecycle(); + test_pipeline_receiver_lifecycle(); +} diff --git a/tests/test_config.h b/tests/test_config.h new file mode 100644 index 0000000..e245193 --- /dev/null +++ b/tests/test_config.h @@ -0,0 +1,6 @@ +#ifndef TEST_CONFIG_H +#define TEST_CONFIG_H + +void test_config(); + +#endif diff --git a/tests/test_queue.c b/tests/test_queue.c new file mode 100644 index 0000000..e04e997 --- /dev/null +++ b/tests/test_queue.c @@ -0,0 +1,206 @@ +#include "test_queue.h" +#include "queue.h" +#include "test_utils.h" +#include +#include +#include +#include + +static void test_queue_basic() { + Queue *q = queue_create(10, NULL); + EXPECT_NOT_NULL(q); + EXPECT_TRUE(queue_is_empty(q)); + EXPECT_FALSE(queue_is_full(q)); + + int *vals[5]; + for (int i = 0; i < 5; i++) { + vals[i] = malloc(sizeof(int)); + *vals[i] = (i + 1) * 10; + queue_enqueue(q, vals[i]); + } + + EXPECT_FALSE(queue_is_empty(q)); + EXPECT_FALSE(queue_is_full(q)); + EXPECT_EQ_INT(q->size, 5); + + int *v1 = (int *)queue_dequeue(q); + EXPECT_NOT_NULL(v1); + EXPECT_EQ_INT(*v1, 10); + free(v1); + + int *v2 = (int *)queue_dequeue(q); + EXPECT_NOT_NULL(v2); + EXPECT_EQ_INT(*v2, 20); + free(v2); + + EXPECT_EQ_INT(q->size, 3); + + int *vals2[3]; + for (int i = 0; i < 3; i++) { + vals2[i] = malloc(sizeof(int)); + *vals2[i] = (i + 6) * 10; + queue_enqueue(q, vals2[i]); + } + + EXPECT_EQ_INT(q->size, 6); + + int expected_vals[] = {30, 40, 50, 60, 70, 80}; + for (int i = 0; i < 6; i++) { + int *v = (int *)queue_dequeue(q); + EXPECT_NOT_NULL(v); + EXPECT_EQ_INT(*v, expected_vals[i]); + free(v); + } + + EXPECT_TRUE(queue_is_empty(q)); + queue_destroy(q); +} + +static void test_queue_resize() { + Queue *q = queue_create(3, NULL); + EXPECT_NOT_NULL(q); + EXPECT_EQ_INT(q->capacity, 3); + + int a = 1, b = 2, c = 3, d = 4, e = 5; + + queue_enqueue(q, &a); + queue_enqueue(q, &b); + queue_enqueue(q, &c); + + EXPECT_TRUE(queue_is_full(q)); + + int *v1 = (int *)queue_dequeue(q); + EXPECT_NOT_NULL(v1); + EXPECT_EQ_INT(*v1, 1); + + // Now front = 1, rear = 0, size = 2 (wrapped state) + queue_enqueue(q, &d); + EXPECT_TRUE(queue_is_full(q)); + + // This enqueue triggers capacity doubling + queue_enqueue(q, &e); + EXPECT_FALSE(queue_is_full(q)); + EXPECT_EQ_INT(q->capacity, 6); + EXPECT_EQ_INT(q->size, 4); + + // Dequeue all and check order: B, C, D, E + int *v2 = (int *)queue_dequeue(q); + EXPECT_NOT_NULL(v2); + EXPECT_EQ_INT(*v2, 2); + + int *v3 = (int *)queue_dequeue(q); + EXPECT_NOT_NULL(v3); + EXPECT_EQ_INT(*v3, 3); + + int *v4 = (int *)queue_dequeue(q); + EXPECT_NOT_NULL(v4); + EXPECT_EQ_INT(*v4, 4); + + int *v5 = (int *)queue_dequeue(q); + EXPECT_NOT_NULL(v5); + EXPECT_EQ_INT(*v5, 5); + + EXPECT_TRUE(queue_is_empty(q)); + queue_destroy(q); +} + +static int destroyer_calls = 0; +static void my_destroyer(void *item) { + destroyer_calls++; + free(item); +} + +static void test_queue_destroyer() { + destroyer_calls = 0; + Queue *q = queue_create(5, my_destroyer); + EXPECT_NOT_NULL(q); + + for (int i = 0; i < 3; i++) { + int *val = malloc(sizeof(int)); + *val = i; + queue_enqueue(q, val); + } + + int *v = (int *)queue_dequeue(q); + EXPECT_NOT_NULL(v); + EXPECT_EQ_INT(*v, 0); + free(v); + + queue_destroy(q); + EXPECT_EQ_INT(destroyer_calls, 2); +} + +typedef struct { + Queue *q; + mtx_t *mutex; + cnd_t *cnd_empty; + cnd_t *cnd_full; + bool done; + int sum; +} ThreadContext; + +static int consumer_func(void *arg) { + ThreadContext *ctx = (ThreadContext *)arg; + while (true) { + int *val = (int *)queue_dequeue_multithreaded(ctx->q, ctx->mutex, ctx->cnd_empty, ctx->cnd_full, &ctx->done); + if (val == NULL) { + break; + } + ctx->sum += *val; + free(val); + } + return 0; +} + +static void test_queue_multithreaded() { + Queue *q = queue_create(2, NULL); + mtx_t mutex; + cnd_t cnd_empty; + cnd_t cnd_full; + + mtx_init(&mutex, mtx_plain); + cnd_init(&cnd_empty); + cnd_init(&cnd_full); + + ThreadContext ctx = { + .q = q, + .mutex = &mutex, + .cnd_empty = &cnd_empty, + .cnd_full = &cnd_full, + .done = false, + .sum = 0 + }; + + thrd_t consumer; + int res = thrd_create(&consumer, consumer_func, &ctx); + EXPECT_EQ_INT(res, thrd_success); + + for (int i = 1; i <= 100; i++) { + int *val = malloc(sizeof(int)); + *val = i; + queue_enqueue_multithreaded(q, val, &mutex, &cnd_empty, &cnd_full); + } + + mtx_lock(&mutex); + ctx.done = true; + cnd_signal(&cnd_empty); + mtx_unlock(&mutex); + + int join_res; + thrd_join(consumer, &join_res); + + EXPECT_EQ_INT(ctx.sum, 5050); + EXPECT_TRUE(queue_is_empty(q)); + + queue_destroy(q); + mtx_destroy(&mutex); + cnd_destroy(&cnd_empty); + cnd_destroy(&cnd_full); +} + +void test_queue() { + test_queue_basic(); + test_queue_resize(); + test_queue_destroyer(); + test_queue_multithreaded(); +} diff --git a/tests/test_queue.h b/tests/test_queue.h new file mode 100644 index 0000000..1f53e69 --- /dev/null +++ b/tests/test_queue.h @@ -0,0 +1,6 @@ +#ifndef TEST_QUEUE_H +#define TEST_QUEUE_H + +void test_queue(); + +#endif diff --git a/tests/test_scanner.c b/tests/test_scanner.c new file mode 100644 index 0000000..a733491 --- /dev/null +++ b/tests/test_scanner.c @@ -0,0 +1,124 @@ +#include "test_utils.h" +#include "scanner.h" +#include "file.h" +#include "utils.h" +#include +#include +#include + +static void create_test_file(const char *path, const char *content) { + to_disk(path, content, strlen(content)); +} + +static void test_scanner_single_file() { + const char *dir = "test_scan_dir_single"; + const char *file1 = "test_scan_dir_single/file1.txt"; + const char *content1 = "hello scanner"; + + mkdir(dir, 0755); + create_test_file(file1, content1); + + DirectoryScanner *scanner = directory_scanner_create((char *)dir, false, 0, NULL, 0); + EXPECT_NOT_NULL(scanner); + + Chunk *chunk = directory_scanner_next(scanner); + EXPECT_NOT_NULL(chunk); + EXPECT_EQ_INT(chunk->element_count, 1); + EXPECT_EQ_STR(chunk->items[0]->path, file1); + + Chunk *next = directory_scanner_next(scanner); + EXPECT_NULL(next); + + chunk_destroy(chunk); + directory_scanner_destroy(scanner); + unlink(file1); + rmdir(dir); +} + +static void test_scanner_multiple_files() { + const char *dir = "test_scan_dir_multi"; + const char *file1 = "test_scan_dir_multi/a.txt"; + const char *file2 = "test_scan_dir_multi/b.txt"; + const char *content1 = "alpha"; + const char *content2 = "beta"; + + mkdir(dir, 0755); + create_test_file(file1, content1); + create_test_file(file2, content2); + + DirectoryScanner *scanner = directory_scanner_create((char *)dir, false, 0, NULL, 0); + EXPECT_NOT_NULL(scanner); + + Chunk *chunk = directory_scanner_next(scanner); + EXPECT_NOT_NULL(chunk); + EXPECT_EQ_INT(chunk->element_count, 2); + + int found1 = 0, found2 = 0; + for (int i = 0; i < chunk->element_count; i++) { + if (strcmp(chunk->items[i]->path, file1) == 0) found1 = 1; + if (strcmp(chunk->items[i]->path, file2) == 0) found2 = 1; + } + EXPECT_TRUE(found1); + EXPECT_TRUE(found2); + + Chunk *next = directory_scanner_next(scanner); + EXPECT_NULL(next); + + chunk_destroy(chunk); + directory_scanner_destroy(scanner); + unlink(file1); + unlink(file2); + rmdir(dir); +} + +static void test_scanner_subdirectory() { + const char *root = "test_scan_sub"; + const char *sub = "test_scan_sub/sub"; + const char *root_file = "test_scan_sub/root.txt"; + const char *sub_file = "test_scan_sub/sub/sub_file.txt"; + const char *content = "nested content"; + + mkdir(root, 0755); + mkdir(sub, 0755); + create_test_file(root_file, content); + create_test_file(sub_file, content); + + DirectoryScanner *scanner = directory_scanner_create((char *)root, false, 0, NULL, 0); + EXPECT_NOT_NULL(scanner); + + int total_files = 0; + Chunk *chunk; + while ((chunk = directory_scanner_next(scanner)) != NULL) { + total_files += chunk->element_count; + chunk_destroy(chunk); + } + EXPECT_EQ_INT(total_files, 2); + + directory_scanner_destroy(scanner); + unlink(root_file); + unlink(sub_file); + rmdir(sub); + rmdir(root); +} + +static void test_scanner_empty_directory() { + const char *dir = "test_scan_empty"; + + mkdir(dir, 0755); + + DirectoryScanner *scanner = directory_scanner_create((char *)dir, false, 0, NULL, 0); + EXPECT_NOT_NULL(scanner); + + Chunk *chunk = directory_scanner_next(scanner); + EXPECT_NULL(chunk); + + directory_scanner_destroy(scanner); + rmdir(dir); +} + +void test_scanner() { + test_scanner_single_file(); + test_scanner_multiple_files(); + test_scanner_subdirectory(); + test_scanner_empty_directory(); +} diff --git a/tests/test_scanner.h b/tests/test_scanner.h new file mode 100644 index 0000000..a8c6276 --- /dev/null +++ b/tests/test_scanner.h @@ -0,0 +1,6 @@ +#ifndef TEST_SCANNER_H +#define TEST_SCANNER_H + +void test_scanner(); + +#endif diff --git a/tests/test_shared_utils.c b/tests/test_shared_utils.c new file mode 100644 index 0000000..3470d8b --- /dev/null +++ b/tests/test_shared_utils.c @@ -0,0 +1,62 @@ +#include "test_shared_utils.h" +#include "utils.h" +#include "test_utils.h" +#include +#include + +void test_shared_utils() { + // Test str_dup + char *dup_null = str_dup(NULL); + EXPECT_NULL(dup_null); + + char *dup_empty = str_dup(""); + EXPECT_NOT_NULL(dup_empty); + EXPECT_EQ_STR(dup_empty, ""); + free(dup_empty); + + char *dup_normal = str_dup("hello world"); + EXPECT_NOT_NULL(dup_normal); + EXPECT_EQ_STR(dup_normal, "hello world"); + free(dup_normal); + + // Test path_cat + char *cat1 = path_cat("/foo", "/bar"); + EXPECT_NOT_NULL(cat1); + EXPECT_EQ_STR(cat1, "/foo/bar"); + free(cat1); + + char *cat2 = path_cat("/foo/", "/bar"); + EXPECT_NOT_NULL(cat2); + EXPECT_EQ_STR(cat2, "/foo/bar"); + free(cat2); + + char *cat3 = path_cat("/foo", "bar"); + EXPECT_NOT_NULL(cat3); + EXPECT_EQ_STR(cat3, "/foo/bar"); + free(cat3); + + char *cat4 = path_cat("/foo/", "bar"); + EXPECT_NOT_NULL(cat4); + EXPECT_EQ_STR(cat4, "/foo/bar"); + free(cat4); + + char *cat_empty1 = path_cat("", "/bar"); + EXPECT_NOT_NULL(cat_empty1); + EXPECT_EQ_STR(cat_empty1, "/bar"); + free(cat_empty1); + + char *cat_empty2 = path_cat("/foo", ""); + EXPECT_NOT_NULL(cat_empty2); + EXPECT_EQ_STR(cat_empty2, "/foo"); + free(cat_empty2); + + char *cat_null1 = path_cat(NULL, "/bar"); + EXPECT_NOT_NULL(cat_null1); + EXPECT_EQ_STR(cat_null1, "/bar"); + free(cat_null1); + + char *cat_null2 = path_cat("/foo", NULL); + EXPECT_NOT_NULL(cat_null2); + EXPECT_EQ_STR(cat_null2, "/foo"); + free(cat_null2); +} diff --git a/tests/test_shared_utils.h b/tests/test_shared_utils.h new file mode 100644 index 0000000..b33b7a2 --- /dev/null +++ b/tests/test_shared_utils.h @@ -0,0 +1,6 @@ +#ifndef TEST_SHARED_UTILS_H +#define TEST_SHARED_UTILS_H + +void test_shared_utils(); + +#endif diff --git a/tests/test_utils.h b/tests/test_utils.h new file mode 100644 index 0000000..dda6bb6 --- /dev/null +++ b/tests/test_utils.h @@ -0,0 +1,94 @@ +#ifndef TEST_UTILS_H +#define TEST_UTILS_H + +#include +#include +#include + +// Global test suite status +extern int tests_run; +extern int tests_failed; +extern bool current_test_failed; + +// Helper to run a test function +#define RUN_TEST(test_func) \ + do { \ + printf("Running %s...\n", #test_func); \ + tests_run++; \ + current_test_failed = false; \ + test_func(); \ + if (current_test_failed) { \ + tests_failed++; \ + printf(" \033[1;31m[FAILED]\033[0m %s\n", #test_func); \ + } else { \ + printf(" \033[1;32m[PASSED]\033[0m %s\n", #test_func); \ + } \ + } while (0) + +// Assertion macros +#define EXPECT_TRUE(condition) \ + do { \ + if (!(condition)) { \ + printf(" \033[1;31m[FAIL]\033[0m %s:%d: Assertion failed: %s is false\n", __FILE__, __LINE__, #condition); \ + current_test_failed = true; \ + return; \ + } \ + } while (0) + +#define EXPECT_FALSE(condition) \ + do { \ + if (condition) { \ + printf(" \033[1;31m[FAIL]\033[0m %s:%d: Assertion failed: %s is true\n", __FILE__, __LINE__, #condition); \ + current_test_failed = true; \ + return; \ + } \ + } while (0) + +#define EXPECT_EQ_INT(actual, expected) \ + do { \ + int act = (actual); \ + int exp = (expected); \ + if (act != exp) { \ + printf(" \033[1;31m[FAIL]\033[0m %s:%d: Expected %d, got %d\n", __FILE__, __LINE__, exp, act); \ + current_test_failed = true; \ + return; \ + } \ + } while (0) + +#define EXPECT_EQ_STR(actual, expected) \ + do { \ + const char *act = (actual); \ + const char *exp = (expected); \ + if (act == NULL || exp == NULL) { \ + if (act != exp) { \ + printf(" \033[1;31m[FAIL]\033[0m %s:%d: Expected %s, got %s\n", __FILE__, __LINE__, \ + exp ? exp : "NULL", act ? act : "NULL"); \ + current_test_failed = true; \ + return; \ + } \ + } else if (strcmp(act, exp) != 0) { \ + printf(" \033[1;31m[FAIL]\033[0m %s:%d: Expected \"%s\", got \"%s\"\n", __FILE__, __LINE__, exp, act); \ + current_test_failed = true; \ + return; \ + } \ + } while (0) + +#define EXPECT_NOT_NULL(ptr) \ + do { \ + if ((ptr) == NULL) { \ + printf(" \033[1;31m[FAIL]\033[0m %s:%d: Expected non-null pointer, got NULL\n", __FILE__, __LINE__); \ + current_test_failed = true; \ + return; \ + } \ + } while (0) + +#define EXPECT_NULL(ptr) \ + do { \ + if ((ptr) != NULL) { \ + printf(" \033[1;31m[FAIL]\033[0m %s:%d: Expected NULL, got %p\n", __FILE__, __LINE__, (void*)(ptr)); \ + current_test_failed = true; \ + return; \ + } \ + } while (0) + +#endif diff --git a/tmux.sh b/tmux.sh new file mode 100644 index 0000000..4240838 --- /dev/null +++ b/tmux.sh @@ -0,0 +1,15 @@ +SESSION="fastSync" + +tmux has-session -t $SESSION 2>/dev/null + +if [ $? != 0 ]; then + tmux new-session -d -s $SESSION -n "Neovim" + tmux send-keys -t $SESSION:0 'nvim .' C-m + tmux new-window -t $SESSION -n "Console" + tmux send-keys -t $SESSION:1 'cd ./build' C-m + tmux split-window -h -t $SESSION:1 + tmux send-keys -t $SESSION:1.1 'cd ./build' C-m + tmux select-window -t $SESSION:0 +fi + +tmux attach-session -t $SESSION diff --git a/to_one_file.py b/to_one_file.py new file mode 100644 index 0000000..0a28b76 --- /dev/null +++ b/to_one_file.py @@ -0,0 +1,14 @@ +from pathlib import Path + +path = Path(".") +text = "" +for file in path.glob("**/*.h"): + text += "--- " + str(file) + " ---\n\n" + text += file.read_text() +for file in path.glob("**/*.c"): + text += "--- " + str(file) + " ---\n\n" + text += file.read_text() +for file in [Path("CMakeLists.txt")]: + text += "--- " + str(file) + " ---\n\n" + text += file.read_text() +Path("all.txt").write_text(text)