Compare commits
85 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0021ca3fcd | |||
| 269ce0749b | |||
| 52d5492dee | |||
| e68e33c0c5 | |||
| 3b621e591a | |||
| d3dca6c2a5 | |||
| 29f4f8cde6 | |||
| c5acd13df2 | |||
| 41485cbe22 | |||
| c6bf7bb84e | |||
| 30239c6f50 | |||
| d21f6c8be9 | |||
| 0bee82e863 | |||
| bfc2b1f6eb | |||
| 7436b3b148 | |||
| a70ce5c4af | |||
| 415adf2077 | |||
| 3b5d89fdb9 | |||
| 2e73c6dc13 | |||
| 909d84b36e | |||
| 959e5eb959 | |||
| df6d78b6ef | |||
| 6c730c9775 | |||
| a6c471a97a | |||
| ab4bb84ed6 | |||
| 4bba1e8c02 | |||
| 3d598040dd | |||
| 09dc449a5c | |||
| 86c1d159cb | |||
| c21950493a | |||
| ce39f2a910 | |||
| 1929e7d7b3 | |||
| 1ce771b553 | |||
| 60ab410a8c | |||
| 7adb82f8ac | |||
| 62f3df94ee | |||
| 61fd410c63 | |||
| 1c3b752a8e | |||
| 1a765e595b | |||
| 94ff55f256 | |||
| ddfd0f1cc2 | |||
| e34ec3d594 | |||
| f932a48910 | |||
| 23af379bfb | |||
| 31da8bf081 | |||
| 0cbc873f5c | |||
| 925760d1bd | |||
| 83c61aadc2 | |||
| 552561146a | |||
| cdd6d1cfed | |||
| e62296d92f | |||
| 5d819f7388 | |||
| 4339d7905d | |||
| 3ffc5c5236 | |||
| 224e7e8599 | |||
| 9a214c46e2 | |||
| 9dd925a87d | |||
| 4b93082139 | |||
| 8234677276 | |||
| 262a436264 | |||
| 70e1c6788a | |||
| 30d4d6870c | |||
| 744ac8e40c | |||
| 9042dfcfa9 | |||
| e2a500e321 | |||
| a91267ca1b | |||
| cf572ece04 | |||
| f486d34b16 | |||
| e3bd7a8cdf | |||
| 266369b1f2 | |||
| a4fc16650e | |||
| eef272fa4e | |||
| 5e8d0a2dbf | |||
| 62ff1d3929 | |||
| ba0afd4152 | |||
| 3923421224 | |||
| 27e3ac11db | |||
| 9e3e0f57a0 | |||
| e5b46bb1c0 | |||
| 86247fe2b5 | |||
| 788d3c7bea | |||
| 7ecba4e0d5 | |||
| 16376bcafc | |||
| d75701270d | |||
| 990c4362af |
@@ -5,5 +5,6 @@ __pycache__/
|
|||||||
build-asan
|
build-asan
|
||||||
coverage.info
|
coverage.info
|
||||||
build-*/
|
build-*/
|
||||||
|
build2/
|
||||||
build3/
|
build3/
|
||||||
build_docker2/
|
build_docker2/
|
||||||
|
|||||||
@@ -72,3 +72,90 @@ git push -u origin <feature-branch-name>
|
|||||||
gh pr create --fill
|
gh pr create --fill
|
||||||
```
|
```
|
||||||
Wait for CI to pass on the PR before merging.
|
Wait for CI to pass on the PR before merging.
|
||||||
|
|
||||||
|
## Batch PR Workflow
|
||||||
|
|
||||||
|
When handling multiple issues split across several PRs that target the same files:
|
||||||
|
|
||||||
|
1. **Group issues by logical category** into separate PR branches (e.g., memory-safety, refactoring, test-coverage).
|
||||||
|
2. **Fix and push** each branch independently. Let CI run on each PR.
|
||||||
|
3. **Run all 3 reviewer types** on each PR and post results to Gitea via `tea pr approve/reject` or the Gitea API:
|
||||||
|
- `reviewer` — general code correctness
|
||||||
|
- `code-quality-guardian` — code quality, duplication, complexity
|
||||||
|
- `security-auditor` — vulnerability assessment
|
||||||
|
4. **Iterate**: if any reviewer requests changes, fix, push, re-review. Repeat until all 3 approve.
|
||||||
|
5. **Merge approved PRs** one at a time into `main`.
|
||||||
|
6. **Create a combined merge branch** for the remaining PRs that conflict with the new `main`:
|
||||||
|
```bash
|
||||||
|
git checkout -b merge-all origin/main
|
||||||
|
for branch in branch1 branch2 branch3; do
|
||||||
|
git merge origin/$branch --no-edit || true
|
||||||
|
# Resolve conflicts, build, test
|
||||||
|
done
|
||||||
|
```
|
||||||
|
7. **Run review again** on the combined branch. Fix issues, push, re-review until approved.
|
||||||
|
8. **Merge** the combined PR, **close** the redundant individual PRs, and **close all resolved issues** via the Gitea API:
|
||||||
|
```bash
|
||||||
|
curl -s -X PATCH -H "Authorization: token $TOKEN" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"state":"closed"}' \
|
||||||
|
"https://gitea.tap-tap.win/api/v1/repos/owner/repo/issues/<number>"
|
||||||
|
```
|
||||||
|
|
||||||
|
## CI Troubleshooting
|
||||||
|
|
||||||
|
### If lint (clang-format) fails
|
||||||
|
Run clang-format in the CI Docker image to match the exact CI version:
|
||||||
|
```bash
|
||||||
|
docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v9 \
|
||||||
|
sh -c 'find src/ tests/ -name "*.c" -o -name "*.h" | xargs clang-format -i'
|
||||||
|
```
|
||||||
|
|
||||||
|
### If cppcheck fails
|
||||||
|
Fix reported issues locally, then verify with:
|
||||||
|
```bash
|
||||||
|
docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v9 \
|
||||||
|
sh -c 'cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/'
|
||||||
|
```
|
||||||
|
|
||||||
|
### If integration tests fail
|
||||||
|
Run locally before pushing:
|
||||||
|
```bash
|
||||||
|
python3 -m pytest tests/ -v --tb=short
|
||||||
|
```
|
||||||
|
|
||||||
|
## Gitea API & tea CLI
|
||||||
|
|
||||||
|
### Check CI status via API
|
||||||
|
```bash
|
||||||
|
TOKEN="<token>"
|
||||||
|
curl -s -H "Authorization: token $TOKEN" \
|
||||||
|
"https://gitea.tap-tap.win/api/v1/repos/TapTap/FastSync/actions/runs?limit=5" \
|
||||||
|
| python3 -c "
|
||||||
|
import json,sys; d=json.load(sys.stdin)
|
||||||
|
for r in d.get('workflow_runs',[]):
|
||||||
|
path = r.get('path','')
|
||||||
|
prn = path.split('@')[1].replace('refs/pull/','').replace('/head','') if '@' in path else ''
|
||||||
|
print(f'PR #{prn}: sha={r[\"head_sha\"][:8]} {r[\"status\"]} {r.get(\"conclusion\",\"\")}')
|
||||||
|
"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Post review comments
|
||||||
|
```bash
|
||||||
|
curl -s -X POST -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
|
||||||
|
-d '{"body":"MARKDOWN_REVIEW_BODY"}' \
|
||||||
|
"https://gitea.tap-tap.win/api/v1/repos/TapTap/FastSync/issues/<PR_NUMBER>/comments"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Use tea for PR operations
|
||||||
|
```bash
|
||||||
|
tea pr list --repo TapTap/FastSync
|
||||||
|
tea pr close <number> --repo TapTap/FastSync
|
||||||
|
```
|
||||||
|
|
||||||
|
## Common pitfalls
|
||||||
|
|
||||||
|
- **`__thread` on shared SSL context**: io_ssl must NOT be thread-local — worker threads inherit the SSL context from the main thread. Use regular `static SSL* io_ssl`.
|
||||||
|
- **SSL WANT_READ/WANT_WRITE retry**: Always retry on `SSL_ERROR_WANT_READ` and `SSL_ERROR_WANT_WRITE` in `send_n_data`/`receive_n_data`. Removing these breaks TLS multithreaded transfers.
|
||||||
|
- **clang-format version**: The CI image uses clang-format 18. Always format inside the CI Docker container for exact match.
|
||||||
|
- **Merge order matters**: Merge the most comprehensive branch first, then smaller ones, to minimize conflicts when creating a combined branch.
|
||||||
|
|||||||
@@ -5,17 +5,26 @@ A high-performance file synchronization system with SSH and TCP transport, TLS e
|
|||||||
## Technical Overview
|
## Technical Overview
|
||||||
|
|
||||||
1. **Dual transport**: custom TCP client-server or SSH subprocess (rsync-style `user@host:/path`)
|
1. **Dual transport**: custom TCP client-server or SSH subprocess (rsync-style `user@host:/path`)
|
||||||
2. **TLS encryption**: OpenSSL-based TLS 1.2+ for encrypted TCP connections
|
2. **TLS encryption**: OpenSSL-based TLS 1.2+ for encrypted TCP connections with optional CA verification
|
||||||
3. **Chunked file transfer**: files grouped into configurable-size chunks (default ~10 MB)
|
3. **Chunked file transfer**: files grouped into configurable-size chunks (default ~10 MB)
|
||||||
4. **Streaming zstd compression** (levels 1–22) using `ZSTD_compressStream2`
|
4. **Streaming zstd compression** (levels 1–22) using `ZSTD_compressStream2`
|
||||||
5. **Multithreading**: producer-consumer pipeline with thread-safe queues (scanner → loader → sender)
|
5. **Multithreading**: producer-consumer pipeline with thread-safe queues (scanner → loader → sender)
|
||||||
6. **Incremental sync**: skip files unchanged since last transfer (compares size + mtime)
|
6. **Incremental sync**: skip files unchanged since last transfer (compares size + mtime)
|
||||||
7. **Metadata preservation**: `mode`, `uid`, `gid`, `mtime` restored on disk when enabled
|
7. **Batch incremental**: send incremental checks in batched groups for reduced round-trips
|
||||||
8. **`sendfile()` zero-copy** on TCP (~2× faster on loopback)
|
8. **Metadata preservation**: `mode`, `uid`, `gid`, `mtime` restored on disk when enabled
|
||||||
9. **SSH ControlMaster** for connection reuse across repeated invocations
|
9. **`sendfile()` zero-copy** on TCP (~2× faster on loopback)
|
||||||
10. **Bandwidth limiting**: token-bucket throttling (`--bwlimit`)
|
10. **SSH ControlMaster** for connection reuse across repeated invocations
|
||||||
11. **`--delete`**: receiver removes files not present in sender manifest
|
11. **Bandwidth limiting**: token-bucket throttling (`--bwlimit`)
|
||||||
12. **`--exclude` / `--include`**: glob-pattern filename filtering
|
12. **`--delete`**: receiver removes files not present in sender manifest
|
||||||
|
13. **`--exclude` / `--include`**: glob-pattern filename filtering
|
||||||
|
14. **Path traversal protection**: `..` sequences in file paths are rejected automatically
|
||||||
|
15. **Connection limits**: server enforces maximum concurrent connections (default 100)
|
||||||
|
16. **Keep-alive**: periodic `STATUS_KEEPALIVE` messages detect stalled connections
|
||||||
|
17. **Abort handling**: `SIGINT` sends `STATUS_ABORT` for clean server-side teardown
|
||||||
|
18. **Atomic writes**: received files are written to a temporary name then atomically renamed
|
||||||
|
19. **Backup mode**: `--backup` preserves overwritten files with optional `--backup-dir`
|
||||||
|
20. **Log file**: `--log-file` redirects log output to a file instead of stderr
|
||||||
|
21. **Transfer statistics**: `--stats` prints summary of transferred bytes, files, and timing
|
||||||
|
|
||||||
## System Architecture
|
## System Architecture
|
||||||
|
|
||||||
@@ -25,20 +34,33 @@ A high-performance file synchronization system with SSH and TCP transport, TLS e
|
|||||||
- Streaming zstd compression with configurable level
|
- Streaming zstd compression with configurable level
|
||||||
- Chunk serialization (compact binary format) or per-file transfer
|
- Chunk serialization (compact binary format) or per-file transfer
|
||||||
- Incremental transfer: sends file metadata to server, skips unchanged files
|
- Incremental transfer: sends file metadata to server, skips unchanged files
|
||||||
|
- Batch incremental: groups incremental checks to minimize round-trips
|
||||||
- Manifests all sent paths when `--delete` is active
|
- Manifests all sent paths when `--delete` is active
|
||||||
- Sends via TCP `sendfile()` or SSH pipe
|
- Sends via TCP `sendfile()` or SSH pipe
|
||||||
- Optional progress display with throughput
|
- Optional progress display with throughput
|
||||||
- Bandwidth limiting via token-bucket algorithm
|
- Bandwidth limiting via token-bucket algorithm
|
||||||
|
- Configurable I/O and connection timeouts (`--timeout`, `--contimeout`)
|
||||||
|
- Quiet mode (`-q`/`--quiet`) suppresses all non-error output
|
||||||
|
- Backup overwritten files (`--backup`) with optional directory (`--backup-dir`)
|
||||||
|
- Transfer statistics summary (`--stats`)
|
||||||
|
- Maximum directory depth control (`--max-depth`)
|
||||||
|
- Log file output (`--log-file`)
|
||||||
|
- Configurable multithreaded queue size (`--queue-size`)
|
||||||
|
- Exclude patterns from file (`--exclude-from`)
|
||||||
|
|
||||||
### Server
|
### Server
|
||||||
- TCP mode: listens on configurable port (default 8080); SSH mode: runs via `--stdio`
|
- TCP mode: listens on configurable port (default 8080); SSH mode: runs via `--stdio`
|
||||||
- TLS mode: wraps TCP connections with OpenSSL
|
- TLS mode: wraps TCP connections with OpenSSL with optional CA verification
|
||||||
- Receives and reassembles files
|
- Receives and reassembles files
|
||||||
- Decompresses (streaming zstd), deserializes, restores metadata
|
- Decompresses (streaming zstd), deserializes, restores metadata
|
||||||
- Handles incremental checks: compares size + mtime against destination files
|
- Handles incremental checks: compares size + mtime against destination files
|
||||||
|
- Handles batch incremental checks for reduced round-trips
|
||||||
- Processes `STATUS_MANIFEST` for `--delete`: walks destination tree, removes extras
|
- Processes `STATUS_MANIFEST` for `--delete`: walks destination tree, removes extras
|
||||||
- Per-connection concurrency via `fork()`
|
- Per-connection concurrency via `fork()` with configurable connection limit (default 100)
|
||||||
- Thread pool for parallel processing
|
- Thread pool for parallel processing
|
||||||
|
- Atomic writes: files written to `.tmp` path then atomically renamed on success
|
||||||
|
- Abort handling: cleanly shuts down on `STATUS_ABORT` from client
|
||||||
|
- Path traversal protection: rejects file paths containing `..`
|
||||||
|
|
||||||
## Protocol Details
|
## Protocol Details
|
||||||
|
|
||||||
@@ -52,6 +74,11 @@ A high-performance file synchronization system with SSH and TCP transport, TLS e
|
|||||||
| `STATUS_CHUNK` | Following data is a serialized chunk |
|
| `STATUS_CHUNK` | Following data is a serialized chunk |
|
||||||
| `STATUS_MANIFEST` | Following data is a file manifest (for `--delete`) |
|
| `STATUS_MANIFEST` | Following data is a file manifest (for `--delete`) |
|
||||||
| `STATUS_CHECK` | Incremental check: client sends file path + size + mtime, server responds with OK (skip) or NEXT (send) |
|
| `STATUS_CHECK` | Incremental check: client sends file path + size + mtime, server responds with OK (skip) or NEXT (send) |
|
||||||
|
| `STATUS_CHECK_BATCH` | Batch incremental check: multiple file checks sent in one message |
|
||||||
|
| `STATUS_KEEPALIVE` | Keep-alive heartbeat to detect stalled connections |
|
||||||
|
| `STATUS_ABORT` | Abort signal: client interrupts, server cleans up and exits |
|
||||||
|
| `STATUS_DELTA_SIGNATURE` | Delta sync: following data is a file signature (rsync-style rolling hash) |
|
||||||
|
| `STATUS_DELTA_DATA` | Delta sync: following data is a delta patch for a file |
|
||||||
|
|
||||||
### Wire Format — Metadata
|
### Wire Format — Metadata
|
||||||
|
|
||||||
@@ -59,12 +86,16 @@ When `use_metadata` is enabled (`-M`), each file entry carries a 4-byte `present
|
|||||||
|
|
||||||
### Transfer Flow
|
### Transfer Flow
|
||||||
```
|
```
|
||||||
Config → (STATUS_NEXT | STATUS_CHUNK | STATUS_CHECK)* → [STATUS_MANIFEST] → STATUS_FINISHED → STATUS_OK
|
Config → (STATUS_NEXT | STATUS_CHUNK | STATUS_CHECK | STATUS_CHECK_BATCH)* → [STATUS_MANIFEST] → STATUS_FINISHED → STATUS_OK
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Keep-alive (`STATUS_KEEPALIVE`) may be sent at any point during the transfer. The receiver resets its inactivity timer on receipt. If no data arrives within the receive timeout, the connection is aborted.
|
||||||
|
|
||||||
|
Abort (`STATUS_ABORT`) may be sent at any point. On receipt the server cleans up temporary files and exits the child process.
|
||||||
|
|
||||||
### Protocol Version
|
### Protocol Version
|
||||||
|
|
||||||
`1.1.0` — server and client must match. Mismatch results in `STATUS_ERROR`.
|
`1.3.0` — server and client must match. Mismatch results in `STATUS_ERROR`.
|
||||||
|
|
||||||
## Command-Line Arguments
|
## Command-Line Arguments
|
||||||
|
|
||||||
@@ -83,15 +114,26 @@ Config → (STATUS_NEXT | STATUS_CHUNK | STATUS_CHECK)* → [STATUS_MANIFEST]
|
|||||||
| `-n, --dry-run` | Scan and print what would be transferred |
|
| `-n, --dry-run` | Scan and print what would be transferred |
|
||||||
| `-p <port>` | SSH port (default: 22) |
|
| `-p <port>` | SSH port (default: 22) |
|
||||||
| `-v, --verbose` | Enable debug logging |
|
| `-v, --verbose` | Enable debug logging |
|
||||||
|
| `-q, --quiet` | Suppress all non-error output |
|
||||||
|
| `--silent` | Alias for `--quiet` |
|
||||||
| `--progress` | Show real-time transfer speed |
|
| `--progress` | Show real-time transfer speed |
|
||||||
| `--delete` | Delete files on receiver not present in source |
|
| `--delete` | Delete files on receiver not present in source |
|
||||||
| `--exclude <pattern>` | Exclude files matching glob pattern (repeatable) |
|
| `--exclude <pattern>` | Exclude files matching glob pattern (repeatable) |
|
||||||
|
| `--exclude-from <file>` | Read exclude patterns from a file (one per line) |
|
||||||
| `--include <pattern>` | Only transfer files matching glob pattern (repeatable, whitelist) |
|
| `--include <pattern>` | Only transfer files matching glob pattern (repeatable, whitelist) |
|
||||||
| `--max-size <n>` | Skip files larger than n bytes |
|
| `--max-size <n>` | Skip files larger than n bytes |
|
||||||
| `--min-size <n>` | Skip files smaller than n bytes |
|
| `--min-size <n>` | Skip files smaller than n bytes |
|
||||||
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `-s`. |
|
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `-s`. |
|
||||||
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
|
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
|
||||||
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
|
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
|
||||||
|
| `--timeout <sec>` | I/O timeout in seconds (default: 30) |
|
||||||
|
| `--contimeout <sec>` | Connection timeout in seconds (default: 10) |
|
||||||
|
| `--backup` | Backup existing destination files before overwriting |
|
||||||
|
| `--backup-dir <dir>` | Target directory for backups (requires `--backup`) |
|
||||||
|
| `--stats` | Print transfer statistics at end (bytes, files, timing) |
|
||||||
|
| `--max-depth <n>` | Maximum directory depth to recurse (0 = unlimited, default: 0) |
|
||||||
|
| `--log-file <path>` | Write log messages to file instead of stderr |
|
||||||
|
| `--queue-size <n>` | Queue capacity for multithreaded mode (default: 100) |
|
||||||
| `--source-dir <path>` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) |
|
| `--source-dir <path>` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) |
|
||||||
| `--dest-dir <path>` | Server destination directory (overrides `FASTSYNC_DEST_DIR`) |
|
| `--dest-dir <path>` | Server destination directory (overrides `FASTSYNC_DEST_DIR`) |
|
||||||
| `--save-to-disk` | Write received files to disk |
|
| `--save-to-disk` | Write received files to disk |
|
||||||
@@ -122,6 +164,12 @@ Config → (STATUS_NEXT | STATUS_CHUNK | STATUS_CHECK)* → [STATUS_MANIFEST]
|
|||||||
| `FASTSYNC_SOURCE_DIR` | — | Source directory fallback |
|
| `FASTSYNC_SOURCE_DIR` | — | Source directory fallback |
|
||||||
| `FASTSYNC_DEST_DIR` | — | Destination directory fallback |
|
| `FASTSYNC_DEST_DIR` | — | Destination directory fallback |
|
||||||
| `FASTSYNC_SAVE_TO_DISK` | `false` | Disk persistence fallback |
|
| `FASTSYNC_SAVE_TO_DISK` | `false` | Disk persistence fallback |
|
||||||
|
| `FASTSYNC_SSH_PORT` | `22` | Default SSH port |
|
||||||
|
| `FASTSYNC_SERVER_HOST` | `127.0.0.1` | Default server host |
|
||||||
|
| `FASTSYNC_SERVER_PORT` | `8080` | Default server port |
|
||||||
|
| `FASTSYNC_TLS_CERT` | — | Default TLS certificate path |
|
||||||
|
| `FASTSYNC_TLS_KEY` | — | Default TLS private key path |
|
||||||
|
| `FASTSYNC_TLS_CA` | — | Default TLS CA certificate path |
|
||||||
|
|
||||||
## Implementation Details
|
## Implementation Details
|
||||||
|
|
||||||
@@ -129,21 +177,44 @@ Config → (STATUS_NEXT | STATUS_CHUNK | STATUS_CHECK)* → [STATUS_MANIFEST]
|
|||||||
1. **Chunk** — collection of files (~10 MB total by default)
|
1. **Chunk** — collection of files (~10 MB total by default)
|
||||||
2. **File** — path, content (`Data`), optional `FileMetadata` pointer
|
2. **File** — path, content (`Data`), optional `FileMetadata` pointer
|
||||||
3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`
|
3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`
|
||||||
4. **Config** — runtime parameters (transported over wire, TLS settings excluded)
|
4. **Config** — runtime parameters (transported over wire, TLS settings excluded). Includes `timeout`, `contimeout`, `quiet`, `backup`, `backup_dir`, `stats`, `max_depth`, `log_file`, `queue_size`.
|
||||||
5. **Queue** — thread-safe bounded queue with condition variables
|
5. **Queue** — thread-safe bounded queue with condition variables
|
||||||
6. **DirectoryScanner** — recursive BFS traversal with exclude and include pattern support
|
6. **DirectoryScanner** — recursive BFS traversal with exclude and include pattern support, max-depth enforcement
|
||||||
|
|
||||||
### Key Algorithms
|
### Key Algorithms
|
||||||
1. **File scanning** — BFS directory traversal; entries matched against exclude and include patterns
|
1. **File scanning** — BFS directory traversal; entries matched against exclude and include patterns, max-depth enforced
|
||||||
2. **Chunking** — files accumulated until `chunk_size` threshold, then flushed
|
2. **Chunking** — files accumulated until `chunk_size` threshold, then flushed
|
||||||
3. **Compression** — streaming zstd via `ZSTD_compressStream2` / `ZSTD_decompressStream`
|
3. **Compression** — streaming zstd via `ZSTD_compressStream2` / `ZSTD_decompressStream`
|
||||||
4. **Network protocol** — status-code-driven exchange with metadata packing
|
4. **Network protocol** — status-code-driven exchange with metadata packing, keep-alive, and abort support
|
||||||
5. **Incremental check** — client sends `STATUS_CHECK` + path + size + mtime; server compares against destination
|
5. **Incremental check** — client sends `STATUS_CHECK` + path + size + mtime; server compares against destination. Can be batched via `STATUS_CHECK_BATCH` for reduced round-trips.
|
||||||
6. **Bandwidth limiting** — token-bucket algorithm with `nanosleep` throttling on 64 KB write chunks
|
6. **Bandwidth limiting** — token-bucket algorithm with `nanosleep` throttling on 64 KB write chunks
|
||||||
7. **Metadata restoration** — `chmod()`, `chown()`, `utimensat()` on the receiving side
|
7. **Metadata restoration** — `chmod()`, `chown()`, `utimensat()` on the receiving side
|
||||||
8. **`--delete`** — sender tracks all sent paths; receiver walks destination tree and removes unlisted files/directories
|
8. **`--delete`** — sender tracks all sent paths; receiver walks destination tree and removes unlisted files/directories
|
||||||
9. **SSH transport** — `socketpair()` + `fork()` + `execvp("ssh", ...)` with `ControlMaster` and port support
|
9. **SSH transport** — `socketpair()` + `fork()` + `execvp("ssh", ...)` with `ControlMaster` and port support
|
||||||
10. **TLS transport** — OpenSSL `SSL_CTX` with TLS 1.2 minimum, optional CA verification, transparent `SSL_read`/`SSL_write` via `io_set_ssl()`
|
10. **TLS transport** — OpenSSL `SSL_CTX` with TLS 1.2 minimum, optional CA verification, transparent `SSL_read`/`SSL_write` via `io_set_ssl()`
|
||||||
|
11. **Path traversal protection** — `has_path_traversal()` rejects any file path containing `..` components, preventing directory escape attacks
|
||||||
|
12. **Connection limiting** — server tracks active connections and rejects new ones beyond `max_connections` (default 100)
|
||||||
|
13. **Keep-alive** — idle connections receive periodic `STATUS_KEEPALIVE` to detect half-open TCP connections
|
||||||
|
14. **Abort handling** — `SIGINT` sets an abort flag; the next protocol operation sends `STATUS_ABORT` for clean server cleanup
|
||||||
|
15. **Atomic writes** — files are written to a `.tmp` suffix then atomically renamed via `rename()`, preventing partial files
|
||||||
|
16. **Backup** — before overwriting, existing files are moved to `--backup-dir` (or same directory with `~` suffix) preserving the original
|
||||||
|
|
||||||
|
## Security Features
|
||||||
|
|
||||||
|
### Path Traversal Protection
|
||||||
|
All received file paths are validated by `has_path_traversal()` before any disk operation. Any path containing `..` components is rejected with `STATUS_ERROR`, preventing directory escape attacks.
|
||||||
|
|
||||||
|
### TLS Certificate Verification
|
||||||
|
When `--ca` is provided, the server performs mutual TLS verification (`SSL_VERIFY_PEER` with depth 4). Without `--ca`, TLS is still encrypted but peer certificates are not verified.
|
||||||
|
|
||||||
|
### Connection Limits
|
||||||
|
The server enforces a maximum of 100 concurrent connections (configurable via `max_connections` in `Server`). When the limit is reached, new connections are immediately rejected and closed.
|
||||||
|
|
||||||
|
### Abort Handling
|
||||||
|
If the client receives `SIGINT` (Ctrl+C) during a transfer, it sends `STATUS_ABORT` to the server. The server then cleans up temporary files and exits the child process, preventing incomplete files from remaining on disk.
|
||||||
|
|
||||||
|
### Atomic Writes
|
||||||
|
Received files are written to a temporary path (suffixed with `.tmp`) and then atomically renamed to the final filename via `rename()`. This prevents partial or corrupted files from appearing at the destination if the transfer is interrupted.
|
||||||
|
|
||||||
## Build Requirements
|
## Build Requirements
|
||||||
|
|
||||||
@@ -223,6 +294,21 @@ Place the `fastsync-server` binary in the remote `$PATH`. The client runs `ssh u
|
|||||||
# Bandwidth limit to 1 MB/s
|
# Bandwidth limit to 1 MB/s
|
||||||
./build/client --bwlimit 1024 /src user@host:/dst
|
./build/client --bwlimit 1024 /src user@host:/dst
|
||||||
|
|
||||||
|
# With timeouts, quiet mode, and stats
|
||||||
|
./build/client --timeout 60 --contimeout 15 --quiet --stats /src user@host:/dst
|
||||||
|
|
||||||
|
# Backup overwritten files to a directory
|
||||||
|
./build/client --backup --backup-dir /backups /src user@host:/dst
|
||||||
|
|
||||||
|
# Exclude patterns from file, limit depth
|
||||||
|
./build/client --exclude-from ignore.txt --max-depth 3 /src user@host:/dst
|
||||||
|
|
||||||
|
# Custom queue size for multithreading
|
||||||
|
./build/client -m --queue-size 200 /src user@host:/dst
|
||||||
|
|
||||||
|
# Log to file
|
||||||
|
./build/client --log-file /tmp/fastsync.log /src user@host:/dst
|
||||||
|
|
||||||
# All features
|
# All features
|
||||||
./build/client -a --progress --chunk-size 5242880 --exclude "*.log" --delete /src /dst
|
./build/client -a --progress --chunk-size 5242880 --exclude "*.log" --delete /src /dst
|
||||||
```
|
```
|
||||||
@@ -230,7 +316,9 @@ Place the `fastsync-server` binary in the remote `$PATH`. The client runs `ssh u
|
|||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Unit tests (7 suites)
|
# Unit tests (18 suites — array_list, chunk, compression, config, data, delta, file, glob,
|
||||||
|
# metadata, property, protocol, queue, robustness, scanner,
|
||||||
|
# shared_utils, stress, transport_tcp, transport_ssh, transport_tls)
|
||||||
./build/tests
|
./build/tests
|
||||||
|
|
||||||
# Integration + benchmark suite
|
# Integration + benchmark suite
|
||||||
@@ -244,12 +332,15 @@ The benchmark prints throughput metrics, best configuration, and speedup vs rsyn
|
|||||||
1. Chunk size (~10 MB default) balances memory and transfer efficiency
|
1. Chunk size (~10 MB default) balances memory and transfer efficiency
|
||||||
2. Compression level trades CPU for bandwidth
|
2. Compression level trades CPU for bandwidth
|
||||||
3. `sendfile()` bypasses userspace — ~2× faster on localhost for large files
|
3. `sendfile()` bypasses userspace — ~2× faster on localhost for large files
|
||||||
4. Multithreading scales with core count
|
4. Multithreading scales with core count; `--queue-size` controls pipeline buffering
|
||||||
5. Metadata transfer adds negligible overhead (~24 bytes per file when enabled)
|
5. Metadata transfer adds negligible overhead (~24 bytes per file when enabled)
|
||||||
6. SSH socketpair buffer set to 1 MB for improved pipe throughput
|
6. SSH socketpair buffer set to 1 MB for improved pipe throughput
|
||||||
7. SSH ControlMaster reuses connections across repeated invocations
|
7. SSH ControlMaster reuses connections across repeated invocations
|
||||||
8. Incremental sync eliminates redundant transfers entirely
|
8. Incremental sync eliminates redundant transfers entirely
|
||||||
9. Bandwidth limiting uses token-bucket with nanosleep for accurate throttling
|
9. Batch incremental reduces round-trips by grouping multiple checks into one message
|
||||||
|
10. Bandwidth limiting uses token-bucket with nanosleep for accurate throttling
|
||||||
|
11. Atomic writes add a single `rename()` per file — negligible overhead
|
||||||
|
12. Path traversal check is O(n) in path length with negligible cost
|
||||||
|
|
||||||
## Benchmark Results
|
## Benchmark Results
|
||||||
|
|
||||||
|
|||||||
+73
-67
@@ -68,6 +68,9 @@ static void print_usage(void) {
|
|||||||
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
|
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
|
||||||
printf(" --log-file <path> Write log messages to file\n");
|
printf(" --log-file <path> Write log messages to file\n");
|
||||||
printf(" --queue-size <n> Queue capacity for multithreaded mode (default: 100)\n");
|
printf(" --queue-size <n> Queue capacity for multithreaded mode (default: 100)\n");
|
||||||
|
printf(" --partial Keep partial files on interrupted transfer\n");
|
||||||
|
printf(" --fastsync-server-path <path>\n");
|
||||||
|
printf(" Path to fastsync-server on remote (default: fastsync-server)\n");
|
||||||
printf(" --help Show this help\n");
|
printf(" --help Show this help\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -102,21 +105,72 @@ static int read_patterns_from_file(const char* filepath, char*** patterns, int*
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool validate_config(Config* config) {
|
||||||
|
if (!config->send_directory || !config->receive_root_directory) {
|
||||||
|
fprintf(stderr, "Error: source and destination directories are required\n");
|
||||||
|
print_usage();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (config->use_sendfile && (config->use_chunk_serialization || config->use_compression)) {
|
||||||
|
fprintf(stderr, "Error: -f/--sendfile cannot be combined with -c (compression) or -s "
|
||||||
|
"(chunk serialization)\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
|
||||||
|
fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (config->use_incremental && config->use_chunk_serialization) {
|
||||||
|
fprintf(stderr, "Error: --incremental is not supported with -s (chunk serialization)\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (config->use_incremental && !config->use_metadata) {
|
||||||
|
log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --incremental");
|
||||||
|
config->use_metadata = true;
|
||||||
|
}
|
||||||
|
if (config->use_delta && !config->use_incremental) {
|
||||||
|
fprintf(stderr, "Error: --delta requires --incremental\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (config->use_delta && config->use_chunk_serialization) {
|
||||||
|
fprintf(stderr, "Error: --delta cannot be combined with -s (chunk serialization)\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (config->use_delta && config->use_sendfile) {
|
||||||
|
fprintf(stderr, "Error: --delta cannot be combined with -f (sendfile)\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (config->use_delta && !config->use_metadata) {
|
||||||
|
log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --delta");
|
||||||
|
config->use_metadata = true;
|
||||||
|
}
|
||||||
|
if (config->use_tls) {
|
||||||
|
if (!config->tls_cert || !config->tls_key) {
|
||||||
|
fprintf(stderr, "Error: --tls requires --cert and --key\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
tls_global_init();
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
int main(int argc, char* argv[]) {
|
int main(int argc, char* argv[]) {
|
||||||
const char* env_source = getenv("FASTSYNC_SOURCE_DIR");
|
const char* env_source = getenv("FASTSYNC_SOURCE_DIR");
|
||||||
const char* env_dest = getenv("FASTSYNC_DEST_DIR");
|
const char* env_dest = getenv("FASTSYNC_DEST_DIR");
|
||||||
const char* env_save = getenv("FASTSYNC_SAVE_TO_DISK");
|
const char* env_save = getenv("FASTSYNC_SAVE_TO_DISK");
|
||||||
|
|
||||||
bool save_to_disk = false;
|
|
||||||
if (env_save && (strcmp(env_save, "true") == 0 || strcmp(env_save, "1") == 0)) {
|
|
||||||
save_to_disk = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
Config* config = config_create(str_dup(PROTOCOL_VERSION), NULL, NULL, save_to_disk, false, false,
|
|
||||||
false, false, 5, false, 0);
|
|
||||||
int exit_code = 0;
|
int exit_code = 0;
|
||||||
|
Config* config = NULL;
|
||||||
bool config_owned_by_pipeline = false;
|
bool config_owned_by_pipeline = false;
|
||||||
|
|
||||||
|
config = config_create();
|
||||||
|
if (!config) {
|
||||||
|
exit_code = 1;
|
||||||
|
goto cleanup;
|
||||||
|
}
|
||||||
|
if (env_save && (strcmp(env_save, "true") == 0 || strcmp(env_save, "1") == 0))
|
||||||
|
config->save_to_disk = true;
|
||||||
|
|
||||||
int positional_args[2];
|
int positional_args[2];
|
||||||
int positional_count = 0;
|
int positional_count = 0;
|
||||||
|
|
||||||
@@ -301,6 +355,11 @@ int main(int argc, char* argv[]) {
|
|||||||
exit_code = 1;
|
exit_code = 1;
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
|
} else if (strcmp(argv[i], "--partial") == 0) {
|
||||||
|
config->partial = true;
|
||||||
|
} else if (strcmp(argv[i], "--fastsync-server-path") == 0 && i + 1 < argc) {
|
||||||
|
free(config->fastsync_server_path);
|
||||||
|
config->fastsync_server_path = str_dup(argv[++i]);
|
||||||
} else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
|
} else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
|
||||||
set_log_level(LOG_LEVEL_DEBUG);
|
set_log_level(LOG_LEVEL_DEBUG);
|
||||||
} else if (argv[i][0] == '-') {
|
} else if (argv[i][0] == '-') {
|
||||||
@@ -326,7 +385,6 @@ int main(int argc, char* argv[]) {
|
|||||||
config->send_directory = str_dup(argv[positional_args[0]]);
|
config->send_directory = str_dup(argv[positional_args[0]]);
|
||||||
config->receive_root_directory = str_dup(argv[positional_args[1]]);
|
config->receive_root_directory = str_dup(argv[positional_args[1]]);
|
||||||
config->save_to_disk = true;
|
config->save_to_disk = true;
|
||||||
|
|
||||||
config_parse_ssh_dest(config);
|
config_parse_ssh_dest(config);
|
||||||
} else if (positional_count == 1) {
|
} else if (positional_count == 1) {
|
||||||
fprintf(stderr, "Error: missing destination argument\n");
|
fprintf(stderr, "Error: missing destination argument\n");
|
||||||
@@ -340,64 +398,10 @@ int main(int argc, char* argv[]) {
|
|||||||
config->receive_root_directory = str_dup((char*)env_dest);
|
config->receive_root_directory = str_dup((char*)env_dest);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!config->send_directory || !config->receive_root_directory) {
|
if (!validate_config(config)) {
|
||||||
fprintf(stderr, "Error: source and destination directories are required\n");
|
|
||||||
print_usage();
|
|
||||||
exit_code = 1;
|
exit_code = 1;
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
if (config->use_sendfile && (config->use_chunk_serialization || config->use_compression)) {
|
|
||||||
fprintf(stderr, "Error: -f/--sendfile cannot be combined with -c (compression) or -s (chunk "
|
|
||||||
"serialization)\n");
|
|
||||||
exit_code = 1;
|
|
||||||
goto cleanup;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
|
|
||||||
fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
|
|
||||||
exit_code = 1;
|
|
||||||
goto cleanup;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config->use_incremental && config->use_chunk_serialization) {
|
|
||||||
fprintf(stderr, "Error: --incremental is not supported with -s (chunk serialization)\n");
|
|
||||||
exit_code = 1;
|
|
||||||
goto cleanup;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config->use_incremental && !config->use_metadata) {
|
|
||||||
log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --incremental");
|
|
||||||
config->use_metadata = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config->use_delta && !config->use_incremental) {
|
|
||||||
fprintf(stderr, "Error: --delta requires --incremental\n");
|
|
||||||
exit_code = 1;
|
|
||||||
goto cleanup;
|
|
||||||
}
|
|
||||||
if (config->use_delta && config->use_chunk_serialization) {
|
|
||||||
fprintf(stderr, "Error: --delta cannot be combined with -s (chunk serialization)\n");
|
|
||||||
exit_code = 1;
|
|
||||||
goto cleanup;
|
|
||||||
}
|
|
||||||
if (config->use_delta && config->use_sendfile) {
|
|
||||||
fprintf(stderr, "Error: --delta cannot be combined with -f (sendfile)\n");
|
|
||||||
exit_code = 1;
|
|
||||||
goto cleanup;
|
|
||||||
}
|
|
||||||
if (config->use_delta && !config->use_metadata) {
|
|
||||||
log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --delta");
|
|
||||||
config->use_metadata = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (config->use_tls) {
|
|
||||||
if (!config->tls_cert || !config->tls_key) {
|
|
||||||
fprintf(stderr, "Error: --tls requires --cert and --key\n");
|
|
||||||
exit_code = 1;
|
|
||||||
goto cleanup;
|
|
||||||
}
|
|
||||||
tls_global_init();
|
|
||||||
}
|
|
||||||
|
|
||||||
tcp_set_timeouts(config->timeout, config->contimeout);
|
tcp_set_timeouts(config->timeout, config->contimeout);
|
||||||
|
|
||||||
@@ -409,9 +413,11 @@ int main(int argc, char* argv[]) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
cleanup:
|
cleanup:
|
||||||
if (config->log_file)
|
if (config) {
|
||||||
fclose(config->log_file);
|
if (config->log_file)
|
||||||
if (!config_owned_by_pipeline)
|
fclose(config->log_file);
|
||||||
config_delete(config);
|
if (!config_owned_by_pipeline)
|
||||||
|
config_delete(config);
|
||||||
|
}
|
||||||
return exit_code;
|
return exit_code;
|
||||||
}
|
}
|
||||||
|
|||||||
+144
-260
@@ -16,22 +16,14 @@
|
|||||||
#include "transport_ssh.h"
|
#include "transport_ssh.h"
|
||||||
#include "transport_tls.h"
|
#include "transport_tls.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <signal.h>
|
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <threads.h>
|
#include <threads.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
static volatile sig_atomic_t g_abort_requested = 0;
|
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
|
||||||
static int g_abort_fd = -1;
|
|
||||||
|
|
||||||
static void handle_sigint(int sig) {
|
|
||||||
(void)sig;
|
|
||||||
g_abort_requested = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
#define KEEPALIVE_INTERVAL 30
|
|
||||||
|
|
||||||
static int incremental_check(Client* client, File* file, DeltaSignature** out_sig) {
|
static int incremental_check(Client* client, File* file, DeltaSignature** out_sig) {
|
||||||
*out_sig = NULL;
|
*out_sig = NULL;
|
||||||
@@ -107,35 +99,6 @@ static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* c
|
|||||||
return ok ? 0 : -1;
|
return ok ? 0 : -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool batch_incremental_check(Client* client, ArrayList* files) {
|
|
||||||
if (!send_status(client->file_descriptor, STATUS_CHECK_BATCH))
|
|
||||||
return false;
|
|
||||||
if (!send_int(client->file_descriptor, files->size))
|
|
||||||
return false;
|
|
||||||
for (int i = 0; i < files->size; i++) {
|
|
||||||
File* file = (File*)files->items[i];
|
|
||||||
if (!send_str(client->file_descriptor, file->path))
|
|
||||||
return false;
|
|
||||||
unsigned long long fsize = file->data ? file->data->size : 0;
|
|
||||||
long long mtime = file->metadata ? file->metadata->mtime_sec : 0;
|
|
||||||
if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize)))
|
|
||||||
return false;
|
|
||||||
if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime)))
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
for (int i = 0; i < files->size; i++) {
|
|
||||||
Status s;
|
|
||||||
if (!receive_status(client->file_descriptor, &s))
|
|
||||||
return false;
|
|
||||||
File* file = (File*)files->items[i];
|
|
||||||
if (s == STATUS_OK)
|
|
||||||
file->skip = true;
|
|
||||||
else if (s == STATUS_ERROR)
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
typedef bool (*file_send_fn)(File*, int, bool, int, bool);
|
typedef bool (*file_send_fn)(File*, int, bool, int, bool);
|
||||||
|
|
||||||
// Send a single file directly (non-incremental path).
|
// Send a single file directly (non-incremental path).
|
||||||
@@ -158,9 +121,6 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
bool use_sendfile) {
|
bool use_sendfile) {
|
||||||
int compression_level = config->use_compression ? config->compression_level : 0;
|
int compression_level = config->use_compression ? config->compression_level : 0;
|
||||||
|
|
||||||
if (file->skip)
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
if (!use_incremental) {
|
if (!use_incremental) {
|
||||||
if (use_sendfile) {
|
if (use_sendfile) {
|
||||||
return send_file_direct_sendfile(file, client->file_descriptor, config->use_metadata) ? 0
|
return send_file_direct_sendfile(file, client->file_descriptor, config->use_metadata) ? 0
|
||||||
@@ -251,10 +211,13 @@ int send_chunk(Client* client, Chunk* chunk, Config* config) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool use_sendfile = config->use_sendfile && !config->use_compression;
|
|
||||||
for (int i = 0; i < chunk->element_count; i++) {
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
int rc =
|
File* f = chunk->items[i];
|
||||||
send_single_file(client, chunk->items[i], config, config->use_incremental, use_sendfile);
|
if (f == NULL)
|
||||||
|
continue;
|
||||||
|
bool stream = f->data->data == NULL && f->data->size > 0;
|
||||||
|
bool use_sendfile = (config->use_sendfile && !config->use_compression) || stream;
|
||||||
|
int rc = send_single_file(client, f, config, config->use_incremental, use_sendfile);
|
||||||
if (rc == 1)
|
if (rc == 1)
|
||||||
continue;
|
continue;
|
||||||
if (rc < 0)
|
if (rc < 0)
|
||||||
@@ -271,7 +234,8 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
|
fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
client = client_connect_ssh(context->config->ssh_destination, context->config->ssh_port);
|
client = client_connect_ssh(context->config->ssh_destination, context->config->ssh_port,
|
||||||
|
context->config->fastsync_server_path);
|
||||||
} else if (context->config->use_tls) {
|
} else if (context->config->use_tls) {
|
||||||
client = client_create();
|
client = client_create();
|
||||||
if (!client || !client_connect_tls(client, context->config->server_host,
|
if (!client || !client_connect_tls(client, context->config->server_host,
|
||||||
@@ -298,30 +262,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
|
|
||||||
g_abort_fd = client->file_descriptor;
|
|
||||||
time_t last_activity = time(NULL);
|
|
||||||
while (true) {
|
while (true) {
|
||||||
if (g_abort_requested) {
|
|
||||||
send_status(client->file_descriptor, STATUS_ABORT);
|
|
||||||
client_disconnect(client);
|
|
||||||
client_delete(client);
|
|
||||||
return thrd_error;
|
|
||||||
}
|
|
||||||
time_t now = time(NULL);
|
|
||||||
if (now - last_activity >= KEEPALIVE_INTERVAL) {
|
|
||||||
if (!send_status(client->file_descriptor, STATUS_KEEPALIVE)) {
|
|
||||||
client_disconnect(client);
|
|
||||||
client_delete(client);
|
|
||||||
return thrd_error;
|
|
||||||
}
|
|
||||||
Status s;
|
|
||||||
if (!receive_status(client->file_descriptor, &s)) {
|
|
||||||
client_disconnect(client);
|
|
||||||
client_delete(client);
|
|
||||||
return thrd_error;
|
|
||||||
}
|
|
||||||
last_activity = now;
|
|
||||||
}
|
|
||||||
Chunk* current_chunk = queue_dequeue_multithreaded(
|
Chunk* current_chunk = queue_dequeue_multithreaded(
|
||||||
context->queue_loader, &context->mutex_loader, &context->condition_not_empty_loader,
|
context->queue_loader, &context->mutex_loader, &context->condition_not_empty_loader,
|
||||||
&context->condition_not_full_loader, &context->loader_done);
|
&context->condition_not_full_loader, &context->loader_done);
|
||||||
@@ -361,16 +302,14 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
|
|
||||||
static int scan_directory_multithreaded(void* pipeline_context) {
|
static int scan_directory_multithreaded(void* pipeline_context) {
|
||||||
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
||||||
mtx_lock(&context->mutex_scanner);
|
ParallelScanner* scanner = parallel_scanner_create(
|
||||||
DirectoryScanner* scanner = directory_scanner_create(
|
|
||||||
context->config->send_directory, context->config->use_metadata, context->config->chunk_size,
|
context->config->send_directory, context->config->use_metadata, context->config->chunk_size,
|
||||||
context->config->exclude_patterns, context->config->exclude_count,
|
context->config->exclude_patterns, context->config->exclude_count,
|
||||||
context->config->include_patterns, context->config->include_count, context->config->max_size,
|
context->config->include_patterns, context->config->include_count, context->config->max_size,
|
||||||
context->config->min_size, context->config->max_depth);
|
context->config->min_size, context->config->max_depth, 4);
|
||||||
mtx_unlock(&context->mutex_scanner);
|
|
||||||
|
|
||||||
Chunk* current_chunk;
|
Chunk* current_chunk;
|
||||||
while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
|
while ((current_chunk = parallel_scanner_next(scanner)) != NULL) {
|
||||||
if (context->config->use_delete) {
|
if (context->config->use_delete) {
|
||||||
mtx_lock(&context->mutex_scanner);
|
mtx_lock(&context->mutex_scanner);
|
||||||
for (int i = 0; i < current_chunk->element_count; i++) {
|
for (int i = 0; i < current_chunk->element_count; i++) {
|
||||||
@@ -390,7 +329,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
|||||||
cnd_signal(&context->condition_not_empty_scanner);
|
cnd_signal(&context->condition_not_empty_scanner);
|
||||||
mtx_unlock(&context->mutex_scanner);
|
mtx_unlock(&context->mutex_scanner);
|
||||||
|
|
||||||
directory_scanner_destroy(scanner);
|
parallel_scanner_destroy(scanner);
|
||||||
return thrd_success;
|
return thrd_success;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -409,9 +348,12 @@ static int load_files_multithreaded(void* pipeline_context) {
|
|||||||
}
|
}
|
||||||
if (!context->config->use_sendfile) {
|
if (!context->config->use_sendfile) {
|
||||||
for (int i = 0; i < chunk->element_count; i++) {
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
if (!file_load_data(chunk->items[i])) {
|
File* f = chunk->items[i];
|
||||||
|
if (f->data->size > STREAM_THRESHOLD)
|
||||||
|
continue;
|
||||||
|
if (!file_load_data(f)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to load file data, skipping");
|
log_message(LOG_LEVEL_ERROR, "Failed to load file data, skipping");
|
||||||
file_destroy(chunk->items[i]);
|
file_destroy(f);
|
||||||
chunk->items[i] = NULL;
|
chunk->items[i] = NULL;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -422,227 +364,175 @@ static int load_files_multithreaded(void* pipeline_context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
int send_files(Config* config) {
|
static int run_dry_run(Config* config) {
|
||||||
if (config->dry_run) {
|
DirectoryScanner* scanner = directory_scanner_create(
|
||||||
DirectoryScanner* scanner = directory_scanner_create(
|
config->send_directory, config->use_metadata, config->chunk_size, config->exclude_patterns,
|
||||||
config->send_directory, config->use_metadata, config->chunk_size, config->exclude_patterns,
|
config->exclude_count, config->include_patterns, config->include_count, config->max_size,
|
||||||
config->exclude_count, config->include_patterns, config->include_count, config->max_size,
|
config->min_size, config->max_depth);
|
||||||
config->min_size, config->max_depth);
|
if (!scanner)
|
||||||
Chunk* chunk;
|
return -1;
|
||||||
int file_count = 0;
|
Chunk* chunk;
|
||||||
unsigned long long total_bytes = 0;
|
int file_count = 0;
|
||||||
if (!config->quiet)
|
unsigned long long total_bytes = 0;
|
||||||
printf("Dry run: files to be transferred\n");
|
printf("Dry run: files to be transferred\n");
|
||||||
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||||
for (int i = 0; i < chunk->element_count; i++) {
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
if (!config->quiet)
|
printf(" %s (%zu bytes)\n", chunk->items[i]->path, chunk->items[i]->data->size);
|
||||||
printf(" %s (%zu bytes)\n", chunk->items[i]->path, chunk->items[i]->data->size);
|
total_bytes += chunk->items[i]->data->size;
|
||||||
total_bytes += chunk->items[i]->data->size;
|
file_count++;
|
||||||
file_count++;
|
|
||||||
}
|
|
||||||
chunk_destroy(chunk);
|
|
||||||
}
|
}
|
||||||
directory_scanner_destroy(scanner);
|
chunk_destroy(chunk);
|
||||||
if (!config->quiet)
|
|
||||||
printf("Total: %d files, %.1f MB\n", file_count, total_bytes / 1048576.0);
|
|
||||||
return 0;
|
|
||||||
}
|
}
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
printf("Total: %d files, %.1f MB\n", file_count, total_bytes / 1048576.0);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
Client* client;
|
static Client* connect_to_server(Config* config) {
|
||||||
if (config->transport == TRANSPORT_SSH) {
|
if (config->transport == TRANSPORT_SSH) {
|
||||||
if (config->use_sendfile) {
|
if (config->use_sendfile) {
|
||||||
fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
|
fprintf(stderr, "Error: -f/--sendfile is not supported with SSH transport\n");
|
||||||
return 1;
|
return NULL;
|
||||||
}
|
|
||||||
client = client_connect_ssh(config->ssh_destination, config->ssh_port);
|
|
||||||
if (!client)
|
|
||||||
return 1;
|
|
||||||
} else if (config->use_tls) {
|
|
||||||
client = client_create();
|
|
||||||
if (!client || !client_connect_tls(client, config->server_host, config->server_port,
|
|
||||||
config->tls_cert, config->tls_key, config->tls_ca)) {
|
|
||||||
if (client)
|
|
||||||
client_delete(client);
|
|
||||||
fprintf(stderr, "Error: could not connect to server via TLS\n");
|
|
||||||
return 1;
|
|
||||||
}
|
}
|
||||||
|
return client_connect_ssh(config->ssh_destination, config->ssh_port,
|
||||||
|
config->fastsync_server_path);
|
||||||
|
}
|
||||||
|
Client* client = client_create();
|
||||||
|
if (!client)
|
||||||
|
return NULL;
|
||||||
|
bool ok;
|
||||||
|
if (config->use_tls) {
|
||||||
|
ok = client_connect_tls(client, config->server_host, config->server_port, config->tls_cert,
|
||||||
|
config->tls_key, config->tls_ca);
|
||||||
} else {
|
} else {
|
||||||
client = client_create();
|
ok = client_connect(client, config->server_host, config->server_port);
|
||||||
if (!client || !client_connect(client, config->server_host, config->server_port)) {
|
|
||||||
if (client)
|
|
||||||
client_delete(client);
|
|
||||||
fprintf(stderr, "Error: could not connect to server\n");
|
|
||||||
return 1;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (!config_send(client->file_descriptor, config)) {
|
if (!ok) {
|
||||||
client_disconnect(client);
|
|
||||||
client_delete(client);
|
client_delete(client);
|
||||||
return 1;
|
fprintf(stderr, "Error: could not connect to server\n");
|
||||||
|
return NULL;
|
||||||
}
|
}
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
|
||||||
g_abort_fd = client->file_descriptor;
|
static bool send_manifest(int fd, ArrayList* manifest) {
|
||||||
struct sigaction sa;
|
if (!send_status(fd, STATUS_MANIFEST))
|
||||||
memset(&sa, 0, sizeof(sa));
|
return false;
|
||||||
sa.sa_handler = handle_sigint;
|
if (!send_int(fd, manifest->size))
|
||||||
sigaction(SIGINT, &sa, NULL);
|
return false;
|
||||||
sigaction(SIGTERM, &sa, NULL);
|
for (int i = 0; i < manifest->size; i++) {
|
||||||
|
if (!send_str(fd, (char*)manifest->items[i]))
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void print_progress(unsigned long long total_bytes, time_t start) {
|
||||||
|
double elapsed = difftime(time(NULL), start);
|
||||||
|
double rate = elapsed > 0 ? total_bytes / (1048576.0 * elapsed) : 0;
|
||||||
|
fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) ", total_bytes / 1048576.0, rate);
|
||||||
|
fflush(stderr);
|
||||||
|
}
|
||||||
|
|
||||||
|
int send_files(Config* config) {
|
||||||
|
if (config->dry_run)
|
||||||
|
return run_dry_run(config);
|
||||||
|
|
||||||
|
Client* client = connect_to_server(config);
|
||||||
|
if (!client)
|
||||||
|
return 1;
|
||||||
|
if (!config_send(client->file_descriptor, config))
|
||||||
|
goto send_fail;
|
||||||
|
|
||||||
DirectoryScanner* scanner = directory_scanner_create(
|
DirectoryScanner* scanner = directory_scanner_create(
|
||||||
config->send_directory, config->use_metadata, config->chunk_size, config->exclude_patterns,
|
config->send_directory, config->use_metadata, config->chunk_size, config->exclude_patterns,
|
||||||
config->exclude_count, config->include_patterns, config->include_count, config->max_size,
|
config->exclude_count, config->include_patterns, config->include_count, config->max_size,
|
||||||
config->min_size, config->max_depth);
|
config->min_size, config->max_depth);
|
||||||
ArrayList* all_files = array_list_create(NULL);
|
|
||||||
ArrayList* manifest = config->use_delete ? array_list_create(free) : NULL;
|
|
||||||
Chunk* current_chunk;
|
Chunk* current_chunk;
|
||||||
|
unsigned long long total_bytes = 0;
|
||||||
|
time_t last_progress = 0;
|
||||||
|
time_t start = time(NULL);
|
||||||
|
ArrayList* manifest = config->use_delete ? array_list_create(free) : NULL;
|
||||||
while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
|
while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
|
||||||
|
unsigned long long chunk_bytes = 0;
|
||||||
for (int i = 0; i < current_chunk->element_count; i++) {
|
for (int i = 0; i < current_chunk->element_count; i++) {
|
||||||
File* f = current_chunk->items[i];
|
chunk_bytes += current_chunk->items[i]->data->size;
|
||||||
array_list_add(all_files, f);
|
|
||||||
current_chunk->items[i] = NULL;
|
|
||||||
if (manifest) {
|
if (manifest) {
|
||||||
const char* p = f->path;
|
const char* p = current_chunk->items[i]->path;
|
||||||
if (*p == '/')
|
if (*p == '/')
|
||||||
p++;
|
p++;
|
||||||
array_list_add(manifest, str_dup(p));
|
array_list_add(manifest, str_dup(p));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
chunk_destroy(current_chunk);
|
if (!config->use_sendfile) {
|
||||||
}
|
for (int i = 0; i < current_chunk->element_count; i++) {
|
||||||
directory_scanner_destroy(scanner);
|
File* f = current_chunk->items[i];
|
||||||
scanner = NULL;
|
if (f->data->size > STREAM_THRESHOLD)
|
||||||
|
continue;
|
||||||
bool batch_ok = true;
|
if (!file_load_data(f)) {
|
||||||
if (config->use_incremental && all_files->size > 0) {
|
log_message(LOG_LEVEL_ERROR, "Failed to load file data");
|
||||||
if (!batch_incremental_check(client, all_files)) {
|
continue;
|
||||||
log_message(LOG_LEVEL_ERROR, "Batch incremental check failed");
|
}
|
||||||
batch_ok = false;
|
}
|
||||||
}
|
}
|
||||||
}
|
if (send_chunk(client, current_chunk, config) != 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
|
||||||
unsigned long long total_bytes = 0;
|
chunk_destroy(current_chunk);
|
||||||
time_t last_progress = 0;
|
|
||||||
time_t last_activity = 0;
|
|
||||||
time_t start = time(NULL);
|
|
||||||
bool use_sendfile = config->use_sendfile && !config->use_compression;
|
|
||||||
for (int i = 0; i < all_files->size; i++) {
|
|
||||||
File* file = (File*)all_files->items[i];
|
|
||||||
if (file->skip)
|
|
||||||
continue;
|
|
||||||
if (g_abort_requested) {
|
|
||||||
send_status(client->file_descriptor, STATUS_ABORT);
|
|
||||||
batch_ok = false;
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
time_t now = time(NULL);
|
|
||||||
if (now - last_activity >= KEEPALIVE_INTERVAL) {
|
|
||||||
if (!send_status(client->file_descriptor, STATUS_KEEPALIVE)) {
|
|
||||||
batch_ok = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Status s;
|
|
||||||
if (!receive_status(client->file_descriptor, &s)) {
|
|
||||||
batch_ok = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
last_activity = now;
|
|
||||||
}
|
|
||||||
int compression_level = config->use_compression ? config->compression_level : 0;
|
|
||||||
if (!send_status(client->file_descriptor, STATUS_NEXT)) {
|
|
||||||
batch_ok = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (use_sendfile) {
|
|
||||||
if (!file_send_sendfile(file, client->file_descriptor, config->use_metadata, 0, true)) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to send file via sendfile");
|
|
||||||
batch_ok = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (!file_load_data(file)) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to load file data");
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (!file_send_single_calls(file, client->file_descriptor, config->use_metadata,
|
|
||||||
compression_level, true)) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to send file");
|
|
||||||
batch_ok = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
total_bytes += file->data ? file->data->size : 0;
|
|
||||||
if (config->show_progress) {
|
if (config->show_progress) {
|
||||||
|
total_bytes += chunk_bytes;
|
||||||
|
time_t now = time(NULL);
|
||||||
if (now - last_progress >= 1) {
|
if (now - last_progress >= 1) {
|
||||||
last_progress = now;
|
last_progress = now;
|
||||||
double elapsed = difftime(now, start);
|
print_progress(total_bytes, start);
|
||||||
double rate = elapsed > 0 ? total_bytes / (1048576.0 * elapsed) : 0;
|
|
||||||
fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) ", total_bytes / 1048576.0, rate);
|
|
||||||
fflush(stderr);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
chunk_destroy(current_chunk);
|
||||||
}
|
}
|
||||||
|
if (config->use_delete) {
|
||||||
if (batch_ok && config->use_delete && manifest) {
|
if (!send_manifest(client->file_descriptor, manifest)) {
|
||||||
if (!send_status(client->file_descriptor, STATUS_MANIFEST))
|
array_list_delete(manifest);
|
||||||
batch_ok = false;
|
goto send_fail;
|
||||||
else if (!send_int(client->file_descriptor, manifest->size))
|
|
||||||
batch_ok = false;
|
|
||||||
else {
|
|
||||||
for (int i = 0; i < manifest->size && batch_ok; i++) {
|
|
||||||
if (!send_str(client->file_descriptor, (char*)manifest->items[i]))
|
|
||||||
batch_ok = false;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
array_list_delete(manifest);
|
||||||
}
|
}
|
||||||
array_list_delete(manifest);
|
if (!send_status(client->file_descriptor, STATUS_FINISHED))
|
||||||
|
goto send_fail;
|
||||||
if (batch_ok && !send_status(client->file_descriptor, STATUS_FINISHED))
|
|
||||||
batch_ok = false;
|
|
||||||
Status s;
|
Status s;
|
||||||
int ok = 0;
|
int ok = receive_status(client->file_descriptor, &s) && s == STATUS_OK;
|
||||||
if (batch_ok)
|
if (config->show_progress)
|
||||||
ok = receive_status(client->file_descriptor, &s) && s == STATUS_OK;
|
print_progress(total_bytes, start);
|
||||||
if (config->show_progress) {
|
if (config->show_progress)
|
||||||
double elapsed = difftime(time(NULL), start);
|
fprintf(stderr, "Done.\n");
|
||||||
double rate = elapsed > 0 ? total_bytes / (1048576.0 * elapsed) : 0;
|
directory_scanner_destroy(scanner);
|
||||||
fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) Done.\n", total_bytes / 1048576.0, rate);
|
|
||||||
}
|
|
||||||
for (int i = 0; i < all_files->size; i++)
|
|
||||||
file_destroy(all_files->items[i]);
|
|
||||||
array_list_delete(all_files);
|
|
||||||
client_disconnect(client);
|
client_disconnect(client);
|
||||||
client_delete(client);
|
client_delete(client);
|
||||||
return (batch_ok && ok) ? 0 : -1;
|
return ok ? 0 : -1;
|
||||||
|
|
||||||
|
send_fail:
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
client_disconnect(client);
|
||||||
|
client_delete(client);
|
||||||
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
int send_files_multithreaded(Config* config) {
|
int send_files_multithreaded(Config* config) {
|
||||||
time_t start_time = time(NULL);
|
if (config->dry_run)
|
||||||
if (config->dry_run) {
|
return run_dry_run(config);
|
||||||
DirectoryScanner* scanner = directory_scanner_create(
|
|
||||||
config->send_directory, config->use_metadata, config->chunk_size, config->exclude_patterns,
|
long pages = sysconf(_SC_AVPHYS_PAGES);
|
||||||
config->exclude_count, config->include_patterns, config->include_count, config->max_size,
|
long page_size = sysconf(_SC_PAGE_SIZE);
|
||||||
config->min_size, config->max_depth);
|
unsigned long long available_memory =
|
||||||
Chunk* chunk;
|
pages > 0 && page_size > 0 ? (unsigned long long)pages * (unsigned long long)page_size
|
||||||
int file_count = 0;
|
: 512ULL * 1024 * 1024;
|
||||||
unsigned long long total_bytes = 0;
|
unsigned long long avg_file_size = 1024 * 1024;
|
||||||
if (!config->quiet)
|
int qsize = (int)(available_memory / avg_file_size);
|
||||||
printf("Dry run: files to be transferred\n");
|
if (qsize < 10)
|
||||||
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
qsize = 10;
|
||||||
for (int i = 0; i < chunk->element_count; i++) {
|
if (qsize > 1000)
|
||||||
if (!config->quiet)
|
qsize = 1000;
|
||||||
printf(" %s (%zu bytes)\n", chunk->items[i]->path, chunk->items[i]->data->size);
|
|
||||||
total_bytes += chunk->items[i]->data->size;
|
|
||||||
file_count++;
|
|
||||||
}
|
|
||||||
chunk_destroy(chunk);
|
|
||||||
}
|
|
||||||
directory_scanner_destroy(scanner);
|
|
||||||
if (!config->quiet)
|
|
||||||
printf("Total: %d files, %.1f MB\n", file_count, total_bytes / 1048576.0);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
int qsize = config->queue_size > 0 ? config->queue_size : 100;
|
|
||||||
Queue* q1 = queue_create(qsize, chunk_destroy);
|
Queue* q1 = queue_create(qsize, chunk_destroy);
|
||||||
Queue* q2 = queue_create(qsize, chunk_destroy);
|
Queue* q2 = queue_create(qsize, chunk_destroy);
|
||||||
if (!q1 || !q2) {
|
if (!q1 || !q2) {
|
||||||
@@ -675,12 +565,6 @@ int send_files_multithreaded(Config* config) {
|
|||||||
thrd_join(loader, NULL);
|
thrd_join(loader, NULL);
|
||||||
thrd_join(sender, &sender_result);
|
thrd_join(sender, &sender_result);
|
||||||
|
|
||||||
if (config->stats && !config->quiet) {
|
|
||||||
double elapsed = difftime(time(NULL), start_time);
|
|
||||||
printf("\nTransfer statistics:\n");
|
|
||||||
printf(" Elapsed time: %.1f sec\n", elapsed);
|
|
||||||
}
|
|
||||||
|
|
||||||
pipeline_context_sender_destroy(context);
|
pipeline_context_sender_destroy(context);
|
||||||
return sender_result == thrd_success ? 0 : -1;
|
return sender_result == thrd_success ? 0 : -1;
|
||||||
}
|
}
|
||||||
|
|||||||
+260
-5
@@ -9,6 +9,7 @@
|
|||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
|
#include <threads.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
@@ -79,7 +80,6 @@ static Chunk* chunk_data_to_chunk(ArrayList* chunk_data) {
|
|||||||
return chunk;
|
return chunk;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns: 1 on success, 0 if no more directories in queue, -1 on opendir failure
|
|
||||||
static int open_next_directory(DirectoryScanner* scanner) {
|
static int open_next_directory(DirectoryScanner* scanner) {
|
||||||
if (scanner->current_dir) {
|
if (scanner->current_dir) {
|
||||||
closedir(scanner->current_dir);
|
closedir(scanner->current_dir);
|
||||||
@@ -138,10 +138,12 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
|
|
||||||
if (S_ISDIR(stats.st_mode)) {
|
if (S_ISDIR(stats.st_mode)) {
|
||||||
int next_depth = scanner->current_depth + 1;
|
int next_depth = scanner->current_depth + 1;
|
||||||
if (scanner->max_depth <= 0 || next_depth < scanner->max_depth)
|
if (scanner->max_depth <= 0 || next_depth < scanner->max_depth) {
|
||||||
queue_enqueue(scanner->directories, dir_entry_create(cur_path, next_depth));
|
DirEntry* de = dir_entry_create(cur_path, next_depth);
|
||||||
else
|
if (!queue_enqueue(scanner->directories, de))
|
||||||
free(cur_path);
|
dir_entry_destroy(de);
|
||||||
|
}
|
||||||
|
free(cur_path);
|
||||||
} else {
|
} else {
|
||||||
if (scanner->max_depth > 0 && scanner->current_depth + 1 > scanner->max_depth) {
|
if (scanner->max_depth > 0 && scanner->current_depth + 1 > scanner->max_depth) {
|
||||||
free(cur_path);
|
free(cur_path);
|
||||||
@@ -202,3 +204,256 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
array_list_delete(chunk_data);
|
array_list_delete(chunk_data);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
ParallelScanner* ps;
|
||||||
|
char** dirs;
|
||||||
|
int dir_count;
|
||||||
|
bool use_metadata;
|
||||||
|
unsigned long long chunk_size;
|
||||||
|
char** exclude_patterns;
|
||||||
|
int exclude_count;
|
||||||
|
char** include_patterns;
|
||||||
|
int include_count;
|
||||||
|
unsigned long long max_size;
|
||||||
|
unsigned long long min_size;
|
||||||
|
int max_depth;
|
||||||
|
} ParallelWorkerArg;
|
||||||
|
|
||||||
|
static int parallel_worker_thread(void* arg) {
|
||||||
|
ParallelWorkerArg* wa = (ParallelWorkerArg*)arg;
|
||||||
|
for (int i = 0; i < wa->dir_count; i++) {
|
||||||
|
DirectoryScanner* ds = directory_scanner_create(
|
||||||
|
wa->dirs[i], wa->use_metadata, wa->chunk_size, wa->exclude_patterns, wa->exclude_count,
|
||||||
|
wa->include_patterns, wa->include_count, wa->max_size, wa->min_size, wa->max_depth);
|
||||||
|
Chunk* chunk;
|
||||||
|
while ((chunk = directory_scanner_next(ds)) != NULL) {
|
||||||
|
queue_enqueue_multithreaded(wa->ps->result_queue, chunk, &wa->ps->result_mutex,
|
||||||
|
&wa->ps->result_not_empty, &wa->ps->result_not_full);
|
||||||
|
}
|
||||||
|
directory_scanner_destroy(ds);
|
||||||
|
free(wa->dirs[i]);
|
||||||
|
}
|
||||||
|
ParallelScanner* ps = wa->ps;
|
||||||
|
free(wa->dirs);
|
||||||
|
free(wa);
|
||||||
|
mtx_lock(&ps->result_mutex);
|
||||||
|
ps->completed++;
|
||||||
|
if (ps->completed >= ps->num_threads) {
|
||||||
|
ps->done = true;
|
||||||
|
cnd_signal(&ps->result_not_empty);
|
||||||
|
}
|
||||||
|
mtx_unlock(&ps->result_mutex);
|
||||||
|
return thrd_success;
|
||||||
|
}
|
||||||
|
|
||||||
|
ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata,
|
||||||
|
unsigned long long chunk_size, char** exclude_patterns,
|
||||||
|
int exclude_count, char** include_patterns,
|
||||||
|
int include_count, unsigned long long max_size,
|
||||||
|
unsigned long long min_size, int max_depth,
|
||||||
|
int num_threads) {
|
||||||
|
ParallelScanner* ps = calloc(1, sizeof(ParallelScanner));
|
||||||
|
if (!ps)
|
||||||
|
return NULL;
|
||||||
|
ps->result_queue = queue_create(100, chunk_destroy);
|
||||||
|
if (!ps->result_queue) {
|
||||||
|
free(ps);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (mtx_init(&ps->result_mutex, mtx_plain) != thrd_success ||
|
||||||
|
cnd_init(&ps->result_not_empty) != thrd_success ||
|
||||||
|
cnd_init(&ps->result_not_full) != thrd_success) {
|
||||||
|
queue_destroy(ps->result_queue);
|
||||||
|
free(ps);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
DIR* dir = opendir(root_directory);
|
||||||
|
if (!dir) {
|
||||||
|
perror("Could not open root directory for parallel scan");
|
||||||
|
parallel_scanner_destroy(ps);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
ArrayList* root_files = array_list_create(file_destroy);
|
||||||
|
ArrayList* subdirs = array_list_create(free);
|
||||||
|
struct dirent* entry;
|
||||||
|
while ((entry = readdir(dir)) != NULL) {
|
||||||
|
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||||
|
continue;
|
||||||
|
char* cur_path = path_cat(root_directory, entry->d_name);
|
||||||
|
if (!cur_path)
|
||||||
|
continue;
|
||||||
|
struct stat st;
|
||||||
|
if (stat(cur_path, &st) != 0) {
|
||||||
|
free(cur_path);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (S_ISDIR(st.st_mode)) {
|
||||||
|
array_list_add(subdirs, cur_path);
|
||||||
|
} else {
|
||||||
|
bool excluded = false;
|
||||||
|
for (int i = 0; i < exclude_count; i++) {
|
||||||
|
if (glob_match(exclude_patterns[i], entry->d_name)) {
|
||||||
|
excluded = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (excluded) {
|
||||||
|
free(cur_path);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (include_count > 0) {
|
||||||
|
bool included = false;
|
||||||
|
for (int i = 0; i < include_count; i++) {
|
||||||
|
if (glob_match(include_patterns[i], entry->d_name)) {
|
||||||
|
included = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!included) {
|
||||||
|
free(cur_path);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ((max_size > 0 && (unsigned long long)st.st_size > max_size) ||
|
||||||
|
(min_size > 0 && (unsigned long long)st.st_size < min_size)) {
|
||||||
|
free(cur_path);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
File* file = file_create(cur_path);
|
||||||
|
free(cur_path);
|
||||||
|
if (!file)
|
||||||
|
continue;
|
||||||
|
file->data->size = st.st_size;
|
||||||
|
if (use_metadata)
|
||||||
|
file->metadata = file_metadata_create(&st);
|
||||||
|
array_list_add(root_files, file);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
closedir(dir);
|
||||||
|
|
||||||
|
unsigned long long cs = chunk_size > 0 ? chunk_size : DESIRED_CHUNK_SIZE;
|
||||||
|
if (root_files->size > 0) {
|
||||||
|
ArrayList* batch = array_list_create(NULL);
|
||||||
|
unsigned long long batch_size = 0;
|
||||||
|
Chunk* first = NULL;
|
||||||
|
for (int i = 0; i < root_files->size; i++) {
|
||||||
|
File* f = (File*)root_files->items[i];
|
||||||
|
array_list_add(batch, f);
|
||||||
|
batch_size += f->data->size;
|
||||||
|
if (batch_size >= cs || i == root_files->size - 1) {
|
||||||
|
void** items = array_list_to_array(batch);
|
||||||
|
Chunk* c = chunk_create((File**)items, batch->size);
|
||||||
|
free(items);
|
||||||
|
batch->item_destroyer = NULL;
|
||||||
|
array_list_delete(batch);
|
||||||
|
batch = NULL;
|
||||||
|
if (!first) {
|
||||||
|
first = c;
|
||||||
|
} else {
|
||||||
|
queue_enqueue_multithreaded(ps->result_queue, c, &ps->result_mutex, &ps->result_not_empty,
|
||||||
|
&ps->result_not_full);
|
||||||
|
}
|
||||||
|
if (i < root_files->size - 1) {
|
||||||
|
batch = array_list_create(NULL);
|
||||||
|
batch_size = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (batch) {
|
||||||
|
batch->item_destroyer = NULL;
|
||||||
|
array_list_delete(batch);
|
||||||
|
}
|
||||||
|
ps->initial_chunk = first;
|
||||||
|
root_files->item_destroyer = NULL;
|
||||||
|
}
|
||||||
|
array_list_delete(root_files);
|
||||||
|
|
||||||
|
int n = num_threads > 0 ? num_threads : 4;
|
||||||
|
if (n > subdirs->size)
|
||||||
|
n = subdirs->size > 0 ? subdirs->size : 1;
|
||||||
|
|
||||||
|
if (subdirs->size > 0) {
|
||||||
|
ps->num_threads = n;
|
||||||
|
ps->threads = calloc(n, sizeof(thrd_t));
|
||||||
|
if (!ps->threads) {
|
||||||
|
array_list_delete(subdirs);
|
||||||
|
parallel_scanner_destroy(ps);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
int dirs_per_thread = subdirs->size / n;
|
||||||
|
int remainder = subdirs->size % n;
|
||||||
|
int start = 0;
|
||||||
|
for (int t = 0; t < n; t++) {
|
||||||
|
int count = dirs_per_thread + (t < remainder ? 1 : 0);
|
||||||
|
if (count == 0)
|
||||||
|
break;
|
||||||
|
ParallelWorkerArg* wa = calloc(1, sizeof(ParallelWorkerArg));
|
||||||
|
if (!wa)
|
||||||
|
break;
|
||||||
|
wa->ps = ps;
|
||||||
|
wa->dirs = calloc(count, sizeof(char*));
|
||||||
|
if (!wa->dirs) {
|
||||||
|
free(wa);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
for (int j = 0; j < count; j++)
|
||||||
|
wa->dirs[j] = str_dup((char*)subdirs->items[start + j]);
|
||||||
|
wa->dir_count = count;
|
||||||
|
wa->use_metadata = use_metadata;
|
||||||
|
wa->chunk_size = cs;
|
||||||
|
wa->exclude_patterns = exclude_patterns;
|
||||||
|
wa->exclude_count = exclude_count;
|
||||||
|
wa->include_patterns = include_patterns;
|
||||||
|
wa->include_count = include_count;
|
||||||
|
wa->max_size = max_size;
|
||||||
|
wa->min_size = min_size;
|
||||||
|
wa->max_depth = max_depth;
|
||||||
|
start += count;
|
||||||
|
if (thrd_create(&ps->threads[t], parallel_worker_thread, wa) != thrd_success) {
|
||||||
|
for (int j = 0; j < count; j++)
|
||||||
|
free(wa->dirs[j]);
|
||||||
|
free(wa->dirs);
|
||||||
|
free(wa);
|
||||||
|
ps->num_threads = t;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
array_list_delete(subdirs);
|
||||||
|
return ps;
|
||||||
|
}
|
||||||
|
|
||||||
|
Chunk* parallel_scanner_next(ParallelScanner* ps) {
|
||||||
|
if (ps->initial_chunk) {
|
||||||
|
Chunk* c = ps->initial_chunk;
|
||||||
|
ps->initial_chunk = NULL;
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
if (ps->num_threads == 0) {
|
||||||
|
ps->done = true;
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
Chunk* chunk = queue_dequeue_multithreaded(
|
||||||
|
ps->result_queue, &ps->result_mutex, &ps->result_not_empty, &ps->result_not_full, &ps->done);
|
||||||
|
return chunk;
|
||||||
|
}
|
||||||
|
|
||||||
|
void parallel_scanner_destroy(ParallelScanner* ps) {
|
||||||
|
if (!ps)
|
||||||
|
return;
|
||||||
|
ps->done = true;
|
||||||
|
cnd_signal(&ps->result_not_empty);
|
||||||
|
for (int i = 0; i < ps->num_threads; i++)
|
||||||
|
thrd_join(ps->threads[i], NULL);
|
||||||
|
free(ps->threads);
|
||||||
|
if (ps->initial_chunk)
|
||||||
|
chunk_destroy(ps->initial_chunk);
|
||||||
|
queue_destroy(ps->result_queue);
|
||||||
|
mtx_destroy(&ps->result_mutex);
|
||||||
|
cnd_destroy(&ps->result_not_empty);
|
||||||
|
cnd_destroy(&ps->result_not_full);
|
||||||
|
free(ps);
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@
|
|||||||
#include "queue.h"
|
#include "queue.h"
|
||||||
#include <dirent.h>
|
#include <dirent.h>
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
|
#include <threads.h>
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
Queue* directories;
|
Queue* directories;
|
||||||
@@ -22,6 +23,18 @@ typedef struct {
|
|||||||
int current_depth;
|
int current_depth;
|
||||||
} DirectoryScanner;
|
} DirectoryScanner;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
Queue* result_queue;
|
||||||
|
mtx_t result_mutex;
|
||||||
|
cnd_t result_not_empty;
|
||||||
|
cnd_t result_not_full;
|
||||||
|
int num_threads;
|
||||||
|
thrd_t* threads;
|
||||||
|
bool done;
|
||||||
|
int completed;
|
||||||
|
Chunk* initial_chunk;
|
||||||
|
} ParallelScanner;
|
||||||
|
|
||||||
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
|
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
|
||||||
unsigned long long chunk_size, char** exclude_patterns,
|
unsigned long long chunk_size, char** exclude_patterns,
|
||||||
int exclude_count, char** include_patterns,
|
int exclude_count, char** include_patterns,
|
||||||
@@ -30,4 +43,13 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
|
|||||||
Chunk* directory_scanner_next(DirectoryScanner* scanner);
|
Chunk* directory_scanner_next(DirectoryScanner* scanner);
|
||||||
void directory_scanner_destroy(DirectoryScanner* scanner);
|
void directory_scanner_destroy(DirectoryScanner* scanner);
|
||||||
|
|
||||||
|
ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata,
|
||||||
|
unsigned long long chunk_size, char** exclude_patterns,
|
||||||
|
int exclude_count, char** include_patterns,
|
||||||
|
int include_count, unsigned long long max_size,
|
||||||
|
unsigned long long min_size, int max_depth,
|
||||||
|
int num_threads);
|
||||||
|
Chunk* parallel_scanner_next(ParallelScanner* scanner);
|
||||||
|
void parallel_scanner_destroy(ParallelScanner* scanner);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -2,10 +2,28 @@
|
|||||||
#include "data.h"
|
#include "data.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "stdlib.h"
|
#include "stdlib.h"
|
||||||
|
#include "string.h"
|
||||||
|
#include <strings.h>
|
||||||
#include "zstd.h"
|
#include "zstd.h"
|
||||||
|
|
||||||
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
|
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
|
||||||
|
|
||||||
|
static const char* SKIP_COMPRESSION_EXTENSIONS[] = {".jpg", ".jpeg", ".png", ".gif", ".mp4", ".mkv",
|
||||||
|
".zip", ".gz", ".xz", ".zst", NULL};
|
||||||
|
|
||||||
|
bool compression_should_skip(const char* path) {
|
||||||
|
if (!path)
|
||||||
|
return false;
|
||||||
|
const char* dot = strrchr(path, '.');
|
||||||
|
if (!dot)
|
||||||
|
return false;
|
||||||
|
for (int i = 0; SKIP_COMPRESSION_EXTENSIONS[i]; i++) {
|
||||||
|
if (strcasecmp(dot, SKIP_COMPRESSION_EXTENSIONS[i]) == 0)
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
Data* data_compress(Data* data_to_compress, int compression_level) {
|
Data* data_compress(Data* data_to_compress, int compression_level) {
|
||||||
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
|
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
|
||||||
size_t dst_size = ZSTD_compressBound(data_to_compress->size);
|
size_t dst_size = ZSTD_compressBound(data_to_compress->size);
|
||||||
|
|||||||
@@ -2,8 +2,10 @@
|
|||||||
#define COMPRESSION_H
|
#define COMPRESSION_H
|
||||||
|
|
||||||
#include "data.h"
|
#include "data.h"
|
||||||
|
#include <stdbool.h>
|
||||||
|
|
||||||
Data* data_compress(Data* data_to_compress, int compression_level);
|
Data* data_compress(Data* data_to_compress, int compression_level);
|
||||||
Data* data_decompress(Data* compressed_data);
|
Data* data_decompress(Data* compressed_data);
|
||||||
|
bool compression_should_skip(const char* path);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+13
-43
@@ -8,54 +8,21 @@
|
|||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
Config* config_create(char* version, char* send_directory, char* receive_directory,
|
Config* config_create(void) {
|
||||||
bool save_to_disk, bool use_multithreading, bool use_chunk_serialization,
|
Config* config = calloc(1, sizeof(Config));
|
||||||
bool use_compression, bool use_metadata, int compression_level,
|
if (!config)
|
||||||
bool use_sendfile, unsigned long long chunk_size) {
|
return NULL;
|
||||||
|
config->version = str_dup(PROTOCOL_VERSION);
|
||||||
Config* config = malloc(sizeof(Config));
|
config->compression_level = 5;
|
||||||
config->version = version;
|
config->chunk_size = DEFAULT_CHUNK_SIZE;
|
||||||
config->send_directory = send_directory;
|
|
||||||
config->receive_root_directory = receive_directory;
|
|
||||||
config->save_to_disk = save_to_disk;
|
|
||||||
config->use_multithreading = use_multithreading;
|
|
||||||
config->use_chunk_serialization = use_chunk_serialization;
|
|
||||||
config->use_compression = use_compression;
|
|
||||||
config->use_metadata = use_metadata;
|
|
||||||
config->show_progress = false;
|
|
||||||
config->dry_run = false;
|
|
||||||
config->use_delete = false;
|
|
||||||
config->compression_level = compression_level;
|
|
||||||
config->use_sendfile = use_sendfile;
|
|
||||||
config->chunk_size = chunk_size > 0 ? chunk_size : DEFAULT_CHUNK_SIZE;
|
|
||||||
config->ssh_port = 22;
|
config->ssh_port = 22;
|
||||||
config->transport = TRANSPORT_TCP;
|
|
||||||
config->ssh_destination = NULL;
|
|
||||||
config->exclude_patterns = NULL;
|
|
||||||
config->exclude_count = 0;
|
|
||||||
config->include_patterns = NULL;
|
|
||||||
config->include_count = 0;
|
|
||||||
config->max_size = 0;
|
|
||||||
config->min_size = 0;
|
|
||||||
config->use_incremental = false;
|
|
||||||
config->use_delta = false;
|
|
||||||
config->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT;
|
|
||||||
config->delta_max_file_size = DELTA_MAX_FILE_SIZE;
|
|
||||||
config->use_tls = false;
|
|
||||||
config->tls_cert = NULL;
|
|
||||||
config->tls_key = NULL;
|
|
||||||
config->tls_ca = NULL;
|
|
||||||
config->server_host = str_dup("127.0.0.1");
|
|
||||||
config->server_port = 8080;
|
config->server_port = 8080;
|
||||||
|
config->server_host = str_dup("127.0.0.1");
|
||||||
config->timeout = 30;
|
config->timeout = 30;
|
||||||
config->contimeout = 10;
|
config->contimeout = 10;
|
||||||
config->quiet = false;
|
|
||||||
config->backup = false;
|
|
||||||
config->backup_dir = NULL;
|
|
||||||
config->stats = false;
|
|
||||||
config->max_depth = 0;
|
|
||||||
config->log_file = NULL;
|
|
||||||
config->queue_size = 100;
|
config->queue_size = 100;
|
||||||
|
config->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT;
|
||||||
|
config->delta_max_file_size = DELTA_MAX_FILE_SIZE;
|
||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -90,6 +57,7 @@ void config_delete(Config* config) {
|
|||||||
free(config->send_directory);
|
free(config->send_directory);
|
||||||
free(config->receive_root_directory);
|
free(config->receive_root_directory);
|
||||||
free(config->ssh_destination);
|
free(config->ssh_destination);
|
||||||
|
free(config->fastsync_server_path);
|
||||||
for (int i = 0; i < config->exclude_count; i++)
|
for (int i = 0; i < config->exclude_count; i++)
|
||||||
free(config->exclude_patterns[i]);
|
free(config->exclude_patterns[i]);
|
||||||
free(config->exclude_patterns);
|
free(config->exclude_patterns);
|
||||||
@@ -225,6 +193,7 @@ Config* config_receive(int file_descriptor) {
|
|||||||
config->ssh_port = 22;
|
config->ssh_port = 22;
|
||||||
config->transport = TRANSPORT_TCP;
|
config->transport = TRANSPORT_TCP;
|
||||||
config->ssh_destination = NULL;
|
config->ssh_destination = NULL;
|
||||||
|
config->fastsync_server_path = NULL;
|
||||||
config->exclude_patterns = NULL;
|
config->exclude_patterns = NULL;
|
||||||
config->exclude_count = 0;
|
config->exclude_count = 0;
|
||||||
config->include_patterns = NULL;
|
config->include_patterns = NULL;
|
||||||
@@ -259,6 +228,7 @@ error:
|
|||||||
free(config->send_directory);
|
free(config->send_directory);
|
||||||
free(config->receive_root_directory);
|
free(config->receive_root_directory);
|
||||||
free(config->server_host);
|
free(config->server_host);
|
||||||
|
free(config->backup_dir);
|
||||||
free(config);
|
free(config);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-4
@@ -25,6 +25,7 @@ typedef struct Config {
|
|||||||
int ssh_port;
|
int ssh_port;
|
||||||
TransportType transport;
|
TransportType transport;
|
||||||
char* ssh_destination;
|
char* ssh_destination;
|
||||||
|
char* fastsync_server_path;
|
||||||
char** exclude_patterns;
|
char** exclude_patterns;
|
||||||
int exclude_count;
|
int exclude_count;
|
||||||
char** include_patterns;
|
char** include_patterns;
|
||||||
@@ -50,15 +51,14 @@ typedef struct Config {
|
|||||||
int max_depth;
|
int max_depth;
|
||||||
FILE* log_file;
|
FILE* log_file;
|
||||||
int queue_size;
|
int queue_size;
|
||||||
|
bool follow_symlinks;
|
||||||
|
bool partial;
|
||||||
} Config;
|
} Config;
|
||||||
|
|
||||||
#define PROTOCOL_VERSION "1.3.0"
|
#define PROTOCOL_VERSION "1.3.0"
|
||||||
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
||||||
|
|
||||||
Config* config_create(char* version, char* send_directory, char* receive_directory,
|
Config* config_create(void);
|
||||||
bool save_to_disk, bool use_multithreading, bool use_chunk_serialization,
|
|
||||||
bool use_compression, bool use_metadata, int compression_level,
|
|
||||||
bool use_sendfile, unsigned long long chunk_size);
|
|
||||||
void config_delete(Config* config);
|
void config_delete(Config* config);
|
||||||
bool config_send(int file_descriptor, const Config* config);
|
bool config_send(int file_descriptor, const Config* config);
|
||||||
Config* config_receive(int file_descriptor);
|
Config* config_receive(int file_descriptor);
|
||||||
|
|||||||
+4
-2
@@ -1,9 +1,11 @@
|
|||||||
#include "data.h"
|
#include "data.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "stdlib.h"
|
#include <stdlib.h>
|
||||||
|
|
||||||
Data* data_create_empty(size_t data_size) {
|
Data* data_create_empty(size_t data_size) {
|
||||||
void* data = malloc(data_size);
|
/* malloc(0) is UB; allocate at least 1 byte but preserve requested size */
|
||||||
|
size_t alloc_size = data_size > 0 ? data_size : 1;
|
||||||
|
void* data = malloc(alloc_size);
|
||||||
if (data == NULL) {
|
if (data == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for empty data");
|
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for empty data");
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|||||||
+1
-1
@@ -1,7 +1,7 @@
|
|||||||
#ifndef DATA_H
|
#ifndef DATA_H
|
||||||
#define DATA_H
|
#define DATA_H
|
||||||
|
|
||||||
#include "stdlib.h"
|
#include <stdlib.h>
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
void* data;
|
void* data;
|
||||||
|
|||||||
+1
-1
@@ -104,7 +104,7 @@ bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
|
|||||||
int compression_level, bool send_path) {
|
int compression_level, bool send_path) {
|
||||||
const Data* data_to_send = file->data;
|
const Data* data_to_send = file->data;
|
||||||
Data* compressed_data = NULL;
|
Data* compressed_data = NULL;
|
||||||
if (compression_level > 0) {
|
if (compression_level > 0 && !compression_should_skip(file->path)) {
|
||||||
compressed_data = data_compress(file->data, compression_level);
|
compressed_data = data_compress(file->data, compression_level);
|
||||||
if (compressed_data == NULL) {
|
if (compressed_data == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
|
log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
|
||||||
|
|||||||
@@ -6,6 +6,8 @@
|
|||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
|
|
||||||
|
typedef enum { FILE_TYPE_REGULAR, FILE_TYPE_SYMLINK, FILE_TYPE_DIR } FileType;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
mode_t mode;
|
mode_t mode;
|
||||||
uid_t uid;
|
uid_t uid;
|
||||||
|
|||||||
+1
-1
@@ -15,7 +15,7 @@ void log_set_file(FILE* fp) {
|
|||||||
log_fp = fp;
|
log_fp = fp;
|
||||||
}
|
}
|
||||||
|
|
||||||
void log_message(LogLevel log_level, char* format, ...) {
|
void log_message(LogLevel log_level, const char* format, ...) {
|
||||||
if (log_level < current_log_level)
|
if (log_level < current_log_level)
|
||||||
return;
|
return;
|
||||||
time_t now = time(NULL);
|
time_t now = time(NULL);
|
||||||
|
|||||||
+1
-1
@@ -5,7 +5,7 @@
|
|||||||
|
|
||||||
typedef enum { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR } LogLevel;
|
typedef enum { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR } LogLevel;
|
||||||
|
|
||||||
void log_message(LogLevel log_level, char* message, ...);
|
void log_message(LogLevel log_level, const char* message, ...);
|
||||||
void set_log_level(LogLevel level);
|
void set_log_level(LogLevel level);
|
||||||
void log_set_file(FILE* fp);
|
void log_set_file(FILE* fp);
|
||||||
|
|
||||||
|
|||||||
+109
-43
@@ -4,67 +4,103 @@
|
|||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
|
#include <stdint.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Wire format serialization (protocol version 2.0.0+):
|
||||||
|
* All metadata fields are serialized as fixed-width integers (int32_t / int64_t)
|
||||||
|
* to ensure cross-platform binary compatiblity. See metadata.h for the
|
||||||
|
* exact wire layout.
|
||||||
|
*
|
||||||
|
* Compile-time assertions verify that the native platform types fit within
|
||||||
|
* the chosen fixed-width representations.
|
||||||
|
*/
|
||||||
|
typedef char static_assert_mode_t_fits[(sizeof(mode_t) <= sizeof(int32_t)) ? 1 : -1];
|
||||||
|
typedef char static_assert_uid_t_fits[(sizeof(uid_t) <= sizeof(int32_t)) ? 1 : -1];
|
||||||
|
typedef char static_assert_gid_t_fits[(sizeof(gid_t) <= sizeof(int32_t)) ? 1 : -1];
|
||||||
|
|
||||||
void metadata_to_buf(char** buf, const FileMetadata* m) {
|
void metadata_to_buf(char** buf, const FileMetadata* m) {
|
||||||
int present = (m != NULL) ? 1 : 0;
|
int32_t present = (m != NULL) ? 1 : 0;
|
||||||
memcpy(*buf, &present, sizeof(int));
|
memcpy(*buf, &present, sizeof(present));
|
||||||
*buf += sizeof(int);
|
*buf += sizeof(present);
|
||||||
if (m == NULL)
|
if (m == NULL)
|
||||||
return;
|
return;
|
||||||
memcpy(*buf, &m->mode, sizeof(mode_t));
|
int32_t mode = (int32_t)m->mode;
|
||||||
*buf += sizeof(mode_t);
|
memcpy(*buf, &mode, sizeof(mode));
|
||||||
memcpy(*buf, &m->uid, sizeof(uid_t));
|
*buf += sizeof(mode);
|
||||||
*buf += sizeof(uid_t);
|
int32_t uid = (int32_t)m->uid;
|
||||||
memcpy(*buf, &m->gid, sizeof(gid_t));
|
memcpy(*buf, &uid, sizeof(uid));
|
||||||
*buf += sizeof(gid_t);
|
*buf += sizeof(uid);
|
||||||
memcpy(*buf, &m->mtime_sec, sizeof(time_t));
|
int32_t gid = (int32_t)m->gid;
|
||||||
*buf += sizeof(time_t);
|
memcpy(*buf, &gid, sizeof(gid));
|
||||||
memcpy(*buf, &m->mtime_nsec, sizeof(long));
|
*buf += sizeof(gid);
|
||||||
*buf += sizeof(long);
|
int64_t mtime_sec = (int64_t)m->mtime_sec;
|
||||||
|
memcpy(*buf, &mtime_sec, sizeof(mtime_sec));
|
||||||
|
*buf += sizeof(mtime_sec);
|
||||||
|
int64_t mtime_nsec = (int64_t)m->mtime_nsec;
|
||||||
|
memcpy(*buf, &mtime_nsec, sizeof(mtime_nsec));
|
||||||
|
*buf += sizeof(mtime_nsec);
|
||||||
}
|
}
|
||||||
|
|
||||||
FileMetadata* metadata_from_buf(char** buf) {
|
FileMetadata* metadata_from_buf(char** buf) {
|
||||||
int present;
|
int32_t present;
|
||||||
memcpy(&present, *buf, sizeof(int));
|
memcpy(&present, *buf, sizeof(present));
|
||||||
*buf += sizeof(int);
|
*buf += sizeof(present);
|
||||||
if (!present)
|
if (!present)
|
||||||
return NULL;
|
return NULL;
|
||||||
FileMetadata* m = malloc(sizeof(FileMetadata));
|
FileMetadata* m = malloc(sizeof(FileMetadata));
|
||||||
memcpy(&m->mode, *buf, sizeof(mode_t));
|
if (m == NULL)
|
||||||
*buf += sizeof(mode_t);
|
return NULL;
|
||||||
memcpy(&m->uid, *buf, sizeof(uid_t));
|
int32_t mode;
|
||||||
*buf += sizeof(uid_t);
|
memcpy(&mode, *buf, sizeof(mode));
|
||||||
memcpy(&m->gid, *buf, sizeof(gid_t));
|
*buf += sizeof(mode);
|
||||||
*buf += sizeof(gid_t);
|
m->mode = (mode_t)mode;
|
||||||
memcpy(&m->mtime_sec, *buf, sizeof(time_t));
|
int32_t uid;
|
||||||
*buf += sizeof(time_t);
|
memcpy(&uid, *buf, sizeof(uid));
|
||||||
memcpy(&m->mtime_nsec, *buf, sizeof(long));
|
*buf += sizeof(uid);
|
||||||
*buf += sizeof(long);
|
m->uid = (uid_t)uid;
|
||||||
|
int32_t gid;
|
||||||
|
memcpy(&gid, *buf, sizeof(gid));
|
||||||
|
*buf += sizeof(gid);
|
||||||
|
m->gid = (gid_t)gid;
|
||||||
|
int64_t mtime_sec;
|
||||||
|
memcpy(&mtime_sec, *buf, sizeof(mtime_sec));
|
||||||
|
*buf += sizeof(mtime_sec);
|
||||||
|
m->mtime_sec = (time_t)mtime_sec;
|
||||||
|
int64_t mtime_nsec;
|
||||||
|
memcpy(&mtime_nsec, *buf, sizeof(mtime_nsec));
|
||||||
|
*buf += sizeof(mtime_nsec);
|
||||||
|
m->mtime_nsec = (long)mtime_nsec;
|
||||||
return m;
|
return m;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool metadata_send(int file_descriptor, FileMetadata* m) {
|
bool metadata_send(int file_descriptor, FileMetadata* m) {
|
||||||
if (m == NULL) {
|
if (m == NULL) {
|
||||||
int zero = 0;
|
int32_t zero = 0;
|
||||||
return send_n_data(file_descriptor, &zero, sizeof(int));
|
return send_n_data(file_descriptor, &zero, sizeof(zero));
|
||||||
}
|
}
|
||||||
int present = 1;
|
int32_t present = 1;
|
||||||
return send_n_data(file_descriptor, &present, sizeof(int)) &&
|
int32_t mode = (int32_t)m->mode;
|
||||||
send_n_data(file_descriptor, &m->mode, sizeof(mode_t)) &&
|
int32_t uid = (int32_t)m->uid;
|
||||||
send_n_data(file_descriptor, &m->uid, sizeof(uid_t)) &&
|
int32_t gid = (int32_t)m->gid;
|
||||||
send_n_data(file_descriptor, &m->gid, sizeof(gid_t)) &&
|
int64_t mtime_sec = (int64_t)m->mtime_sec;
|
||||||
send_n_data(file_descriptor, &m->mtime_sec, sizeof(time_t)) &&
|
int64_t mtime_nsec = (int64_t)m->mtime_nsec;
|
||||||
send_n_data(file_descriptor, &m->mtime_nsec, sizeof(long));
|
return send_n_data(file_descriptor, &present, sizeof(present)) &&
|
||||||
|
send_n_data(file_descriptor, &mode, sizeof(mode)) &&
|
||||||
|
send_n_data(file_descriptor, &uid, sizeof(uid)) &&
|
||||||
|
send_n_data(file_descriptor, &gid, sizeof(gid)) &&
|
||||||
|
send_n_data(file_descriptor, &mtime_sec, sizeof(mtime_sec)) &&
|
||||||
|
send_n_data(file_descriptor, &mtime_nsec, sizeof(mtime_nsec));
|
||||||
}
|
}
|
||||||
|
|
||||||
FileMetadata* metadata_receive(int file_descriptor, int* ok) {
|
FileMetadata* metadata_receive(int file_descriptor, int* ok) {
|
||||||
int present;
|
int32_t present;
|
||||||
if (!receive_n_data(file_descriptor, &present, sizeof(int))) {
|
if (!receive_n_data(file_descriptor, &present, sizeof(present))) {
|
||||||
if (ok)
|
if (ok)
|
||||||
*ok = 0;
|
*ok = 0;
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -80,16 +116,46 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok) {
|
|||||||
*ok = 0;
|
*ok = 0;
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
if (!receive_n_data(file_descriptor, &m->mode, sizeof(mode_t)) ||
|
int32_t mode;
|
||||||
!receive_n_data(file_descriptor, &m->uid, sizeof(uid_t)) ||
|
if (!receive_n_data(file_descriptor, &mode, sizeof(mode))) {
|
||||||
!receive_n_data(file_descriptor, &m->gid, sizeof(gid_t)) ||
|
|
||||||
!receive_n_data(file_descriptor, &m->mtime_sec, sizeof(time_t)) ||
|
|
||||||
!receive_n_data(file_descriptor, &m->mtime_nsec, sizeof(long))) {
|
|
||||||
free(m);
|
free(m);
|
||||||
if (ok)
|
if (ok)
|
||||||
*ok = 0;
|
*ok = 0;
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
m->mode = (mode_t)mode;
|
||||||
|
int32_t uid;
|
||||||
|
if (!receive_n_data(file_descriptor, &uid, sizeof(uid))) {
|
||||||
|
free(m);
|
||||||
|
if (ok)
|
||||||
|
*ok = 0;
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
m->uid = (uid_t)uid;
|
||||||
|
int32_t gid;
|
||||||
|
if (!receive_n_data(file_descriptor, &gid, sizeof(gid))) {
|
||||||
|
free(m);
|
||||||
|
if (ok)
|
||||||
|
*ok = 0;
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
m->gid = (gid_t)gid;
|
||||||
|
int64_t mtime_sec;
|
||||||
|
if (!receive_n_data(file_descriptor, &mtime_sec, sizeof(mtime_sec))) {
|
||||||
|
free(m);
|
||||||
|
if (ok)
|
||||||
|
*ok = 0;
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
m->mtime_sec = (time_t)mtime_sec;
|
||||||
|
int64_t mtime_nsec;
|
||||||
|
if (!receive_n_data(file_descriptor, &mtime_nsec, sizeof(mtime_nsec))) {
|
||||||
|
free(m);
|
||||||
|
if (ok)
|
||||||
|
*ok = 0;
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
m->mtime_nsec = (long)mtime_nsec;
|
||||||
if (ok)
|
if (ok)
|
||||||
*ok = 1;
|
*ok = 1;
|
||||||
return m;
|
return m;
|
||||||
@@ -98,7 +164,7 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok) {
|
|||||||
void file_restore_metadata(const char* path, FileMetadata* metadata) {
|
void file_restore_metadata(const char* path, FileMetadata* metadata) {
|
||||||
if (metadata == NULL)
|
if (metadata == NULL)
|
||||||
return;
|
return;
|
||||||
if (chmod(path, metadata->mode & 07777) != 0)
|
if (chmod(path, metadata->mode & 07777 & ~(S_ISUID | S_ISGID)) != 0)
|
||||||
log_message(LOG_LEVEL_WARNING, "Failed to chmod %s: %s", path, strerror(errno));
|
log_message(LOG_LEVEL_WARNING, "Failed to chmod %s: %s", path, strerror(errno));
|
||||||
if (chown(path, metadata->uid, metadata->gid) != 0)
|
if (chown(path, metadata->uid, metadata->gid) != 0)
|
||||||
log_message(LOG_LEVEL_WARNING, "Failed to chown %s: %s", path, strerror(errno));
|
log_message(LOG_LEVEL_WARNING, "Failed to chown %s: %s", path, strerror(errno));
|
||||||
|
|||||||
+19
-2
@@ -3,10 +3,27 @@
|
|||||||
|
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
|
#include <stdint.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
|
|
||||||
#define FILE_METADATA_WIRE_SIZE \
|
/*
|
||||||
(sizeof(mode_t) + sizeof(uid_t) + sizeof(gid_t) + sizeof(time_t) + sizeof(long))
|
* Wire format (introduced in protocol version 2.0.0):
|
||||||
|
* int32_t present
|
||||||
|
* int32_t mode (was mode_t, platform-dependent)
|
||||||
|
* int32_t uid (was uid_t, platform-dependent)
|
||||||
|
* int32_t gid (was gid_t, platform-dependent)
|
||||||
|
* int64_t mtime_sec (was time_t, platform-dependent)
|
||||||
|
* int64_t mtime_nsec (was long, platform-dependent)
|
||||||
|
*
|
||||||
|
* Prior to 2.0.0 the wire format used the raw platform-dependent types,
|
||||||
|
* which broke compatiblity across different systems. All fields are now
|
||||||
|
* serialized as fixed-width integers.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/* Size of metadata fields on wire, excluding the int32_t `present` field that
|
||||||
|
* is always sent first. The total wire size for present metadata is
|
||||||
|
* sizeof(int32_t) + FILE_METADATA_WIRE_SIZE (32 bytes on most platforms). */
|
||||||
|
#define FILE_METADATA_WIRE_SIZE (sizeof(int32_t) * 3 + sizeof(int64_t) * 2)
|
||||||
|
|
||||||
void metadata_to_buf(char** buf, const FileMetadata* m);
|
void metadata_to_buf(char** buf, const FileMetadata* m);
|
||||||
FileMetadata* metadata_from_buf(char** buf);
|
FileMetadata* metadata_from_buf(char** buf);
|
||||||
|
|||||||
+17
-7
@@ -2,6 +2,7 @@
|
|||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <openssl/ssl.h>
|
#include <openssl/ssl.h>
|
||||||
|
#include <poll.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
@@ -14,7 +15,7 @@
|
|||||||
|
|
||||||
static __thread int io_read_fd = -1;
|
static __thread int io_read_fd = -1;
|
||||||
static __thread int io_write_fd = -1;
|
static __thread int io_write_fd = -1;
|
||||||
static SSL* io_ssl = NULL;
|
static SSL* io_ssl;
|
||||||
|
|
||||||
static unsigned long long io_bwlimit = 0;
|
static unsigned long long io_bwlimit = 0;
|
||||||
static long long bw_tokens = 0;
|
static long long bw_tokens = 0;
|
||||||
@@ -52,12 +53,11 @@ static void bw_throttle(size_t bytes_written) {
|
|||||||
bw_tokens -= (long long)bytes_written;
|
bw_tokens -= (long long)bytes_written;
|
||||||
|
|
||||||
if (bw_tokens < 0) {
|
if (bw_tokens < 0) {
|
||||||
long long deficit_ns = (long long)((double)(-bw_tokens) / io_bwlimit * 1000000000.0);
|
long long deficit_us = (long long)((double)(-bw_tokens) / io_bwlimit * 1000000.0);
|
||||||
struct timespec sleep_time, remaining;
|
if (deficit_us >= 1000)
|
||||||
sleep_time.tv_sec = deficit_ns / 1000000000LL;
|
poll(NULL, 0, (int)(deficit_us / 1000));
|
||||||
sleep_time.tv_nsec = deficit_ns % 1000000000LL;
|
else
|
||||||
while (nanosleep(&sleep_time, &remaining) < 0 && errno == EINTR)
|
usleep((useconds_t)deficit_us);
|
||||||
sleep_time = remaining;
|
|
||||||
bw_tokens = 0;
|
bw_tokens = 0;
|
||||||
clock_gettime(CLOCK_MONOTONIC, &bw_last_refill);
|
clock_gettime(CLOCK_MONOTONIC, &bw_last_refill);
|
||||||
}
|
}
|
||||||
@@ -85,6 +85,11 @@ bool send_n_data(int file_descriptor, const void* data, size_t data_size) {
|
|||||||
else
|
else
|
||||||
bytes_send = write(fd, (const char*)data + total_bytes_send, chunk);
|
bytes_send = write(fd, (const char*)data + total_bytes_send, chunk);
|
||||||
if (bytes_send <= 0) {
|
if (bytes_send <= 0) {
|
||||||
|
if (io_ssl) {
|
||||||
|
int ssl_err = SSL_get_error(io_ssl, (int)bytes_send);
|
||||||
|
if (ssl_err == SSL_ERROR_WANT_WRITE || ssl_err == SSL_ERROR_WANT_READ)
|
||||||
|
continue;
|
||||||
|
}
|
||||||
log_message(LOG_LEVEL_ERROR, "Could not send data");
|
log_message(LOG_LEVEL_ERROR, "Could not send data");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -121,6 +126,11 @@ bool receive_n_data(int file_descriptor, void* data, size_t data_size) {
|
|||||||
bytes_received =
|
bytes_received =
|
||||||
read(fd, (char*)data + total_bytes_received, data_size - total_bytes_received);
|
read(fd, (char*)data + total_bytes_received, data_size - total_bytes_received);
|
||||||
if (bytes_received <= 0) {
|
if (bytes_received <= 0) {
|
||||||
|
if (io_ssl) {
|
||||||
|
int ssl_err = SSL_get_error(io_ssl, (int)bytes_received);
|
||||||
|
if (ssl_err == SSL_ERROR_WANT_WRITE || ssl_err == SSL_ERROR_WANT_READ)
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (bytes_received == 0)
|
if (bytes_received == 0)
|
||||||
log_message(LOG_LEVEL_ERROR, "Connection closed while receiving data");
|
log_message(LOG_LEVEL_ERROR, "Connection closed while receiving data");
|
||||||
else
|
else
|
||||||
|
|||||||
@@ -5,6 +5,9 @@
|
|||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
|
|
||||||
|
/* Maximum allowed string size for receive_str (10 MB) */
|
||||||
|
#define MAX_STRING_SIZE (10 * 1024 * 1024)
|
||||||
|
|
||||||
typedef struct ssl_st SSL;
|
typedef struct ssl_st SSL;
|
||||||
|
|
||||||
typedef int Status;
|
typedef int Status;
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ static int parse_remote_dest(const char* dest, RemoteDest* r) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
Client* client_connect_ssh(const char* destination, int port) {
|
Client* client_connect_ssh(const char* destination, int port, const char* server_path) {
|
||||||
RemoteDest r;
|
RemoteDest r;
|
||||||
if (parse_remote_dest(destination, &r) != 0) {
|
if (parse_remote_dest(destination, &r) != 0) {
|
||||||
fprintf(stderr, "Invalid remote destination: %s\n", destination);
|
fprintf(stderr, "Invalid remote destination: %s\n", destination);
|
||||||
@@ -147,7 +147,7 @@ Client* client_connect_ssh(const char* destination, int port) {
|
|||||||
ssh_argv[ac++] = port_str;
|
ssh_argv[ac++] = port_str;
|
||||||
}
|
}
|
||||||
ssh_argv[ac++] = ssh_user;
|
ssh_argv[ac++] = ssh_user;
|
||||||
ssh_argv[ac++] = "fastsync-server";
|
ssh_argv[ac++] = (char*)(server_path ? server_path : "fastsync-server");
|
||||||
ssh_argv[ac++] = "--stdio";
|
ssh_argv[ac++] = "--stdio";
|
||||||
ssh_argv[ac] = NULL;
|
ssh_argv[ac] = NULL;
|
||||||
execvp("ssh", ssh_argv);
|
execvp("ssh", ssh_argv);
|
||||||
@@ -168,7 +168,8 @@ Client* client_connect_ssh(const char* destination, int port) {
|
|||||||
close(sv[0]);
|
close(sv[0]);
|
||||||
waitpid(pid, NULL, 0);
|
waitpid(pid, NULL, 0);
|
||||||
remote_dest_destroy(&r);
|
remote_dest_destroy(&r);
|
||||||
fprintf(stderr, "Error: could not launch 'fastsync-server --stdio' on remote\n");
|
fprintf(stderr, "Error: could not launch '%s --stdio' on remote\n",
|
||||||
|
server_path ? server_path : "fastsync-server");
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,6 @@
|
|||||||
|
|
||||||
#include "transport_tcp.h"
|
#include "transport_tcp.h"
|
||||||
|
|
||||||
Client* client_connect_ssh(const char* destination, int port);
|
Client* client_connect_ssh(const char* destination, int port, const char* server_path);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
#include <sys/wait.h>
|
#include <sys/wait.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
static volatile unsigned int g_active_connections = 0;
|
static volatile sig_atomic_t g_active_connections = 0;
|
||||||
|
|
||||||
static void sigchld_handler(int sig) {
|
static void sigchld_handler(int sig) {
|
||||||
(void)sig;
|
(void)sig;
|
||||||
@@ -92,7 +92,7 @@ static void accept_loop(Server* server, void (*child_fn)(int, void*), void* chil
|
|||||||
perror("Could not accept the connection");
|
perror("Could not accept the connection");
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (g_active_connections >= server->max_connections) {
|
if ((unsigned int)g_active_connections >= server->max_connections) {
|
||||||
log_message(LOG_LEVEL_WARNING, "Max connections (%u) reached, rejecting",
|
log_message(LOG_LEVEL_WARNING, "Max connections (%u) reached, rejecting",
|
||||||
server->max_connections);
|
server->max_connections);
|
||||||
close(fd);
|
close(fd);
|
||||||
|
|||||||
+28
-8
@@ -61,6 +61,19 @@ char* str_dup(const char* string) {
|
|||||||
bool glob_match(const char* pattern, const char* str) {
|
bool glob_match(const char* pattern, const char* str) {
|
||||||
while (*pattern) {
|
while (*pattern) {
|
||||||
if (*pattern == '*') {
|
if (*pattern == '*') {
|
||||||
|
if (*(pattern + 1) == '*') {
|
||||||
|
pattern += 2;
|
||||||
|
if (*pattern == '\0')
|
||||||
|
return true;
|
||||||
|
if (*pattern == '/')
|
||||||
|
pattern++;
|
||||||
|
while (*str) {
|
||||||
|
if (glob_match(pattern, str))
|
||||||
|
return true;
|
||||||
|
str++;
|
||||||
|
}
|
||||||
|
return glob_match(pattern, str);
|
||||||
|
}
|
||||||
pattern++;
|
pattern++;
|
||||||
while (*str && *str != '/') {
|
while (*str && *str != '/') {
|
||||||
if (glob_match(pattern, str))
|
if (glob_match(pattern, str))
|
||||||
@@ -74,8 +87,15 @@ bool glob_match(const char* pattern, const char* str) {
|
|||||||
pattern++;
|
pattern++;
|
||||||
str++;
|
str++;
|
||||||
} else {
|
} else {
|
||||||
if (*pattern != *str)
|
if (*pattern != *str) {
|
||||||
|
if (*pattern == '/' && *(pattern + 1) == '*' && *(pattern + 2) == '*') {
|
||||||
|
const char* rest = pattern + 3;
|
||||||
|
if (*rest == '/')
|
||||||
|
rest++;
|
||||||
|
return glob_match(rest, str);
|
||||||
|
}
|
||||||
return false;
|
return false;
|
||||||
|
}
|
||||||
pattern++;
|
pattern++;
|
||||||
str++;
|
str++;
|
||||||
}
|
}
|
||||||
@@ -99,7 +119,7 @@ static void delete_extras_walk(const char* abs_path, const char* rel_path, Array
|
|||||||
if (!dir)
|
if (!dir)
|
||||||
return;
|
return;
|
||||||
bool all_removed = true;
|
bool all_removed = true;
|
||||||
struct dirent* entry;
|
const struct dirent* entry;
|
||||||
while ((entry = readdir(dir)) != NULL) {
|
while ((entry = readdir(dir)) != NULL) {
|
||||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||||
continue;
|
continue;
|
||||||
@@ -168,18 +188,18 @@ bool has_path_traversal(const char* path) {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
char* path_cat(const char* path1, char* path2) {
|
char* path_cat(const char* path1, const char* path2) {
|
||||||
if (path1 == NULL || *path1 == '\0')
|
if (path1 == NULL || *path1 == '\0')
|
||||||
return str_dup(path2);
|
return str_dup(path2);
|
||||||
if (path2 == NULL || *path2 == '\0')
|
if (path2 == NULL || *path2 == '\0')
|
||||||
return str_dup(path1);
|
return str_dup(path1);
|
||||||
int path1_len = strlen(path1);
|
size_t path1_len = strlen(path1);
|
||||||
int path2_len = strlen(path2);
|
size_t path2_len = strlen(path2);
|
||||||
char* path2_pointer = path2;
|
size_t offset = 0;
|
||||||
if (path1[path1_len - 1] == '/')
|
if (path1[path1_len - 1] == '/')
|
||||||
path1_len -= 1;
|
path1_len -= 1;
|
||||||
if (path2[0] == '/') {
|
if (path2[0] == '/') {
|
||||||
path2_pointer += 1;
|
offset = 1;
|
||||||
path2_len -= 1;
|
path2_len -= 1;
|
||||||
}
|
}
|
||||||
char* new_path = malloc(path1_len + path2_len + 2);
|
char* new_path = malloc(path1_len + path2_len + 2);
|
||||||
@@ -187,7 +207,7 @@ char* path_cat(const char* path1, char* path2) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
memcpy(new_path, path1, path1_len);
|
memcpy(new_path, path1, path1_len);
|
||||||
new_path[path1_len] = '/';
|
new_path[path1_len] = '/';
|
||||||
memcpy(new_path + path1_len + 1, path2_pointer, path2_len);
|
memcpy(new_path + path1_len + 1, path2 + offset, path2_len);
|
||||||
new_path[path1_len + path2_len + 1] = '\0';
|
new_path[path1_len + path2_len + 1] = '\0';
|
||||||
return new_path;
|
return new_path;
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
bool mkdir_r(const char* path);
|
bool mkdir_r(const char* path);
|
||||||
char* str_dup(const char* string);
|
char* str_dup(const char* string);
|
||||||
char* path_cat(const char* path1, char* path2);
|
char* path_cat(const char* path1, const char* path2);
|
||||||
bool glob_match(const char* pattern, const char* str);
|
bool glob_match(const char* pattern, const char* str);
|
||||||
void delete_extras(const char* dest_root, ArrayList* manifest);
|
void delete_extras(const char* dest_root, ArrayList* manifest);
|
||||||
bool has_path_traversal(const char* path);
|
bool has_path_traversal(const char* path);
|
||||||
|
|||||||
@@ -5,8 +5,11 @@
|
|||||||
#include "test_data.h"
|
#include "test_data.h"
|
||||||
#include "test_delta.h"
|
#include "test_delta.h"
|
||||||
#include "test_file.h"
|
#include "test_file.h"
|
||||||
|
#include "test_file_sendfile.h"
|
||||||
#include "test_glob.h"
|
#include "test_glob.h"
|
||||||
|
#include "test_log.h"
|
||||||
#include "test_metadata.h"
|
#include "test_metadata.h"
|
||||||
|
#include "test_multiprocessing.h"
|
||||||
#include "test_property.h"
|
#include "test_property.h"
|
||||||
#include "test_protocol.h"
|
#include "test_protocol.h"
|
||||||
#include "test_queue.h"
|
#include "test_queue.h"
|
||||||
@@ -14,6 +17,9 @@
|
|||||||
#include "test_scanner.h"
|
#include "test_scanner.h"
|
||||||
#include "test_shared_utils.h"
|
#include "test_shared_utils.h"
|
||||||
#include "test_stress.h"
|
#include "test_stress.h"
|
||||||
|
#include "test_transport_tcp.h"
|
||||||
|
#include "test_transport_ssh.h"
|
||||||
|
#include "test_transport_tls.h"
|
||||||
#include "test_utils.h"
|
#include "test_utils.h"
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
|
||||||
@@ -38,9 +44,15 @@ int main() {
|
|||||||
RUN_TEST(test_metadata);
|
RUN_TEST(test_metadata);
|
||||||
RUN_TEST(test_glob);
|
RUN_TEST(test_glob);
|
||||||
RUN_TEST(test_file);
|
RUN_TEST(test_file);
|
||||||
|
RUN_TEST(test_file_sendfile);
|
||||||
|
RUN_TEST(test_multiprocessing);
|
||||||
|
RUN_TEST(test_log);
|
||||||
RUN_TEST(test_robustness);
|
RUN_TEST(test_robustness);
|
||||||
RUN_TEST(test_stress);
|
RUN_TEST(test_stress);
|
||||||
RUN_TEST(test_property);
|
RUN_TEST(test_property);
|
||||||
|
RUN_TEST(test_transport_tcp);
|
||||||
|
RUN_TEST(test_transport_ssh);
|
||||||
|
RUN_TEST(test_transport_tls);
|
||||||
|
|
||||||
printf("\n\033[1;36m=== TEST SUMMARY ===\033[0m\n");
|
printf("\n\033[1;36m=== TEST SUMMARY ===\033[0m\n");
|
||||||
printf("Total Tests Run: %d\n", tests_run);
|
printf("Total Tests Run: %d\n", tests_run);
|
||||||
|
|||||||
+51
-12
@@ -7,9 +7,17 @@
|
|||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
|
|
||||||
static void test_config_lifecycle() {
|
static void test_config_lifecycle() {
|
||||||
Config* cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("/dst"), true, true, false,
|
Config* cfg = config_create();
|
||||||
false, false, 1, false, 0);
|
|
||||||
EXPECT_NOT_NULL(cfg);
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
free(cfg->version);
|
||||||
|
cfg->version = str_dup("1.0");
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
cfg->save_to_disk = true;
|
||||||
|
cfg->use_multithreading = true;
|
||||||
|
cfg->use_chunk_serialization = false;
|
||||||
|
cfg->use_compression = false;
|
||||||
|
cfg->compression_level = 1;
|
||||||
EXPECT_EQ_STR(cfg->version, "1.0");
|
EXPECT_EQ_STR(cfg->version, "1.0");
|
||||||
EXPECT_EQ_STR(cfg->send_directory, "/src");
|
EXPECT_EQ_STR(cfg->send_directory, "/src");
|
||||||
EXPECT_EQ_STR(cfg->receive_root_directory, "/dst");
|
EXPECT_EQ_STR(cfg->receive_root_directory, "/dst");
|
||||||
@@ -23,9 +31,14 @@ static void test_config_lifecycle() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void test_config_ssh_dest() {
|
static void test_config_ssh_dest() {
|
||||||
Config* cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("user@host:/dst"), true,
|
Config* cfg = config_create();
|
||||||
false, false, false, false, 1, false, 0);
|
|
||||||
EXPECT_NOT_NULL(cfg);
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
free(cfg->version);
|
||||||
|
cfg->version = str_dup("1.0");
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("user@host:/dst");
|
||||||
|
cfg->save_to_disk = true;
|
||||||
|
cfg->compression_level = 1;
|
||||||
EXPECT_EQ_INT(cfg->transport, TRANSPORT_TCP);
|
EXPECT_EQ_INT(cfg->transport, TRANSPORT_TCP);
|
||||||
EXPECT_NULL(cfg->ssh_destination);
|
EXPECT_NULL(cfg->ssh_destination);
|
||||||
EXPECT_EQ_STR(cfg->receive_root_directory, "user@host:/dst");
|
EXPECT_EQ_STR(cfg->receive_root_directory, "user@host:/dst");
|
||||||
@@ -38,8 +51,14 @@ static void test_config_ssh_dest() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void test_config_ssh_dest_local_path() {
|
static void test_config_ssh_dest_local_path() {
|
||||||
Config* cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("/local/path"), true, false,
|
Config* cfg = config_create();
|
||||||
false, false, false, 1, false, 0);
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
free(cfg->version);
|
||||||
|
cfg->version = str_dup("1.0");
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/local/path");
|
||||||
|
cfg->save_to_disk = true;
|
||||||
|
cfg->compression_level = 1;
|
||||||
config_parse_ssh_dest(cfg);
|
config_parse_ssh_dest(cfg);
|
||||||
EXPECT_EQ_INT(cfg->transport, TRANSPORT_TCP);
|
EXPECT_EQ_INT(cfg->transport, TRANSPORT_TCP);
|
||||||
EXPECT_NULL(cfg->ssh_destination);
|
EXPECT_NULL(cfg->ssh_destination);
|
||||||
@@ -48,8 +67,14 @@ static void test_config_ssh_dest_local_path() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void test_config_ssh_dest_no_user() {
|
static void test_config_ssh_dest_no_user() {
|
||||||
Config* cfg = config_create(str_dup("1.0"), str_dup("/src"), str_dup("host:/remote"), true, false,
|
Config* cfg = config_create();
|
||||||
false, false, false, 1, false, 0);
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
free(cfg->version);
|
||||||
|
cfg->version = str_dup("1.0");
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("host:/remote");
|
||||||
|
cfg->save_to_disk = true;
|
||||||
|
cfg->compression_level = 1;
|
||||||
config_parse_ssh_dest(cfg);
|
config_parse_ssh_dest(cfg);
|
||||||
EXPECT_EQ_INT(cfg->transport, TRANSPORT_SSH);
|
EXPECT_EQ_INT(cfg->transport, TRANSPORT_SSH);
|
||||||
EXPECT_EQ_STR(cfg->ssh_destination, "host:/remote");
|
EXPECT_EQ_STR(cfg->ssh_destination, "host:/remote");
|
||||||
@@ -58,8 +83,14 @@ static void test_config_ssh_dest_no_user() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void test_pipeline_sender_lifecycle() {
|
static void test_pipeline_sender_lifecycle() {
|
||||||
Config* cfg = config_create(str_dup("2.0"), str_dup("/src2"), str_dup("/dst2"), false, false,
|
Config* cfg = config_create();
|
||||||
true, true, false, 1, false, 0);
|
free(cfg->version);
|
||||||
|
cfg->version = str_dup("2.0");
|
||||||
|
cfg->send_directory = str_dup("/src2");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst2");
|
||||||
|
cfg->use_chunk_serialization = true;
|
||||||
|
cfg->use_compression = true;
|
||||||
|
cfg->compression_level = 1;
|
||||||
Queue* q1 = queue_create(5, NULL);
|
Queue* q1 = queue_create(5, NULL);
|
||||||
Queue* q2 = queue_create(15, NULL);
|
Queue* q2 = queue_create(15, NULL);
|
||||||
|
|
||||||
@@ -75,8 +106,16 @@ static void test_pipeline_sender_lifecycle() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void test_pipeline_receiver_lifecycle() {
|
static void test_pipeline_receiver_lifecycle() {
|
||||||
Config* cfg = config_create(str_dup("3.0"), str_dup("/src3"), str_dup("/dst3"), true, true, true,
|
Config* cfg = config_create();
|
||||||
true, false, 1, false, 0);
|
free(cfg->version);
|
||||||
|
cfg->version = str_dup("3.0");
|
||||||
|
cfg->send_directory = str_dup("/src3");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst3");
|
||||||
|
cfg->save_to_disk = true;
|
||||||
|
cfg->use_multithreading = true;
|
||||||
|
cfg->use_chunk_serialization = true;
|
||||||
|
cfg->use_compression = true;
|
||||||
|
cfg->compression_level = 1;
|
||||||
Queue* q = queue_create(20, NULL);
|
Queue* q = queue_create(20, NULL);
|
||||||
|
|
||||||
PipelineContextReceiver* pcr = pipeline_context_receiver_create(cfg, q, 42);
|
PipelineContextReceiver* pcr = pipeline_context_receiver_create(cfg, q, 42);
|
||||||
|
|||||||
@@ -23,6 +23,14 @@ static void test_data_create_empty() {
|
|||||||
data_destroy(d);
|
data_destroy(d);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_data_create_empty_zero() {
|
||||||
|
Data* d = data_create_empty(0);
|
||||||
|
EXPECT_NOT_NULL(d);
|
||||||
|
EXPECT_NOT_NULL(d->data);
|
||||||
|
EXPECT_EQ_INT((int)d->size, 0);
|
||||||
|
data_destroy(d);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_data_create_reserve() {
|
static void test_data_create_reserve() {
|
||||||
Data* d = data_create_reserve(1024);
|
Data* d = data_create_reserve(1024);
|
||||||
EXPECT_NOT_NULL(d);
|
EXPECT_NOT_NULL(d);
|
||||||
@@ -44,6 +52,7 @@ static void test_data_destroy_normal() {
|
|||||||
void test_data() {
|
void test_data() {
|
||||||
test_data_create();
|
test_data_create();
|
||||||
test_data_create_empty();
|
test_data_create_empty();
|
||||||
|
test_data_create_empty_zero();
|
||||||
test_data_create_reserve();
|
test_data_create_reserve();
|
||||||
test_data_destroy_null();
|
test_data_destroy_null();
|
||||||
test_data_destroy_normal();
|
test_data_destroy_normal();
|
||||||
|
|||||||
+4
-3
@@ -154,8 +154,9 @@ static void test_file_send_receive() {
|
|||||||
memcpy(file->data->data, content, len);
|
memcpy(file->data->data, content, len);
|
||||||
file->data->size = len;
|
file->data->size = len;
|
||||||
|
|
||||||
Config* cfg = config_create(str_dup(PROTOCOL_VERSION), str_dup("/tmp"), str_dup("/tmp"), false,
|
Config* cfg = config_create();
|
||||||
false, false, false, false, 0, false, 0);
|
cfg->send_directory = str_dup("/tmp");
|
||||||
|
cfg->receive_root_directory = str_dup("/tmp");
|
||||||
|
|
||||||
int p[2];
|
int p[2];
|
||||||
EXPECT_EQ_INT(pipe(p), 0);
|
EXPECT_EQ_INT(pipe(p), 0);
|
||||||
@@ -271,7 +272,7 @@ void test_file() {
|
|||||||
test_to_disk_basic();
|
test_to_disk_basic();
|
||||||
test_to_disk_creates_dirs();
|
test_to_disk_creates_dirs();
|
||||||
test_file_content_to_buffer();
|
test_file_content_to_buffer();
|
||||||
if (!getenv("FASTSYNC_UNDER_VALGRIND")) {
|
if (!is_running_under_valgrind()) {
|
||||||
// Fork tests are skipped under valgrind because the parent process runs
|
// Fork tests are skipped under valgrind because the parent process runs
|
||||||
// orders of magnitude slower than the child (parent is instrumented, child
|
// orders of magnitude slower than the child (parent is instrumented, child
|
||||||
// is not), which causes pipe-based protocol handshake timeouts. The parent
|
// is not), which causes pipe-based protocol handshake timeouts. The parent
|
||||||
|
|||||||
@@ -0,0 +1,276 @@
|
|||||||
|
#include "test_file_sendfile.h"
|
||||||
|
#include "file.h"
|
||||||
|
#include "config.h"
|
||||||
|
#include "protocol.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include "test_utils.h"
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/wait.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
|
/* Test basic sendfile transfer: create a file, send it via file_send_sendfile,
|
||||||
|
* receive via file_receive, and verify contents. */
|
||||||
|
static void test_sendfile_basic() {
|
||||||
|
const char* content = "Hello from sendfile test!";
|
||||||
|
size_t len = strlen(content);
|
||||||
|
EXPECT_TRUE(to_disk("test_sendfile_basic.txt", content, len));
|
||||||
|
|
||||||
|
File* file = file_create("test_sendfile_basic.txt");
|
||||||
|
EXPECT_NOT_NULL(file);
|
||||||
|
/* Set the size so file_send_sendfile can report it */
|
||||||
|
file->data->size = len;
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->send_directory = str_dup("/tmp");
|
||||||
|
cfg->receive_root_directory = str_dup("/tmp");
|
||||||
|
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(pipe(p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
/* Child: receive */
|
||||||
|
close(p[1]);
|
||||||
|
File* received = file_receive(cfg, p[0]);
|
||||||
|
close(p[0]);
|
||||||
|
|
||||||
|
bool ok = true;
|
||||||
|
if (!received)
|
||||||
|
ok = false;
|
||||||
|
else {
|
||||||
|
if (!received->path || strcmp(received->path, "test_sendfile_basic.txt") != 0)
|
||||||
|
ok = false;
|
||||||
|
if (!received->data || received->data->size != len)
|
||||||
|
ok = false;
|
||||||
|
else if (memcmp(received->data->data, content, len) != 0)
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
file_destroy(received);
|
||||||
|
config_delete(cfg);
|
||||||
|
_exit(ok ? 0 : 1);
|
||||||
|
} else {
|
||||||
|
/* Parent: send via sendfile */
|
||||||
|
close(p[0]);
|
||||||
|
bool sent = file_send_sendfile(file, p[1], false, 0, true);
|
||||||
|
close(p[1]);
|
||||||
|
|
||||||
|
int status;
|
||||||
|
waitpid(pid, &status, 0);
|
||||||
|
|
||||||
|
file_destroy(file);
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink("test_sendfile_basic.txt");
|
||||||
|
|
||||||
|
EXPECT_TRUE(sent);
|
||||||
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Test sendfile with an empty file */
|
||||||
|
static void test_sendfile_empty_file() {
|
||||||
|
const char* content = "";
|
||||||
|
size_t len = 0;
|
||||||
|
EXPECT_TRUE(to_disk("test_sendfile_empty.txt", content, len));
|
||||||
|
|
||||||
|
File* file = file_create("test_sendfile_empty.txt");
|
||||||
|
EXPECT_NOT_NULL(file);
|
||||||
|
file->data->size = 0;
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->send_directory = str_dup("/tmp");
|
||||||
|
cfg->receive_root_directory = str_dup("/tmp");
|
||||||
|
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(pipe(p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
/* Child: receive */
|
||||||
|
close(p[1]);
|
||||||
|
File* received = file_receive(cfg, p[0]);
|
||||||
|
close(p[0]);
|
||||||
|
|
||||||
|
bool ok = true;
|
||||||
|
if (!received)
|
||||||
|
ok = false;
|
||||||
|
else {
|
||||||
|
if (strcmp(received->path, "test_sendfile_empty.txt") != 0)
|
||||||
|
ok = false;
|
||||||
|
if (received->data->size != 0)
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
file_destroy(received);
|
||||||
|
config_delete(cfg);
|
||||||
|
_exit(ok ? 0 : 1);
|
||||||
|
} else {
|
||||||
|
close(p[0]);
|
||||||
|
bool sent = file_send_sendfile(file, p[1], false, 0, true);
|
||||||
|
close(p[1]);
|
||||||
|
|
||||||
|
int status;
|
||||||
|
waitpid(pid, &status, 0);
|
||||||
|
|
||||||
|
file_destroy(file);
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink("test_sendfile_empty.txt");
|
||||||
|
|
||||||
|
EXPECT_TRUE(sent);
|
||||||
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Test error path: file does not exist on disk */
|
||||||
|
static void test_sendfile_missing_file() {
|
||||||
|
File* file = file_create("nonexistent_sendfile_test_file.txt");
|
||||||
|
EXPECT_NOT_NULL(file);
|
||||||
|
file->data->size = 100; /* fake size */
|
||||||
|
|
||||||
|
/* Use a pipe that we can write to but sendfile should fail */
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(pipe(p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
|
/* file_send_sendfile will try to open the nonexistent file -> should return false */
|
||||||
|
bool sent = file_send_sendfile(file, p[1], false, 0, true);
|
||||||
|
|
||||||
|
close(p[0]);
|
||||||
|
close(p[1]);
|
||||||
|
file_destroy(file);
|
||||||
|
|
||||||
|
EXPECT_FALSE(sent);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Test compression level > 0 falls back to file_send_single_calls */
|
||||||
|
static void test_sendfile_compression_fallback() {
|
||||||
|
const char* content = "Compression fallback content";
|
||||||
|
size_t len = strlen(content);
|
||||||
|
EXPECT_TRUE(to_disk("test_sendfile_comp.txt", content, len));
|
||||||
|
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat("test_sendfile_comp.txt", &st), 0);
|
||||||
|
|
||||||
|
File* file = file_create("test_sendfile_comp.txt");
|
||||||
|
EXPECT_NOT_NULL(file);
|
||||||
|
/* Load the file data into memory (required by file_send_single_calls fallback) */
|
||||||
|
file->data->size = (size_t)st.st_size;
|
||||||
|
EXPECT_TRUE(file_load_data(file));
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->send_directory = str_dup("/tmp");
|
||||||
|
cfg->receive_root_directory = str_dup("/tmp");
|
||||||
|
cfg->use_compression = true;
|
||||||
|
cfg->compression_level = 3;
|
||||||
|
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(pipe(p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
/* Child: receive */
|
||||||
|
close(p[1]);
|
||||||
|
File* received = file_receive(cfg, p[0]);
|
||||||
|
close(p[0]);
|
||||||
|
|
||||||
|
bool ok = true;
|
||||||
|
if (!received)
|
||||||
|
ok = false;
|
||||||
|
else {
|
||||||
|
if (received->data->size != len)
|
||||||
|
ok = false;
|
||||||
|
else if (memcmp(received->data->data, content, len) != 0)
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
file_destroy(received);
|
||||||
|
config_delete(cfg);
|
||||||
|
_exit(ok ? 0 : 1);
|
||||||
|
} else {
|
||||||
|
close(p[0]);
|
||||||
|
/* compression_level = 3 triggers fallback to file_send_single_calls */
|
||||||
|
bool sent = file_send_sendfile(file, p[1], false, 3, true);
|
||||||
|
close(p[1]);
|
||||||
|
|
||||||
|
int status;
|
||||||
|
waitpid(pid, &status, 0);
|
||||||
|
|
||||||
|
file_destroy(file);
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink("test_sendfile_comp.txt");
|
||||||
|
|
||||||
|
EXPECT_TRUE(sent);
|
||||||
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Test sendfile without path (send_path = false) */
|
||||||
|
static void test_sendfile_no_path() {
|
||||||
|
const char* content = "No path sendfile test";
|
||||||
|
size_t len = strlen(content);
|
||||||
|
EXPECT_TRUE(to_disk("test_sendfile_nopath.txt", content, len));
|
||||||
|
|
||||||
|
File* file = file_create("test_sendfile_nopath.txt");
|
||||||
|
EXPECT_NOT_NULL(file);
|
||||||
|
file->data->size = len;
|
||||||
|
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(pipe(p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
close(p[1]);
|
||||||
|
Data* received = receive_data(p[0]);
|
||||||
|
close(p[0]);
|
||||||
|
|
||||||
|
bool ok = true;
|
||||||
|
if (!received)
|
||||||
|
ok = false;
|
||||||
|
else if (received->size != len)
|
||||||
|
ok = false;
|
||||||
|
else if (memcmp(received->data, content, len) != 0)
|
||||||
|
ok = false;
|
||||||
|
|
||||||
|
data_destroy(received);
|
||||||
|
_exit(ok ? 0 : 1);
|
||||||
|
} else {
|
||||||
|
close(p[0]);
|
||||||
|
bool sent = file_send_sendfile(file, p[1], false, 0, false);
|
||||||
|
close(p[1]);
|
||||||
|
|
||||||
|
int status;
|
||||||
|
waitpid(pid, &status, 0);
|
||||||
|
|
||||||
|
file_destroy(file);
|
||||||
|
unlink("test_sendfile_nopath.txt");
|
||||||
|
|
||||||
|
EXPECT_TRUE(sent);
|
||||||
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void test_file_sendfile() {
|
||||||
|
if (!is_running_under_valgrind()) {
|
||||||
|
// Fork tests are skipped under valgrind because the parent process runs
|
||||||
|
// orders of magnitude slower than the child (parent is instrumented, child
|
||||||
|
// is not), which causes pipe-based protocol handshake timeouts. The parent
|
||||||
|
// process itself has zero valgrind errors -- the failures are all in the
|
||||||
|
// forked children where inherited allocations are reported as leaks.
|
||||||
|
test_sendfile_basic();
|
||||||
|
test_sendfile_empty_file();
|
||||||
|
test_sendfile_compression_fallback();
|
||||||
|
test_sendfile_no_path();
|
||||||
|
}
|
||||||
|
test_sendfile_missing_file(); // no fork, safe under valgrind
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#ifndef TEST_FILE_SENDFILE_H
|
||||||
|
#define TEST_FILE_SENDFILE_H
|
||||||
|
|
||||||
|
void test_file_sendfile();
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -49,6 +49,33 @@ static void test_glob_question_star() {
|
|||||||
EXPECT_TRUE(glob_match("?*.txt", "a.txt"));
|
EXPECT_TRUE(glob_match("?*.txt", "a.txt"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_glob_doublestar_match_all() {
|
||||||
|
EXPECT_TRUE(glob_match("**", "anything"));
|
||||||
|
EXPECT_TRUE(glob_match("**", "path/to/file"));
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_glob_doublestar_prefix() {
|
||||||
|
EXPECT_TRUE(glob_match("**/foo", "foo"));
|
||||||
|
EXPECT_TRUE(glob_match("**/foo", "bar/foo"));
|
||||||
|
EXPECT_TRUE(glob_match("**/foo", "a/b/c/foo"));
|
||||||
|
EXPECT_FALSE(glob_match("**/foo", "foobar"));
|
||||||
|
EXPECT_FALSE(glob_match("**/foo", "bar/foobar"));
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_glob_doublestar_suffix() {
|
||||||
|
EXPECT_TRUE(glob_match("foo/**", "foo"));
|
||||||
|
EXPECT_TRUE(glob_match("foo/**", "foo/bar"));
|
||||||
|
EXPECT_TRUE(glob_match("foo/**", "foo/bar/baz"));
|
||||||
|
EXPECT_FALSE(glob_match("foo/**", "foobar"));
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_glob_doublestar_mid() {
|
||||||
|
EXPECT_TRUE(glob_match("a/**/b", "a/b"));
|
||||||
|
EXPECT_TRUE(glob_match("a/**/b", "a/x/b"));
|
||||||
|
EXPECT_TRUE(glob_match("a/**/b", "a/x/y/z/b"));
|
||||||
|
EXPECT_FALSE(glob_match("a/**/b", "a/x/bad"));
|
||||||
|
}
|
||||||
|
|
||||||
void test_glob() {
|
void test_glob() {
|
||||||
test_glob_exact_match();
|
test_glob_exact_match();
|
||||||
test_glob_question_mark();
|
test_glob_question_mark();
|
||||||
@@ -60,4 +87,8 @@ void test_glob() {
|
|||||||
test_glob_slash_not_matched();
|
test_glob_slash_not_matched();
|
||||||
test_glob_complex();
|
test_glob_complex();
|
||||||
test_glob_question_star();
|
test_glob_question_star();
|
||||||
|
test_glob_doublestar_match_all();
|
||||||
|
test_glob_doublestar_prefix();
|
||||||
|
test_glob_doublestar_suffix();
|
||||||
|
test_glob_doublestar_mid();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
#include "test_log.h"
|
||||||
|
#include "log.h"
|
||||||
|
#include "test_utils.h"
|
||||||
|
|
||||||
|
/* Test default log level: WARNING and ERROR should print, DEBUG and INFO should not.
|
||||||
|
* We can't easily capture stderr in unit tests, so we verify the functions don't crash
|
||||||
|
* and that set_log_level changes behavior. */
|
||||||
|
|
||||||
|
static void test_log_message_debug() {
|
||||||
|
set_log_level(LOG_LEVEL_WARNING);
|
||||||
|
log_message(LOG_LEVEL_DEBUG, "debug message: %d", 42);
|
||||||
|
/* crash regression test — stderr capture would need infrastructure changes */
|
||||||
|
EXPECT_TRUE(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_log_message_info() {
|
||||||
|
set_log_level(LOG_LEVEL_WARNING);
|
||||||
|
log_message(LOG_LEVEL_INFO, "info message: %s", "test");
|
||||||
|
/* crash regression test — stderr capture would need infrastructure changes */
|
||||||
|
EXPECT_TRUE(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_log_message_warning() {
|
||||||
|
set_log_level(LOG_LEVEL_WARNING);
|
||||||
|
log_message(LOG_LEVEL_WARNING, "warning message: %d %s", 1, "test");
|
||||||
|
/* crash regression test — stderr capture would need infrastructure changes */
|
||||||
|
EXPECT_TRUE(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_log_message_error() {
|
||||||
|
set_log_level(LOG_LEVEL_WARNING);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "error message: %s", "critical");
|
||||||
|
/* crash regression test — stderr capture would need infrastructure changes */
|
||||||
|
EXPECT_TRUE(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_log_set_level_debug() {
|
||||||
|
set_log_level(LOG_LEVEL_DEBUG);
|
||||||
|
|
||||||
|
/* After setting to DEBUG, all levels should be shown */
|
||||||
|
log_message(LOG_LEVEL_DEBUG, "debug after set");
|
||||||
|
log_message(LOG_LEVEL_INFO, "info after set");
|
||||||
|
log_message(LOG_LEVEL_WARNING, "warning after set");
|
||||||
|
log_message(LOG_LEVEL_ERROR, "error after set");
|
||||||
|
|
||||||
|
/* crash regression test — stderr capture would need infrastructure changes */
|
||||||
|
EXPECT_TRUE(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_log_set_level_info() {
|
||||||
|
set_log_level(LOG_LEVEL_INFO);
|
||||||
|
|
||||||
|
/* INFO level should show INFO, WARNING, ERROR but not DEBUG */
|
||||||
|
log_message(LOG_LEVEL_DEBUG, "debug should be filtered"); /* filtered */
|
||||||
|
log_message(LOG_LEVEL_INFO, "info should show");
|
||||||
|
log_message(LOG_LEVEL_WARNING, "warning should show");
|
||||||
|
log_message(LOG_LEVEL_ERROR, "error should show");
|
||||||
|
|
||||||
|
/* crash regression test — stderr capture would need infrastructure changes */
|
||||||
|
EXPECT_TRUE(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_log_set_level_error() {
|
||||||
|
set_log_level(LOG_LEVEL_ERROR);
|
||||||
|
|
||||||
|
/* ERROR level: only ERROR should show */
|
||||||
|
log_message(LOG_LEVEL_DEBUG, "debug filtered");
|
||||||
|
log_message(LOG_LEVEL_INFO, "info filtered");
|
||||||
|
log_message(LOG_LEVEL_WARNING, "warning filtered");
|
||||||
|
log_message(LOG_LEVEL_ERROR, "error should show");
|
||||||
|
|
||||||
|
/* crash regression test — stderr capture would need infrastructure changes */
|
||||||
|
EXPECT_TRUE(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Test that set_log_level with default WARNING filters correctly */
|
||||||
|
static void test_log_filtering() {
|
||||||
|
/* Reset to default */
|
||||||
|
set_log_level(LOG_LEVEL_WARNING);
|
||||||
|
|
||||||
|
/* These should be filtered */
|
||||||
|
log_message(LOG_LEVEL_DEBUG, "filtered debug");
|
||||||
|
log_message(LOG_LEVEL_INFO, "filtered info");
|
||||||
|
|
||||||
|
/* These should be shown */
|
||||||
|
log_message(LOG_LEVEL_WARNING, "visible warning");
|
||||||
|
log_message(LOG_LEVEL_ERROR, "visible error");
|
||||||
|
|
||||||
|
/* crash regression test — stderr capture would need infrastructure changes */
|
||||||
|
EXPECT_TRUE(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Test that log_message handles various format strings */
|
||||||
|
static void test_log_message_formats() {
|
||||||
|
set_log_level(LOG_LEVEL_DEBUG);
|
||||||
|
|
||||||
|
log_message(LOG_LEVEL_DEBUG, "simple string");
|
||||||
|
log_message(LOG_LEVEL_INFO, "integer: %d", -1);
|
||||||
|
log_message(LOG_LEVEL_WARNING, "string: %s", "hello");
|
||||||
|
log_message(LOG_LEVEL_ERROR, "multiple: %d %s %d", 1, "two", 3);
|
||||||
|
|
||||||
|
/* crash regression test — stderr capture would need infrastructure changes */
|
||||||
|
EXPECT_TRUE(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
void test_log() {
|
||||||
|
test_log_message_debug();
|
||||||
|
test_log_message_info();
|
||||||
|
test_log_message_warning();
|
||||||
|
test_log_message_error();
|
||||||
|
test_log_set_level_debug();
|
||||||
|
test_log_set_level_info();
|
||||||
|
test_log_set_level_error();
|
||||||
|
test_log_filtering();
|
||||||
|
test_log_message_formats();
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#ifndef TEST_LOG_H
|
||||||
|
#define TEST_LOG_H
|
||||||
|
|
||||||
|
void test_log();
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
#include "test_multiprocessing.h"
|
||||||
|
#include "multiprocessing.h"
|
||||||
|
#include "config.h"
|
||||||
|
#include "queue.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include "test_utils.h"
|
||||||
|
#include <stdlib.h>
|
||||||
|
|
||||||
|
/* Test pipeline_context_sender_create/destroy with valid arguments */
|
||||||
|
static void test_sender_create_destroy() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
free(cfg->version);
|
||||||
|
cfg->version = str_dup("1.0");
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
|
||||||
|
Queue* q_scanner = queue_create(5, NULL);
|
||||||
|
EXPECT_NOT_NULL(q_scanner);
|
||||||
|
|
||||||
|
Queue* q_loader = queue_create(10, NULL);
|
||||||
|
EXPECT_NOT_NULL(q_loader);
|
||||||
|
|
||||||
|
PipelineContextSender* ctx = pipeline_context_sender_create(cfg, q_scanner, q_loader);
|
||||||
|
EXPECT_NOT_NULL(ctx);
|
||||||
|
EXPECT_EQ_STR(ctx->config->version, "1.0");
|
||||||
|
EXPECT_EQ_INT(ctx->queue_scanner->capacity, 5);
|
||||||
|
EXPECT_EQ_INT(ctx->queue_loader->capacity, 10);
|
||||||
|
EXPECT_FALSE(ctx->scanner_done);
|
||||||
|
EXPECT_FALSE(ctx->loader_done);
|
||||||
|
EXPECT_NULL(ctx->manifest);
|
||||||
|
|
||||||
|
pipeline_context_sender_destroy(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Test pipeline_context_receiver_create/destroy with valid arguments */
|
||||||
|
static void test_receiver_create_destroy() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
free(cfg->version);
|
||||||
|
cfg->version = str_dup("2.0");
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
cfg->save_to_disk = true;
|
||||||
|
cfg->use_multithreading = true;
|
||||||
|
|
||||||
|
Queue* q = queue_create(20, NULL);
|
||||||
|
EXPECT_NOT_NULL(q);
|
||||||
|
|
||||||
|
PipelineContextReceiver* ctx = pipeline_context_receiver_create(cfg, q, 42);
|
||||||
|
EXPECT_NOT_NULL(ctx);
|
||||||
|
EXPECT_EQ_STR(ctx->config->version, "2.0");
|
||||||
|
EXPECT_EQ_INT(ctx->queue->capacity, 20);
|
||||||
|
EXPECT_EQ_INT(ctx->file_descriptor, 42);
|
||||||
|
EXPECT_FALSE(ctx->receiver_done);
|
||||||
|
|
||||||
|
pipeline_context_receiver_destroy(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Test that create handles various queue capacities */
|
||||||
|
static void test_sender_queue_capacities() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
free(cfg->version);
|
||||||
|
cfg->version = str_dup("3.0");
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
|
||||||
|
/* Single-element queues */
|
||||||
|
Queue* q1 = queue_create(1, NULL);
|
||||||
|
Queue* q2 = queue_create(1, NULL);
|
||||||
|
PipelineContextSender* ctx = pipeline_context_sender_create(cfg, q1, q2);
|
||||||
|
EXPECT_NOT_NULL(ctx);
|
||||||
|
EXPECT_EQ_INT(ctx->queue_scanner->capacity, 1);
|
||||||
|
EXPECT_EQ_INT(ctx->queue_loader->capacity, 1);
|
||||||
|
pipeline_context_sender_destroy(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Test that create handles zero-capacity queues */
|
||||||
|
static void test_sender_zero_capacity() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
free(cfg->version);
|
||||||
|
cfg->version = str_dup("4.0");
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
|
||||||
|
Queue* q1 = queue_create(0, NULL);
|
||||||
|
Queue* q2 = queue_create(0, NULL);
|
||||||
|
PipelineContextSender* ctx = pipeline_context_sender_create(cfg, q1, q2);
|
||||||
|
EXPECT_NOT_NULL(ctx);
|
||||||
|
EXPECT_EQ_INT(ctx->queue_scanner->capacity, 0);
|
||||||
|
EXPECT_EQ_INT(ctx->queue_loader->capacity, 0);
|
||||||
|
pipeline_context_sender_destroy(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Test receiver with zero file_descriptor */
|
||||||
|
static void test_receiver_fd_zero() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
free(cfg->version);
|
||||||
|
cfg->version = str_dup("5.0");
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
Queue* q = queue_create(5, NULL);
|
||||||
|
PipelineContextReceiver* ctx = pipeline_context_receiver_create(cfg, q, 0);
|
||||||
|
EXPECT_NOT_NULL(ctx);
|
||||||
|
EXPECT_EQ_INT(ctx->file_descriptor, 0);
|
||||||
|
EXPECT_FALSE(ctx->receiver_done);
|
||||||
|
pipeline_context_receiver_destroy(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
void test_multiprocessing() {
|
||||||
|
test_sender_create_destroy();
|
||||||
|
test_receiver_create_destroy();
|
||||||
|
test_sender_queue_capacities();
|
||||||
|
test_sender_zero_capacity();
|
||||||
|
test_receiver_fd_zero();
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#ifndef TEST_MULTIPROCESSING_H
|
||||||
|
#define TEST_MULTIPROCESSING_H
|
||||||
|
|
||||||
|
void test_multiprocessing();
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -128,8 +128,9 @@ static void test_send_receive_status() {
|
|||||||
io_set_fds(p[0], p[1]);
|
io_set_fds(p[0], p[1]);
|
||||||
io_set_bwlimit(0);
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
Status statuses[] = {STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT,
|
Status statuses[] = {STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT,
|
||||||
STATUS_CHUNK, STATUS_CHECK, STATUS_DELTA_SIGNATURE, STATUS_DELTA_DATA};
|
STATUS_CHUNK, STATUS_CHECK, STATUS_DELTA_SIGNATURE, STATUS_DELTA_DATA,
|
||||||
|
STATUS_KEEPALIVE, STATUS_ABORT, STATUS_CHECK_BATCH};
|
||||||
int count = sizeof(statuses) / sizeof(statuses[0]);
|
int count = sizeof(statuses) / sizeof(statuses[0]);
|
||||||
|
|
||||||
for (int i = 0; i < count; i++) {
|
for (int i = 0; i < count; i++) {
|
||||||
|
|||||||
+277
-5
@@ -15,7 +15,7 @@ static void test_scanner_single_file() {
|
|||||||
const char* file1 = "test_scan_dir_single/file1.txt";
|
const char* file1 = "test_scan_dir_single/file1.txt";
|
||||||
const char* content1 = "hello scanner";
|
const char* content1 = "hello scanner";
|
||||||
|
|
||||||
mkdir(dir, 0755);
|
EXPECT_EQ_INT(mkdir(dir, 0755), 0);
|
||||||
create_test_file(file1, content1);
|
create_test_file(file1, content1);
|
||||||
|
|
||||||
DirectoryScanner* scanner =
|
DirectoryScanner* scanner =
|
||||||
@@ -43,7 +43,7 @@ static void test_scanner_multiple_files() {
|
|||||||
const char* content1 = "alpha";
|
const char* content1 = "alpha";
|
||||||
const char* content2 = "beta";
|
const char* content2 = "beta";
|
||||||
|
|
||||||
mkdir(dir, 0755);
|
EXPECT_EQ_INT(mkdir(dir, 0755), 0);
|
||||||
create_test_file(file1, content1);
|
create_test_file(file1, content1);
|
||||||
create_test_file(file2, content2);
|
create_test_file(file2, content2);
|
||||||
|
|
||||||
@@ -82,8 +82,8 @@ static void test_scanner_subdirectory() {
|
|||||||
const char* sub_file = "test_scan_sub/sub/sub_file.txt";
|
const char* sub_file = "test_scan_sub/sub/sub_file.txt";
|
||||||
const char* content = "nested content";
|
const char* content = "nested content";
|
||||||
|
|
||||||
mkdir(root, 0755);
|
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||||
mkdir(sub, 0755);
|
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
|
||||||
create_test_file(root_file, content);
|
create_test_file(root_file, content);
|
||||||
create_test_file(sub_file, content);
|
create_test_file(sub_file, content);
|
||||||
|
|
||||||
@@ -109,7 +109,7 @@ static void test_scanner_subdirectory() {
|
|||||||
static void test_scanner_empty_directory() {
|
static void test_scanner_empty_directory() {
|
||||||
const char* dir = "test_scan_empty";
|
const char* dir = "test_scan_empty";
|
||||||
|
|
||||||
mkdir(dir, 0755);
|
EXPECT_EQ_INT(mkdir(dir, 0755), 0);
|
||||||
|
|
||||||
DirectoryScanner* scanner =
|
DirectoryScanner* scanner =
|
||||||
directory_scanner_create((char*)dir, false, 0, NULL, 0, NULL, 0, 0, 0, 0);
|
directory_scanner_create((char*)dir, false, 0, NULL, 0, NULL, 0, 0, 0, 0);
|
||||||
@@ -122,9 +122,281 @@ static void test_scanner_empty_directory() {
|
|||||||
rmdir(dir);
|
rmdir(dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --- Exclude/include pattern and size filter edge cases (Issue #56) --- */
|
||||||
|
|
||||||
|
static void test_scanner_exclude_pattern() {
|
||||||
|
const char* dir = "test_scan_excl";
|
||||||
|
const char* f_txt = "test_scan_excl/keep.txt";
|
||||||
|
const char* f_tmp = "test_scan_excl/remove.tmp";
|
||||||
|
const char* content = "data";
|
||||||
|
|
||||||
|
EXPECT_EQ_INT(mkdir(dir, 0755), 0);
|
||||||
|
create_test_file(f_txt, content);
|
||||||
|
create_test_file(f_tmp, content);
|
||||||
|
|
||||||
|
char* exclude[] = {"*.tmp"};
|
||||||
|
DirectoryScanner* scanner =
|
||||||
|
directory_scanner_create((char*)dir, false, 0, exclude, 1, NULL, 0, 0, 0, 0);
|
||||||
|
EXPECT_NOT_NULL(scanner);
|
||||||
|
|
||||||
|
Chunk* chunk = directory_scanner_next(scanner);
|
||||||
|
EXPECT_NOT_NULL(chunk);
|
||||||
|
EXPECT_EQ_INT(chunk->element_count, 1);
|
||||||
|
EXPECT_EQ_STR(chunk->items[0]->path, f_txt);
|
||||||
|
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
EXPECT_NULL(directory_scanner_next(scanner));
|
||||||
|
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
unlink(f_txt);
|
||||||
|
unlink(f_tmp);
|
||||||
|
rmdir(dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_scanner_exclude_subdirectory() {
|
||||||
|
const char* root = "test_scan_excl_sub";
|
||||||
|
const char* sub = "test_scan_excl_sub/sub";
|
||||||
|
const char* root_txt = "test_scan_excl_sub/root.txt";
|
||||||
|
const char* sub_txt = "test_scan_excl_sub/sub/data.txt";
|
||||||
|
const char* sub_tmp = "test_scan_excl_sub/sub/temp.tmp";
|
||||||
|
const char* content = "data";
|
||||||
|
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||||
|
EXPECT_EQ_INT(mkdir(sub, 0755), 0);
|
||||||
|
create_test_file(root_txt, content);
|
||||||
|
create_test_file(sub_txt, content);
|
||||||
|
create_test_file(sub_tmp, content);
|
||||||
|
|
||||||
|
char* exclude[] = {"*.tmp"};
|
||||||
|
DirectoryScanner* scanner =
|
||||||
|
directory_scanner_create((char*)root, false, 0, exclude, 1, NULL, 0, 0, 0, 0);
|
||||||
|
EXPECT_NOT_NULL(scanner);
|
||||||
|
|
||||||
|
int total = 0;
|
||||||
|
Chunk* chunk;
|
||||||
|
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||||
|
total += chunk->element_count;
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
size_t len = strlen(chunk->items[i]->path);
|
||||||
|
EXPECT_TRUE(len < 4 || strcmp(chunk->items[i]->path + len - 4, ".tmp") != 0);
|
||||||
|
}
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
}
|
||||||
|
EXPECT_EQ_INT(total, 2);
|
||||||
|
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
unlink(root_txt);
|
||||||
|
unlink(sub_txt);
|
||||||
|
unlink(sub_tmp);
|
||||||
|
rmdir(sub);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_scanner_include_and_exclude() {
|
||||||
|
const char* dir = "test_scan_inc_exc";
|
||||||
|
const char* f_txt = "test_scan_inc_exc/a.txt";
|
||||||
|
const char* f_log = "test_scan_inc_exc/b.log";
|
||||||
|
const char* f_bak = "test_scan_inc_exc/c.bak";
|
||||||
|
const char* content = "filter";
|
||||||
|
|
||||||
|
EXPECT_EQ_INT(mkdir(dir, 0755), 0);
|
||||||
|
create_test_file(f_txt, content);
|
||||||
|
create_test_file(f_log, content);
|
||||||
|
create_test_file(f_bak, content);
|
||||||
|
|
||||||
|
char* exclude[] = {"*.bak"};
|
||||||
|
char* include[] = {"*.txt", "*.log"};
|
||||||
|
DirectoryScanner* scanner =
|
||||||
|
directory_scanner_create((char*)dir, false, 0, exclude, 1, include, 2, 0, 0, 0);
|
||||||
|
EXPECT_NOT_NULL(scanner);
|
||||||
|
|
||||||
|
Chunk* chunk = directory_scanner_next(scanner);
|
||||||
|
EXPECT_NOT_NULL(chunk);
|
||||||
|
EXPECT_EQ_INT(chunk->element_count, 2);
|
||||||
|
|
||||||
|
int found_txt = 0, found_log = 0;
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
if (strstr(chunk->items[i]->path, "a.txt"))
|
||||||
|
found_txt = 1;
|
||||||
|
if (strstr(chunk->items[i]->path, "b.log"))
|
||||||
|
found_log = 1;
|
||||||
|
}
|
||||||
|
EXPECT_TRUE(found_txt);
|
||||||
|
EXPECT_TRUE(found_log);
|
||||||
|
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
EXPECT_NULL(directory_scanner_next(scanner));
|
||||||
|
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
unlink(f_txt);
|
||||||
|
unlink(f_log);
|
||||||
|
unlink(f_bak);
|
||||||
|
rmdir(dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_scanner_max_size() {
|
||||||
|
const char* dir = "test_scan_max";
|
||||||
|
const char* small = "test_scan_max/small.txt";
|
||||||
|
const char* large = "test_scan_max/large.txt";
|
||||||
|
EXPECT_EQ_INT(mkdir(dir, 0755), 0);
|
||||||
|
create_test_file(small, "tiny");
|
||||||
|
create_test_file(large, "this_content_is_longer_than_ten_chars");
|
||||||
|
|
||||||
|
/* max_size = 10 — only files <= 10 bytes */
|
||||||
|
DirectoryScanner* scanner =
|
||||||
|
directory_scanner_create((char*)dir, false, 0, NULL, 0, NULL, 0, 10, 0, 0);
|
||||||
|
EXPECT_NOT_NULL(scanner);
|
||||||
|
|
||||||
|
Chunk* chunk = directory_scanner_next(scanner);
|
||||||
|
EXPECT_NOT_NULL(chunk);
|
||||||
|
EXPECT_EQ_INT(chunk->element_count, 1);
|
||||||
|
EXPECT_EQ_STR(chunk->items[0]->path, small);
|
||||||
|
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
EXPECT_NULL(directory_scanner_next(scanner));
|
||||||
|
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
unlink(small);
|
||||||
|
unlink(large);
|
||||||
|
rmdir(dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_scanner_min_size() {
|
||||||
|
const char* dir = "test_scan_min";
|
||||||
|
const char* empty_f = "test_scan_min/empty.txt";
|
||||||
|
const char* data_f = "test_scan_min/data.txt";
|
||||||
|
|
||||||
|
EXPECT_EQ_INT(mkdir(dir, 0755), 0);
|
||||||
|
create_test_file(empty_f, "");
|
||||||
|
create_test_file(data_f, "some content here");
|
||||||
|
|
||||||
|
/* min_size = 1 — only files >= 1 byte */
|
||||||
|
DirectoryScanner* scanner =
|
||||||
|
directory_scanner_create((char*)dir, false, 0, NULL, 0, NULL, 0, 0, 1, 0);
|
||||||
|
EXPECT_NOT_NULL(scanner);
|
||||||
|
|
||||||
|
Chunk* chunk = directory_scanner_next(scanner);
|
||||||
|
EXPECT_NOT_NULL(chunk);
|
||||||
|
EXPECT_EQ_INT(chunk->element_count, 1);
|
||||||
|
EXPECT_EQ_STR(chunk->items[0]->path, data_f);
|
||||||
|
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
EXPECT_NULL(directory_scanner_next(scanner));
|
||||||
|
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
unlink(empty_f);
|
||||||
|
unlink(data_f);
|
||||||
|
rmdir(dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_scanner_size_range() {
|
||||||
|
const char* dir = "test_scan_range";
|
||||||
|
const char* tiny = "test_scan_range/tiny.txt";
|
||||||
|
const char* medium = "test_scan_range/med.txt";
|
||||||
|
const char* huge = "test_scan_range/huge.txt";
|
||||||
|
|
||||||
|
EXPECT_EQ_INT(mkdir(dir, 0755), 0);
|
||||||
|
create_test_file(tiny, "ab");
|
||||||
|
create_test_file(medium, "hello world");
|
||||||
|
create_test_file(huge, "this is a much larger file for testing size filters");
|
||||||
|
|
||||||
|
/* Only files between 3 and 20 bytes */
|
||||||
|
DirectoryScanner* scanner =
|
||||||
|
directory_scanner_create((char*)dir, false, 0, NULL, 0, NULL, 0, 20, 3, 0);
|
||||||
|
EXPECT_NOT_NULL(scanner);
|
||||||
|
|
||||||
|
Chunk* chunk = directory_scanner_next(scanner);
|
||||||
|
EXPECT_NOT_NULL(chunk);
|
||||||
|
EXPECT_EQ_INT(chunk->element_count, 1);
|
||||||
|
EXPECT_EQ_STR(chunk->items[0]->path, medium);
|
||||||
|
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
EXPECT_NULL(directory_scanner_next(scanner));
|
||||||
|
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
unlink(tiny);
|
||||||
|
unlink(medium);
|
||||||
|
unlink(huge);
|
||||||
|
rmdir(dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_scanner_mixed_patterns() {
|
||||||
|
/* Combine exclude, include, and size filters together */
|
||||||
|
const char* dir = "test_scan_mixed";
|
||||||
|
const char* a_txt = "test_scan_mixed/a.txt"; /* size ~= 5 */
|
||||||
|
const char* b_bin = "test_scan_mixed/b.bin"; /* size ~= 13 */
|
||||||
|
const char* c_txt = "test_scan_mixed/c.txt"; /* size ~= 5 */
|
||||||
|
const char* d_bak = "test_scan_mixed/d.bak"; /* size ~= 42 */
|
||||||
|
|
||||||
|
EXPECT_EQ_INT(mkdir(dir, 0755), 0);
|
||||||
|
create_test_file(a_txt, "aaaaa");
|
||||||
|
create_test_file(b_bin, "bbbbbbbbbbbbb");
|
||||||
|
create_test_file(c_txt, "ccccc");
|
||||||
|
create_test_file(d_bak, "dddddddddddddddddddddddddddddddddddddddddd");
|
||||||
|
|
||||||
|
/* Exclude *.bak, include *.txt, min_size=3, max_size=10 */
|
||||||
|
char* exclude[] = {"*.bak"};
|
||||||
|
char* include[] = {"*.txt"};
|
||||||
|
DirectoryScanner* scanner =
|
||||||
|
directory_scanner_create((char*)dir, false, 0, exclude, 1, include, 1, 10, 3, 0);
|
||||||
|
EXPECT_NOT_NULL(scanner);
|
||||||
|
|
||||||
|
Chunk* chunk = directory_scanner_next(scanner);
|
||||||
|
EXPECT_NOT_NULL(chunk);
|
||||||
|
/* Both a.txt and c.txt meet the criteria: .txt extension, size 5 <= 10 and >= 3 */
|
||||||
|
EXPECT_EQ_INT(chunk->element_count, 2);
|
||||||
|
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
EXPECT_NULL(directory_scanner_next(scanner));
|
||||||
|
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
unlink(a_txt);
|
||||||
|
unlink(b_bin);
|
||||||
|
unlink(c_txt);
|
||||||
|
unlink(d_bak);
|
||||||
|
rmdir(dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_scanner_no_patterns() {
|
||||||
|
/* Explicit test with no exclude/include patterns and no size filters.
|
||||||
|
* This verifies that NULL/0 for all pattern parameters works correctly. */
|
||||||
|
const char* dir = "test_scan_none";
|
||||||
|
const char* f1 = "test_scan_none/f1.txt";
|
||||||
|
const char* f2 = "test_scan_none/f2.txt";
|
||||||
|
|
||||||
|
EXPECT_EQ_INT(mkdir(dir, 0755), 0);
|
||||||
|
create_test_file(f1, "first");
|
||||||
|
create_test_file(f2, "second");
|
||||||
|
|
||||||
|
DirectoryScanner* scanner =
|
||||||
|
directory_scanner_create((char*)dir, false, 0, NULL, 0, NULL, 0, 0, 0, 0);
|
||||||
|
EXPECT_NOT_NULL(scanner);
|
||||||
|
|
||||||
|
Chunk* chunk = directory_scanner_next(scanner);
|
||||||
|
EXPECT_NOT_NULL(chunk);
|
||||||
|
EXPECT_EQ_INT(chunk->element_count, 2);
|
||||||
|
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
EXPECT_NULL(directory_scanner_next(scanner));
|
||||||
|
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
unlink(f1);
|
||||||
|
unlink(f2);
|
||||||
|
rmdir(dir);
|
||||||
|
}
|
||||||
|
|
||||||
void test_scanner() {
|
void test_scanner() {
|
||||||
test_scanner_single_file();
|
test_scanner_single_file();
|
||||||
test_scanner_multiple_files();
|
test_scanner_multiple_files();
|
||||||
test_scanner_subdirectory();
|
test_scanner_subdirectory();
|
||||||
test_scanner_empty_directory();
|
test_scanner_empty_directory();
|
||||||
|
/* Issue #56: scanner pattern edge cases */
|
||||||
|
test_scanner_exclude_pattern();
|
||||||
|
test_scanner_exclude_subdirectory();
|
||||||
|
test_scanner_include_and_exclude();
|
||||||
|
test_scanner_max_size();
|
||||||
|
test_scanner_min_size();
|
||||||
|
test_scanner_size_range();
|
||||||
|
test_scanner_mixed_patterns();
|
||||||
|
test_scanner_no_patterns();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
#include "test_transport_ssh.h"
|
||||||
|
#include "test_utils.h"
|
||||||
|
#include "transport_ssh.h"
|
||||||
|
|
||||||
|
static void test_ssh_connect_invalid_dest_no_colon() {
|
||||||
|
/* cppcheck-suppress constVariablePointer */
|
||||||
|
Client* client = client_connect_ssh("invalid-destination-no-colon", 22, NULL);
|
||||||
|
EXPECT_NULL(client);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_ssh_connect_invalid_dest_empty() {
|
||||||
|
/* cppcheck-suppress constVariablePointer */
|
||||||
|
Client* client = client_connect_ssh("", 22, NULL);
|
||||||
|
EXPECT_NULL(client);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Test client_connect_ssh with malformed destination (just a colon).
|
||||||
|
* parse_remote_dest succeeds, ssh is exec'd and fails, but the function
|
||||||
|
* creates a Client that must be cleaned up. */
|
||||||
|
static void test_ssh_connect_malformed() {
|
||||||
|
Client* client = client_connect_ssh(":", 22, NULL);
|
||||||
|
/* ssh binary exists, so exec succeeds; the function returns a Client.
|
||||||
|
* We just verify it doesn't crash and clean up properly. */
|
||||||
|
if (client != NULL) {
|
||||||
|
client_disconnect(client);
|
||||||
|
client_delete(client);
|
||||||
|
}
|
||||||
|
EXPECT_TRUE(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Test client_connect_ssh with valid format but unreachable host.
|
||||||
|
* The function launches ssh which will fail to connect, returns a Client. */
|
||||||
|
static void test_ssh_connect_unreachable() {
|
||||||
|
Client* client = client_connect_ssh("nonexistent.invalid:/remote/path", 22, NULL);
|
||||||
|
if (client != NULL) {
|
||||||
|
client_disconnect(client);
|
||||||
|
client_delete(client);
|
||||||
|
}
|
||||||
|
EXPECT_TRUE(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
void test_transport_ssh() {
|
||||||
|
test_ssh_connect_invalid_dest_no_colon();
|
||||||
|
test_ssh_connect_invalid_dest_empty();
|
||||||
|
test_ssh_connect_malformed();
|
||||||
|
test_ssh_connect_unreachable();
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#ifndef TEST_TRANSPORT_SSH_H
|
||||||
|
#define TEST_TRANSPORT_SSH_H
|
||||||
|
|
||||||
|
void test_transport_ssh();
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
#include "test_transport_tcp.h"
|
||||||
|
#include "test_utils.h"
|
||||||
|
#include "transport_tcp.h"
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
|
static void test_server_create_ephemeral() {
|
||||||
|
Server* s = server_create(0);
|
||||||
|
EXPECT_NOT_NULL(s);
|
||||||
|
EXPECT_TRUE(s->file_descriptor >= 0);
|
||||||
|
EXPECT_EQ_INT(s->address.sin_family, AF_INET);
|
||||||
|
server_delete(&s);
|
||||||
|
EXPECT_NULL(s);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_server_delete_null() {
|
||||||
|
Server* s = NULL;
|
||||||
|
server_delete(&s);
|
||||||
|
EXPECT_NULL(s);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_client_create() {
|
||||||
|
Client* c = client_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
EXPECT_TRUE(c->file_descriptor >= 0);
|
||||||
|
EXPECT_EQ_INT(c->address.sin_family, AF_INET);
|
||||||
|
EXPECT_EQ_INT(c->ssh_child_pid, -1);
|
||||||
|
EXPECT_NULL(c->ssl);
|
||||||
|
EXPECT_NULL(c->ssl_ctx);
|
||||||
|
client_disconnect(c);
|
||||||
|
client_delete(c);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_client_delete_null() {
|
||||||
|
Client* c = NULL;
|
||||||
|
client_delete(c);
|
||||||
|
}
|
||||||
|
|
||||||
|
void test_transport_tcp() {
|
||||||
|
test_server_create_ephemeral();
|
||||||
|
test_server_delete_null();
|
||||||
|
test_client_create();
|
||||||
|
test_client_delete_null();
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#ifndef TEST_TRANSPORT_TCP_H
|
||||||
|
#define TEST_TRANSPORT_TCP_H
|
||||||
|
|
||||||
|
void test_transport_tcp();
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
#include "test_transport_tls.h"
|
||||||
|
#include "test_utils.h"
|
||||||
|
#include "transport_tcp.h"
|
||||||
|
#include "transport_tls.h"
|
||||||
|
|
||||||
|
static void test_tls_global_init() {
|
||||||
|
bool ok = tls_global_init();
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_server_create_tls_without_certs() {
|
||||||
|
Server* s = server_create(0);
|
||||||
|
EXPECT_NOT_NULL(s);
|
||||||
|
bool ok = server_create_tls(s, NULL, NULL, NULL);
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
EXPECT_NOT_NULL(s->ssl_ctx);
|
||||||
|
server_delete(&s);
|
||||||
|
EXPECT_NULL(s);
|
||||||
|
}
|
||||||
|
|
||||||
|
void test_transport_tls() {
|
||||||
|
test_tls_global_init();
|
||||||
|
test_server_create_tls_without_certs();
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#ifndef TEST_TRANSPORT_TLS_H
|
||||||
|
#define TEST_TRANSPORT_TLS_H
|
||||||
|
|
||||||
|
void test_transport_tls();
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -2,9 +2,24 @@
|
|||||||
#define TEST_UTILS_H
|
#define TEST_UTILS_H
|
||||||
|
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
|
|
||||||
|
// Detect if running under valgrind by checking /proc/self/maps for vgpreload.
|
||||||
|
// This is used to skip fork-based tests that are incompatible with valgrind
|
||||||
|
// (the instrumented parent runs too slowly, causing pipe timeouts).
|
||||||
|
static inline bool is_running_under_valgrind(void) {
|
||||||
|
FILE* f = fopen("/proc/self/maps", "r");
|
||||||
|
if (!f)
|
||||||
|
return false;
|
||||||
|
char buf[4096];
|
||||||
|
size_t n = fread(buf, 1, sizeof(buf) - 1, f);
|
||||||
|
fclose(f);
|
||||||
|
buf[n] = '\0';
|
||||||
|
return strstr(buf, "vgpreload") != NULL;
|
||||||
|
}
|
||||||
|
|
||||||
// Global test suite status
|
// Global test suite status
|
||||||
extern int tests_run;
|
extern int tests_run;
|
||||||
extern int tests_failed;
|
extern int tests_failed;
|
||||||
|
|||||||
Reference in New Issue
Block a user