88 Commits
Author SHA1 Message Date
TapTap 9f8b58893b Apply clang-format 18 to merge-conflict resolution code
CI / lint (push) Successful in 15s
CI / sanitizers (address) (push) Successful in 37s
CI / sanitizers (undefined) (push) Successful in 37s
CI / fuzz-build (push) Successful in 16s
CI / coverage (push) Successful in 33s
CI / build-and-test (push) Successful in 1m18s
CI / valgrind (push) Successful in 34s
The Phase 1 merge conflict resolutions introduced formatting that failed
the CI lint job (clang-format 18.1.3). Reformatted with the exact CI
version; no functional changes.
2026-09-05 10:57:41 +02:00
TapTap 48dfd5dfa0 Fix incremental skip regression from modify-window nsec comparison
CI / lint (push) Failing after 3s
CI / build-and-test (push) Skipped
CI / sanitizers (address) (push) Skipped
CI / sanitizers (undefined) (push) Skipped
CI / fuzz-build (push) Skipped
CI / coverage (push) Skipped
CI / valgrind (push) Skipped
metadata_mtime_matches() required exact nanosecond equality at the default
modify_window=0, but destination write-time nsecs never match source
creation-time nsecs unless -M preserves metadata. Compare whole seconds
(rsync's default quick-check) at window 0; keep the subsecond-refined
tolerance for explicit window values.
2026-09-05 10:43:58 +02:00
TapTap 8384a1997b Merge remote-tracking branch 'origin/feat/rsync-delete-during-alias' into dev 2026-09-05 02:53:59 +02:00
TapTap a1981a78ad Merge remote-tracking branch 'origin/feat/rsync-old-dirs-aliases' into dev 2026-09-05 02:48:48 +02:00
TapTap b5646a0b9e Merge remote-tracking branch 'origin/feat/rsync-checksum-choice-alias' into dev 2026-09-05 02:44:04 +02:00
TapTap c6a341bd1e Merge remote-tracking branch 'origin/feat/rsync-partial-progress' into dev 2026-09-05 02:39:03 +02:00
TapTap c1e9509f16 Merge remote-tracking branch 'origin/feat/rsync-old-args' into dev 2026-09-05 02:34:29 +02:00
TapTap 8689778233 Merge remote-tracking branch 'origin/feat/rsync-no-options' into dev 2026-09-05 02:30:50 +02:00
TapTap cf0d10fccb Merge remote-tracking branch 'origin/feat/rsync-info' into dev 2026-09-05 02:25:36 +02:00
TapTap 90112a7585 Merge remote-tracking branch 'origin/feat/max-alloc' into dev 2026-09-05 02:13:35 +02:00
TapTap 3304541337 Merge remote-tracking branch 'origin/feat/compress-threads' into dev 2026-09-05 02:01:43 +02:00
TapTap 6a95efbe41 Merge remote-tracking branch 'origin/feat/skip-compress' into dev 2026-09-04 18:35:19 +02:00
TapTap 3b013bf9d2 Merge remote-tracking branch 'origin/feat/rsync-compression-aliases' into dev 2026-09-04 18:24:21 +02:00
TapTap 4f21498ee6 Merge remote-tracking branch 'origin/feat/chmod' into dev 2026-09-04 18:20:08 +02:00
TapTap 315e572d18 Merge remote-tracking branch 'origin/feat/executability' into dev 2026-09-04 18:13:34 +02:00
TapTap 55172ff6de Merge remote-tracking branch 'origin/feat/ignore-existing' into dev 2026-09-04 17:59:50 +02:00
TapTap c650f9a3d0 Merge remote-tracking branch 'origin/feat/existing' into dev 2026-09-04 17:50:55 +02:00
TapTap 192aff8c46 Merge remote-tracking branch 'origin/feat/update' into dev
# Conflicts:
#	src/client/client_cli.c
#	src/client/usage.c
#	src/shared/file.c
#	src/shared/file.h
#	src/shared/file_receive.c
#	tests/test_client_cli.c
2026-09-04 17:44:07 +02:00
TapTap 6d10116d50 Merge remote-tracking branch 'origin/feat/modify-window' into dev
# Conflicts:
#	RSYNC_COMPAT.md
#	src/client/client_cli.c
#	src/client/usage.c
#	src/server/receiver.c
#	src/server/server.c
#	src/shared/config.c
#	src/shared/config.h
#	src/shared/file_receive.c
#	tests/integration/test_features.py
#	tests/test_client_cli.c
#	tests/test_config.c
2026-09-04 17:40:16 +02:00
TapTap e42df1bde3 Merge remote-tracking branch 'origin/feat/size-only' into dev
# Conflicts:
#	src/client/client_cli.c
#	src/client/usage.c
#	src/server/receiver.c
#	src/server/server.c
#	src/shared/config.c
#	src/shared/config.h
#	tests/integration/test_features.py
#	tests/test_config.c
2026-09-04 17:30:14 +02:00
TapTap 9fc1e72972 Merge remote-tracking branch 'origin/feat/ignore-times' into dev
# Conflicts:
#	src/shared/config.c
#	tests/integration/test_features.py
#	tests/test_client_cli.c
#	tests/test_config.c
2026-09-04 17:27:04 +02:00
TapTap 71f1529222 Merge remote-tracking branch 'origin/feat/whole-file' into dev
# Conflicts:
#	tests/test_client_cli.c
#	tests/test_config.c
2026-09-04 17:24:22 +02:00
TapTap 2adfa5a03b Merge remote-tracking branch 'origin/feat/rsync-secluded-args' into dev
# Conflicts:
#	RSYNC_COMPAT.md
#	tests/test_client_cli.c
2026-09-04 17:22:53 +02:00
TapTap 47d1cbdae8 Merge remote-tracking branch 'origin/feat/rsync-debug' into dev
# Conflicts:
#	RSYNC_COMPAT.md
#	src/client/client_cli.c
#	src/client/usage.c
#	src/shared/log.h
#	src/shared/protocol.c
2026-09-04 17:21:48 +02:00
TapTap b59139589d Merge remote-tracking branch 'origin/feat/rsync-stderr-mode' into dev
# Conflicts:
#	RSYNC_COMPAT.md
#	src/shared/log.c
#	src/shared/log.h
#	tests/test_client_cli.c
2026-09-04 17:20:36 +02:00
TapTap 49e4af275f Merge remote-tracking branch 'origin/feat/8-bit-output' into dev
# Conflicts:
#	src/client/client_cli.c
#	src/client/client_send.c
#	src/shared/config.c
#	tests/test_client_cli.c
#	tests/test_config.c
#	tests/test_shared_utils.c
2026-09-04 17:18:01 +02:00
TapTap 8e1bc9bb9c Merge remote-tracking branch 'origin/feat/human-readable' into dev
# Conflicts:
#	RSYNC_COMPAT.md
#	src/client/client_send.c
#	tests/test_client_cli.c
2026-09-04 17:16:24 +02:00
TapTap 421a94773f Merge remote-tracking branch 'origin/feat/quiet' into dev
# Conflicts:
#	src/client/client_send.c
2026-09-04 17:14:45 +02:00
TapTap ead4651d12 Merge remote-tracking branch 'origin/feat/remove-source-files' into dev 2026-09-04 17:14:23 +02:00
TapTap 638d953b1d fix: bump protocol version for max alloc wire format
CI / lint (pull_request) Successful in 12s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-04 02:29:06 +02:00
TapTap 04cea295b5 fix: enforce max alloc in delta deserialization
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-04 02:25:26 +02:00
TapTap 820afd334a fix: reject malformed max-alloc sizes
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 38s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-04 02:22:00 +02:00
TapTap 2f553a2a43 fix: honor explicit protocol allocation sessions
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 36s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 34s
2026-09-04 02:18:44 +02:00
TapTap f990e86309 fix: make protocol allocation accounting atomic
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 34s
2026-09-04 02:12:22 +02:00
TapTap 0d306940e1 fix: bind allocation sessions in worker threads
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 36s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-04 01:56:34 +02:00
TapTap 552c19d3fe fix: guard skip-compress cleanup
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 38s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m17s
CI / valgrind (pull_request) Successful in 34s
2026-09-04 01:49:38 +02:00
TapTap 14c5da98d8 fix: free old snapshot on full transfer fallback
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 36s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 22:51:42 +02:00
TapTap aab45873ac test: cover compression option equals forms
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 33s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 22:47:41 +02:00
TapTap 605f79a450 fix: make rsync info none override verbose
CI / lint (pull_request) Successful in 12s
CI / sanitizers (address) (pull_request) Successful in 36s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 16s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 22:44:44 +02:00
TapTap 066c7ed1af fix: address 8-bit output re-review findings
CI / lint (pull_request) Successful in 10s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 22:40:30 +02:00
TapTap 97a628c3fd docs: document delete-during aliases
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 34s
2026-09-03 22:34:52 +02:00
TapTap 9f23b23893 fix: clarify unsupported directory option diagnostics
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 36s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 22:32:09 +02:00
TapTap 2ace6416a7 test: cover configured fsync save path
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 22:29:40 +02:00
TapTap e491e811e3 fix: enforce max alloc across protocol workers
CI / lint (pull_request) Successful in 12s
CI / sanitizers (address) (pull_request) Successful in 36s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 22:26:29 +02:00
TapTap ec02ee6dde fix: bound compression worker threads
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 34s
2026-09-03 22:22:04 +02:00
TapTap 98cbf3495d fix: complete skip-compress protocol handling
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m17s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 22:19:00 +02:00
TapTap 245a34fa11 fix: guard source removal against file replacement
CI / lint (pull_request) Successful in 11s
CI / sanitizers (undefined) (pull_request) Successful in 38s
CI / sanitizers (address) (pull_request) Successful in 39s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 22:11:18 +02:00
TapTap 68aaf89da7 fix: address update option review findings
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 34s
2026-09-03 21:59:55 +02:00
TapTap e37d5b9438 fix: address modify-window review findings
CI / lint (pull_request) Successful in 13s
CI / sanitizers (undefined) (pull_request) Successful in 38s
CI / sanitizers (address) (pull_request) Successful in 39s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 21:54:46 +02:00
TapTap 5b8cdf457b fix: bump protocol version for size-only field
CI / lint (pull_request) Successful in 12s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 21:51:29 +02:00
TapTap 1ebe261e05 fix: complete ignore-times incremental handling
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 21:49:10 +02:00
TapTap 05313f00ea fix: prioritize whole-file validation
CI / lint (pull_request) Successful in 11s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 34s
2026-09-03 21:46:50 +02:00
TapTap 4152bf379e fix: honor compression choice aliases
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 36s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 21:44:21 +02:00
TapTap f8291d895d fix: classify partial progress as incomplete
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 38s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m14s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 21:40:07 +02:00
TapTap ba236d5b2e fix: clarify secluded args compatibility semantics
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 21:37:45 +02:00
TapTap ec6160a8b9 fix: propagate SSH child setup failures
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 21:34:32 +02:00
TapTap 302c3cb664 fix: preserve explicit rsync option negations
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 36s
CI / sanitizers (undefined) (pull_request) Successful in 35s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 21:30:55 +02:00
TapTap caabd2e314 fix: reject unsupported rsync debug categories
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 38s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 21:28:21 +02:00
TapTap 21e6b1c1bc fix: address rsync info review findings
CI / lint (pull_request) Successful in 12s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 21:25:56 +02:00
TapTap 5350290f52 fix: reject unsupported rsync client stderr mode
CI / lint (pull_request) Successful in 12s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 21:21:22 +02:00
TapTap 5b997d5d25 fix: complete 8-bit output negotiation
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 35s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 33s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 21:18:35 +02:00
TapTap fb96c21d93 test: complete human-readable progress coverage
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 36s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 16s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 21:14:49 +02:00
TapTap 17b5b31845 fix: complete quiet option handling
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 34s
2026-09-03 21:12:43 +02:00
TapTap 48a36b0038 feat: add rsync delete-during alias
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 38s
CI / fuzz-build (pull_request) Successful in 16s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 18:34:06 +02:00
TapTap ce64fcd4b1 feat: recognize rsync old-dirs aliases
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 16s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 18:31:09 +02:00
TapTap 4982e15969 feat: add rsync-compatible fsync
CI / lint (pull_request) Successful in 12s
CI / sanitizers (address) (pull_request) Successful in 40s
CI / sanitizers (undefined) (pull_request) Successful in 38s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 18:28:02 +02:00
TapTap d69f5db652 feat: add rsync-compatible max-alloc limit
CI / lint (pull_request) Successful in 11s
CI / sanitizers (undefined) (pull_request) Successful in 38s
CI / sanitizers (address) (pull_request) Successful in 39s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 18:23:45 +02:00
TapTap f443b2986d feat: add compression worker threads
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 38s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 18:15:14 +02:00
TapTap 2a08a7ba5c feat: add skip-compress option
CI / lint (pull_request) Successful in 12s
CI / sanitizers (address) (pull_request) Successful in 36s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 17:56:27 +02:00
TapTap 36a373ff5d Add executability preservation option
CI / lint (pull_request) Successful in 12s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 17:39:50 +02:00
TapTap 3b85658024 Add remove-source-files option
CI / lint (pull_request) Successful in 11s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 17:33:52 +02:00
TapTap 5e67a721a4 feat: add rsync-compatible update option
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 39s
CI / sanitizers (undefined) (pull_request) Successful in 39s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 17:07:02 +02:00
TapTap 0b5b1a8643 feat: add rsync modify-window option
CI / lint (pull_request) Successful in 14s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 38s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Failing after 1m15s
CI / valgrind (pull_request) Successful in 32s
2026-09-03 17:01:45 +02:00
TapTap 2afb1d9649 feat: add rsync-compatible size-only option
CI / lint (pull_request) Successful in 12s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 16:54:07 +02:00
TapTap 230401c629 feat: add ignore-times incremental option
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 16:50:07 +02:00
TapTap c91db3266a feat: add whole-file transfer mode
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 38s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 16:47:02 +02:00
TapTap 44ce154e37 feat: add checksum choice alias
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 36s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 16:42:29 +02:00
TapTap cd27886dde feat: add rsync compression aliases
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 34s
2026-09-03 16:38:12 +02:00
TapTap a0aff000ad feat: add rsync partial progress alias
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 32s
2026-09-03 16:34:18 +02:00
TapTap 50f9ebb55b feat: add secluded args compatibility option
CI / lint (pull_request) Successful in 11s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 16:31:37 +02:00
TapTap 393a440c78 feat: add rsync old-args compatibility mode
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 34s
2026-09-03 16:27:45 +02:00
TapTap 2ff2688ef1 feat: support rsync no-option handling
CI / lint (pull_request) Successful in 12s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 16:22:18 +02:00
TapTap 5c7d82b79c feat: add rsync debug flags
CI / lint (pull_request) Successful in 12s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 16:18:46 +02:00
TapTap 4872d425a3 feat: add rsync info flags
CI / lint (pull_request) Successful in 12s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 16:13:56 +02:00
TapTap 60eb7f11fe feat: add rsync stderr output modes
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 38s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 16:07:51 +02:00
TapTap eb4e9fba1f feat: add 8-bit output option
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 16s
CI / coverage (pull_request) Successful in 33s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 16:03:19 +02:00
TapTap 8ab5026224 feat: add human-readable output flag
CI / lint (pull_request) Successful in 12s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 34s
2026-09-03 15:58:48 +02:00
TapTap dbacc4f0e5 feat: add quiet client option
CI / lint (pull_request) Successful in 11s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-09-03 15:52:36 +02:00
57 changed files with 3934 additions and 332 deletions

No files matched your search

+24 -6
View File
@@ -66,8 +66,12 @@ replacement for every rsync feature or protocol mode.
- Owner/group, ACL, xattr, hard-link, device, and special-file handling is
incomplete or unavailable.
- Sparse-file handling does not yet preserve all holes correctly.
- `--partial`, `--partial-dir`, `--append`, and `--append-verify` are not yet
full rsync-style resumable transfers.
- `--partial`, `--partial-dir`, `-P`, `--append`, and `--append-verify` are not
yet full rsync-style resumable transfers. Interrupted files are not retained
for resumption.
- `--dirs` is not implemented. Its compatibility aliases `--old-dirs` and
`--old-d` are recognized but rejected explicitly rather than silently using
FastSync's recursive directory behavior.
- Several rsync short options currently have FastSync-specific meanings. Do
not assume every short option is interchangeable yet.
@@ -89,18 +93,22 @@ partial, alternate, and planned behavior.
| `-a, --archive` | Archive mode: enables `-c -m -M` (no `-s`) |
| `-m` | Multithreading mode |
| `-s` | Chunk serialization (batch all files per chunk) |
| `--secluded-args` | Accepted as an rsync compatibility option with no effect; `-s` remains chunk serialization. |
| `-f, --sendfile` | Sendfile zero-copy. Incompatible with `-c` / `-s`. TCP only. |
| `-M, --preserve` | Preserve supported file metadata (mode and mtime; ownership and atime are unsupported) |
| `-n, --dry-run` | Scan and print what would be transferred |
| `-p <port>` | SSH port (default: 22) |
| `-v, --verbose` | Enable debug logging |
| `-q, --quiet` | Suppress non-error output |
| `--progress` | Show real-time transfer speed |
| `-P` | Enables partial-transfer mode and progress output (partial retention is incomplete) |
| `--delete` | Delete files on receiver not present in source |
| `--exclude <pattern>` | Exclude files matching glob pattern (repeatable) |
| `--exclude-from <file>` | Read exclude patterns from a file (one per line) |
| `--include <pattern>` | Only transfer files matching glob pattern (repeatable, whitelist) |
| `--max-size <n>` | Skip files larger than n bytes |
| `--min-size <n>` | Skip files smaller than n bytes |
| `--max-alloc <SIZE>` | Maximum single allocation (binary units: B, K, M, G, T, P, E; default 1G) |
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `-s`. |
| `--existing` | Skip files not already present at the destination; update existing files normally. |
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
@@ -110,6 +118,7 @@ partial, alternate, and planned behavior.
| `--backup` | Backup existing destination files before overwriting |
| `--backup-dir <dir>` | Target directory for backups (requires `--backup`) |
| `--stats` | Print transfer statistics at end (bytes, files, timing) |
| `-h, --human-readable` | Format transfer byte sizes with binary units |
| `--max-depth <n>` | Maximum directory depth to recurse (0 = unlimited, default: 0) |
| `--log-file <path>` | Write log messages to file instead of stderr |
| `--source-dir <path>` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) |
@@ -340,6 +349,10 @@ features without changing the meaning of ordinary compatibility options.
| `-m` | Enable the multithreaded scanner/loader/sender pipeline. |
| `-c [level]`, `-z [level]` | Enable streaming zstd compression, levels 1-22. |
| `--compress-level <n>` | Set the zstd compression level. |
| `--zc <alg>` | Alias for `--compress-choice`. FastSync supports `zstd` and `none`. |
| `--zl <n>` | Alias for `--compress-level`. |
| `--skip-compress <list>` | Skip compression for comma-separated suffixes; incompatible with `-s`. |
| `--compress-threads <n>` | Use `n` zstd compression workers. Requires compression and a zstd build with threaded support; the setting affects sender CPU work only. |
| `--chunk-size <bytes>` | Set the transfer chunk size. |
| `-s` | Enable FastSync chunk serialization. |
| `-f`, `--sendfile` | Use TCP `sendfile()` zero-copy transfer. Incompatible with compression and chunk serialization. |
@@ -360,6 +373,11 @@ particular, FastSync currently uses `-p` for SSH port, `-s` for chunk
serialization, and `-S` for sparse handling. These meanings must be reconciled
before FastSync can claim full rsync CLI compatibility.
`--secluded-args` is accepted as a long-form compatibility no-op. It does not
change FastSync's transport or protocol behavior. The rsync short form `-s` is
intentionally not aliased because it remains FastSync's chunk-serialization
option.
## Client Options
### Selection and transfer
@@ -466,11 +484,11 @@ defaults to the current directory. |
## Protocol and Security
FastSync protocol version `2.3.0` is shared by the client and server. The
FastSync protocol version `2.4.0` is shared by the client and server. The
current protocol is sender-driven and includes configuration negotiation,
incremental checks, checksums, manifests, keep-alives, abort handling, and
FastSync-native delta messages. Client and server versions must currently
match exactly.
including the maximum allocation limit, incremental checks, checksums,
manifests, keep-alives, abort handling, and FastSync-native delta messages.
Client and server versions must currently match exactly.
TLS provides encrypted TCP transport. Supplying `--ca` enables certificate
verification; without it, traffic is encrypted but peer identity is not
+33 -31
View File
@@ -7,10 +7,11 @@ This document maps rsync's full feature set to FastSync's current implementation
| Status | Count | Description |
|--------|-------|-------------|
| ✅ Implemented | 35 | Feature works end-to-end |
| 🔀 Alt Arg | 3 | Functionality exists but under different flag/semantics |
| ⚠️ Partial | 3 | Flag parsed/stored but behavior incomplete |
| ❌ Not Implemented | 97 | Flag not recognized or no behavior |
| **Total** | **138** | |
| 🔀 Alt Arg | 4 | Functionality exists but under different flag/semantics |
| ⚠️ Partial | 5 | Flag parsed/stored but behavior incomplete |
| 🔄 Compatibility No-op | 1 | Flag is accepted for CLI compatibility but has no effect |
| ❌ Not Implemented | 95 | Flag not recognized or no behavior |
| **Total** | **141** | |
---
@@ -20,12 +21,12 @@ This document maps rsync's full feature set to FastSync's current implementation
|------|-------------------|-----------------|-------|
| `-a`, `--archive` | Archive mode is -rlptgoD | 🔀 Alt Arg | Maps to -c -m -M (compression + multithread + metadata) |
| `-v`, `--verbose` | Increase verbosity | ✅ Implemented | Sets `log_level=DEBUG` |
| `-q`, `--quiet` | Suppress non-error messages | ❌ Not Implemented | Removed because it had no effect |
| `-q`, `--quiet` | Suppress non-error messages | ✅ Implemented | Suppresses client output while preserving errors |
| `--help` | Show help | ✅ Implemented | Prints usage and exits; `-h` is not accepted |
| `-V`, `--version` | Print version | ✅ Implemented | |
| `--info=FLAGS` | Fine-grained info verbosity | ❌ Not Implemented | Removed because it had no effect |
| `--debug=FLAGS` | Fine-grained debug verbosity | ❌ Not Implemented | Removed because it had no effect |
| `--stderr=MODE` | Change stderr output mode | ❌ Not Implemented | |
| `--info=FLAGS` | Fine-grained info verbosity | ✅ Implemented | Supports `copy`, `misc`, `skip`, `stats`, `all`, and `none`; explicit flags override `--verbose`, and `none` suppresses info output; unsupported names are rejected |
| `--debug=FLAGS` | Fine-grained debug verbosity | ✅ Implemented | `io`, `proto`, `pack`, and `util` are supported; `--debug=help` lists flags; other rsync categories are rejected |
| `--stderr=MODE` | Change stderr output mode | ⚠️ Partial | `errors` (default) and `all` are supported; `client` is rejected because FastSync has no rsync message channel |
| `--no-motd` | Suppress daemon MOTD | ❌ Not Implemented | |
| `--exclude=PATTERN` | Exclude files matching pattern | ✅ Implemented | Glob matching in scanner |
| `--include=PATTERN` | Include files matching pattern | ✅ Implemented | Glob matching in scanner |
@@ -36,14 +37,14 @@ This document maps rsync's full feature set to FastSync's current implementation
| Flag | Rsync Description | FastSync Status | Notes |
|------|-------------------|-----------------|-------|
| `--stats` | Give transfer stats | ✅ Implemented | Prints file/byte counts |
| `-h`, `--human-readable` | Human-readable numbers | ❌ Not Implemented | Removed because it had no effect |
| `-h`, `--human-readable` | Human-readable numbers | ✅ Implemented | Formats transfer byte sizes using binary units |
| `-i`, `--itemize-changes` | Per-file change summary | ❌ Not Implemented | Removed because it had no effect |
| `--progress` | Show progress | ✅ Implemented | Progress callback in sender |
| `-P` | Same as --partial --progress | ❌ Not Implemented | |
| `-P` | Same as --partial --progress | ⚠️ Partial | Parses and enables progress, but interrupted files are not retained for resumable transfers |
| `--out-format=FORMAT` | Custom output format | ❌ Not Implemented | Removed because it had no effect |
| `--log-file=FILE` | Log to file | ✅ Implemented | `log_file` config field |
| `--log-file-format=FMT` | Log format | ❌ Not Implemented | |
| `--8-bit-output` | Leave high-bit chars unescaped | ❌ Not Implemented | |
| `--8-bit-output`, `-8` | Leave high-bit chars unescaped | ✅ Implemented | Applies to displayed paths and protocol debug output |
| `--list-only` | List files instead of copying | ❌ Not Implemented | Removed because it had no effect |
## 3. File Selection
@@ -58,11 +59,11 @@ This document maps rsync's full feature set to FastSync's current implementation
| `--max-size=SIZE` | Skip files larger than SIZE | ✅ Implemented | `max_size` in scanner |
| `--min-size=SIZE` | Skip files smaller than SIZE | ✅ Implemented | `min_size` in scanner |
| `-I`, `--ignore-times` | Don't skip files matching size+time | ❌ Not Implemented | |
| `--size-only` | Skip based on size only | ❌ Not Implemented | |
| `-@`, `--modify-window=NUM` | Mod-time comparison accuracy | ❌ Not Implemented | |
| `--size-only` | Skip based on size only | ✅ Implemented | With `--incremental`, ignores mtime |
| `-@`, `--modify-window=NUM` | Mod-time comparison accuracy | ✅ Implemented | Whole-second tolerance with nanosecond-aware comparisons |
| `--existing` | Skip creating new files on receiver | ✅ Implemented | Existing destination files continue through normal update handling |
| `--ignore-existing` | Skip updating existing files | ❌ Not Implemented | |
| `--remove-source-files` | Sender removes synced files | ❌ Not Implemented | |
| `--remove-source-files` | Sender removes regular files after confirmed transfer | ✅ Implemented | |
## 4. Directory Options
@@ -71,7 +72,7 @@ This document maps rsync's full feature set to FastSync's current implementation
| `-r`, `--recursive` | Recurse into directories | ✅ Implemented | Default behavior |
| `-R`, `--relative` | Use relative path names | ❌ Not Implemented | Removed because it had no effect |
| `--no-implied-dirs` | Don't send implied dirs with -R | ❌ Not Implemented | |
| `-d`, `--dirs` | Transfer dirs without recursing | ❌ Not Implemented | |
| `-d`, `--dirs`, `--old-dirs`, `--old-d` | Transfer dirs without recursing | ❌ Not Implemented | The aliases are recognized and rejected explicitly; they depend on the unimplemented `--dirs` behavior |
| `--mkpath` | Create missing path components | ❌ Not Implemented | |
## 5. Transfer Modifications
@@ -101,7 +102,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|------|-------------------|-----------------|-------|
| `--delete` | Delete extraneous files from dest | ✅ Implemented | `use_delete` config field |
| `--delete-before` | Delete before transfer | ❌ Not Implemented | Removed because it had no effect |
| `--delete-during` | Delete during transfer | ❌ Not Implemented | |
| `--del`, `--delete-during` | Delete during transfer | ❌ Not Implemented | Both flags are recognized but rejected; delete timing is not implemented |
| `--delete-delay` | Find deletions during, delete after | ❌ Not Implemented | |
| `--delete-after` | Delete after transfer | ❌ Not Implemented | Removed because it had no effect |
| `--delete-excluded` | Also delete excluded files | ❌ Not Implemented | Removed because it had no effect |
@@ -119,8 +120,8 @@ This document maps rsync's full feature set to FastSync's current implementation
| `-o`, `--owner` | Preserve owner | ✅ Implemented | Part of -M |
| `-g`, `--group` | Preserve group | ✅ Implemented | Part of -M |
| `-t`, `--times` | Preserve modification times | ✅ Implemented | Part of -M |
| `-E`, `--executability` | Preserve executability | ❌ Not Implemented | |
| `--chmod=CHMOD` | Affect file permissions | ❌ Not Implemented | |
| `-E`, `--executability` | Preserve executability | ✅ Implemented | Preserves executable permission bits (implies metadata preservation) |
| `--chmod=CHMOD` | Affect file permissions | ✅ Implemented | Supports numeric and symbolic `ugo` `rwx` changes; retains receiver safety masking |
| `-A`, `--acls` | Preserve ACLs | ❌ Not Implemented | Removed because it had no effect |
| `-X`, `--xattrs` | Preserve extended attributes | ❌ Not Implemented | Removed because it had no effect |
| `-H`, `--hard-links` | Preserve hard links | ❌ Not Implemented | Removed because it had no effect |
@@ -171,10 +172,10 @@ This document maps rsync's full feature set to FastSync's current implementation
| Flag | Rsync Description | FastSync Status | Notes |
|------|-------------------|-----------------|-------|
| `-z`, `--compress` | Compress file data | 🔀 Alt Arg | Always uses zstd (rsync supports multiple algorithms) |
| `--compress-choice=STR` | Choose compression algorithm | ❌ Not Implemented | Removed because it had no effect; FastSync always uses zstd |
| `--compress-level=NUM` | Set compression level | ✅ Implemented | 1-22, default 5 |
| `--compress-choice=STR`, `--zc=STR` | Choose compression algorithm | ✅ Implemented | FastSync supports `zstd` and `none` |
| `--compress-level=NUM`, `--zl=NUM` | Set compression level | ✅ Implemented | 1-22, default 5 |
| `--compress-threads=NUM` | Set compression threads | ❌ Not Implemented | |
| `--skip-compress=LIST` | Skip compress for suffixes | ❌ Not Implemented | Internal skip for hardcoded types; not user-configurable |
| `--skip-compress=LIST` | Skip compress for suffixes | ✅ Implemented | Comma-separated, case-insensitive suffix list; empty list skips none; incompatible with FastSync chunk serialization (`-s`) |
## 13. Connectivity
@@ -211,7 +212,7 @@ This document maps rsync's full feature set to FastSync's current implementation
| Max data/string/chunk sizes | Prevent OOM attacks | ✅ Implemented | Per-message limits |
| Per-connection memory limit | 1GB per connection | ✅ Implemented | `MAX_CONNECTION_MEMORY` |
| `--trust-sender` | Trust remote sender's file list | ❌ Not Implemented | |
| `--old-args` | Disable modern arg protection | ❌ Not Implemented | |
| `--old-args` | Disable modern arg protection | ✅ Implemented | SSH-only legacy mode; restores raw remote command construction and permits shell interpretation of the configured server path |
| `--ignore-missing-args` | Ignore missing source args | ❌ Not Implemented | |
| `--delete-missing-args` | Delete missing source args | ❌ Not Implemented | |
@@ -229,12 +230,12 @@ This document maps rsync's full feature set to FastSync's current implementation
|------|-------------------|-----------------|-------|
| `--stop-after=MINS` | Stop after N minutes | ❌ Not Implemented | |
| `--stop-at=TIME` | Stop at specified time | ❌ Not Implemented | |
| `--fsync` | Fsync every written file | ❌ Not Implemented | |
| `--fsync` | Fsync every written file before publication | ✅ Implemented | |
| `--protocol=NUM` | Force older protocol version | ❌ Not Implemented | |
| `--iconv=CONVERT_SPEC` | Charset conversion | ❌ Not Implemented | |
| `--checksum-seed=NUM` | Set checksum seed | ❌ Not Implemented | |
| `-s`, `--secluded-args` | Use protocol to send args | ❌ Not Implemented | |
| `--no-OPTION` | Turn off implied option | ❌ Not Implemented | |
| `--secluded-args` | Use protocol to send args | 🔄 Compatibility No-op | Accepted for CLI compatibility; it does not change FastSync transport or protocol behavior. `-s` remains chunk serialization. |
| `--no-OPTION` | Turn off implied option | ✅ Supported | Supported boolean FastSync options and archive-implied options; unsafe or value-taking options are rejected. |
---
@@ -248,12 +249,13 @@ Ranked by user demand, implementation complexity, and interoperability impact:
| 2 | `--ignore-times` / `-I` | Low | Medium — useful for forcing re-transfer |
| 3 | `--size-only` | Low | Medium — common migration scenario |
| 4 | `--ignore-existing` | Low | Medium — common sync patterns |
| 5 | `--remove-source-files` | Low | High — common for moves/backup |
| 6 | `--delete-during` | Medium | High — performance improvement |
| 7 | `--delay-updates` | Medium | High — atomic updates |
| 8 | `--chmod` | Low | Medium — permission flexibility |
| 9 | `--executability` / `-E` | Low | Low — simple flag |
| 10 | `--skip-compress` | Low | Medium — performance tuning |
| 5 | `--existing` | Low | Medium — common sync patterns |
| 6 | `--remove-source-files` | Low | High — common for moves/backup |
| 7 | `--delete-during` | Medium | High — performance improvement |
| 8 | `--delay-updates` | Medium | High — atomic updates |
| 9 | `--chmod` | Low | Medium — permission flexibility |
| 10 | `--executability` / `-E` | Low | Low — simple flag |
| 11 | `--skip-compress` | Low | Medium — performance tuning |
---
+517 -36
View File
@@ -1,5 +1,7 @@
#include "client_send.h"
#include "client_validation.h"
#include "chmod.h"
#include "compression.h"
#include "config.h"
#include "delta.h"
#include "log.h"
@@ -77,6 +79,29 @@ static int set_positive_int_option(int* dest, const char* value, const char* opt
return 0;
}
/* Set and validate the compression algorithm selected by the client. */
static int set_compression_choice(Config* config, const char* value) {
if (strcmp(value, "zstd") != 0 && strcmp(value, "none") != 0) {
log_message(LOG_LEVEL_ERROR, "--compress-choice must be zstd or none");
return -1;
}
if (set_string_option(&config->compress_choice, value, "--compress-choice") != 0)
return -1;
config->use_compression = strcmp(value, "zstd") == 0;
return 0;
}
static int set_compression_threads_option(int* dest, const char* value) {
if (set_positive_int_option(dest, value, "--compress-threads") != 0)
return -1;
if (*dest > COMPRESSION_MAX_THREADS) {
log_message(LOG_LEVEL_ERROR, "--compress-threads must be between 1 and %d",
COMPRESSION_MAX_THREADS);
return -1;
}
return 0;
}
/* Parse a string as a non-negative integer into *dest. Returns true on success, false on error. */
static int set_nonneg_int_option(int* dest, const char* value, const char* option_name) {
if (!parse_nonneg_int(value, dest)) {
@@ -86,8 +111,119 @@ static int set_nonneg_int_option(int* dest, const char* value, const char* optio
return 0;
}
static int set_stderr_mode(const char* value) {
if (strcmp(value, "errors") == 0 || strcmp(value, "e") == 0)
log_set_stderr_mode(LOG_STDERR_ERRORS);
else if (strcmp(value, "all") == 0 || strcmp(value, "a") == 0)
log_set_stderr_mode(LOG_STDERR_ALL);
else if (strcmp(value, "client") == 0 || strcmp(value, "c") == 0) {
log_message(LOG_LEVEL_ERROR,
"--stderr=client is not supported: FastSync has no client message channel");
return -1;
} else {
log_message(LOG_LEVEL_ERROR, "--stderr must be errors or all");
return -1;
}
return 0;
}
static int read_patterns_from_file(const char* filepath, char*** patterns, int* count);
static int parse_debug_flags(const char* value, Config* config) {
if (!value || value[0] == '\0' || value[0] == ',' || value[strlen(value) - 1] == ',' ||
strstr(value, ",,")) {
log_message(LOG_LEVEL_ERROR, "--debug requires at least one flag");
return -1;
}
char* flags = str_dup(value);
if (!flags) {
log_message(LOG_LEVEL_ERROR, "memory allocation failed for --debug");
return -1;
}
uint32_t parsed = (uint32_t)config->debug_level;
char* saveptr = NULL;
for (char* token = strtok_r(flags, ",", &saveptr); token != NULL;
token = strtok_r(NULL, ",", &saveptr)) {
uint32_t flag = 0;
if (strcmp(token, "help") == 0) {
print_debug_usage();
free(flags);
return 1;
} else if (strcmp(token, "all") == 0) {
parsed = LOG_DEBUG_ALL;
continue;
} else if (strcmp(token, "none") == 0) {
parsed = 0;
continue;
} else if (strcmp(token, "io") == 0) {
flag = LOG_DEBUG_IO;
} else if (strcmp(token, "proto") == 0) {
flag = LOG_DEBUG_PROTO;
} else if (strcmp(token, "pack") == 0) {
flag = LOG_DEBUG_PACK;
} else if (strcmp(token, "util") == 0) {
flag = LOG_DEBUG_UTIL;
} else {
log_message(LOG_LEVEL_ERROR, "unsupported --debug flag: %s", token);
free(flags);
return -1;
}
parsed |= flag;
}
free(flags);
config->debug_level = (int)parsed;
set_log_debug_flags(parsed);
set_log_level(LOG_LEVEL_DEBUG);
return 0;
}
static int parse_info_flags(const char* value, Config* config) {
if (!value || value[0] == '\0' || value[0] == ',' || value[strlen(value) - 1] == ',' ||
strstr(value, ",,")) {
log_message(LOG_LEVEL_ERROR, "--info requires at least one flag");
return -1;
}
char* flags = str_dup(value);
if (!flags) {
log_message(LOG_LEVEL_ERROR, "memory allocation failed for --info");
return -1;
}
uint32_t parsed = (uint32_t)config->info_level;
char* saveptr = NULL;
for (char* token = strtok_r(flags, ",", &saveptr); token != NULL;
token = strtok_r(NULL, ",", &saveptr)) {
uint32_t flag = 0;
if (strcmp(token, "all") == 0) {
parsed = LOG_INFO_ALL;
continue;
}
if (strcmp(token, "none") == 0) {
parsed = 0;
continue;
}
if (strcmp(token, "copy") == 0)
flag = LOG_INFO_COPY;
else if (strcmp(token, "misc") == 0)
flag = LOG_INFO_MISC;
else if (strcmp(token, "skip") == 0)
flag = LOG_INFO_SKIP;
else if (strcmp(token, "stats") == 0)
flag = LOG_INFO_STATS;
else {
log_message(LOG_LEVEL_ERROR, "unsupported --info flag: %s", token);
free(flags);
return -1;
}
parsed |= flag;
}
free(flags);
config->info_level = (int)parsed;
set_log_info_flags(parsed);
return 0;
}
/* Parse a string as an unsigned long long. Returns 0 on success, -1 on error. */
static int parse_ull_arg(const char* val, unsigned long long* out, const char* optname) {
char* end;
@@ -101,6 +237,56 @@ static int parse_ull_arg(const char* val, unsigned long long* out, const char* o
return 0;
}
static int parse_size_arg(const char* value, unsigned long long* out) {
if (!value || *value < '0' || *value > '9')
return -1;
char* end;
errno = 0;
unsigned long long number = strtoull(value, &end, 10);
if (errno != 0 || end == value)
return -1;
unsigned long long multiplier = 1;
if (*end != '\0') {
if (end[1] != '\0')
return -1;
switch (*end) {
case 'b':
case 'B':
break;
case 'k':
case 'K':
multiplier = 1024ULL;
break;
case 'm':
case 'M':
multiplier = 1024ULL * 1024;
break;
case 'g':
case 'G':
multiplier = 1024ULL * 1024 * 1024;
break;
case 't':
case 'T':
multiplier = 1024ULL * 1024 * 1024 * 1024;
break;
case 'p':
case 'P':
multiplier = 1024ULL * 1024 * 1024 * 1024 * 1024;
break;
case 'e':
case 'E':
multiplier = 1024ULL * 1024 * 1024 * 1024 * 1024 * 1024;
break;
default:
return -1;
}
}
if (number == 0 || number > ULLONG_MAX / multiplier)
return -1;
*out = number * multiplier;
return 0;
}
/* Append a duplicated pattern to a growable pattern array. Returns 0 on success, -1 on error. */
static int config_add_pattern(char*** patterns, int* count, const char* value,
const char* optname) {
@@ -119,41 +305,89 @@ static int config_add_pattern(char*** patterns, int* count, const char* value,
return 0;
}
static int parse_skip_compress(Config* config, const char* value) {
char* list = str_dup(value);
if (!list)
return -1;
config->skip_compress_set = true;
for (char* token = strtok(list, ","); token; token = strtok(NULL, ",")) {
while (*token == ' ' || *token == '\t')
token++;
size_t len = strlen(token);
while (len > 0 && (token[len - 1] == ' ' || token[len - 1] == '\t'))
token[--len] = '\0';
if (len == 0)
continue;
if (config_add_pattern(&config->skip_compress_suffixes, &config->skip_compress_count, token,
"--skip-compress") != 0) {
free(list);
return -1;
}
}
free(list);
return 0;
}
typedef enum {
OPT_FLAG,
OPT_NOOP,
OPT_STRING,
OPT_POS_INT,
OPT_NONNEG_INT,
OPT_ULL,
OPT_UNSUPPORTED,
} OptKind;
typedef struct {
const char* name;
const char* alias;
OptKind kind;
size_t offset; /* offsetof of the target field in Config */
size_t offset; /* offsetof of the target field in Config, or 0 for OPT_NOOP */
} OptionEntry;
/* Options parsed directly into Config, plus compatibility options with no effect. */
typedef struct {
const char* name;
const char* alias;
size_t offset; /* offsetof of the boolean target field in Config */
} NegatableOption;
/* Options that map directly onto a Config field with no side effects. */
static const OptionEntry OPTION_TABLE[] = {
{"--dry-run", "-n", OPT_FLAG, offsetof(Config, dry_run)},
{"--remove-source-files", NULL, OPT_FLAG, offsetof(Config, remove_source_files)},
{"--delete", NULL, OPT_FLAG, offsetof(Config, use_delete)},
{"--incremental", NULL, OPT_FLAG, offsetof(Config, use_incremental)},
{"--size-only", NULL, OPT_FLAG, offsetof(Config, size_only)},
{"--ignore-times", "-I", OPT_FLAG, offsetof(Config, ignore_times)},
{"--modify-window", "-@", OPT_NONNEG_INT, offsetof(Config, modify_window)},
{"--delta", NULL, OPT_FLAG, offsetof(Config, use_delta)},
{"--whole-file", "-W", OPT_FLAG, offsetof(Config, whole_file)},
{"--save-to-disk", NULL, OPT_FLAG, offsetof(Config, save_to_disk)},
{"--progress", NULL, OPT_FLAG, offsetof(Config, show_progress)},
{"--tls", NULL, OPT_FLAG, offsetof(Config, use_tls)},
{"--backup", NULL, OPT_FLAG, offsetof(Config, backup)},
{"--stats", NULL, OPT_FLAG, offsetof(Config, stats)},
{"--human-readable", "-h", OPT_FLAG, offsetof(Config, human_readable)},
{"--partial", NULL, OPT_FLAG, offsetof(Config, partial)},
{"--secluded-args", NULL, OPT_NOOP, 0},
{"--update", "-u", OPT_FLAG, offsetof(Config, update)},
{"--old-args", NULL, OPT_FLAG, offsetof(Config, old_args)},
{"--links", "-l", OPT_FLAG, offsetof(Config, follow_symlinks)},
{"--copy-links", NULL, OPT_FLAG, offsetof(Config, copy_links)},
{"--safe-links", NULL, OPT_FLAG, offsetof(Config, safe_links)},
{"--copy-unsafe-links", NULL, OPT_FLAG, offsetof(Config, copy_unsafe_links)},
{"--sparse", "-S", OPT_FLAG, offsetof(Config, preserve_sparse)},
{"--inplace", NULL, OPT_FLAG, offsetof(Config, inplace)},
{"--fsync", NULL, OPT_FLAG, offsetof(Config, use_fsync)},
{"--checksum", NULL, OPT_FLAG, offsetof(Config, checksum)},
{"--8-bit-output", "-8", OPT_FLAG, offsetof(Config, eight_bit_output)},
{"--existing", NULL, OPT_FLAG, offsetof(Config, existing)},
{"--ignore-existing", NULL, OPT_FLAG, offsetof(Config, ignore_existing)},
{"--chmod", NULL, OPT_STRING, offsetof(Config, chmod_spec)},
{"--dirs", "--old-dirs", OPT_UNSUPPORTED, 0},
{"--old-d", NULL, OPT_UNSUPPORTED, 0},
{"--delete-during", "--del", OPT_UNSUPPORTED, 0},
{"--source-dir", NULL, OPT_STRING, offsetof(Config, send_directory)},
{"--dest-dir", NULL, OPT_STRING, offsetof(Config, receive_root_directory)},
@@ -165,6 +399,8 @@ static const OptionEntry OPTION_TABLE[] = {
{"--fastsync-server-path", NULL, OPT_STRING, offsetof(Config, fastsync_server_path)},
{"--partial-dir", NULL, OPT_STRING, offsetof(Config, partial_dir)},
{"--suffix", NULL, OPT_STRING, offsetof(Config, suffix)},
{"--compress-choice", "--zc", OPT_STRING, offsetof(Config, compress_choice)},
{"--compress-level", "--zl", OPT_POS_INT, offsetof(Config, compression_level)},
{"--timeout", NULL, OPT_POS_INT, offsetof(Config, timeout)},
{"--contimeout", NULL, OPT_POS_INT, offsetof(Config, contimeout)},
@@ -174,6 +410,35 @@ static const OptionEntry OPTION_TABLE[] = {
{"--min-size", NULL, OPT_ULL, offsetof(Config, min_size)},
};
/* Only boolean options with no required argument are safe to negate. */
static const NegatableOption NEGATABLE_OPTIONS[] = {
{"dry-run", "n", offsetof(Config, dry_run)},
{"delete", NULL, offsetof(Config, use_delete)},
{"incremental", NULL, offsetof(Config, use_incremental)},
{"delta", NULL, offsetof(Config, use_delta)},
{"save-to-disk", NULL, offsetof(Config, save_to_disk)},
{"progress", NULL, offsetof(Config, show_progress)},
{"tls", NULL, offsetof(Config, use_tls)},
{"backup", NULL, offsetof(Config, backup)},
{"stats", NULL, offsetof(Config, stats)},
{"partial", NULL, offsetof(Config, partial)},
{"links", "l", offsetof(Config, follow_symlinks)},
{"copy-links", NULL, offsetof(Config, copy_links)},
{"safe-links", NULL, offsetof(Config, safe_links)},
{"copy-unsafe-links", NULL, offsetof(Config, copy_unsafe_links)},
{"sparse", "S", offsetof(Config, preserve_sparse)},
{"inplace", NULL, offsetof(Config, inplace)},
{"checksum", NULL, offsetof(Config, checksum)},
/* These options are also implied by --archive or handled outside the table. */
{"compress", "c", offsetof(Config, use_compression)},
{"compress", "z", offsetof(Config, use_compression)},
{"multithreading", "m", offsetof(Config, use_multithreading)},
{"preserve", "M", offsetof(Config, use_metadata)},
{"sendfile", "f", offsetof(Config, use_sendfile)},
{"chunk-serialization", "s", offsetof(Config, use_chunk_serialization)},
};
static bool opt_is(const char* arg, const char* name, const char* alias) {
return strcmp(arg, name) == 0 || (alias && strcmp(arg, alias) == 0);
}
@@ -185,11 +450,63 @@ static const OptionEntry* find_table_option(const char* arg) {
return NULL;
}
static int apply_table_option(Config* config, const OptionEntry* entry, const char* value) {
static const OptionEntry* find_table_option_with_equals(const char* arg, const char** value) {
const char* equals = strchr(arg, '=');
if (!equals || equals == arg)
return NULL;
size_t name_len = (size_t)(equals - arg);
for (size_t i = 0; i < sizeof(OPTION_TABLE) / sizeof(OPTION_TABLE[0]); i++) {
const OptionEntry* entry = &OPTION_TABLE[i];
if ((strlen(entry->name) == name_len && strncmp(arg, entry->name, name_len) == 0) ||
(entry->alias && strlen(entry->alias) == name_len &&
strncmp(arg, entry->alias, name_len) == 0)) {
if (strcmp(entry->name, "--compress-choice") == 0 ||
strcmp(entry->name, "--compress-level") == 0) {
*value = equals + 1;
return entry;
}
}
}
return NULL;
}
static const NegatableOption* find_negatable_option(const char* name) {
for (size_t i = 0; i < sizeof(NEGATABLE_OPTIONS) / sizeof(NEGATABLE_OPTIONS[0]); i++)
if (strcmp(name, NEGATABLE_OPTIONS[i].name) == 0 ||
(NEGATABLE_OPTIONS[i].alias && strcmp(name, NEGATABLE_OPTIONS[i].alias) == 0))
return &NEGATABLE_OPTIONS[i];
return NULL;
}
static int apply_negation(Config* config, const char* arg) {
const char* name = arg + strlen("--no-");
if (*name == '\0') {
fprintf(stderr, "Cannot negate an empty option name: %s\n", arg);
return -1;
}
const NegatableOption* entry = find_negatable_option(name);
if (!entry) {
fprintf(stderr, "Cannot negate unsupported or unsafe option: %s\n", arg);
return -1;
}
*(bool*)((char*)config + entry->offset) = false;
if (entry->offset == offsetof(Config, use_metadata))
config->metadata_explicitly_disabled = true;
return 0;
}
static int apply_table_option(Config* config, const OptionEntry* entry, const char* option_name,
const char* value) {
if (entry->kind == OPT_NOOP)
return 0;
void* field = (char*)config + entry->offset;
switch (entry->kind) {
case OPT_FLAG:
*(bool*)field = true;
if (entry->offset == offsetof(Config, update))
config->use_metadata = true;
return 0;
case OPT_NOOP:
return 0;
case OPT_STRING:
return set_string_option((char**)field, value, entry->name);
@@ -204,6 +521,13 @@ static int apply_table_option(Config* config, const OptionEntry* entry, const ch
*(unsigned long long*)field = v;
return 0;
}
case OPT_UNSUPPORTED: {
const char* reason = "directory-only transfer is not implemented";
if (strcmp(entry->name, "--delete-during") == 0)
reason = "delete-during is not implemented";
log_message(LOG_LEVEL_ERROR, "%s: %s; refusing to ignore option", option_name, reason);
return -1;
}
}
return -1;
}
@@ -211,19 +535,123 @@ static int apply_table_option(Config* config, const OptionEntry* entry, const ch
/* Parse CLI arguments into config. Returns 0 on success, -1 on error, 1 for help/clean-exit. */
int parse_args(Config* config, int argc, char* argv[], int* positional_args,
int* positional_count) {
bool verbose = false;
protocol_set_8_bit_output(config->eight_bit_output);
/* Apply output controls before processing other options so their order is irrelevant. */
for (int i = 1; i < argc; i++) {
const OptionEntry* entry = find_table_option(argv[i]);
if (entry) {
if (entry->kind != OPT_FLAG) {
if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
set_log_level(LOG_LEVEL_DEBUG);
} else if (strncmp(argv[i], "--info=", 7) == 0) {
if (parse_info_flags(argv[i] + 7, config) != 0)
return -1;
} else if (strcmp(argv[i], "--info") == 0) {
if (i + 1 >= argc || parse_info_flags(argv[++i], config) != 0)
return -1;
}
}
for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-P") == 0) {
config->partial = true;
config->show_progress = true;
continue;
}
if (strncmp(argv[i], "--no-", strlen("--no-")) == 0) {
if (apply_negation(config, argv[i]) != 0)
return -1;
continue;
}
const char* modify_window_prefix = "--modify-window=";
if (strncmp(argv[i], modify_window_prefix, strlen(modify_window_prefix)) == 0) {
if (set_nonneg_int_option(&config->modify_window, argv[i] + strlen(modify_window_prefix),
"--modify-window") != 0)
return -1;
continue;
}
if (strncmp(argv[i], "-@", 2) == 0 && argv[i][2] != '\0') {
if (set_nonneg_int_option(&config->modify_window, argv[i] + 2, "-@") != 0)
return -1;
continue;
}
const char* threads_prefix = "--compress-threads=";
if (strncmp(argv[i], threads_prefix, strlen(threads_prefix)) == 0) {
if (set_compression_threads_option(&config->compression_threads,
argv[i] + strlen(threads_prefix)) != 0)
return -1;
continue;
}
if (strncmp(argv[i], "--max-alloc=", 12) == 0 || strcmp(argv[i], "--max-alloc") == 0) {
const char* value = strcmp(argv[i], "--max-alloc") == 0 ? "" : argv[i] + 12;
if (*value == '\0') {
if (i + 1 >= argc) {
log_message(LOG_LEVEL_ERROR, "missing argument for --max-alloc");
return -1;
}
value = argv[++i];
}
if (parse_size_arg(value, &config->max_alloc) != 0) {
log_message(LOG_LEVEL_ERROR,
"--max-alloc must be a positive size (B, K, M, G, T, P, or E)");
return -1;
}
continue;
}
const OptionEntry* entry = find_table_option(argv[i]);
const char* inline_value = NULL;
if (!entry)
entry = find_table_option_with_equals(argv[i], &inline_value);
if (entry) {
const char* option_name = argv[i];
const char* value = NULL;
if (entry->kind != OPT_FLAG) {
if (entry->kind != OPT_UNSUPPORTED) {
value = inline_value;
if (!value && i + 1 < argc)
value = argv[++i];
if (!value) {
log_message(LOG_LEVEL_ERROR, "missing argument for %s", entry->name);
return -1;
}
if (apply_table_option(config, entry, argv[++i]) != 0)
}
if (strcmp(entry->name, "--compress-choice") == 0) {
if (set_compression_choice(config, value) != 0)
return -1;
} else if (apply_table_option(config, entry, NULL) != 0) {
} else {
if (apply_table_option(config, entry, option_name, value) != 0)
return -1;
if (strcmp(entry->name, "--compress-level") == 0 &&
(config->compression_level < 1 || config->compression_level > 22)) {
log_message(LOG_LEVEL_ERROR, "--compress-level must be between 1 and 22");
return -1;
}
if (entry->offset == offsetof(Config, chmod_spec)) {
mode_t ignored;
if (!chmod_apply(0, config->chmod_spec, &ignored)) {
log_message(LOG_LEVEL_ERROR, "--chmod has invalid permission changes");
return -1;
}
config->use_metadata = true;
}
}
} else if (apply_table_option(config, entry, option_name, NULL) != 0) {
return -1;
}
if (entry->offset == offsetof(Config, eight_bit_output))
protocol_set_8_bit_output(true);
continue;
}
if (strncmp(argv[i], "--chmod=", 8) == 0) {
if (set_string_option(&config->chmod_spec, argv[i] + 8, "--chmod") != 0)
return -1;
mode_t ignored;
if (!chmod_apply(0, config->chmod_spec, &ignored)) {
log_message(LOG_LEVEL_ERROR, "--chmod has invalid permission changes");
return -1;
}
config->use_metadata = true;
continue;
}
@@ -234,10 +662,11 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
printf("fastsync version %s\n", PROTOCOL_VERSION);
return 1;
} else if (opt_is(argv[i], "-a", "--archive")) {
config->use_compression = true;
config->use_compression =
!config->compress_choice || strcmp(config->compress_choice, "zstd") == 0;
config->use_multithreading = true;
config->use_metadata = true;
log_message(LOG_LEVEL_INFO, "Enabled archive mode (-c -m -M)");
log_info_message(LOG_INFO_MISC, "Enabled archive mode (-c -m -M)");
} else if (opt_is(argv[i], "-p", NULL) && i + 1 < argc) {
if (set_positive_int_option(&config->ssh_port, argv[++i], "-p") != 0)
return -1;
@@ -270,8 +699,9 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
else
log_message(LOG_LEVEL_WARNING, "--delta-max value %llu too small, using default", val);
} else if (opt_is(argv[i], "-c", "-z")) {
config->use_compression = true;
log_message(LOG_LEVEL_INFO, "Enabled Compression");
config->use_compression =
!config->compress_choice || strcmp(config->compress_choice, "zstd") == 0;
log_info_message(LOG_INFO_MISC, "Enabled Compression");
if (i + 1 < argc) {
char* end_ptr;
long level = strtol(argv[i + 1], &end_ptr, 10);
@@ -281,25 +711,32 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
return -1;
}
config->compression_level = (int)level;
log_message(LOG_LEVEL_INFO, "Set Compression level to %ld", level);
log_info_message(LOG_INFO_MISC, "Set Compression level to %ld", level);
i++;
}
}
} else if (opt_is(argv[i], "-M", "--preserve")) {
config->use_metadata = true;
log_message(LOG_LEVEL_INFO, "Enabled metadata preservation");
log_info_message(LOG_INFO_MISC, "Enabled metadata preservation");
} else if (opt_is(argv[i], "-E", "--executability")) {
config->use_metadata = true;
config->use_executability = true;
log_info_message(LOG_INFO_MISC, "Enabled executable permission preservation");
} else if (opt_is(argv[i], "-f", "--sendfile")) {
config->use_sendfile = true;
log_message(LOG_LEVEL_INFO, "Enabled sendfile");
log_info_message(LOG_INFO_MISC, "Enabled sendfile");
} else if (opt_is(argv[i], "-m", NULL)) {
config->use_multithreading = true;
log_message(LOG_LEVEL_INFO, "Enabled Multithreading");
log_info_message(LOG_INFO_MISC, "Enabled Multithreading");
} else if (opt_is(argv[i], "-s", NULL)) {
config->use_chunk_serialization = true;
log_message(LOG_LEVEL_INFO, "Enabled Chunk Serialization");
log_info_message(LOG_INFO_MISC, "Enabled Chunk Serialization");
} else if (opt_is(argv[i], "--server-port", NULL) && i + 1 < argc) {
if (!parse_positive_int(argv[++i], &config->server_port)) {
log_message(LOG_LEVEL_ERROR, "invalid --server-port value: %s", argv[i]);
char* escaped = output_escape(argv[i], false);
log_message(LOG_LEVEL_ERROR, "invalid --server-port value: %s",
escaped ? escaped : "<allocation failed>");
free(escaped);
return -1;
}
if (config->server_port > 65535) {
@@ -319,7 +756,7 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
return -1;
}
io_set_bwlimit(kbps * 1024);
log_message(LOG_LEVEL_INFO, "Set bandwidth limit to %llu KB/s", kbps);
log_info_message(LOG_INFO_MISC, "Set bandwidth limit to %llu KB/s", kbps);
} else if (opt_is(argv[i], "--chunk-size", NULL) && i + 1 < argc) {
unsigned long long val;
if (parse_ull_arg(argv[++i], &val, "--chunk-size") != 0)
@@ -337,11 +774,20 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
}
FILE* lf = fopen(argv[++i], "a");
if (!lf) {
log_message(LOG_LEVEL_ERROR, "could not open log file '%s': %s", argv[i], strerror(errno));
char* escaped = output_escape(argv[i], false);
log_message(LOG_LEVEL_ERROR, "could not open log file '%s': %s",
escaped ? escaped : "<allocation failed>", strerror(errno));
free(escaped);
return -1;
}
config->log_file = lf;
log_set_file(lf);
} else if (strncmp(argv[i], "--stderr=", 9) == 0) {
if (set_stderr_mode(argv[i] + 9) != 0)
return -1;
} else if (opt_is(argv[i], "--stderr", NULL)) {
if (i + 1 >= argc || set_stderr_mode(argv[++i]) != 0)
return -1;
} else if (opt_is(argv[i], "--exclude-from", NULL) && i + 1 < argc) {
if (read_patterns_from_file(argv[++i], &config->exclude_patterns, &config->exclude_count) !=
0)
@@ -351,38 +797,83 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
0)
return -1;
} else if (opt_is(argv[i], "-v", "--verbose")) {
verbose = true;
set_log_level(LOG_LEVEL_DEBUG);
} else if (opt_is(argv[i], "-q", "--quiet")) {
config->quiet = true;
} else if (strncmp(argv[i], "--debug=", 8) == 0) {
int debug_ret = parse_debug_flags(argv[i] + 8, config);
if (debug_ret != 0)
return debug_ret;
} else if (opt_is(argv[i], "--debug", NULL)) {
if (i + 1 >= argc)
return parse_debug_flags(NULL, config);
int debug_ret = parse_debug_flags(argv[++i], config);
if (debug_ret != 0)
return debug_ret;
} else if (strncmp(argv[i], "--info=", 7) == 0) {
if (parse_info_flags(argv[i] + 7, config) != 0)
return -1;
} else if (opt_is(argv[i], "--info", NULL)) {
if (i + 1 >= argc || parse_info_flags(argv[++i], config) != 0)
return -1;
} else if (opt_is(argv[i], "-T", NULL) && i + 1 < argc) {
if (set_positive_int_option(&config->timeout, argv[++i], "-T") != 0)
return -1;
} else if (opt_is(argv[i], "--compress-level", NULL) && i + 1 < argc) {
if (set_positive_int_option(&config->compression_level, argv[++i], "--compress-level") != 0)
} else if (strncmp(argv[i], "--skip-compress=", 16) == 0) {
if (parse_skip_compress(config, argv[i] + 16) != 0)
return -1;
if (config->compression_level < 1 || config->compression_level > 22) {
log_message(LOG_LEVEL_ERROR, "--compress-level must be between 1 and 22");
} else if (opt_is(argv[i], "--skip-compress", NULL) && i + 1 < argc) {
if (parse_skip_compress(config, argv[++i]) != 0)
return -1;
} else if (opt_is(argv[i], "--compress-threads", NULL) && i + 1 < argc) {
if (set_compression_threads_option(&config->compression_threads, argv[++i]) != 0)
return -1;
} else if (opt_is(argv[i], "--checksum-choice", "--cc")) {
if (i + 1 >= argc) {
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
return -1;
}
log_message(LOG_LEVEL_ERROR, "%s is not supported yet (xxHash64 is used)", argv[i]);
return -1;
} else if (argv[i][0] == '-') {
fprintf(stderr, "Unknown option: %s\n", argv[i]);
char* escaped = output_escape(argv[i], false);
fprintf(stderr, "Unknown option: %s\n", escaped ? escaped : "<allocation failed>");
free(escaped);
print_usage();
return -1;
} else {
if (*positional_count < 2)
positional_args[(*positional_count)++] = i;
else {
fprintf(stderr, "Unexpected argument: %s\n", argv[i]);
char* escaped = output_escape(argv[i], false);
fprintf(stderr, "Unexpected argument: %s\n", escaped ? escaped : "<allocation failed>");
free(escaped);
print_usage();
return -1;
}
}
}
set_log_level(config->quiet ? LOG_LEVEL_ERROR : (verbose ? LOG_LEVEL_DEBUG : LOG_LEVEL_WARNING));
if (config->compress_choice)
config->use_compression = strcmp(config->compress_choice, "zstd") == 0;
/* Incremental and delta transfers need metadata unless the user disabled it. */
if ((config->use_incremental || config->use_delta) && !config->use_metadata &&
!config->metadata_explicitly_disabled) {
log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for incremental/delta transfer");
config->use_metadata = true;
}
return 0;
}
static int read_patterns_from_file(const char* filepath, char*** patterns, int* count) {
FILE* fp = fopen(filepath, "r");
if (!fp) {
log_message(LOG_LEVEL_ERROR, "could not open pattern file '%s': %s", filepath, strerror(errno));
char* escaped = output_escape(filepath, false);
log_message(LOG_LEVEL_ERROR, "could not open pattern file '%s': %s",
escaped ? escaped : "<allocation failed>", strerror(errno));
free(escaped);
return -1;
}
char* line = NULL;
@@ -482,16 +973,6 @@ int main(int argc, char* argv[]) {
goto cleanup;
}
/* Enable implicit flags */
if (config->use_incremental && !config->use_metadata) {
log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --incremental");
config->use_metadata = true;
}
if (config->use_delta && !config->use_metadata) {
log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --delta");
config->use_metadata = true;
}
/* Initialize TLS if needed */
if (config->use_tls)
tls_global_init();
+253 -40
View File
@@ -16,11 +16,14 @@
#include "transport_ssh.h"
#include "transport_tls.h"
#include "utils.h"
#include <fcntl.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <threads.h>
#include <time.h>
#include <sys/stat.h>
#include <unistd.h>
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
@@ -28,6 +31,14 @@
/* Forward declaration for progress-reporting thread used in multithreaded send. */
static int progress_thread_fn(void* arg);
static const char* display_bytes(unsigned long long bytes, bool human_readable, char* buffer,
size_t buffer_size) {
if (human_readable && format_human_bytes(bytes, buffer, buffer_size))
return buffer;
snprintf(buffer, buffer_size, "%.1f MB", bytes / 1048576.0);
return buffer;
}
static ScannerOptions scanner_options_from_config(const Config* config, int num_threads) {
ScannerOptions options = {
config->use_metadata, config->chunk_size, config->exclude_patterns,
@@ -46,7 +57,7 @@ static Client* connect_transfer_client(const Config* config) {
return NULL;
}
return client_connect_ssh(config->ssh_destination, config->ssh_port,
config->fastsync_server_path);
config->fastsync_server_path, config->old_args);
}
Client* client = client_create();
@@ -104,6 +115,88 @@ static bool finalize_transfer(Client* client) {
receive_status(client->file_descriptor, &status) && status == STATUS_OK;
}
typedef struct {
char* path;
dev_t device;
ino_t inode;
} SourceFile;
static void source_file_destroy(void* item) {
SourceFile* source = item;
if (source) {
free(source->path);
free(source);
}
}
/* Remove only the same regular source file that was sent. */
static void remove_transferred_sources(const Config* config, ArrayList* paths) {
if (!config->remove_source_files || !paths)
return;
for (int i = 0; i < paths->size; i++) {
SourceFile* source = paths->items[i];
const char* slash = strrchr(source->path, '/');
const char* leaf = slash ? slash + 1 : source->path;
char parent[PATH_MAX];
if (slash) {
size_t parent_length = (size_t)(slash - source->path);
if (parent_length == 0)
parent_length = 1;
if (parent_length >= sizeof(parent))
continue;
memcpy(parent, source->path, parent_length);
parent[parent_length] = '\0';
} else {
(void)snprintf(parent, sizeof(parent), ".");
}
int dirfd = open(parent, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
if (dirfd < 0)
continue;
struct stat st;
if (fstatat(dirfd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0 || !S_ISREG(st.st_mode) ||
st.st_dev != source->device || st.st_ino != source->inode) {
close(dirfd);
continue;
}
if (unlinkat(dirfd, leaf, 0) != 0)
log_message(LOG_LEVEL_WARNING, "Could not remove source file %s", source->path);
close(dirfd);
}
}
static SourceFile* source_file_create(const File* file) {
if (!file || !file->path)
return NULL;
struct stat st;
if (lstat(file->path, &st) != 0 || !S_ISREG(st.st_mode))
return NULL;
SourceFile* source = malloc(sizeof(*source));
if (!source)
return NULL;
source->path = str_dup(file->path);
source->device = st.st_dev;
source->inode = st.st_ino;
if (!source->path) {
source_file_destroy(source);
return NULL;
}
return source;
}
static bool remember_source_file(ArrayList* paths, const File* file) {
if (!paths || !file || !file->path)
return true;
SourceFile* source = source_file_create(file);
if (!source)
return true;
if (!array_list_add(paths, source)) {
source_file_destroy(source);
return false;
}
return true;
}
static void mark_sender_done(PipelineContextSender* context) {
mtx_lock(&context->mutex_progress);
context->sender_done = true;
@@ -134,17 +227,39 @@ static int send_dry_run_manifest(const Config* config) {
Chunk* chunk;
int file_count = 0;
unsigned long long total_bytes = 0;
char size_buffer[32];
if (!config->quiet)
printf("Dry run: files to be transferred\n");
while ((chunk = directory_scanner_next(scanner)) != NULL) {
for (int i = 0; i < chunk->element_count; i++) {
printf(" %s (%zu bytes)\n", chunk->items[i]->path, chunk->items[i]->data->size);
if (!config->quiet) {
char* escaped_path = output_escape(chunk->items[i]->path, config->eight_bit_output);
if (!escaped_path) {
chunk_destroy(chunk);
directory_scanner_destroy(scanner);
return -1;
}
if (config->human_readable)
printf(
" %s (%s)\n", escaped_path,
display_bytes(chunk->items[i]->data->size, true, size_buffer, sizeof(size_buffer)));
else
printf(" %s (%zu bytes)\n", escaped_path, chunk->items[i]->data->size);
free(escaped_path);
}
total_bytes += chunk->items[i]->data->size;
file_count++;
}
chunk_destroy(chunk);
}
directory_scanner_destroy(scanner);
if (!config->quiet) {
if (config->human_readable)
printf("Total: %d files, %s\n", file_count,
display_bytes(total_bytes, true, size_buffer, sizeof(size_buffer)));
else
printf("Total: %d files, %.1f MB\n", file_count, total_bytes / 1048576.0);
}
return 0;
}
@@ -172,10 +287,13 @@ static int incremental_check(Client* client, File* file, const Config* config,
return -1;
unsigned long long fsize = file->data->size;
long long mtime = file->metadata ? file->metadata->mtime_sec : 0;
long long mtime_nsec = file->metadata ? file->metadata->mtime_nsec : 0;
if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize)))
return -1;
if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime)))
return -1;
if (!send_n_data(client->file_descriptor, &mtime_nsec, sizeof(mtime_nsec)))
return -1;
if (config->checksum) {
uint64_t checksum;
if (!file_checksum(file, &checksum) ||
@@ -234,8 +352,11 @@ static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* c
return send_status(client->file_descriptor, STATUS_NEXT) ? 1 : -1;
Data* to_send = delta_data;
if (config->use_compression) {
to_send = data_compress(delta_data, config->compression_level);
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
if (config->use_compression && !compression_should_skip_with_suffixes(
file->path, config->skip_compress_suffixes, skip_count)) {
to_send = data_compress_with_threads(delta_data, config->compression_level,
config->compression_threads);
data_destroy(delta_data);
if (!to_send)
return send_status(client->file_descriptor, STATUS_NEXT) ? 1 : -1;
@@ -251,20 +372,25 @@ static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* c
return ok ? 0 : -1;
}
typedef bool (*file_send_fn)(File*, int, bool, int, bool);
// Send a single file directly (non-incremental path).
static bool send_file_direct(File* file, int fd, bool use_metadata, int compression_level) {
static bool send_file_direct(File* file, int fd, bool use_metadata, int compression_level,
const Config* config) {
if (!send_status(fd, STATUS_NEXT))
return false;
return file_send_single_calls(file, fd, use_metadata, compression_level, true);
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
return file_send_single_calls_with_skip(file, fd, use_metadata, compression_level, true,
config->skip_compress_suffixes, skip_count,
config->compression_threads);
}
// Send a single file directly via sendfile (non-incremental path).
static bool send_file_direct_sendfile(File* file, int fd, bool use_metadata) {
static bool send_file_direct_sendfile(File* file, int fd, bool use_metadata, const Config* config) {
if (!send_status(fd, STATUS_NEXT))
return false;
return file_send_sendfile(file, fd, use_metadata, 0, true);
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
return file_send_sendfile_with_skip(file, fd, use_metadata, 0, true,
config->skip_compress_suffixes, skip_count,
config->compression_threads);
}
// Process one file in a chunk: either via incremental check or direct send.
@@ -272,13 +398,16 @@ static bool send_file_direct_sendfile(File* file, int fd, bool use_metadata) {
static int send_single_file(Client* client, File* file, Config* config, bool use_incremental,
bool use_sendfile) {
int compression_level = config->use_compression ? config->compression_level : 0;
log_info_message(LOG_INFO_COPY, "Transferring %s", file->path);
if (!use_incremental) {
if (use_sendfile) {
return send_file_direct_sendfile(file, client->file_descriptor, config->use_metadata) ? 0
return send_file_direct_sendfile(file, client->file_descriptor, config->use_metadata, config)
? 0
: -1;
}
return send_file_direct(file, client->file_descriptor, config->use_metadata, compression_level)
return send_file_direct(file, client->file_descriptor, config->use_metadata, compression_level,
config)
? 0
: -1;
}
@@ -288,6 +417,7 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
DeltaSignature* sig = NULL;
int rc = incremental_check(client, file, config, &sig);
if (rc == 1) {
log_info_message(LOG_INFO_SKIP, "Skipping unchanged %s", file->path);
delta_signature_destroy(sig);
return 1;
}
@@ -304,13 +434,15 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
return -1;
}
// Fall through: send full file via sendfile (pass 0 for compression_level)
if (!file_send_sendfile(file, client->file_descriptor, config->use_metadata, 0, false))
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
if (!file_send_sendfile_with_skip(file, client->file_descriptor, config->use_metadata, 0, false,
config->skip_compress_suffixes, skip_count,
config->compression_threads))
return -1;
return 0;
}
// Incremental path with single_calls (supports compression and delta)
file_send_fn send_fn = (file_send_fn)file_send_single_calls;
DeltaSignature* sig = NULL;
int rc = incremental_check(client, file, config, &sig);
if (rc < 0) {
@@ -318,10 +450,11 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
return -1;
}
if (rc == 1) {
log_info_message(LOG_INFO_SKIP, "Skipping unchanged %s", file->path);
delta_signature_destroy(sig);
return 1;
}
if (rc == 2 && config->use_delta) {
if (rc == 2 && config->use_delta && !config->whole_file) {
int drc = send_delta(client, file, sig, config);
delta_signature_destroy(sig);
if (drc == 0)
@@ -338,18 +471,29 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
return -1;
}
}
if (!send_fn(file, client->file_descriptor, config->use_metadata, compression_level, false))
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
if (!file_send_single_calls_with_skip(file, client->file_descriptor, config->use_metadata,
compression_level, false, config->skip_compress_suffixes,
skip_count, config->compression_threads))
return -1;
return 0;
}
int send_chunk(Client* client, Chunk* chunk, Config* config) {
static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
ArrayList* remove_sources) {
if (config->use_chunk_serialization) {
if (remove_sources) {
for (int i = 0; i < chunk->element_count; i++) {
if (!remember_source_file(remove_sources, chunk->items[i]))
return -1;
}
}
if (!send_status(client->file_descriptor, STATUS_CHUNK))
return -1;
Data* data;
if (config->use_compression) {
data = chunk_compress(chunk, config->compression_level, config->use_metadata);
data = chunk_compress_with_threads(chunk, config->compression_level, config->use_metadata,
config->compression_threads);
} else {
data = chunk_serialize(chunk, config->use_metadata);
}
@@ -370,15 +514,28 @@ int send_chunk(Client* client, Chunk* chunk, Config* config) {
bool stream = f->data->data == NULL && f->data->size > 0;
bool use_sendfile =
(config->use_sendfile && !config->use_compression) || (stream && !config->use_compression);
SourceFile* source = remove_sources ? source_file_create(f) : NULL;
int rc = send_single_file(client, f, config, config->use_incremental, use_sendfile);
if (rc == 1)
if (rc == 1) {
source_file_destroy(source);
continue;
if (rc < 0)
}
if (rc < 0) {
source_file_destroy(source);
return -1;
}
if (source && !array_list_add(remove_sources, source)) {
source_file_destroy(source);
return -1;
}
}
return 0;
}
int send_chunk(Client* client, Chunk* chunk, Config* config) {
return send_chunk_with_removal(client, chunk, config, NULL);
}
static int send_chunks_multithreaded(void* pipeline_context) {
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
Client* client = connect_transfer_client(context->config);
@@ -419,6 +576,16 @@ static int send_chunks_multithreaded(void* pipeline_context) {
goto send_fail;
}
bool ok = finalize_transfer(client);
if (ok)
remove_transferred_sources(context->config, context->remove_source_files);
mtx_lock(&context->mutex_progress);
int total_files = context->total_files;
unsigned long long total_bytes = context->total_bytes;
mtx_unlock(&context->mutex_progress);
if (context->config->stats)
fprintf(stderr, "Stats: %d files, %.1f MB\n", total_files, total_bytes / 1048576.0);
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
total_bytes / 1048576.0);
disconnect_transfer_client(client);
mark_sender_done(context);
protocol_session_unbind();
@@ -431,7 +598,8 @@ static int send_chunks_multithreaded(void* pipeline_context) {
protocol_session_unbind();
return thrd_error;
}
if (send_chunk(client, current_chunk, context->config) != 0) {
if (send_chunk_with_removal(client, current_chunk, context->config,
context->remove_source_files) != 0) {
log_message(LOG_LEVEL_ERROR, "unexpected error while sending chunk");
chunk_destroy(current_chunk);
pipeline_cancel(context);
@@ -440,30 +608,35 @@ static int send_chunks_multithreaded(void* pipeline_context) {
protocol_session_unbind();
return thrd_error;
}
if (context->config->show_progress) {
unsigned long long chunk_bytes = 0;
int chunk_files = 0;
for (int i = 0; i < current_chunk->element_count; i++) {
if (current_chunk->items[i] && current_chunk->items[i]->data)
if (current_chunk->items[i] && current_chunk->items[i]->data) {
chunk_files++;
chunk_bytes += current_chunk->items[i]->data->size;
}
mtx_lock(&context->mutex_progress);
context->progress_bytes += chunk_bytes;
mtx_unlock(&context->mutex_progress);
}
mtx_lock(&context->mutex_progress);
context->total_files += chunk_files;
context->total_bytes += chunk_bytes;
context->progress_bytes = context->total_bytes;
mtx_unlock(&context->mutex_progress);
chunk_destroy(current_chunk);
}
}
static int scan_directory_multithreaded(void* pipeline_context) {
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
protocol_session_bind(&context->allocation_session);
ScannerOptions options = scanner_options_from_config(context->config, 4);
ParallelScanner* scanner =
parallel_scanner_create_with_options(context->config->send_directory, &options);
ParallelScanner* scanner = parallel_scanner_create_with_options(
context->config->send_directory, &options, &context->allocation_session);
Chunk* current_chunk;
if (scanner == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to create parallel scanner");
pipeline_cancel(context);
protocol_session_unbind();
return thrd_error;
}
while ((current_chunk = parallel_scanner_next(scanner)) != NULL) {
@@ -475,6 +648,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
pipeline_cancel(context);
chunk_destroy(current_chunk);
parallel_scanner_destroy(scanner);
protocol_session_unbind();
return thrd_error;
}
}
@@ -485,6 +659,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
chunk_destroy(current_chunk);
pipeline_cancel(context);
parallel_scanner_destroy(scanner);
protocol_session_unbind();
return thrd_error;
}
}
@@ -496,6 +671,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
cnd_broadcast(&context->condition_not_full_scanner);
mtx_unlock(&context->mutex_scanner);
pipeline_cancel(context);
protocol_session_unbind();
return thrd_error;
}
mtx_lock(&context->mutex_scanner);
@@ -504,11 +680,13 @@ static int scan_directory_multithreaded(void* pipeline_context) {
mtx_unlock(&context->mutex_scanner);
parallel_scanner_destroy(scanner);
protocol_session_unbind();
return thrd_success;
}
static int load_files_multithreaded(void* pipeline_context) {
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
protocol_session_bind(&context->allocation_session);
while (true) {
Chunk* chunk = queue_dequeue_multithreaded(
context->queue_scanner, &context->mutex_scanner, &context->condition_not_empty_scanner,
@@ -518,6 +696,7 @@ static int load_files_multithreaded(void* pipeline_context) {
context->loader_done = true;
cnd_signal(&context->condition_not_empty_loader);
mtx_unlock(&context->mutex_loader);
protocol_session_unbind();
return thrd_success;
}
if (!context->config->use_sendfile) {
@@ -529,6 +708,7 @@ static int load_files_multithreaded(void* pipeline_context) {
log_message(LOG_LEVEL_ERROR, "Failed to load file data");
chunk_destroy(chunk);
pipeline_cancel(context);
protocol_session_unbind();
return thrd_error;
}
}
@@ -539,6 +719,7 @@ static int load_files_multithreaded(void* pipeline_context) {
&context->cancelled)) {
chunk_destroy(chunk);
pipeline_cancel(context);
protocol_session_unbind();
return thrd_error;
}
}
@@ -548,10 +729,20 @@ static int load_files_multithreaded(void* pipeline_context) {
line (e.g. "Done.\n") or is "" for in-place refresh. Shared by the
single-threaded loop and the multithreaded progress thread. */
static void print_transfer_progress(unsigned long long total_bytes, time_t start,
const char* suffix) {
const char* suffix, bool human_readable) {
double elapsed = difftime(time(NULL), start);
double rate = elapsed > 0.0 ? total_bytes / (1048576.0 * elapsed) : 0.0;
if (human_readable) {
char total_buffer[32];
char rate_buffer[32];
fprintf(stderr, "\rSent %s (%s/s) %s",
display_bytes(total_bytes, true, total_buffer, sizeof(total_buffer)),
display_bytes((unsigned long long)(rate * 1048576.0), true, rate_buffer,
sizeof(rate_buffer)),
suffix);
} else {
fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) %s", total_bytes / 1048576.0, rate, suffix);
}
fflush(stderr);
}
@@ -569,14 +760,14 @@ static int progress_thread_fn(void* arg) {
mtx_unlock(&context->mutex_progress);
if (done) {
print_transfer_progress(total, start, "Done.\n");
print_transfer_progress(total, start, "Done.\n", context->config->human_readable);
break;
}
time_t now = time(NULL);
if (now - last_progress >= 1) {
last_progress = now;
print_transfer_progress(total, start, "");
print_transfer_progress(total, start, "", context->config->human_readable);
}
struct timespec ts = {0, 100 * 1000000L}; /* 100 ms */
@@ -603,12 +794,16 @@ int send_files(Config* config) {
int ret = 1;
DirectoryScanner* scanner = NULL;
ArrayList* manifest = NULL;
ArrayList* remove_sources = NULL;
if (!config_send(client->file_descriptor, config))
goto send_fail;
ScannerOptions scanner_options = scanner_options_from_config(config, 0);
scanner = directory_scanner_create_with_options(config->send_directory, &scanner_options);
manifest = create_transfer_manifest(config);
if (!scanner || (config->use_delete && !manifest))
if (config->remove_source_files)
remove_sources = array_list_create(source_file_destroy);
if (!scanner || (config->use_delete && !manifest) ||
(config->remove_source_files && !remove_sources))
goto send_fail;
Chunk* current_chunk;
unsigned long long total_bytes = 0;
@@ -642,7 +837,7 @@ int send_files(Config* config) {
goto send_fail;
}
}
if (send_chunk(client, current_chunk, config) != 0) {
if (send_chunk_with_removal(client, current_chunk, config, remove_sources) != 0) {
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
chunk_destroy(current_chunk);
if (manifest)
@@ -650,12 +845,12 @@ int send_files(Config* config) {
manifest = NULL;
break;
}
if (config->show_progress) {
total_bytes += chunk_bytes;
if (config->show_progress && !config->quiet) {
time_t now = time(NULL);
if (now - last_progress >= 1) {
last_progress = now;
print_transfer_progress(total_bytes, start, "");
print_transfer_progress(total_bytes, start, "", config->human_readable);
}
}
chunk_destroy(current_chunk);
@@ -672,14 +867,27 @@ int send_files(Config* config) {
manifest = NULL;
}
bool ok = finalize_transfer(client);
if (config->show_progress)
print_transfer_progress(total_bytes, start, "Done.\n");
if (config->stats) {
if (ok)
remove_transferred_sources(config, remove_sources);
if (config->show_progress && !config->quiet)
print_transfer_progress(total_bytes, start, "Done.\n", config->human_readable);
if (config->stats && !config->quiet) {
double elapsed_total = difftime(time(NULL), start);
double rate = elapsed_total > 0 ? total_bytes / (1048576.0 * elapsed_total) : 0;
if (config->human_readable) {
char total_buffer[32];
char rate_buffer[32];
fprintf(stderr, "Stats: %d files, %s, %s/s\n", total_files,
display_bytes(total_bytes, true, total_buffer, sizeof(total_buffer)),
display_bytes((unsigned long long)(rate * 1048576.0), true, rate_buffer,
sizeof(rate_buffer)));
} else {
fprintf(stderr, "Stats: %d files, %.1f MB, %.1f MB/s\n", total_files, total_bytes / 1048576.0,
rate);
}
}
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
total_bytes / 1048576.0);
ret = ok ? 0 : 1;
send_fail:
@@ -687,6 +895,8 @@ send_fail:
here even on success without --delete, fixing a pre-existing leak. */
if (manifest)
array_list_delete(manifest);
if (remove_sources)
array_list_delete(remove_sources);
if (scanner)
directory_scanner_destroy(scanner);
disconnect_transfer_client(client);
@@ -731,7 +941,10 @@ int send_files_multithreaded(Config** config_ptr) {
*config_ptr = NULL; /* context now owns config through all remaining paths */
if (config->use_delete)
context->manifest = array_list_create(free);
if (config->use_delete && !context->manifest) {
if (config->remove_source_files)
context->remove_source_files = array_list_create(source_file_destroy);
if ((config->use_delete && !context->manifest) ||
(config->remove_source_files && !context->remove_source_files)) {
pipeline_context_sender_destroy(context);
return 1;
}
@@ -765,7 +978,7 @@ int send_files_multithreaded(Config** config_ptr) {
thrd_t progress;
bool progress_created = false;
if (config->show_progress) {
if (config->show_progress && !config->quiet) {
progress_created = (thrd_create(&progress, progress_thread_fn, context) == thrd_success);
if (!progress_created) {
log_perror("Error creating progress thread");
+12 -3
View File
@@ -15,6 +15,10 @@ bool validate_config(const Config* config) {
"(chunk serialization)");
return false;
}
if (config->compression_threads > 0 && !config->use_compression) {
log_message(LOG_LEVEL_ERROR, "--compress-threads requires compression (-c or -z)");
return false;
}
if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
log_message(LOG_LEVEL_ERROR, "-f/--sendfile is not supported with SSH transport");
return false;
@@ -23,15 +27,20 @@ bool validate_config(const Config* config) {
log_message(LOG_LEVEL_ERROR, "--incremental is not supported with -s (chunk serialization)");
return false;
}
if (config->use_delta && !config->use_incremental) {
if (config->skip_compress_set && config->use_chunk_serialization) {
log_message(LOG_LEVEL_ERROR,
"--skip-compress cannot be combined with -s (chunk serialization)");
return false;
}
if (config->use_delta && !config->whole_file && !config->use_incremental) {
log_message(LOG_LEVEL_ERROR, "--delta requires --incremental");
return false;
}
if (config->use_delta && config->use_chunk_serialization) {
if (config->use_delta && !config->whole_file && config->use_chunk_serialization) {
log_message(LOG_LEVEL_ERROR, "--delta cannot be combined with -s (chunk serialization)");
return false;
}
if (config->use_delta && config->use_sendfile) {
if (config->use_delta && !config->whole_file && config->use_sendfile) {
log_message(LOG_LEVEL_ERROR, "--delta cannot be combined with -f (sendfile)");
return false;
}
+10 -1
View File
@@ -351,10 +351,14 @@ typedef struct {
char** dirs;
int dir_count;
ScannerOptions options;
ProtocolSession* allocation_session;
} ParallelWorkerArg;
static int parallel_worker_thread(void* arg) {
ParallelWorkerArg* wa = (ParallelWorkerArg*)arg;
ProtocolSession* allocation_session = wa->allocation_session;
if (allocation_session)
protocol_session_bind(allocation_session);
for (int i = 0; i < wa->dir_count; i++) {
DirectoryScanner* ds = directory_scanner_create_with_options(wa->dirs[i], &wa->options);
if (!ds) {
@@ -398,6 +402,8 @@ static int parallel_worker_thread(void* arg) {
cnd_signal(&ps->result_not_empty);
}
mtx_unlock(&ps->result_mutex);
if (allocation_session)
protocol_session_unbind();
return thrd_success;
}
@@ -633,6 +639,7 @@ static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs,
wa->dir_count = count;
wa->options = *options;
wa->options.chunk_size = cs;
wa->allocation_session = ps->allocation_session;
start += count;
if (thrd_create(&ps->threads[t], parallel_worker_thread, wa) != thrd_success) {
for (int j = 0; j < count; j++)
@@ -648,7 +655,8 @@ static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs,
}
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
const ScannerOptions* options) {
const ScannerOptions* options,
ProtocolSession* allocation_session) {
if (!root_directory || !options)
return NULL;
ParallelScanner* ps = calloc(1, sizeof(ParallelScanner));
@@ -658,6 +666,7 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
free(ps);
return NULL;
}
ps->allocation_session = allocation_session;
ArrayList* root_files = array_list_create(file_destroy);
ArrayList* subdirs = array_list_create(free);
+4 -1
View File
@@ -2,6 +2,7 @@
#define SCANNER_H
#include "chunk.h"
#include "protocol.h"
#include "queue.h"
#include <dirent.h>
#include <stdbool.h>
@@ -62,6 +63,7 @@ typedef struct {
atomic_bool cancelled;
int completed;
Chunk* initial_chunk;
ProtocolSession* allocation_session;
} ParallelScanner;
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
@@ -78,7 +80,8 @@ bool directory_scanner_failed(const DirectoryScanner* scanner);
void directory_scanner_destroy(DirectoryScanner* scanner);
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
const ScannerOptions* options);
const ScannerOptions* options,
ProtocolSession* allocation_session);
Chunk* parallel_scanner_next(ParallelScanner* scanner);
bool parallel_scanner_failed(const ParallelScanner* scanner);
void parallel_scanner_destroy(ParallelScanner* scanner);
+39
View File
@@ -18,27 +18,51 @@ void print_usage(void) {
printf(" -z [level] Alias for -c\n");
printf(" -a, --archive Archive mode (-c -m -M)\n");
printf(" -n, --dry-run Show what would be transferred\n");
printf(" --remove-source-files Remove regular source files after successful transfer\n");
printf(" -p <port> SSH port (default: 22)\n");
printf(" --progress Show transfer progress\n");
printf(" -P Partial mode with progress (retention incomplete)\n");
printf(" -8, --8-bit-output Leave high-bit characters unescaped in output\n");
printf(" --delete Delete files on receiver not in source\n");
printf(" --ignore-existing Skip files that already exist on receiver\n");
printf(
" --dirs, --old-dirs, --old-d Transfer directories without recursing (not implemented)\n");
printf(" --del Alias for --delete-during (not implemented)\n");
printf(" --exclude <pattern> Exclude files matching pattern\n");
printf(" --include <pattern> Only include files matching pattern\n");
printf(" --exclude-from <file> Read exclude patterns from file\n");
printf(" --include-from <file> Read include patterns from file\n");
printf(" --max-size <n> Skip files larger than n bytes\n");
printf(" --min-size <n> Skip files smaller than n bytes\n");
printf(" --max-alloc <SIZE> Maximum single allocation (default: 1G)\n");
printf(" --incremental Skip files unchanged since last transfer\n");
printf(" --size-only Skip incremental files matching in size, ignoring mtime\n");
printf(" -I, --ignore-times Transfer files even when size and mtime match\n");
printf(" -@, --modify-window <sec> Modification time tolerance\n");
printf(" -u, --update Skip files newer than the source on receiver\n");
printf(" --existing Skip files not already present at destination\n");
printf(" --checksum-choice, --cc <alg> Checksum algorithm (not supported yet; xxHash64 is "
"used)\n");
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
printf(" -W, --whole-file Transfer changed files without delta processing\n");
printf(" --delta-block <n> Delta block size in bytes (default: %d)\n",
DELTA_BLOCK_SIZE_DEFAULT);
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
DELTA_MAX_FILE_SIZE);
printf(" -m Enable multithreading\n");
printf(" -s Enable chunk serialization\n");
printf(" --secluded-args Accept rsync compatibility option (no effect)\n");
printf(" -f Enable sendfile (TCP only, not with -c or -s)\n");
printf(" --compress-choice <alg> Compression algorithm (default: zstd)\n");
printf(" --zc <alg> Alias for --compress-choice\n");
printf(" -v, --verbose Enable debug logging\n");
printf(" -q, --quiet Suppress non-error output\n");
printf(" --debug=FLAGS Fine-grained debug logging (use --debug=help for flags)\n");
printf(" --info=FLAGS Fine-grained info: copy,misc,skip,stats,all,none\n");
printf(" none suppresses info even with --verbose\n");
printf(" -M, --preserve Preserve file metadata\n");
printf(" -E, --executability Preserve executable permission bits\n");
printf(" --chmod <changes> Modify transferred permissions (rsync syntax)\n");
printf(" --chunk-size <n> Chunk size in bytes (default: %d)\n", DEFAULT_CHUNK_SIZE);
printf(" --source-dir <path> Source directory\n");
printf(" --dest-dir <path> Destination directory\n");
@@ -57,19 +81,34 @@ void print_usage(void) {
printf(" --backup-dir <dir> Directory for backups (requires --backup)\n");
printf(" --suffix <str> Backup suffix (default: ~)\n");
printf(" --stats Print transfer statistics at end\n");
printf(" -h, --human-readable Print byte sizes in human-readable form\n");
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
printf(" --log-file <path> Write log messages to file\n");
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
printf(" --partial Keep partial files on interrupted transfer\n");
printf(" --partial-dir <dir> Directory for partial files\n");
printf(" --fastsync-server-path <path>\n");
printf(" Path to fastsync-server on remote (default: fastsync-server)\n");
printf(
" --old-args Disable safe SSH command argument quoting (legacy compatibility)\n");
printf(" -l, --links Copy symlinks as symlinks\n");
printf(" --copy-links Transform symlinks into referent files\n");
printf(" --safe-links Skip symlinks that point outside transfer tree\n");
printf(" --copy-unsafe-links Only transform unsafe symlinks into referent files\n");
printf(" -S, --sparse Handle sparse files efficiently\n");
printf(" --inplace Update files in-place (no temp+rename)\n");
printf(" --fsync Fsync every written file before publication\n");
printf(" --compress-level <n> Compression level (default: 5)\n");
printf(" --zl <n> Alias for --compress-level\n");
printf(" --skip-compress=LIST Skip compression for comma-separated suffixes\n");
printf(" --compress-threads <n> Compression worker threads (requires zstd threaded support)\n");
printf(" --no-OPTION Disable a supported boolean option\n");
printf(" --help Show this help\n");
printf(" -V, --version Show version\n");
}
void print_debug_usage(void) {
printf("Supported debug flags: IO,PROTO,PACK,UTIL,ALL,NONE\n");
printf("Flags may be comma-separated, for example: --debug=io,proto\n");
printf("Other rsync debug flags are unsupported and rejected.\n");
}
+1
View File
@@ -2,5 +2,6 @@
#define USAGE_H
void print_usage(void);
void print_debug_usage(void);
#endif
+15 -3
View File
@@ -2,6 +2,7 @@
#include "chunk.h"
#include "log.h"
#include "metadata.h"
#include "protocol.h"
#include "utils.h"
#include <stdlib.h>
@@ -37,9 +38,13 @@ static bool receiver_process_batch(Config* config, int file_descriptor) {
return false;
unsigned long long check_size;
long long check_mtime;
long long check_mtime_nsec;
if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) ||
!receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) {
!receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime)) ||
!receive_n_data(file_descriptor, &check_mtime_nsec, sizeof(check_mtime_nsec)) ||
check_mtime_nsec < 0 || check_mtime_nsec >= 1000000000LL) {
free(check_path);
send_status(file_descriptor, STATUS_ERROR);
return false;
}
if (!utils_valid_batch_path(check_path)) {
@@ -60,8 +65,15 @@ static bool receiver_process_batch(Config* config, int file_descriptor) {
}
struct stat st;
bool has_old = file_stat_secure(full_path, &st);
bool match = has_old && (unsigned long long)st.st_size == check_size &&
(long long)st.st_mtime == check_mtime;
long long old_mtime_nsec = 0;
if (has_old) {
#ifdef __linux__
old_mtime_nsec = st.st_mtim.tv_nsec;
#endif
}
bool match = !config->ignore_times && has_old && (unsigned long long)st.st_size == check_size &&
metadata_mtime_matches(st.st_mtime, old_mtime_nsec, (time_t)check_mtime,
(long)check_mtime_nsec, config->modify_window);
bool sent = send_status(file_descriptor, match ? STATUS_OK : STATUS_NEXT);
free(full_path);
free(check_path);
+34 -6
View File
@@ -2,6 +2,7 @@
#include "chunk.h"
#include "file.h"
#include "log.h"
#include "metadata.h"
#include "multiprocessing.h"
#include "queue.h"
#include "receiver.h"
@@ -161,6 +162,13 @@ int receive_files(Config* config, int fd) {
free(check_path);
return -1;
}
long long check_mtime_nsec;
if (!receive_n_data(fd, &check_mtime_nsec, sizeof(check_mtime_nsec)) ||
check_mtime_nsec < 0 || check_mtime_nsec >= 1000000000LL) {
free(check_path);
send_status(fd, STATUS_ERROR);
return -1;
}
if (!utils_valid_batch_path(check_path)) {
free(check_path);
send_status(fd, STATUS_ERROR);
@@ -174,8 +182,16 @@ int receive_files(Config* config, int fd) {
return -1;
}
bool has_old = full_path && file_stat_secure(full_path, &st);
bool match = has_old && (unsigned long long)st.st_size == check_size &&
(long long)st.st_mtime == check_mtime;
long long old_mtime_nsec = 0;
if (has_old) {
#ifdef __linux__
old_mtime_nsec = st.st_mtim.tv_nsec;
#endif
}
bool match = !config->ignore_times && has_old &&
(unsigned long long)st.st_size == check_size &&
metadata_mtime_matches(st.st_mtime, old_mtime_nsec, (time_t)check_mtime,
(long)check_mtime_nsec, config->modify_window);
bool sent = send_status(fd, match ? STATUS_OK : STATUS_NEXT);
free(full_path);
free(check_path);
@@ -232,6 +248,7 @@ void handler(int file_descriptor) {
protocol_session_unbind();
return;
}
protocol_set_8_bit_output(config->eight_bit_output);
if (!authorized_root) {
log_message(LOG_LEVEL_ERROR, "No server-side destination root configured");
config_delete(config);
@@ -294,7 +311,9 @@ void handler(int file_descriptor) {
protocol_session_unbind();
return;
}
context->session.total_allocated_bytes = session.total_allocated_bytes;
protocol_session_set_max_alloc(&context->session, config->max_alloc);
atomic_store(&context->session.total_allocated_bytes,
atomic_load(&session.total_allocated_bytes));
thrd_t receiver, writer;
bool receiver_created = thrd_create(&receiver, receive_thread, context) == thrd_success;
bool writer_created = false;
@@ -380,6 +399,7 @@ int main(int argc, char* argv[]) {
stdio_mode = true;
} else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
set_log_level(LOG_LEVEL_DEBUG);
set_log_debug_flags(LOG_DEBUG_ALL);
} else if (strcmp(argv[i], "--tls") == 0) {
use_tls = true;
} else if (strcmp(argv[i], "--cert") == 0 && i + 1 < argc) {
@@ -400,12 +420,17 @@ int main(int argc, char* argv[]) {
char* end;
long p = strtol(argv[++i], &end, 10);
if (*end || p <= 0 || p > 65535) {
fprintf(stderr, "Error: invalid port '%s' (must be 1-65535)\n", argv[i]);
char* escaped = output_escape(argv[i], false);
fprintf(stderr, "Error: invalid port '%s' (must be 1-65535)\n",
escaped ? escaped : "<allocation failed>");
free(escaped);
return 1;
}
port = (int)p;
} else if (argv[i][0] == '-') {
fprintf(stderr, "Unknown option: %s\n", argv[i]);
char* escaped = output_escape(argv[i], false);
fprintf(stderr, "Unknown option: %s\n", escaped ? escaped : "<allocation failed>");
free(escaped);
print_server_usage();
return 1;
}
@@ -415,7 +440,10 @@ int main(int argc, char* argv[]) {
signal(SIGINT, cleanup);
signal(SIGTERM, cleanup);
if (!configure_authorization(destination_root)) {
fprintf(stderr, "Error: invalid destination root '%s'\n", destination_root);
char* escaped = output_escape(destination_root, false);
fprintf(stderr, "Error: invalid destination root '%s'\n",
escaped ? escaped : "<allocation failed>");
free(escaped);
return 1;
}
if (stdio_mode) {
+5 -4
View File
@@ -1,17 +1,18 @@
#include "log.h"
#include "array_list.h"
#include "protocol.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
ArrayList* array_list_create(void (*item_destroyer)(void* item)) {
ArrayList* list = (ArrayList*)malloc(sizeof(ArrayList));
ArrayList* list = (ArrayList*)protocol_alloc(sizeof(ArrayList));
if (list == NULL) {
log_perror("ERROR: Could not allocate memory for array list struct");
return NULL;
}
list->items = malloc(INITIAL_ARRAY_SIZE * sizeof(void*));
list->items = protocol_alloc(INITIAL_ARRAY_SIZE * sizeof(void*));
if (list->items == NULL) {
free(list);
return NULL;
@@ -41,7 +42,7 @@ static bool array_list_extend(ArrayList* array_list) {
int new_capacity = array_list->capacity * 2;
if (new_capacity == 0)
new_capacity = INITIAL_ARRAY_SIZE;
void* new_items = realloc(array_list->items, new_capacity * sizeof(void*));
void* new_items = protocol_realloc(array_list->items, new_capacity * sizeof(void*));
if (new_items == NULL) {
log_perror("ERROR: Could not reallocate memory for array list items");
return false;
@@ -67,7 +68,7 @@ void** array_list_to_array(const ArrayList* array_list) {
if (array_list == NULL) {
return NULL;
}
void** array = malloc(array_list->size * sizeof(void*));
void** array = protocol_alloc(array_list->size * sizeof(void*));
if (array == NULL) {
log_perror("Could not malloc space for array from array list!");
return NULL;
+90
View File
@@ -0,0 +1,90 @@
#include "chmod.h"
#include <stddef.h>
#include <string.h>
static bool parse_clause(mode_t* mode, const char* begin, const char* end) {
const char* p = begin;
unsigned who = 0;
while (p < end && strchr("ugoa", *p)) {
if (*p == 'a')
who = 7;
else
who |= *p == 'u' ? 1U : (*p == 'g' ? 2U : 4U);
p++;
}
if (who == 0)
who = 7;
if (p == end || (*p != '+' && *p != '-' && *p != '='))
return false;
char operation = *p++;
mode_t bits = 0;
while (p < end) {
mode_t bit;
switch (*p++) {
case 'r':
bit = 4;
break;
case 'w':
bit = 2;
break;
case 'x':
bit = 1;
break;
default:
return false;
}
bits |= bit;
}
for (unsigned class_index = 0; class_index < 3; class_index++) {
unsigned class_bit = 1U << class_index;
if (!(who & class_bit))
continue;
mode_t shift = (mode_t)((2U - class_index) * 3U);
mode_t mask = (mode_t)(7U << shift);
mode_t class_bits = (mode_t)(bits << shift);
if (operation == '+')
*mode |= class_bits;
else if (operation == '-')
*mode &= ~class_bits;
else
*mode = (*mode & ~mask) | class_bits;
}
return true;
}
bool chmod_apply(mode_t mode, const char* spec, mode_t* result) {
if (!spec || !*spec || !result)
return false;
bool numeric = true;
size_t length = strlen(spec);
if (length > 4)
numeric = false;
for (size_t i = 0; i < length && numeric; i++)
numeric = spec[i] >= '0' && spec[i] <= '7';
if (numeric) {
if (length == 0 || length > 4)
return false;
mode_t parsed = 0;
for (size_t i = 0; i < length; i++)
parsed = (mode_t)((parsed << 3) | (spec[i] - '0'));
*result = parsed;
return true;
}
mode_t changed = mode;
const char* begin = spec;
while (*begin) {
const char* end = strchr(begin, ',');
if (!end)
end = begin + strlen(begin);
if (!parse_clause(&changed, begin, end))
return false;
if (*end == '\0')
break;
begin = end + 1;
if (!*begin)
return false;
}
*result = changed;
return true;
}
+10
View File
@@ -0,0 +1,10 @@
#ifndef CHMOD_H
#define CHMOD_H
#include <stdbool.h>
#include <sys/stat.h>
/* Apply the supported rsync --chmod syntax to a permission mode. */
bool chmod_apply(mode_t mode, const char* spec, mode_t* result);
#endif
+11 -6
View File
@@ -22,7 +22,7 @@
Chunk* chunk_create(File** items, int element_count) {
if (element_count < 0 || (element_count > 0 && items == NULL))
return NULL;
Chunk* chunk = (Chunk*)malloc(sizeof(Chunk));
Chunk* chunk = (Chunk*)protocol_alloc(sizeof(Chunk));
if (chunk == NULL) {
log_perror("ERROR: Could not allocate memory for chunk structure");
return NULL;
@@ -35,7 +35,7 @@ Chunk* chunk_create(File** items, int element_count) {
free(chunk);
return NULL;
}
chunk->items = (File**)malloc((size_t)element_count * sizeof(File*));
chunk->items = (File**)protocol_alloc((size_t)element_count * sizeof(File*));
if (chunk->items == NULL) {
free(chunk);
return NULL;
@@ -158,7 +158,7 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
array_list_delete(files);
return NULL;
}
char* path = malloc(path_len + 1);
char* path = protocol_alloc(path_len + 1);
if (path == NULL) {
log_perror("Could not allocate memory for file path");
array_list_delete(files);
@@ -245,7 +245,7 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
}
size_t allocation_size = file_data_size > 0 ? file_data_size : 1;
void* file_data = malloc(allocation_size);
void* file_data = protocol_alloc(allocation_size);
if (file_data == NULL) {
log_perror("Could not allocate memory for file data");
file_destroy(file);
@@ -290,15 +290,20 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
}
Data* chunk_compress(Chunk* chunk, int compression_level, bool use_metadata) {
return chunk_compress_with_threads(chunk, compression_level, use_metadata, 0);
}
Data* chunk_compress_with_threads(Chunk* chunk, int compression_level, bool use_metadata,
int compression_threads) {
log_message(LOG_LEVEL_DEBUG, "Starting to compress chunk");
Data* serialized = chunk_serialize(chunk, use_metadata);
if (serialized == NULL)
return NULL;
Data* compressed = data_compress(serialized, compression_level);
Data* compressed = data_compress_with_threads(serialized, compression_level, compression_threads);
data_destroy(serialized);
if (compressed == NULL)
return NULL;
log_message(LOG_LEVEL_DEBUG, "Chunk successfully compressed");
log_debug_message(LOG_DEBUG_PACK, "Chunk successfully compressed");
return compressed;
}
+2
View File
@@ -19,6 +19,8 @@ void chunk_destroy(void* chunk);
Data* chunk_serialize(Chunk* chunk, bool use_metadata);
Chunk* chunk_deserialize(Data* data, bool use_metadata);
Data* chunk_compress(Chunk* chunk, int compression_level, bool use_metadata);
Data* chunk_compress_with_threads(Chunk* chunk, int compression_level, bool use_metadata,
int compression_threads);
Chunk* receive_chunk_data(int fd, const Config* config);
#endif
+51 -7
View File
@@ -1,33 +1,53 @@
#include "compression.h"
#include "data.h"
#include "log.h"
#include "protocol.h"
#include <stdlib.h>
#include <limits.h>
#include <stdint.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
#include <zstd.h>
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
#define MAX_DECOMPRESSED_SIZE (100ULL * 1024 * 1024) /* 100 MB hard ceiling */
static const char* SKIP_COMPRESSION_EXTENSIONS[] = {".jpg", ".jpeg", ".png", ".gif", ".mp4", ".mkv",
static char* SKIP_COMPRESSION_EXTENSIONS[] = {".jpg", ".jpeg", ".png", ".gif", ".mp4", ".mkv",
".zip", ".gz", ".xz", ".zst", NULL};
bool compression_should_skip(const char* path) {
return compression_should_skip_with_suffixes(path, NULL, -1);
}
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count) {
if (!path)
return false;
const char* dot = strrchr(path, '.');
if (!dot)
return false;
for (int i = 0; SKIP_COMPRESSION_EXTENSIONS[i]; i++) {
if (strcasecmp(dot, SKIP_COMPRESSION_EXTENSIONS[i]) == 0)
if (count < 0) {
suffixes = SKIP_COMPRESSION_EXTENSIONS;
count = 0;
while (SKIP_COMPRESSION_EXTENSIONS[count])
count++;
}
for (int i = 0; i < count; i++) {
if (strcasecmp(dot, suffixes[i]) == 0)
return true;
}
return false;
}
Data* data_compress(Data* data_to_compress, int compression_level) {
return data_compress_with_threads(data_to_compress, compression_level, 0);
}
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
int compression_threads) {
if (!data_to_compress || (!data_to_compress->data && data_to_compress->size != 0) ||
compression_threads < 0 || compression_threads > COMPRESSION_MAX_THREADS)
return NULL;
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
size_t dst_size = ZSTD_compressBound(data_to_compress->size);
Data* compressed_data = data_create_empty(dst_size);
@@ -49,6 +69,30 @@ Data* data_compress(Data* data_to_compress, int compression_level) {
return NULL;
}
if (compression_threads > 0) {
long online_cpus = sysconf(_SC_NPROCESSORS_ONLN);
int available_threads = online_cpus > 0 && online_cpus < compression_threads
? (int)online_cpus
: compression_threads;
zret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, available_threads);
if (ZSTD_isError(zret)) {
log_message(LOG_LEVEL_ERROR, "Failed to set compression threads: %s",
ZSTD_getErrorName(zret));
ZSTD_freeCCtx(cctx);
data_destroy(compressed_data);
return NULL;
}
/* Streaming compression needs the source size before threaded mode can end a frame. */
zret = ZSTD_CCtx_setPledgedSrcSize(cctx, data_to_compress->size);
if (ZSTD_isError(zret)) {
log_message(LOG_LEVEL_ERROR, "Failed to set compression source size: %s",
ZSTD_getErrorName(zret));
ZSTD_freeCCtx(cctx);
data_destroy(compressed_data);
return NULL;
}
}
ZSTD_inBuffer input = {data_to_compress->data, data_to_compress->size, 0};
ZSTD_outBuffer output = {compressed_data->data, dst_size, 0};
@@ -66,7 +110,7 @@ Data* data_compress(Data* data_to_compress, int compression_level) {
compressed_data->size = output.pos;
ZSTD_freeCCtx(cctx);
log_message(LOG_LEVEL_DEBUG, "Data succesfully compressed from %zu to %zu",
log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu",
data_to_compress->size, compressed_data->size);
return compressed_data;
}
@@ -75,7 +119,7 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
if (!compressed_data || (!compressed_data->data && compressed_data->size != 0) ||
maximum_size == 0)
return NULL;
log_message(LOG_LEVEL_DEBUG, "Start to decompress data");
log_debug_message(LOG_DEBUG_UTIL, "Start to decompress data");
unsigned long long dst_size =
ZSTD_getFrameContentSize(compressed_data->data, compressed_data->size);
if (ZSTD_isError(dst_size)) {
@@ -139,7 +183,7 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
buf_size *= 2;
if (buf_size > hard_limit)
buf_size = (size_t)hard_limit;
void* new_data = realloc(uncompressed_data->data, buf_size);
void* new_data = protocol_realloc(uncompressed_data->data, buf_size);
if (!new_data) {
log_message(LOG_LEVEL_ERROR, "Failed to grow decompression buffer");
ZSTD_freeDCtx(dctx);
@@ -155,7 +199,7 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
uncompressed_data->size = output.pos;
ZSTD_freeDCtx(dctx);
log_message(LOG_LEVEL_DEBUG, "Decompressed data successfully");
log_debug_message(LOG_DEBUG_UTIL, "Decompressed data successfully");
return uncompressed_data;
}
+5
View File
@@ -4,9 +4,14 @@
#include "data.h"
#include <stdbool.h>
#define COMPRESSION_MAX_THREADS 64
Data* data_compress(Data* data_to_compress, int compression_level);
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
int compression_threads);
Data* data_decompress(Data* compressed_data);
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
bool compression_should_skip(const char* path);
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
#endif
+115 -27
View File
@@ -1,4 +1,5 @@
#include "config.h"
#include "chmod.h"
#include "delta.h"
#include "log.h"
#include "protocol.h"
@@ -17,10 +18,14 @@ static void config_set_defaults(Config* config) {
config->use_chunk_serialization = false;
config->use_compression = false;
config->use_metadata = false;
config->use_executability = false;
config->metadata_explicitly_disabled = false;
config->show_progress = false;
config->dry_run = false;
config->remove_source_files = false;
config->use_delete = false;
config->compression_level = 5;
config->compression_threads = 0;
config->use_sendfile = false;
config->chunk_size = DEFAULT_CHUNK_SIZE;
config->ssh_port = 22;
@@ -33,8 +38,13 @@ static void config_set_defaults(Config* config) {
config->include_count = 0;
config->max_size = 0;
config->min_size = 0;
config->max_alloc = DEFAULT_MAX_ALLOC;
config->use_incremental = false;
config->ignore_times = false;
config->size_only = false;
config->use_delta = false;
config->whole_file = false;
config->modify_window = 0;
config->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT;
config->delta_max_file_size = DELTA_MAX_FILE_SIZE;
config->use_tls = false;
@@ -68,9 +78,12 @@ static void config_set_defaults(Config* config) {
config->debug_level = 0;
config->list_only = false;
config->human_readable = false;
config->eight_bit_output = false;
config->existing = false;
config->ignore_existing = false;
config->update = false;
config->inplace = false;
config->use_fsync = false;
config->append = false;
config->append_verify = false;
config->delete_excluded = false;
@@ -83,6 +96,7 @@ static void config_set_defaults(Config* config) {
config->relative = false;
config->rsh_command = NULL;
config->rsync_path = NULL;
config->old_args = false;
config->temp_dir = NULL;
config->compare_dest = NULL;
config->copy_dest = NULL;
@@ -99,6 +113,10 @@ static void config_set_defaults(Config* config) {
config->server_mode = false;
config->checksum = false;
config->compress_choice = NULL;
config->chmod_spec = NULL;
config->skip_compress_suffixes = NULL;
config->skip_compress_count = 0;
config->skip_compress_set = false;
}
static bool valid_wire_bool(int value) {
@@ -117,26 +135,33 @@ static bool validate_received_config(const Config* config) {
return valid_wire_bool(config->save_to_disk) && valid_wire_bool(config->use_multithreading) &&
valid_wire_bool(config->use_chunk_serialization) &&
valid_wire_bool(config->use_compression) && valid_wire_bool(config->use_metadata) &&
valid_wire_bool(config->use_sendfile) && valid_wire_bool(config->use_delete) &&
valid_wire_bool(config->use_incremental) && valid_wire_bool(config->use_delta) &&
valid_wire_bool(config->backup) && valid_wire_bool(config->follow_symlinks) &&
valid_wire_bool(config->copy_links) && valid_wire_bool(config->safe_links) &&
valid_wire_bool(config->copy_unsafe_links) &&
valid_wire_bool(config->use_executability) && valid_wire_bool(config->use_sendfile) &&
valid_wire_bool(config->use_delete) && valid_wire_bool(config->use_incremental) &&
valid_wire_bool(config->size_only) && valid_wire_bool(config->ignore_times) &&
valid_wire_bool(config->use_delta) && valid_wire_bool(config->backup) &&
valid_wire_bool(config->follow_symlinks) && valid_wire_bool(config->copy_links) &&
valid_wire_bool(config->safe_links) && valid_wire_bool(config->copy_unsafe_links) &&
valid_wire_bool(config->preserve_hard_links) && valid_wire_bool(config->preserve_acls) &&
valid_wire_bool(config->preserve_xattrs) && valid_wire_bool(config->preserve_devices) &&
valid_wire_bool(config->preserve_sparse) && valid_wire_bool(config->existing) &&
valid_wire_bool(config->update) && valid_wire_bool(config->inplace) &&
valid_wire_bool(config->append) && valid_wire_bool(config->append_verify) &&
valid_wire_bool(config->preserve_sparse) && valid_wire_bool(config->ignore_existing) &&
valid_wire_bool(config->existing) && valid_wire_bool(config->update) &&
valid_wire_bool(config->inplace) && valid_wire_bool(config->append) &&
valid_wire_bool(config->use_fsync) && valid_wire_bool(config->append_verify) &&
valid_wire_bool(config->delete_excluded) && valid_wire_bool(config->delete_after) &&
valid_wire_bool(config->relative) && valid_wire_bool(config->prune_empty_dirs) &&
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
valid_wire_bool(config->checksum) &&
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
!(config->skip_compress_set && config->use_chunk_serialization) &&
(!config->use_compression ||
(config->compression_level >= 1 && config->compression_level <= 22)) &&
config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE &&
config->delta_block_size >= DELTA_BLOCK_SIZE_MIN &&
config->delta_block_size <= DELTA_BLOCK_SIZE_MAX &&
config->delta_max_file_size <= DELTA_MAX_FILE_SIZE && config->max_delete >= 0;
config->delta_max_file_size <= DELTA_MAX_FILE_SIZE && config->modify_window >= 0 &&
config->max_delete >= 0 && config->skip_compress_count >= 0 &&
config->skip_compress_count <= 10000 && config->max_alloc > 0 &&
(!config->chmod_spec || !*config->chmod_spec ||
chmod_apply(0, config->chmod_spec, &(mode_t){0}));
}
Config* config_create(void) {
@@ -210,6 +235,12 @@ void config_delete(Config* config) {
free(config->bind_address);
free(config->daemon_config);
free(config->compress_choice);
free(config->chmod_spec);
if (config->skip_compress_suffixes) {
for (int i = 0; i < config->skip_compress_count; i++)
free(config->skip_compress_suffixes[i]);
free(config->skip_compress_suffixes);
}
if (config->filters) {
array_list_delete(config->filters);
}
@@ -220,17 +251,23 @@ void config_delete(Config* config) {
* helper call order in config_send and config_receive unchanged when adding
* fields. */
static bool send_core_fields(int fd, const Config* c) {
return send_str(fd, c->version) && send_str(fd, c->send_directory) &&
send_str(fd, c->receive_root_directory) && send_int(fd, c->save_to_disk) &&
send_int(fd, c->use_multithreading) && send_int(fd, c->use_chunk_serialization) &&
send_int(fd, c->use_compression) && send_int(fd, c->use_metadata) &&
if (!send_str(fd, c->version) || !send_int(fd, c->eight_bit_output))
return false;
protocol_set_8_bit_output(c->eight_bit_output);
if (!send_n_data(fd, &c->max_alloc, sizeof(c->max_alloc)))
return false;
return send_str(fd, c->send_directory) && send_str(fd, c->receive_root_directory) &&
send_int(fd, c->save_to_disk) && send_int(fd, c->use_multithreading) &&
send_int(fd, c->use_chunk_serialization) && send_int(fd, c->use_compression) &&
send_int(fd, c->use_metadata) && send_int(fd, c->use_executability) &&
send_int(fd, c->compression_level) &&
send_n_data(fd, &c->chunk_size, sizeof(c->chunk_size)) && send_int(fd, c->use_sendfile);
}
static bool send_delta_fields(int fd, const Config* c) {
return send_int(fd, c->use_delete) && send_int(fd, c->use_incremental) &&
send_int(fd, c->use_delta) &&
send_int(fd, c->size_only) && send_int(fd, c->ignore_times) &&
send_int(fd, c->use_delta && !c->whole_file) &&
send_n_data(fd, &c->delta_block_size, sizeof(c->delta_block_size)) &&
send_n_data(fd, &c->delta_max_file_size, sizeof(unsigned long long));
}
@@ -245,29 +282,50 @@ static bool send_file_options(int fd, const Config* c) {
}
static bool send_selection_options(int fd, const Config* c) {
return send_int(fd, c->existing) && send_int(fd, c->update) && send_int(fd, c->inplace) &&
send_int(fd, c->append) && send_int(fd, c->append_verify) &&
send_int(fd, c->delete_excluded) && send_int(fd, c->delete_after) &&
send_n_data(fd, &c->max_delete, sizeof(c->max_delete)) && send_int(fd, c->relative) &&
send_int(fd, c->prune_empty_dirs);
return send_int(fd, c->ignore_existing) && send_int(fd, c->existing) && send_int(fd, c->update) &&
send_int(fd, c->inplace) && send_int(fd, c->append) && send_int(fd, c->use_fsync) &&
send_int(fd, c->append_verify) && send_int(fd, c->delete_excluded) &&
send_int(fd, c->delete_after) && send_n_data(fd, &c->max_delete, sizeof(c->max_delete)) &&
send_int(fd, c->relative) && send_int(fd, c->prune_empty_dirs);
}
static bool send_skip_compress_options(int fd, const Config* c) {
if (!send_int(fd, c->skip_compress_set) || !send_int(fd, c->skip_compress_count))
return false;
for (int i = 0; i < c->skip_compress_count; i++) {
if (!send_str(fd, c->skip_compress_suffixes[i]))
return false;
}
return true;
}
static bool send_resume_options(int fd, const Config* c) {
return send_str(fd, c->temp_dir ? c->temp_dir : "") && send_int(fd, c->partial) &&
send_str(fd, c->partial_dir ? c->partial_dir : "") &&
send_str(fd, c->suffix ? c->suffix : "") && send_int(fd, c->delete_before) &&
send_int(fd, c->checksum) && send_str(fd, c->compress_choice ? c->compress_choice : "");
send_int(fd, c->checksum) && send_int(fd, c->modify_window) &&
send_str(fd, c->compress_choice ? c->compress_choice : "") &&
send_str(fd, c->chmod_spec ? c->chmod_spec : "") && send_skip_compress_options(fd, c);
}
static bool receive_core_fields(int fd, Config* c) {
int value;
if (!receive_wire_bool(fd, &c->eight_bit_output))
return false;
protocol_set_8_bit_output(c->eight_bit_output);
if (!receive_n_data(fd, &c->max_alloc, sizeof(c->max_alloc)) || c->max_alloc == 0)
return false;
if (c->max_alloc > MAX_SERVER_ALLOC)
c->max_alloc = MAX_SERVER_ALLOC;
protocol_session_set_max_alloc(NULL, c->max_alloc);
c->send_directory = receive_str(fd);
c->receive_root_directory = receive_str(fd);
if (!c->send_directory || !c->receive_root_directory)
return false;
if (!receive_wire_bool(fd, &c->save_to_disk) || !receive_wire_bool(fd, &c->use_multithreading) ||
!receive_wire_bool(fd, &c->use_chunk_serialization) ||
!receive_wire_bool(fd, &c->use_compression) || !receive_wire_bool(fd, &c->use_metadata))
!receive_wire_bool(fd, &c->use_compression) || !receive_wire_bool(fd, &c->use_metadata) ||
!receive_wire_bool(fd, &c->use_executability))
return false;
if (!receive_int(fd, &value))
return false;
@@ -284,6 +342,10 @@ static bool receive_delta_fields(int fd, Config* c) {
return false;
if (!receive_wire_bool(fd, &c->use_incremental))
return false;
if (!receive_wire_bool(fd, &c->size_only))
return false;
if (!receive_wire_bool(fd, &c->ignore_times))
return false;
if (!receive_wire_bool(fd, &c->use_delta))
return false;
return receive_n_data(fd, &c->delta_block_size, sizeof(c->delta_block_size)) &&
@@ -307,7 +369,8 @@ static bool receive_file_options(int fd, Config* c) {
}
static bool receive_selection_options(int fd, Config* c) {
bool* flags[] = {&c->existing, &c->update, &c->inplace, &c->append,
bool* flags[] = {&c->ignore_existing, &c->existing, &c->update,
&c->inplace, &c->append, &c->use_fsync,
&c->append_verify, &c->delete_excluded, &c->delete_after};
for (size_t i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
if (!receive_wire_bool(fd, flags[i]))
@@ -332,11 +395,31 @@ static bool receive_resume_options(int fd, Config* c) {
return false;
if (!receive_wire_bool(fd, &c->checksum))
return false;
if (!receive_n_data(fd, &c->modify_window, sizeof(c->modify_window)))
return false;
c->compress_choice = receive_str(fd);
return c->compress_choice != NULL;
if (!c->compress_choice)
return false;
c->chmod_spec = receive_str(fd);
if (!c->chmod_spec || !receive_wire_bool(fd, &c->skip_compress_set) ||
!receive_int(fd, &c->skip_compress_count) || c->skip_compress_count < 0 ||
c->skip_compress_count > 10000)
return false;
if (c->skip_compress_count > 0) {
c->skip_compress_suffixes = calloc((size_t)c->skip_compress_count, sizeof(char*));
if (!c->skip_compress_suffixes)
return false;
for (int i = 0; i < c->skip_compress_count; i++) {
c->skip_compress_suffixes[i] = receive_str(fd);
if (!c->skip_compress_suffixes[i])
return false;
}
}
return true;
}
bool config_send(int file_descriptor, const Config* config) {
protocol_session_set_max_alloc(NULL, config->max_alloc);
if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) ||
!send_file_options(file_descriptor, config) ||
!send_selection_options(file_descriptor, config) ||
@@ -361,8 +444,10 @@ Config* config_receive(int file_descriptor) {
if (!config->version)
goto error;
if (strcmp(config->version, PROTOCOL_VERSION) != 0) {
fprintf(stderr, "Protocol version mismatch: client=%s, server=%s\n", config->version,
PROTOCOL_VERSION);
char* escaped_version = output_escape(config->version, false);
fprintf(stderr, "Protocol version mismatch: client=%s, server=%s\n",
escaped_version ? escaped_version : "<allocation failed>", PROTOCOL_VERSION);
free(escaped_version);
send_status(file_descriptor, STATUS_ERROR);
goto error;
}
@@ -374,7 +459,10 @@ Config* config_receive(int file_descriptor) {
goto error;
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
strcmp(config->compress_choice, "none") != 0) {
fprintf(stderr, "Unsupported compression choice: %s\n", config->compress_choice);
char* escaped_choice = output_escape(config->compress_choice, config->eight_bit_output);
fprintf(stderr, "Unsupported compression choice: %s\n",
escaped_choice ? escaped_choice : "<allocation failed>");
free(escaped_choice);
send_status(file_descriptor, STATUS_ERROR);
goto error;
}
+19 -1
View File
@@ -18,10 +18,14 @@ typedef struct Config {
bool use_compression;
bool use_sendfile;
bool use_metadata;
bool use_executability;
bool metadata_explicitly_disabled;
bool show_progress;
bool dry_run;
bool remove_source_files;
bool use_delete;
int compression_level;
int compression_threads;
unsigned long long chunk_size;
int ssh_port;
TransportType transport;
@@ -33,8 +37,13 @@ typedef struct Config {
int include_count;
unsigned long long max_size;
unsigned long long min_size;
unsigned long long max_alloc;
bool use_incremental;
bool ignore_times;
bool size_only;
bool use_delta;
bool whole_file;
int modify_window;
uint32_t delta_block_size;
unsigned long long delta_max_file_size;
bool use_tls;
@@ -74,11 +83,14 @@ typedef struct Config {
int debug_level;
bool list_only;
bool human_readable;
bool eight_bit_output;
// Issue #127: Transfer modes
bool existing;
bool ignore_existing;
bool update;
bool inplace;
bool use_fsync;
bool append;
bool append_verify;
@@ -97,6 +109,7 @@ typedef struct Config {
// Issue #130: Remote shell/connection options
char* rsh_command;
char* rsync_path;
bool old_args;
char* temp_dir;
char* compare_dest;
char* copy_dest;
@@ -127,9 +140,14 @@ typedef struct Config {
// PR #184: Compression algorithm negotiation
char* compress_choice;
char* chmod_spec;
char** skip_compress_suffixes;
int skip_compress_count;
bool skip_compress_set;
} Config;
#define PROTOCOL_VERSION "2.3.0"
#define PROTOCOL_VERSION "2.4.0"
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
Config* config_create(void);
+4 -3
View File
@@ -1,11 +1,12 @@
#include "data.h"
#include "log.h"
#include "protocol.h"
#include <stdlib.h>
Data* data_create_empty(size_t data_size) {
/* malloc(0) is UB; allocate at least 1 byte but preserve requested size */
size_t alloc_size = data_size > 0 ? data_size : 1;
void* data = malloc(alloc_size);
void* data = protocol_alloc(alloc_size);
if (data == NULL) {
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for empty data");
return NULL;
@@ -14,7 +15,7 @@ Data* data_create_empty(size_t data_size) {
}
Data* data_create_reserve(size_t size) {
Data* d = malloc(sizeof(Data));
Data* d = protocol_alloc(sizeof(Data));
if (d == NULL) {
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for data");
return NULL;
@@ -26,7 +27,7 @@ Data* data_create_reserve(size_t size) {
}
Data* data_create(void* data, size_t data_size) {
Data* new_data = malloc(sizeof(Data));
Data* new_data = protocol_alloc(sizeof(Data));
if (new_data == NULL) {
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for data");
free(data);
+17 -15
View File
@@ -1,5 +1,6 @@
#include "delta.h"
#include "log.h"
#include "protocol.h"
#include <stdint.h>
#include <limits.h>
#include <stdlib.h>
@@ -43,7 +44,7 @@ DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_f
uint32_t block_count = (uint32_t)((old_file_size + block_size - 1) / block_size);
DeltaSignature* sig = malloc(sizeof(DeltaSignature));
DeltaSignature* sig = protocol_alloc(sizeof(DeltaSignature));
if (!sig)
return NULL;
@@ -54,7 +55,7 @@ DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_f
free(sig);
return NULL;
}
sig->blocks = malloc((size_t)block_count * sizeof(DeltaBlockSig));
sig->blocks = protocol_alloc((size_t)block_count * sizeof(DeltaBlockSig));
if (!sig->blocks) {
free(sig);
return NULL;
@@ -82,7 +83,7 @@ Data* delta_signature_serialize(const DeltaSignature* sig) {
total > SIZE_MAX)
return NULL;
uint8_t* buf = malloc((size_t)total);
uint8_t* buf = protocol_alloc((size_t)total);
if (!buf)
return NULL;
@@ -111,7 +112,7 @@ DeltaSignature* delta_signature_deserialize(const Data* data) {
const uint8_t* buf = (const uint8_t*)data->data;
size_t pos = 0;
DeltaSignature* sig = malloc(sizeof(DeltaSignature));
DeltaSignature* sig = protocol_alloc(sizeof(DeltaSignature));
if (!sig)
return NULL;
@@ -149,7 +150,7 @@ DeltaSignature* delta_signature_deserialize(const Data* data) {
free(sig);
return NULL;
}
sig->blocks = malloc((size_t)blocks_size);
sig->blocks = protocol_alloc((size_t)blocks_size);
if (!sig->blocks) {
free(sig);
return NULL;
@@ -178,7 +179,7 @@ static bool ensure_capacity(DeltaInstruction** instrs, uint32_t* capacity, uint3
if (*capacity > MAX_DELTA_INSTRUCTIONS / 2)
return false;
uint32_t new_cap = *capacity * 2;
DeltaInstruction* tmp = realloc(*instrs, (size_t)new_cap * sizeof(DeltaInstruction));
DeltaInstruction* tmp = protocol_realloc(*instrs, (size_t)new_cap * sizeof(DeltaInstruction));
if (!tmp)
return false;
*instrs = tmp;
@@ -195,7 +196,7 @@ static bool flush_literal(DeltaInstruction** instrs, uint32_t* capacity, uint32_
uint32_t lit_len = (uint32_t)(end - start);
if (!ensure_capacity(instrs, capacity, *count))
return false;
uint8_t* lit_data = malloc(lit_len);
uint8_t* lit_data = protocol_alloc(lit_len);
if (!lit_data)
return false;
memcpy(lit_data, data + start, lit_len);
@@ -225,7 +226,7 @@ Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const De
uint32_t capacity = 64;
uint32_t count = 0;
DeltaInstruction* instrs = malloc((size_t)capacity * sizeof(DeltaInstruction));
DeltaInstruction* instrs = protocol_alloc((size_t)capacity * sizeof(DeltaInstruction));
if (!instrs)
return NULL;
@@ -309,7 +310,7 @@ Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const De
}
}
Delta* delta = malloc(sizeof(Delta));
Delta* delta = protocol_alloc(sizeof(Delta));
if (!delta) {
free_instructions(instrs, count);
return NULL;
@@ -355,7 +356,7 @@ Data* delta_serialize(const Delta* delta) {
if (delta->delta_size > UINT64_MAX - header_size || header_size + delta->delta_size > SIZE_MAX)
return NULL;
uint64_t total = header_size + delta->delta_size;
uint8_t* buf = malloc((size_t)total);
uint8_t* buf = protocol_alloc((size_t)total);
if (!buf)
return NULL;
@@ -395,7 +396,7 @@ Delta* delta_deserialize(const Data* data) {
const uint8_t* buf = (const uint8_t*)data->data;
size_t pos = 0;
Delta* delta = malloc(sizeof(Delta));
Delta* delta = protocol_alloc(sizeof(Delta));
if (!delta)
return NULL;
@@ -412,9 +413,10 @@ Delta* delta_deserialize(const Data* data) {
return NULL;
}
delta->instructions = delta->instruction_count == 0
delta->instructions =
delta->instruction_count == 0
? NULL
: malloc((size_t)delta->instruction_count * sizeof(DeltaInstruction));
: protocol_alloc((size_t)delta->instruction_count * sizeof(DeltaInstruction));
if (delta->instruction_count > 0 && !delta->instructions) {
free(delta);
return NULL;
@@ -465,7 +467,7 @@ Delta* delta_deserialize(const Data* data) {
free(delta);
return NULL;
}
delta->instructions[i].literal.data = malloc(lit_len ? lit_len : 1);
delta->instructions[i].literal.data = protocol_alloc(lit_len ? lit_len : 1);
if (!delta->instructions[i].literal.data) {
log_message(LOG_LEVEL_ERROR, "Failed to allocate %u bytes for literal data", lit_len);
free_instructions(delta->instructions, i);
@@ -492,7 +494,7 @@ void* delta_apply(const void* old_data, uint64_t old_size, const Delta* delta,
delta->new_file_size > DELTA_MAX_FILE_SIZE || delta->new_file_size > SIZE_MAX)
return NULL;
void* output = malloc(delta->new_file_size ? (size_t)delta->new_file_size : 1);
void* output = protocol_alloc(delta->new_file_size ? (size_t)delta->new_file_size : 1);
if (!output)
return NULL;
+108 -16
View File
@@ -14,6 +14,7 @@
#include "log.h"
#include "metadata.h"
#include "utils.h"
#include "protocol.h"
static bool write_all(int fd, const void* data, unsigned long long size) {
const unsigned char* p = data;
@@ -45,14 +46,14 @@ bool file_checksum(File* file, uint64_t* checksum) {
File* file_create(const char* path) {
if (!path)
return NULL;
File* file = (File*)malloc(sizeof(File));
File* file = (File*)protocol_alloc(sizeof(File));
if (file == NULL) {
log_perror("ERROR: Could not allocate memory for file struct");
return NULL;
}
size_t path_len = strlen(path);
file->path = (char*)malloc(path_len + 1);
file->path = (char*)protocol_alloc(path_len + 1);
if (file->path == NULL) {
free(file);
return NULL;
@@ -85,7 +86,7 @@ void file_destroy(void* item) {
}
FileMetadata* file_metadata_create(const struct stat* stats) {
FileMetadata* m = malloc(sizeof(FileMetadata));
FileMetadata* m = protocol_alloc(sizeof(FileMetadata));
if (m == NULL) {
log_perror("ERROR: Could not allocate memory for file metadata");
return NULL;
@@ -112,7 +113,7 @@ bool file_load_data(File* file) {
if (file->data->data == NULL) {
if (file->data->size == 0)
return true;
file->data->data = malloc(file->data->size);
file->data->data = protocol_alloc(file->data->size);
if (file->data->data == NULL) {
log_perror("Could not allocate memory for file data");
return false;
@@ -172,8 +173,17 @@ bool file_set_authorized_root(int fd, const char* canonical_path) {
}
bool file_path_exists_secure(const char* path) {
if (!path)
return false;
char* leaf = NULL;
int parent_fd = file_open_secure_parent(path, &leaf, false);
if (parent_fd < 0)
return false;
struct stat st;
return file_stat_secure(path, &st);
bool exists = fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) == 0;
close(parent_fd);
free(leaf);
return exists;
}
bool file_stat_secure(const char* path, struct stat* st) {
@@ -183,15 +193,29 @@ bool file_stat_secure(const char* path, struct stat* st) {
int parent_fd = file_open_secure_parent(path, &leaf, false);
if (parent_fd < 0)
return false;
int fd = openat(parent_fd, leaf, O_RDONLY | O_NONBLOCK | O_CLOEXEC | O_NOFOLLOW);
bool exists = fd >= 0 && fstat(fd, st) == 0 && S_ISREG(st->st_mode);
if (fd >= 0)
close(fd);
bool exists = fstatat(parent_fd, leaf, st, AT_SYMLINK_NOFOLLOW) == 0 && S_ISREG(st->st_mode);
close(parent_fd);
free(leaf);
return exists;
}
static bool stat_is_newer(const struct stat* st, const FileMetadata* metadata) {
if (!st || !metadata)
return false;
#ifdef __linux__
long mtime_nsec = st->st_mtim.tv_nsec;
#else
long mtime_nsec = 0;
#endif
return st->st_mtime > metadata->mtime_sec ||
(st->st_mtime == metadata->mtime_sec && mtime_nsec > metadata->mtime_nsec);
}
bool file_destination_is_newer_secure(const char* path, const FileMetadata* metadata) {
struct stat st;
return file_stat_secure(path, &st) && stat_is_newer(&st, metadata);
}
int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs) {
char* copy = str_dup(path);
if (!copy)
@@ -302,8 +326,10 @@ bool file_rename_secure(const char* old_path, const char* new_path) {
return ok;
}
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse, const FileMetadata* metadata) {
static bool file_to_disk_secure_impl(const char* path, const void* data,
unsigned long long data_size, bool inplace, bool sparse,
const FileMetadata* metadata, bool preserve_executability,
bool update, bool no_replace, bool use_fsync) {
char* leaf = NULL;
int dirfd = file_open_secure_parent(path, &leaf, true);
if (dirfd < 0)
@@ -311,15 +337,38 @@ bool file_to_disk_secure(const char* path, const void* data, unsigned long long
int fd = -1;
bool ok = false;
if (inplace) {
fd = openat(dirfd, leaf, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW, 0644);
fd = openat(dirfd, leaf, O_WRONLY | O_CREAT | O_CLOEXEC | O_NOFOLLOW, 0644);
if (fd >= 0) {
struct stat destination_stat;
bool newer = false;
if (update && metadata && fstat(fd, &destination_stat) == 0 &&
S_ISREG(destination_stat.st_mode)) {
newer = stat_is_newer(&destination_stat, metadata);
}
if (newer) {
ok = true;
} else {
if (!sparse || data_size == 0 || ftruncate(fd, (off_t)data_size) == 0)
ok = write_all(fd, data, data_size);
if (ok && metadata)
ok = file_restore_metadata_fd(fd, metadata);
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
if (ok && use_fsync)
ok = fsync(fd) == 0;
}
}
} else {
char tmp[NAME_MAX];
if (update && metadata) {
/* This check protects the normal atomic path as far as possible. A
concurrent replacement can still occur before the final rename. */
struct stat destination_stat;
if (fstatat(dirfd, leaf, &destination_stat, AT_SYMLINK_NOFOLLOW) == 0 &&
S_ISREG(destination_stat.st_mode) && stat_is_newer(&destination_stat, metadata)) {
close(dirfd);
free(leaf);
return true;
}
}
for (unsigned int i = 0; i < 100 && !ok; ++i) {
snprintf(tmp, sizeof(tmp), ".%s.tmp.%ld.%u", leaf, (long)getpid(), i);
fd = openat(dirfd, tmp, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600);
@@ -330,12 +379,26 @@ bool file_to_disk_secure(const char* path, const void* data, unsigned long long
if (ok || (!sparse || data_size == 0))
ok = write_all(fd, data, data_size);
if (ok && metadata)
ok = file_restore_metadata_fd(fd, metadata);
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
if (ok && use_fsync)
ok = fsync(fd) == 0;
if (close(fd) != 0)
ok = false;
fd = -1;
if (ok && renameat(dirfd, tmp, dirfd, leaf) != 0)
if (ok) {
if (no_replace) {
/* The probe and commit cannot be one operation. A concurrent
creator may win; EEXIST is then the requested skip. */
if (linkat(dirfd, tmp, dirfd, leaf, 0) == 0 || errno == EEXIST) {
if (unlinkat(dirfd, tmp, 0) != 0 && errno != ENOENT)
ok = false;
} else {
ok = false;
}
} else if (renameat(dirfd, tmp, dirfd, leaf) != 0) {
ok = false;
}
}
if (!ok)
unlinkat(dirfd, tmp, 0);
}
@@ -347,9 +410,38 @@ bool file_to_disk_secure(const char* path, const void* data, unsigned long long
return ok;
}
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse, const FileMetadata* metadata,
bool preserve_executability) {
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, metadata,
preserve_executability, false, false, false);
}
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse, const FileMetadata* metadata,
bool preserve_executability) {
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, metadata,
preserve_executability, true, false, false);
}
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
unsigned long long data_size, bool inplace, bool sparse,
const FileMetadata* metadata, bool preserve_executability,
bool use_fsync) {
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, metadata,
preserve_executability, false, false, use_fsync);
}
bool file_to_disk_secure_no_replace(const char* path, const void* data,
unsigned long long data_size, bool sparse,
const FileMetadata* metadata, bool preserve_executability) {
return file_to_disk_secure_impl(path, data, data_size, false, sparse, metadata,
preserve_executability, false, true, false);
}
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse) {
if (!path || (!data && data_size != 0) || has_path_traversal(path))
return false;
return file_to_disk_secure(path, data, data_size, inplace, sparse, NULL);
return file_to_disk_secure(path, data, data_size, inplace, sparse, NULL, false);
}
+16 -1
View File
@@ -27,10 +27,25 @@ bool file_set_authorized_root(int fd, const char* canonical_path);
/* Secure path/filesystem primitives (symlink-safe, O_NOFOLLOW, root-confined). */
bool file_path_exists_secure(const char* path);
bool file_stat_secure(const char* path, struct stat* st);
bool file_destination_is_newer_secure(const char* path, const FileMetadata* metadata);
int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs);
bool file_ensure_directory_secure(const char* path);
bool file_rename_secure(const char* old_path, const char* new_path);
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse, const FileMetadata* metadata);
bool inplace, bool sparse, const FileMetadata* metadata,
bool preserve_executability);
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
unsigned long long data_size, bool inplace, bool sparse,
const FileMetadata* metadata, bool preserve_executability,
bool use_fsync);
/* With update enabled, an existing newer destination is left untouched. The
check is descriptor-based for inplace writes; atomic replacement still has
an unavoidable final rename race without filesystem locking. */
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse, const FileMetadata* metadata,
bool preserve_executability);
bool file_to_disk_secure_no_replace(const char* path, const void* data,
unsigned long long data_size, bool sparse,
const FileMetadata* metadata, bool preserve_executability);
#endif
+82 -21
View File
@@ -8,6 +8,7 @@
#include <unistd.h>
#include "array_list.h"
#include "chmod.h"
#include "compression.h"
#include "config.h"
#include "data.h"
@@ -22,9 +23,12 @@
#define MAX_FILE_DATA_SIZE MAX_RECEIVE_FILE_SIZE
bool file_save_to_disk(const char* root_directory, const File* file, const Config* config) {
bool backup_enabled = config && config->backup;
/* Backups are incompatible with ignore-existing: moving the entry first
would make a concurrent no-replace commit overwrite its old name. */
bool backup_enabled = config && config->backup && !config->ignore_existing;
bool inplace = config && config->inplace;
bool sparse = config && config->preserve_sparse;
bool preserve_executability = config && config->use_executability;
const char* backup_suffix = (config && config->suffix) ? config->suffix : "~";
const char* backup_dir = (config && config->backup_dir) ? config->backup_dir : NULL;
const char* partial_dir = (config && config->partial_dir) ? config->partial_dir : NULL;
@@ -73,14 +77,12 @@ bool file_save_to_disk(const char* root_directory, const File* file, const Confi
free(disk_path);
return true;
}
free(destination_path);
destination_path = NULL;
/* --update is receiver-side policy: never replace a newer destination. */
if (config && config->update) {
struct stat destination_stat;
if (file_stat_secure(disk_path, &destination_stat) && file->metadata &&
destination_stat.st_mtime > file->metadata->mtime_sec) {
/* --ignore-existing checks the final destination before partial files or
overwrite policies can modify it. */
if (config && config->ignore_existing) {
bool exists = file_path_exists_secure(destination_path);
if (exists) {
free(confined_backup);
free(confined_partial);
free(destination_path);
@@ -88,6 +90,17 @@ bool file_save_to_disk(const char* root_directory, const File* file, const Confi
return true;
}
}
free(destination_path);
destination_path = NULL;
/* --update is receiver-side policy: never replace a newer destination.
The secure stat does not require read permission on the destination. */
if (config && config->update && file_destination_is_newer_secure(disk_path, file->metadata)) {
free(confined_backup);
free(confined_partial);
free(disk_path);
return true;
}
if (backup_enabled) {
struct stat backup_stat;
@@ -119,8 +132,23 @@ bool file_save_to_disk(const char* root_directory, const File* file, const Confi
}
}
bool ok = file_to_disk_secure(disk_path, file->data->data, file->data->size, inplace, sparse,
file->metadata);
FileMetadata adjusted_metadata;
const FileMetadata* metadata = file->metadata;
if (metadata && config && config->chmod_spec && *config->chmod_spec) {
adjusted_metadata = *metadata;
if (!chmod_apply(adjusted_metadata.mode, config->chmod_spec, &adjusted_metadata.mode))
goto fail;
metadata = &adjusted_metadata;
}
bool ok = config && config->ignore_existing
? file_to_disk_secure_no_replace(disk_path, file->data->data, file->data->size,
sparse, metadata, preserve_executability)
: config && config->update
? file_to_disk_secure_update(disk_path, file->data->data, file->data->size, inplace,
sparse, metadata, preserve_executability)
: file_to_disk_secure_with_fsync(disk_path, file->data->data, file->data->size,
inplace, sparse, metadata, preserve_executability,
config && config->use_fsync);
free(parent_copy);
free(backup_path);
free(confined_backup);
@@ -187,7 +215,10 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
}
Data* raw_delta = delta_data;
if (config->use_compression) {
if (config->use_compression &&
!compression_should_skip_with_suffixes(
check_path, config->skip_compress_suffixes,
config->skip_compress_set ? config->skip_compress_count : -1)) {
raw_delta = data_decompress_limited(delta_data, MAX_RECEIVE_FILE_SIZE);
data_destroy(delta_data);
if (!raw_delta) {
@@ -290,7 +321,10 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
return NULL;
}
if (config->use_compression) {
if (config->use_compression &&
!compression_should_skip_with_suffixes(
file->path, config->skip_compress_suffixes,
config->skip_compress_set ? config->skip_compress_count : -1)) {
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_FILE_SIZE);
data_destroy(file_data);
if (uncompressed == NULL) {
@@ -332,12 +366,19 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
unsigned long long check_size;
long long check_mtime;
long long check_mtime_nsec;
uint64_t check_checksum = 0;
if (!receive_n_data(fd, &check_size, sizeof(check_size)) ||
!receive_n_data(fd, &check_mtime, sizeof(check_mtime))) {
free(check_path);
return NULL;
}
if (!receive_n_data(fd, &check_mtime_nsec, sizeof(check_mtime_nsec)) || check_mtime_nsec < 0 ||
check_mtime_nsec >= 1000000000LL) {
free(check_path);
send_status(fd, STATUS_ERROR);
return NULL;
}
if (config->checksum && !receive_n_data(fd, &check_checksum, sizeof(check_checksum))) {
free(check_path);
return NULL;
@@ -350,7 +391,10 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
}
if (has_path_traversal(check_path)) {
log_message(LOG_LEVEL_ERROR, "Path traversal detected: %s", check_path);
char* escaped_path = output_escape(check_path, log_get_8_bit_output());
log_message(LOG_LEVEL_ERROR, "Path traversal detected: %s",
escaped_path ? escaped_path : "<allocation failed>");
free(escaped_path);
free(check_path);
return NULL;
}
@@ -375,7 +419,7 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
unsigned long long old_size = has_old_file ? (unsigned long long)st.st_size : 0;
void* old_data = NULL;
if (has_old_file && old_size > 0 && old_size <= MAX_RECEIVE_FILE_SIZE && old_size <= SIZE_MAX) {
old_data = malloc((size_t)old_size);
old_data = protocol_alloc((size_t)old_size);
if (old_data) {
size_t got = 0;
while (got < (size_t)old_size) {
@@ -393,7 +437,8 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
close(old_fd);
}
bool match = has_old_file && (unsigned long long)st.st_size == check_size;
bool match =
!config->ignore_times && has_old_file && (unsigned long long)st.st_size == check_size;
if (match && config->checksum) {
uint64_t old_checksum = old_size == 0 ? delta_xxhash64("", 0) : 0;
if (old_data)
@@ -401,8 +446,13 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
match = (old_size == 0 || old_data) && old_checksum == check_checksum;
free(old_data);
old_data = NULL;
} else if (match) {
match = (long long)st.st_mtime == check_mtime;
} else if (match && !config->size_only) {
long long old_mtime_nsec = 0;
#ifdef __linux__
old_mtime_nsec = st.st_mtim.tv_nsec;
#endif
match = metadata_mtime_matches(st.st_mtime, old_mtime_nsec, (time_t)check_mtime,
(long)check_mtime_nsec, config->modify_window);
}
if (match) {
@@ -418,7 +468,7 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
return NULL;
}
bool try_delta = config->use_delta && has_old_file && old_data != NULL &&
bool try_delta = config->use_delta && !config->whole_file && has_old_file && old_data != NULL &&
delta_should_attempt(old_size, check_size, config->delta_max_file_size);
if (try_delta) {
@@ -442,6 +492,8 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
}
if (!try_delta) {
free(old_data);
old_data = NULL;
if (!send_status(fd, STATUS_NEXT)) {
free(full_path);
free(check_path);
@@ -471,7 +523,10 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
return NULL;
}
if (config->use_compression) {
if (config->use_compression &&
!compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
config->skip_compress_set ? config->skip_compress_count
: -1)) {
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_FILE_SIZE);
data_destroy(file_data);
if (uncompressed == NULL) {
@@ -497,7 +552,10 @@ File* file_receive(const Config* config, int file_descriptor) {
if (path == NULL)
return NULL;
if (path[0] == '\0' || has_path_traversal(path)) {
log_message(LOG_LEVEL_ERROR, "Invalid received file path: %s", path);
char* escaped_path = output_escape(path, log_get_8_bit_output());
log_message(LOG_LEVEL_ERROR, "Invalid received file path: %s",
escaped_path ? escaped_path : "<allocation failed>");
free(escaped_path);
free(path);
return NULL;
}
@@ -518,7 +576,10 @@ File* file_receive(const Config* config, int file_descriptor) {
file_destroy(file);
return NULL;
}
if (config->use_compression && !compression_should_skip(file->path)) {
if (config->use_compression &&
!compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
config->skip_compress_set ? config->skip_compress_count
: -1)) {
Data* file_data_uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_FILE_SIZE);
data_destroy(file_data);
if (file_data_uncompressed == NULL) {
+22 -4
View File
@@ -19,12 +19,22 @@
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path) {
return file_send_single_calls_with_skip(file, file_descriptor, use_metadata, compression_level,
send_path, NULL, -1, 0);
}
bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path,
char* const* skip_suffixes, int skip_count,
int compression_threads) {
if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data))
return false;
const Data* data_to_send = file->data;
Data* compressed_data = NULL;
if (compression_level > 0 && !compression_should_skip(file->path)) {
compressed_data = data_compress(file->data, compression_level);
if (compression_level > 0 &&
!compression_should_skip_with_suffixes(file->path, skip_suffixes, skip_count)) {
compressed_data =
data_compress_with_threads(file->data, compression_level, compression_threads);
if (compressed_data == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
return false;
@@ -49,11 +59,19 @@ bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
bool send_path) {
return file_send_sendfile_with_skip(file, file_descriptor, use_metadata, compression_level,
send_path, NULL, -1, 0);
}
bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path, char* const* skip_suffixes,
int skip_count, int compression_threads) {
if (!file || !file->path || !file->data)
return false;
if (compression_level > 0)
return file_send_single_calls(file, file_descriptor, use_metadata, compression_level,
send_path);
return file_send_single_calls_with_skip(file, file_descriptor, use_metadata, compression_level,
send_path, skip_suffixes, skip_count,
compression_threads);
if (send_path && !send_str(file_descriptor, file->path))
return false;
+7
View File
@@ -8,7 +8,14 @@
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path);
bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path,
char* const* skip_suffixes, int skip_count,
int compression_threads);
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
bool send_path);
bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path, char* const* skip_suffixes,
int skip_count, int compression_threads);
#endif
+4 -3
View File
@@ -140,7 +140,8 @@ static bool write_all(int fd, const void* data, unsigned long long size) {
}
bool file_store_write_secure(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse, const FileMetadata* metadata) {
bool inplace, bool sparse, const FileMetadata* metadata,
bool preserve_executability) {
char* leaf = NULL;
int dirfd = file_store_open_secure_parent(path, &leaf);
if (dirfd < 0)
@@ -153,7 +154,7 @@ bool file_store_write_secure(const char* path, const void* data, unsigned long l
if (!sparse || data_size == 0 || ftruncate(fd, (off_t)data_size) == 0)
ok = write_all(fd, data, data_size);
if (ok && metadata)
ok = file_restore_metadata_fd(fd, metadata);
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
}
} else {
int tmp_size = snprintf(NULL, 0, ".%s.tmp.%ld.%u", leaf, (long)getpid(), 99U);
@@ -178,7 +179,7 @@ bool file_store_write_secure(const char* path, const void* data, unsigned long l
if (ok || (!sparse || data_size == 0))
ok = write_all(fd, data, data_size);
if (ok && metadata)
ok = file_restore_metadata_fd(fd, metadata);
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
if (close(fd) != 0)
ok = false;
fd = -1;
+2 -1
View File
@@ -8,6 +8,7 @@ bool file_store_set_authorized_root(int fd, const char* canonical_path);
int file_store_open_secure_parent(const char* path, char** leaf_out);
bool file_store_rename_secure(const char* old_path, const char* new_path);
bool file_store_write_secure(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse, const FileMetadata* metadata);
bool inplace, bool sparse, const FileMetadata* metadata,
bool preserve_executability);
#endif
+83 -1
View File
@@ -1,5 +1,6 @@
#include "log.h"
#include <errno.h>
#include <stdbool.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
@@ -7,16 +8,54 @@
static const char* log_level_strings[] = {"DEBUG", "INFO", "WARN", "ERROR"};
static LogLevel current_log_level = LOG_LEVEL_WARNING;
static uint32_t current_debug_flags = 0;
static uint32_t info_flags = 0;
static bool info_flags_explicit = false;
static FILE* log_fp = NULL;
static _Thread_local bool eight_bit_output;
static LogStderrMode stderr_mode = LOG_STDERR_ERRORS;
void set_log_level(LogLevel level) {
current_log_level = level;
}
void set_log_debug_flags(uint32_t flags) {
current_debug_flags = flags;
}
uint32_t get_log_debug_flags(void) {
return current_debug_flags;
}
void set_log_info_flags(uint32_t flags) {
info_flags = flags;
info_flags_explicit = true;
}
uint32_t get_log_info_flags(void) {
return info_flags;
}
void log_set_file(FILE* fp) {
log_fp = fp;
}
void log_set_8_bit_output(bool enabled) {
eight_bit_output = enabled;
}
bool log_get_8_bit_output(void) {
return eight_bit_output;
}
void log_set_stderr_mode(LogStderrMode mode) {
stderr_mode = mode;
}
LogStderrMode log_get_stderr_mode(void) {
return stderr_mode;
}
static inline void write_message(FILE* dest_io, LogLevel log_level, struct tm t, const char* format,
va_list args) {
fprintf(dest_io, "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t.tm_year + 1900, t.tm_mon + 1,
@@ -37,7 +76,7 @@ void log_message(LogLevel log_level, const char* format, ...) {
return;
FILE* dest_io = stdout;
if (log_level == LOG_LEVEL_ERROR) {
if (stderr_mode == LOG_STDERR_ALL || log_level == LOG_LEVEL_ERROR) {
dest_io = stderr;
}
@@ -53,6 +92,49 @@ void log_message(LogLevel log_level, const char* format, ...) {
}
}
void log_debug_message(LogDebugFlag flag, const char* format, ...) {
if (current_log_level > LOG_LEVEL_DEBUG || !(current_debug_flags & flag))
return;
time_t now = time(NULL);
struct tm t;
if (!localtime_r(&now, &t))
return;
va_list args;
va_start(args, format);
write_message(stdout, LOG_LEVEL_DEBUG, t, format, args);
va_end(args);
if (log_fp) {
va_start(args, format);
write_message(log_fp, LOG_LEVEL_DEBUG, t, format, args);
va_end(args);
}
}
void log_info_message(LogInfoFlag flag, const char* format, ...) {
if ((info_flags_explicit && (info_flags & flag) == 0) ||
(!info_flags_explicit && current_log_level > LOG_LEVEL_DEBUG))
return;
time_t now = time(NULL);
struct tm t;
if (!localtime_r(&now, &t))
return;
va_list args;
va_start(args, format);
write_message(stdout, LOG_LEVEL_INFO, t, format, args);
va_end(args);
if (log_fp) {
va_start(args, format);
write_message(log_fp, LOG_LEVEL_INFO, t, format, args);
va_end(args);
}
}
void log_perror(const char* context) {
log_message(LOG_LEVEL_ERROR, "%s: %s", context, strerror(errno));
}
+29
View File
@@ -2,12 +2,41 @@
#define LOG_H
#include <stdio.h>
#include <stdbool.h>
#include <stdint.h>
typedef enum { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR } LogLevel;
typedef enum { LOG_STDERR_ERRORS, LOG_STDERR_ALL } LogStderrMode;
typedef enum {
LOG_DEBUG_IO = 1u << 0,
LOG_DEBUG_PROTO = 1u << 1,
LOG_DEBUG_PACK = 1u << 2,
LOG_DEBUG_UTIL = 1u << 3,
LOG_DEBUG_ALL = (1u << 4) - 1,
} LogDebugFlag;
typedef enum {
LOG_INFO_COPY = 1u << 0,
LOG_INFO_MISC = 1u << 1,
LOG_INFO_SKIP = 1u << 2,
LOG_INFO_STATS = 1u << 3,
LOG_INFO_ALL = LOG_INFO_COPY | LOG_INFO_MISC | LOG_INFO_SKIP | LOG_INFO_STATS,
} LogInfoFlag;
void log_message(LogLevel log_level, const char* message, ...);
void log_perror(const char* context);
void set_log_level(LogLevel level);
void set_log_debug_flags(uint32_t flags);
uint32_t get_log_debug_flags(void);
void log_debug_message(LogDebugFlag flag, const char* message, ...);
void set_log_info_flags(uint32_t flags);
uint32_t get_log_info_flags(void);
void log_info_message(LogInfoFlag flag, const char* message, ...);
void log_set_file(FILE* fp);
void log_set_8_bit_output(bool enabled);
bool log_get_8_bit_output(void);
void log_set_stderr_mode(LogStderrMode mode);
LogStderrMode log_get_stderr_mode(void);
#endif
+56 -10
View File
@@ -2,6 +2,7 @@
#include "file.h"
#include "log.h"
#include "protocol.h"
#include "utils.h"
#include <errno.h>
#include <fcntl.h>
#include <stdint.h>
@@ -24,6 +25,29 @@ typedef char static_assert_mode_t_fits[(sizeof(mode_t) <= sizeof(int32_t)) ? 1 :
typedef char static_assert_uid_t_fits[(sizeof(uid_t) <= sizeof(int32_t)) ? 1 : -1];
typedef char static_assert_gid_t_fits[(sizeof(gid_t) <= sizeof(int32_t)) ? 1 : -1];
bool metadata_mtime_matches(time_t left_sec, long left_nsec, time_t right_sec, long right_nsec,
int modify_window) {
int64_t left = (int64_t)left_sec;
int64_t right = (int64_t)right_sec;
int64_t seconds;
int64_t nanoseconds;
if (left > right || (left == right && left_nsec >= right_nsec)) {
seconds = left - right;
nanoseconds = (int64_t)left_nsec - (int64_t)right_nsec;
} else {
seconds = right - left;
nanoseconds = (int64_t)right_nsec - (int64_t)left_nsec;
}
if (nanoseconds < 0) {
seconds--;
nanoseconds += 1000000000LL;
}
if (modify_window == 0)
return left == right;
return seconds < modify_window || (seconds == modify_window && nanoseconds == 0);
}
void metadata_to_buf(char** buf, const FileMetadata* m) {
int32_t present = (m != NULL) ? 1 : 0;
memcpy(*buf, &present, sizeof(present));
@@ -55,7 +79,7 @@ FileMetadata* metadata_from_buf(char** buf) {
return NULL;
if (!present)
return NULL;
FileMetadata* m = malloc(sizeof(FileMetadata));
FileMetadata* m = protocol_alloc(sizeof(FileMetadata));
if (m == NULL)
return NULL;
int32_t mode;
@@ -122,7 +146,7 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok) {
*ok = 0;
return NULL;
}
FileMetadata* m = malloc(sizeof(FileMetadata));
FileMetadata* m = protocol_alloc(sizeof(FileMetadata));
if (m == NULL) {
if (ok)
*ok = 0;
@@ -179,12 +203,27 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok) {
return m;
}
void file_restore_metadata(const char* path, const FileMetadata* metadata) {
static mode_t metadata_mode(const FileMetadata* metadata, mode_t current_mode,
bool preserve_executability) {
const mode_t execute_bits = S_IXUSR | S_IXGRP | S_IXOTH;
if (preserve_executability)
return (current_mode & 0777 & ~execute_bits) | (metadata->mode & execute_bits);
return metadata->mode & 0777 & ~(S_IWGRP | S_IWOTH);
}
void file_restore_metadata(const char* path, const FileMetadata* metadata,
bool preserve_executability) {
if (metadata == NULL)
return;
mode_t safe_mode = metadata->mode & 0777 & ~(S_IWGRP | S_IWOTH);
if (chmod(path, safe_mode) != 0)
log_message(LOG_LEVEL_WARNING, "Failed to chmod %s: %s", path, strerror(errno));
struct stat current;
mode_t current_mode = stat(path, &current) == 0 ? current.st_mode : 0;
mode_t safe_mode = metadata_mode(metadata, current_mode, preserve_executability);
if (chmod(path, safe_mode) != 0) {
char* escaped_path = output_escape(path, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING, "Failed to chmod %s: %s",
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
free(escaped_path);
}
/* Never apply client-supplied ownership. The descriptor API below is the
receiver write path; retain this legacy API only for compatibility. */
struct timespec times[2];
@@ -192,15 +231,22 @@ void file_restore_metadata(const char* path, const FileMetadata* metadata) {
times[0].tv_nsec = UTIME_OMIT;
times[1].tv_sec = metadata->mtime_sec;
times[1].tv_nsec = metadata->mtime_nsec;
if (utimensat(AT_FDCWD, path, times, 0) != 0)
log_message(LOG_LEVEL_WARNING, "Failed to set timestamps on %s: %s", path, strerror(errno));
if (utimensat(AT_FDCWD, path, times, 0) != 0) {
char* escaped_path = output_escape(path, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING, "Failed to set timestamps on %s: %s",
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
free(escaped_path);
}
}
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata) {
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserve_executability) {
if (fd < 0 || metadata == NULL)
return metadata == NULL;
bool ok = true;
mode_t safe_mode = metadata->mode & 0777 & ~(S_IWGRP | S_IWOTH);
struct stat current;
if (fstat(fd, &current) != 0)
return false;
mode_t safe_mode = metadata_mode(metadata, current.st_mode, preserve_executability);
if (fchmod(fd, safe_mode) != 0)
ok = false;
/* Client uid/gid values are deliberately not authoritative. */
+8 -2
View File
@@ -5,6 +5,7 @@
#include <stdbool.h>
#include <stdint.h>
#include <sys/stat.h>
#include <time.h>
/*
* Wire format (introduced in protocol version 2.0.0):
@@ -29,7 +30,12 @@ void metadata_to_buf(char** buf, const FileMetadata* m);
FileMetadata* metadata_from_buf(char** buf);
bool metadata_send(int file_descriptor, const FileMetadata* m);
FileMetadata* metadata_receive(int file_descriptor, int* ok);
void file_restore_metadata(const char* path, const FileMetadata* metadata);
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata);
void file_restore_metadata(const char* path, const FileMetadata* metadata,
bool preserve_executability);
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserve_executability);
/* Compare timestamps using rsync's whole-second modification window. */
bool metadata_mtime_matches(time_t left_sec, long left_nsec, time_t right_sec, long right_nsec,
int modify_window);
#endif
+11
View File
@@ -26,9 +26,14 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
context->scanner_done = false;
context->loader_done = false;
context->manifest = NULL;
context->remove_source_files = NULL;
context->total_files = 0;
context->progress_bytes = 0;
context->total_bytes = 0;
context->sender_done = false;
atomic_init(&context->cancelled, false);
protocol_session_init(&context->allocation_session, -1, -1);
protocol_session_set_max_alloc(&context->allocation_session, config->max_alloc);
int init = 0;
if (mtx_init(&context->mutex_scanner, mtx_plain) != thrd_success)
goto fail;
@@ -76,6 +81,8 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) {
if (context->manifest) {
array_list_delete(context->manifest);
}
if (context->remove_source_files)
array_list_delete(context->remove_source_files);
config_delete(context->config);
queue_destroy(context->queue_scanner);
queue_destroy(context->queue_loader);
@@ -179,6 +186,7 @@ int receive_thread(void* pipeline_context) {
int write_thread(void* pipeline_context) {
PipelineContextReceiver* context = (PipelineContextReceiver*)pipeline_context;
protocol_session_bind(&context->session);
mtx_lock(&context->mutex);
bool save_to_disk = context->config->save_to_disk;
char* root_directory = str_dup(context->config->receive_root_directory);
@@ -190,6 +198,7 @@ int write_thread(void* pipeline_context) {
cnd_broadcast(&context->condition_not_full);
cnd_broadcast(&context->condition_not_empty);
mtx_unlock(&context->mutex);
protocol_session_unbind();
return thrd_error;
}
@@ -199,6 +208,7 @@ int write_thread(void* pipeline_context) {
&context->condition_not_full, &context->receiver_done);
if (file == NULL) {
free(root_directory);
protocol_session_unbind();
return thrd_success;
}
if (save_to_disk && !file_save_to_disk(root_directory, file, context->config)) {
@@ -210,6 +220,7 @@ int write_thread(void* pipeline_context) {
cnd_broadcast(&context->condition_not_empty);
mtx_unlock(&context->mutex);
free(root_directory);
protocol_session_unbind();
return thrd_error;
}
file_destroy(file);
+4
View File
@@ -24,10 +24,14 @@ typedef struct {
cnd_t condition_not_empty_loader;
bool loader_done;
ArrayList* manifest;
ArrayList* remove_source_files;
mtx_t mutex_progress;
int total_files;
unsigned long long progress_bytes;
unsigned long long total_bytes;
bool sender_done;
atomic_bool cancelled;
ProtocolSession allocation_session;
} PipelineContextSender;
typedef struct PipelineContextReceiver {
+105 -32
View File
@@ -1,5 +1,6 @@
#include "protocol.h"
#include "log.h"
#include "utils.h"
#include <errno.h>
#include <limits.h>
#include <openssl/ssl.h>
@@ -19,7 +20,8 @@ static __thread int io_read_fd = -1;
static __thread int io_write_fd = -1;
static __thread SSL* io_ssl;
static __thread ProtocolSession* bound_session;
static __thread ProtocolSession legacy_io_session = {.read_fd = -1, .write_fd = -1};
static __thread ProtocolSession legacy_io_session = {
.read_fd = -1, .write_fd = -1, .max_alloc = DEFAULT_MAX_ALLOC};
static unsigned long long io_bwlimit = 0;
static mtx_t bw_mutex;
@@ -27,12 +29,30 @@ static once_flag bw_mutex_once = ONCE_FLAG_INIT;
static unsigned long long global_bwlimit(void);
static bool protocol_reserve_memory(ProtocolSession* session, size_t charge) {
unsigned long long allocated = atomic_load(&session->total_allocated_bytes);
while (true) {
if (allocated > MAX_CONNECTION_MEMORY ||
(unsigned long long)charge > MAX_CONNECTION_MEMORY - allocated)
return false;
if (atomic_compare_exchange_weak(&session->total_allocated_bytes, &allocated,
allocated + (unsigned long long)charge))
return true;
}
}
static void protocol_release_memory_for_session(ProtocolSession* session, size_t charge) {
unsigned long long allocated = atomic_load(&session->total_allocated_bytes);
while (true) {
unsigned long long remaining = (unsigned long long)charge >= allocated ? 0 : allocated - charge;
if (atomic_compare_exchange_weak(&session->total_allocated_bytes, &allocated, remaining))
break;
}
}
void protocol_release_memory(size_t charge) {
ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
if ((unsigned long long)charge >= session->total_allocated_bytes)
session->total_allocated_bytes = 0;
else
session->total_allocated_bytes -= charge;
protocol_release_memory_for_session(session, charge);
}
void io_set_fds(int read_fd, int write_fd) {
bound_session = NULL;
@@ -44,7 +64,9 @@ void io_set_fds(int read_fd, int write_fd) {
legacy_io_session.read_fd = read_fd;
legacy_io_session.write_fd = write_fd;
legacy_io_session.ssl = NULL;
legacy_io_session.total_allocated_bytes = 0;
legacy_io_session.eight_bit_output = false;
atomic_store(&legacy_io_session.total_allocated_bytes, 0);
legacy_io_session.max_alloc = DEFAULT_MAX_ALLOC;
protocol_session_set_bwlimit(&legacy_io_session, global_bwlimit());
}
@@ -54,11 +76,46 @@ void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd)
memset(session, 0, sizeof(*session));
session->read_fd = read_fd;
session->write_fd = write_fd;
session->max_alloc = DEFAULT_MAX_ALLOC;
atomic_init(&session->total_allocated_bytes, 0);
protocol_session_set_bwlimit(session, global_bwlimit());
}
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc) {
if (!session)
session = bound_session ? bound_session : &legacy_io_session;
session->max_alloc = max_alloc;
}
static bool allocation_allowed(const ProtocolSession* session, size_t size) {
return (unsigned long long)size <= session->max_alloc;
}
static void* protocol_alloc_for_session(const ProtocolSession* session, size_t size) {
if (!allocation_allowed(session, size))
return NULL;
return malloc(size);
}
static void* protocol_realloc_for_session(const ProtocolSession* session, void* ptr, size_t size) {
if (!allocation_allowed(session, size))
return NULL;
return realloc(ptr, size);
}
void* protocol_alloc(size_t size) {
const ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
return protocol_alloc_for_session(session, size);
}
void* protocol_realloc(void* ptr, size_t size) {
const ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
return protocol_realloc_for_session(session, ptr, size);
}
void protocol_session_bind(ProtocolSession* session) {
bound_session = session;
log_set_8_bit_output(session && session->eight_bit_output);
}
void protocol_session_unbind(void) {
@@ -103,6 +160,19 @@ void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long b
session->bw_last_refill_nsec = now.tv_nsec;
}
void protocol_session_set_8_bit_output(ProtocolSession* session, bool enabled) {
if (!session)
return;
session->eight_bit_output = enabled;
if (session == bound_session)
log_set_8_bit_output(enabled);
}
void protocol_set_8_bit_output(bool enabled) {
ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
protocol_session_set_8_bit_output(session, enabled);
}
static void bw_throttle_session(ProtocolSession* session, size_t bytes_written) {
if (session->bwlimit == 0)
return;
@@ -153,7 +223,8 @@ static ProtocolSession* legacy_session(int read_fd, int write_fd) {
legacy_io_session.write_fd != target_write_fd) {
legacy_io_session.read_fd = target_read_fd;
legacy_io_session.write_fd = target_write_fd;
legacy_io_session.total_allocated_bytes = 0;
atomic_store(&legacy_io_session.total_allocated_bytes, 0);
legacy_io_session.max_alloc = DEFAULT_MAX_ALLOC;
protocol_session_set_bwlimit(&legacy_io_session, global_bwlimit());
} else if (legacy_io_session.bwlimit != global_bwlimit()) {
protocol_session_set_bwlimit(&legacy_io_session, global_bwlimit());
@@ -184,7 +255,7 @@ static int deadline_remaining_ms(const struct timespec* deadline) {
bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t data_size) {
if (!data && data_size != 0)
return false;
log_message(LOG_LEVEL_DEBUG, " Sending n Data: %zu", data_size);
log_debug_message(LOG_DEBUG_IO, " Sending n Data: %zu", data_size);
if (!session)
return false;
int fd = session->write_fd;
@@ -228,12 +299,12 @@ bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t dat
if (session->ssl)
wait_events = POLLOUT;
}
log_message(LOG_LEVEL_DEBUG, " Send n Data: %zu", total_bytes_send);
log_debug_message(LOG_DEBUG_IO, " Send n Data: %zu", total_bytes_send);
return true;
}
bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_size) {
log_message(LOG_LEVEL_DEBUG, " Receiving n Data: %zu", data_size);
log_debug_message(LOG_DEBUG_IO, " Receiving n Data: %zu", data_size);
if (!session)
return false;
int fd = session->read_fd;
@@ -287,7 +358,7 @@ bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_s
if (session->ssl)
wait_events = POLLIN;
}
log_message(LOG_LEVEL_DEBUG, " Received n Data: %zu", total_bytes_received);
log_debug_message(LOG_DEBUG_IO, " Received n Data: %zu", total_bytes_received);
return true;
}
@@ -328,7 +399,7 @@ bool protocol_send_str(ProtocolSession* session, const char* data) {
return false;
if (!protocol_send_n_data(session, data, size))
return false;
log_message(LOG_LEVEL_DEBUG, "Send String: %s", data);
log_debug_message(LOG_DEBUG_PROTO, "Send String: %s", data);
return true;
}
@@ -336,13 +407,12 @@ char* protocol_receive_str(ProtocolSession* session) {
size_t size;
if (!protocol_receive_n_data(session, &size, sizeof(size_t)))
return NULL;
if (size > MAX_STRING_SIZE || size > SIZE_MAX - 1 ||
size + 1 > MAX_CONNECTION_MEMORY - session->total_allocated_bytes) {
if (size > MAX_STRING_SIZE || size > SIZE_MAX - 1) {
log_message(LOG_LEVEL_ERROR, "String size %zu exceeds maximum %llu", size,
(unsigned long long)MAX_STRING_SIZE);
return NULL;
}
char* data = (char*)malloc(size + 1);
char* data = (char*)protocol_alloc_for_session(session, size + 1);
if (data == NULL)
return NULL;
if (!protocol_receive_n_data(session, data, size)) {
@@ -355,8 +425,7 @@ char* protocol_receive_str(ProtocolSession* session) {
return NULL;
}
data[size] = '\0';
session->total_allocated_bytes += size + 1;
log_message(LOG_LEVEL_DEBUG, "Received String: %s", data);
log_debug_message(LOG_DEBUG_PROTO, "Received String: %s", data);
return data;
}
@@ -370,7 +439,7 @@ bool protocol_send_data(ProtocolSession* session, const Data* data) {
return false;
if (!protocol_send_n_data(session, data->data, data_size))
return false;
log_message(LOG_LEVEL_DEBUG, "Send %lld data", data_size);
log_debug_message(LOG_DEBUG_PROTO, "Send %lld data", data_size);
return true;
}
@@ -385,25 +454,29 @@ Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long
(unsigned long long)MAX_DATA_PAYLOAD_SIZE);
return NULL;
}
if (size > SIZE_MAX)
return NULL;
size_t allocation_size = size == 0 ? 1 : (size_t)size;
if (allocation_size > MAX_CONNECTION_MEMORY - session->total_allocated_bytes) {
if (!protocol_reserve_memory(session, allocation_size)) {
log_message(LOG_LEVEL_ERROR, "Per-connection memory limit exceeded (%llu + %llu > %llu)",
(unsigned long long)session->total_allocated_bytes, size,
(unsigned long long)atomic_load(&session->total_allocated_bytes), size,
(unsigned long long)MAX_CONNECTION_MEMORY);
return NULL;
}
void* data = malloc(allocation_size);
if (data == NULL)
return NULL;
if (!protocol_receive_n_data(session, data, (size_t)size)) {
free(data);
void* data = protocol_alloc_for_session(session, allocation_size);
if (data == NULL) {
protocol_release_memory_for_session(session, allocation_size);
return NULL;
}
session->total_allocated_bytes += allocation_size;
log_message(LOG_LEVEL_DEBUG, "Received %lld data", size);
if (!protocol_receive_n_data(session, data, (size_t)size)) {
free(data);
protocol_release_memory_for_session(session, allocation_size);
return NULL;
}
log_debug_message(LOG_DEBUG_PROTO, "Received %lld data", size);
Data* result = data_create(data, (size_t)size);
if (!result) {
session->total_allocated_bytes -= allocation_size;
protocol_release_memory_for_session(session, allocation_size);
return NULL;
}
result->protocol_charge = allocation_size;
@@ -417,28 +490,28 @@ Data* protocol_receive_data(ProtocolSession* session) {
bool protocol_send_int(ProtocolSession* session, int data) {
if (!protocol_send_n_data(session, &data, sizeof(int)))
return false;
log_message(LOG_LEVEL_DEBUG, "Send Int: %d", data);
log_debug_message(LOG_DEBUG_PROTO, "Send Int: %d", data);
return true;
}
bool protocol_receive_int(ProtocolSession* session, int* data) {
if (!protocol_receive_n_data(session, data, sizeof(int)))
return false;
log_message(LOG_LEVEL_DEBUG, "Received Int: %d", *data);
log_debug_message(LOG_DEBUG_PROTO, "Received Int: %d", *data);
return true;
}
bool protocol_send_status(ProtocolSession* session, Status status) {
if (!protocol_send_n_data(session, &status, sizeof(Status)))
return false;
log_message(LOG_LEVEL_DEBUG, "Send Status: %s", status_to_string(status));
log_debug_message(LOG_DEBUG_PROTO, "Send Status: %s", status_to_string(status));
return true;
}
bool protocol_receive_status(ProtocolSession* session, Status* status) {
if (!protocol_receive_n_data(session, status, sizeof(Status)))
return false;
log_message(LOG_LEVEL_DEBUG, "Received Status: %s", status_to_string(*status));
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
return true;
}
+12 -1
View File
@@ -4,6 +4,7 @@
#include "data.h"
#include <stdbool.h>
#include <stddef.h>
#include <stdatomic.h>
/* Maximum allowed string size for receive_str (64 KB) */
#define MAX_STRING_SIZE (64 * 1024)
@@ -18,6 +19,9 @@
#define MAX_MANIFEST_ENTRIES (1024 * 1024)
/* Aggregate bytes retained by one received deletion manifest. */
#define MAX_MANIFEST_BYTES (16ULL * 1024 * 1024)
#define DEFAULT_MAX_ALLOC (1ULL * 1024 * 1024 * 1024)
/* Server policy ceiling for a client-provided allocation limit. */
#define MAX_SERVER_ALLOC (256ULL * 1024 * 1024)
typedef struct ssl_st SSL;
@@ -35,7 +39,9 @@ typedef struct ProtocolSession {
long long bw_tokens;
long long bw_last_refill_sec;
long bw_last_refill_nsec;
unsigned long long total_allocated_bytes;
atomic_ullong total_allocated_bytes;
bool eight_bit_output;
unsigned long long max_alloc;
} ProtocolSession;
typedef int Status;
@@ -65,6 +71,11 @@ void protocol_session_bind(ProtocolSession* session);
void protocol_session_unbind(void);
void protocol_session_set_ssl(ProtocolSession* session, SSL* ssl);
void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec);
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc);
void* protocol_alloc(size_t size);
void* protocol_realloc(void* ptr, size_t size);
void protocol_session_set_8_bit_output(ProtocolSession* session, bool enabled);
void protocol_set_8_bit_output(bool enabled);
bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t data_size);
bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_size);
bool protocol_send_str(ProtocolSession* session, const char* data);
+73 -15
View File
@@ -3,6 +3,7 @@
#include "utils.h"
#include <fcntl.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
@@ -15,6 +16,12 @@ typedef struct {
char* remote_path;
} RemoteDest;
static void ssh_child_setup_failed(int status_fd) {
ssize_t wret = write(status_fd, "x", 1);
(void)wret;
_exit(1);
}
static void remote_dest_destroy(RemoteDest* r) {
free(r->user);
free(r->host);
@@ -68,10 +75,57 @@ static int parse_remote_dest(const char* dest, RemoteDest* r) {
return 0;
}
Client* client_connect_ssh(const char* destination, int port, const char* server_path) {
char* ssh_build_remote_command(const char* server_path, bool old_args) {
const char* path = server_path ? server_path : "fastsync-server";
const char* suffix = " --stdio";
size_t path_len = strlen(path);
size_t suffix_len = strlen(suffix);
if (old_args) {
if (path_len > SIZE_MAX - suffix_len - 1)
return NULL;
char* command = malloc(path_len + suffix_len + 1);
if (!command)
return NULL;
memcpy(command, path, path_len);
memcpy(command + path_len, suffix, suffix_len + 1);
return command;
}
/* Quote the executable as one remote-shell word. This is the default safety boundary. */
size_t quote_count = 0;
for (const char* p = path; *p; p++)
if (*p == '\'')
quote_count++;
if (path_len > SIZE_MAX - suffix_len - 4 ||
quote_count > (SIZE_MAX - path_len - suffix_len - 4) / 4)
return NULL;
size_t command_len = path_len + quote_count * 4 + suffix_len + 4;
char* command = malloc(command_len + 1);
if (!command)
return NULL;
char* out = command;
*out++ = '\'';
for (const char* p = path; *p; p++) {
if (*p == '\'') {
memcpy(out, "'\\''", 4);
out += 4;
} else {
*out++ = *p;
}
}
*out++ = '\'';
memcpy(out, suffix, suffix_len + 1);
return command;
}
Client* client_connect_ssh(const char* destination, int port, const char* server_path,
bool old_args) {
RemoteDest r;
if (parse_remote_dest(destination, &r) != 0) {
fprintf(stderr, "Invalid remote destination: %s\n", destination);
char* escaped = output_escape(destination, false);
fprintf(stderr, "Invalid remote destination: %s\n", escaped ? escaped : "<allocation failed>");
free(escaped);
return NULL;
}
@@ -111,11 +165,12 @@ Client* client_connect_ssh(const char* destination, int port, const char* server
if (pid == 0) {
close(sv[0]);
close(exec_pipe[0]);
fcntl(exec_pipe[1], F_SETFD, FD_CLOEXEC);
if (sv[1] != STDIN_FILENO)
dup2(sv[1], STDIN_FILENO);
if (sv[1] != STDOUT_FILENO)
dup2(sv[1], STDOUT_FILENO);
if (fcntl(exec_pipe[1], F_SETFD, FD_CLOEXEC) < 0)
ssh_child_setup_failed(exec_pipe[1]);
if (sv[1] != STDIN_FILENO && dup2(sv[1], STDIN_FILENO) < 0)
ssh_child_setup_failed(exec_pipe[1]);
if (sv[1] != STDOUT_FILENO && dup2(sv[1], STDOUT_FILENO) < 0)
ssh_child_setup_failed(exec_pipe[1]);
if (sv[1] > 1)
close(sv[1]);
@@ -126,7 +181,7 @@ Client* client_connect_ssh(const char* destination, int port, const char* server
ssh_user_len = strlen(r.host) + 1;
char* ssh_user = malloc(ssh_user_len);
if (!ssh_user)
_exit(1);
ssh_child_setup_failed(exec_pipe[1]);
if (r.user && r.user[0] != '\0')
snprintf(ssh_user, ssh_user_len, "%s@%s", r.user, r.host);
else
@@ -135,6 +190,9 @@ Client* client_connect_ssh(const char* destination, int port, const char* server
char* ssh_argv[16];
int ac = 0;
char port_str[16];
char* remote_command = ssh_build_remote_command(server_path, old_args);
if (!remote_command)
ssh_child_setup_failed(exec_pipe[1]);
ssh_argv[ac++] = "ssh";
ssh_argv[ac++] = "-o";
ssh_argv[ac++] = "Compression=no";
@@ -148,14 +206,11 @@ Client* client_connect_ssh(const char* destination, int port, const char* server
ssh_argv[ac++] = port_str;
}
ssh_argv[ac++] = ssh_user;
ssh_argv[ac++] = (char*)(server_path ? server_path : "fastsync-server");
ssh_argv[ac++] = "--stdio";
ssh_argv[ac++] = remote_command;
ssh_argv[ac] = NULL;
execvp("ssh", ssh_argv);
log_perror("exec of ssh failed");
ssize_t wret = write(exec_pipe[1], "x", 1);
(void)wret;
_exit(1);
ssh_child_setup_failed(exec_pipe[1]);
}
close(sv[1]);
@@ -165,12 +220,15 @@ Client* client_connect_ssh(const char* destination, int port, const char* server
ssize_t n = read(exec_pipe[0], &exec_status, 1);
close(exec_pipe[0]);
if (n > 0) {
if (n != 0) {
close(sv[0]);
waitpid(pid, NULL, 0);
remote_dest_destroy(&r);
const char* path = server_path ? server_path : "fastsync-server";
char* escaped = output_escape(path, false);
fprintf(stderr, "Error: could not launch '%s --stdio' on remote\n",
server_path ? server_path : "fastsync-server");
escaped ? escaped : "<allocation failed>");
free(escaped);
return NULL;
}
+3 -1
View File
@@ -3,6 +3,8 @@
#include "transport_tcp.h"
Client* client_connect_ssh(const char* destination, int port, const char* server_path);
Client* client_connect_ssh(const char* destination, int port, const char* server_path,
bool old_args);
char* ssh_build_remote_command(const char* server_path, bool old_args);
#endif
+5 -1
View File
@@ -1,6 +1,7 @@
#include "transport_tcp.h"
#include "log.h"
#include "protocol.h"
#include "utils.h"
#include <arpa/inet.h>
#include <errno.h>
#include <netdb.h>
@@ -190,7 +191,10 @@ bool tcp_connect_socket(Client* client, char* host, int port) {
int err = getaddrinfo(host, port_str, &hints, &result);
if (err != 0 || result == NULL) {
fprintf(stderr, "Could not resolve host: %s (%s)\n", host, gai_strerror(err));
char* escaped_host = output_escape(host, false);
fprintf(stderr, "Could not resolve host: %s (%s)\n",
escaped_host ? escaped_host : "<allocation failed>", gai_strerror(err));
free(escaped_host);
return false;
}
+13 -3
View File
@@ -2,6 +2,7 @@
#include "log.h"
#include "protocol.h"
#include "transport_tcp.h"
#include "utils.h"
#include <arpa/inet.h>
#include <openssl/err.h>
#include <openssl/ssl.h>
@@ -65,13 +66,19 @@ static SSL_CTX* create_ssl_ctx(bool is_server, const char* cert, const char* key
return NULL;
}
if (SSL_CTX_use_certificate_file(ctx, cert, SSL_FILETYPE_PEM) <= 0) {
log_message(LOG_LEVEL_ERROR, "Failed to load certificate: %s", cert);
char* escaped = output_escape(cert, false);
log_message(LOG_LEVEL_ERROR, "Failed to load certificate: %s",
escaped ? escaped : "<allocation failed>");
free(escaped);
log_ssl_errors();
SSL_CTX_free(ctx);
return NULL;
}
if (SSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM) <= 0) {
log_message(LOG_LEVEL_ERROR, "Failed to load private key: %s", key);
char* escaped = output_escape(key, false);
log_message(LOG_LEVEL_ERROR, "Failed to load private key: %s",
escaped ? escaped : "<allocation failed>");
free(escaped);
log_ssl_errors();
SSL_CTX_free(ctx);
return NULL;
@@ -85,7 +92,10 @@ static SSL_CTX* create_ssl_ctx(bool is_server, const char* cert, const char* key
if (ca_path) {
if (!SSL_CTX_load_verify_locations(ctx, ca_path, NULL)) {
log_message(LOG_LEVEL_ERROR, "Failed to load CA: %s", ca_path);
char* escaped = output_escape(ca_path, false);
log_message(LOG_LEVEL_ERROR, "Failed to load CA: %s",
escaped ? escaped : "<allocation failed>");
free(escaped);
log_ssl_errors();
SSL_CTX_free(ctx);
return NULL;
+49 -1
View File
@@ -1,6 +1,7 @@
#include "utils.h"
#include "array_list.h"
#include "libgen.h"
#include "log.h"
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
@@ -132,6 +133,32 @@ char* str_dup(const char* string) {
return new_string;
}
char* output_escape(const char* string, bool eight_bit_output) {
if (!string)
return NULL;
size_t length = strlen(string);
if (length > (SIZE_MAX - 1) / 5)
return NULL;
char* escaped = malloc(length * 5 + 1);
if (!escaped)
return NULL;
size_t out = 0;
for (size_t i = 0; i < length; i++) {
unsigned char byte = (unsigned char)string[i];
if ((byte >= 32 && byte <= 126) || (eight_bit_output && byte >= 128)) {
escaped[out++] = (char)byte;
} else {
escaped[out++] = '\\';
escaped[out++] = '#';
escaped[out++] = (char)('0' + ((byte >> 6) & 7));
escaped[out++] = (char)('0' + ((byte >> 3) & 7));
escaped[out++] = (char)('0' + (byte & 7));
}
}
escaped[out] = '\0';
return escaped;
}
/* Match a glob pattern against a string. Supported wildcards:
* ? matches any single character except '/'.
* * matches any sequence of characters within one path component (no '/').
@@ -187,6 +214,25 @@ bool glob_match(const char* pattern, const char* str) {
return *str == '\0';
}
bool format_human_bytes(unsigned long long bytes, char* buffer, size_t buffer_size) {
static const char* const units[] = {"B", "KB", "MB", "GB", "TB", "PB", "EB"};
double value = (double)bytes;
size_t unit = 0;
int written;
if (!buffer || buffer_size == 0)
return false;
while (value >= 1024.0 && unit < sizeof(units) / sizeof(units[0]) - 1) {
value /= 1024.0;
unit++;
}
if (unit == 0)
written = snprintf(buffer, buffer_size, "%llu %s", bytes, units[unit]);
else
written = snprintf(buffer, buffer_size, "%.1f %s", value, units[unit]);
return written >= 0 && (size_t)written < buffer_size;
}
static bool is_dir_in_manifest(const char* rel_path, ArrayList* manifest) {
size_t len = strlen(rel_path);
for (int i = 0; i < manifest->size; i++) {
@@ -274,7 +320,9 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifes
} else {
(*deleted_count)++;
}
fprintf(stderr, " Deleted: %s\n", child_rel);
char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
free(escaped_path);
}
}
free(child_rel);
+2
View File
@@ -7,6 +7,7 @@
bool mkdir_r(const char* path);
char* str_dup(const char* string);
char* output_escape(const char* string, bool eight_bit_output);
char* path_cat(const char* path1, const char* path2);
bool glob_match(const char* pattern, const char* str);
bool delete_extras(const char* dest_root, ArrayList* manifest);
@@ -17,5 +18,6 @@ bool utils_set_authorized_root(int fd, const char* canonical_path);
void utils_set_authorized_root_fd(int fd);
bool has_path_traversal(const char* path);
bool utils_valid_batch_path(const char* path);
bool format_human_bytes(unsigned long long bytes, char* buffer, size_t buffer_size);
#endif
+508
View File
@@ -26,6 +26,12 @@ def setup_test_data():
class TestDryRun:
def test_human_readable_dry_run(self):
result, dur = run_client(SOURCE_DIR, DEST_DIR, flags=["-h", "--dry-run"])
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
assert "Total:" in result.stdout
assert "KB" in result.stdout
def test_dry_run(self):
clean_dir(DEST_DIR)
result, dur = run_client(
@@ -35,6 +41,118 @@ class TestDryRun:
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
assert "Dry run:" in result.stdout, f"No dry run output: {result.stdout[:200]}"
def test_quiet_suppresses_dry_run_output(self):
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-q", "-n", "--progress", "--stats"],
)
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
assert result.stdout == ""
assert result.stderr == ""
def test_quiet_preserves_errors(self):
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--quiet", "--server-port", "1"],
)
assert result.returncode != 0
assert result.stderr != ""
@pytest.mark.parametrize("flags", [["-q", "-v"], ["-v", "-q"]])
def test_quiet_successful_transfer_and_verbose_order(self, shared_server, flags):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=flags,
port=shared_server.port,
)
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
assert result.stdout == ""
assert result.stderr == ""
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
class TestRemoveSourceFiles:
def test_removes_only_transferred_regular_files(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "remove_source")
dest = os.path.join(TEST_DATA_DIR, "remove_dest")
clean_dir(source)
clean_dir(dest)
with open(os.path.join(source, "one.txt"), "wb") as f:
f.write(b"one")
with open(os.path.join(source, "two.txt"), "wb") as f:
f.write(b"two")
os.makedirs(os.path.join(source, "directory"))
os.symlink("one.txt", os.path.join(source, "link.txt"))
result, _ = run_client(source, dest, flags=["--remove-source-files", "-m"],
port=shared_server.port)
assert result.returncode == 0, f"Remove-source sync failed: {result.stderr[:200]}"
assert not os.path.exists(os.path.join(source, "one.txt"))
assert not os.path.exists(os.path.join(source, "two.txt"))
assert os.path.isdir(os.path.join(source, "directory"))
assert os.path.islink(os.path.join(source, "link.txt"))
def test_single_threaded_removes_transferred_file(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "remove_single_source")
dest = os.path.join(TEST_DATA_DIR, "remove_single_dest")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "file.txt")
with open(source_file, "wb") as f:
f.write(b"single threaded")
result, _ = run_client(source, dest, flags=["--remove-source-files"],
port=shared_server.port)
assert result.returncode == 0
assert not os.path.exists(source_file)
def test_dry_run_preserves_source_files(self):
source = os.path.join(TEST_DATA_DIR, "remove_dry_source")
dest = os.path.join(TEST_DATA_DIR, "remove_dry_dest")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "file.txt")
with open(source_file, "wb") as f:
f.write(b"keep")
result, _ = run_client(source, dest, flags=["--remove-source-files", "--dry-run"])
assert result.returncode == 0
assert os.path.isfile(source_file)
def test_failed_connection_preserves_source_files(self):
source = os.path.join(TEST_DATA_DIR, "remove_failed_source")
dest = os.path.join(TEST_DATA_DIR, "remove_failed_dest")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "file.txt")
with open(source_file, "wb") as f:
f.write(b"keep after failure")
result, _ = run_client(source, dest, flags=["--remove-source-files"], port=1)
assert result.returncode != 0
assert os.path.isfile(source_file)
def test_incremental_skip_preserves_source_file(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "remove_skipped_source")
dest = os.path.join(TEST_DATA_DIR, "remove_skipped_dest")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "file.txt")
with open(source_file, "wb") as f:
f.write(b"keep after skip")
result, _ = run_client(source, dest, port=shared_server.port)
assert result.returncode == 0
result, _ = run_client(source, dest,
flags=["--remove-source-files", "--incremental"],
port=shared_server.port)
assert result.returncode == 0
assert os.path.isfile(source_file)
class TestArchiveMode:
def test_archive_mode(self, shared_server):
@@ -51,6 +169,136 @@ class TestArchiveMode:
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
def test_archive_implied_options_can_be_negated(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--archive", "--no-compress", "--no-m", "--no-preserve"],
port=shared_server.port,
)
if result.returncode != 0:
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
class TestExecutability:
def test_preserves_only_executable_bits(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "executability_source")
dest = os.path.join(TEST_DATA_DIR, "executability_dest")
clean_dir(source)
clean_dir(dest)
source_file = os.path.join(source, "tool.sh")
with open(source_file, "w") as f:
f.write("#!/bin/sh\necho test\n")
os.chmod(source_file, 0o751)
result, _ = run_client(source, dest, flags=["-E"], port=shared_server.port)
assert result.returncode == 0, f"Executability sync failed: {result.stderr[:200]}"
received_file = os.path.join(get_dest_received_dir(dest, source), "tool.sh")
received_mode = os.stat(received_file).st_mode
assert received_mode & 0o111 == 0o111
assert received_mode & 0o600 == 0o600
assert received_mode & 0o077 == 0o011
class TestChmod:
def test_chmod_applies_to_transferred_files(self, shared_server):
clean_dir(DEST_DIR)
source_file = os.path.join(SOURCE_DIR, "small.txt")
os.chmod(source_file, 0o777)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--chmod=u=rw,go=r"],
port=shared_server.port,
)
if result.returncode != 0:
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
assert (os.stat(os.path.join(received, "small.txt")).st_mode & 0o777) == 0o644
class TestCompressionChoice:
def test_zstd_choice_compresses(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--zc", "zstd"],
port=shared_server.port)
assert result.returncode == 0, f"zstd sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
def test_none_choice_disables_compression(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-z", "--compress-choice", "none"],
port=shared_server.port)
assert result.returncode == 0, f"none sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
class TestSkipCompress:
def test_skip_compress_case_insensitive(self, shared_server):
clean_dir(DEST_DIR)
with open(os.path.join(SOURCE_DIR, "skip-case.TXT"), "wb") as f:
f.write((b"skip compression case test\n" * 100))
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-c", "--skip-compress=.txt"],
port=shared_server.port,
)
assert result.returncode == 0, f"Skip-compress sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
with open(os.path.join(received, "skip-case.TXT"), "rb") as f:
assert f.read() == b"skip compression case test\n" * 100
def test_skip_compress_empty_list(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-c", "--skip-compress="],
port=shared_server.port,
)
assert result.returncode == 0, f"Empty skip-compress sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
def test_skip_compress_incremental_full_fallback(self, shared_server):
clean_dir(DEST_DIR)
path = os.path.join(SOURCE_DIR, "incremental-skip.TXT")
with open(path, "wb") as f:
f.write(b"original skipped content\n")
flags = ["-c", "-M", "--skip-compress=.txt"]
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"Initial sync failed: {(result.stderr or result.stdout)[:200]}"
with open(path, "wb") as f:
f.write(b"updated skipped content\n")
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=flags + ["--incremental"],
port=shared_server.port,
)
assert result.returncode == 0, f"Incremental sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
with open(os.path.join(received, "incremental-skip.TXT"), "rb") as f:
assert f.read() == b"updated skipped content\n"
def test_skip_compress_rejects_chunk_serialization(self, shared_server):
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-c", "-s", "--skip-compress=.txt"],
port=shared_server.port,
)
assert result.returncode != 0
assert "cannot be combined" in (result.stderr or result.stdout)
class TestExclude:
def test_exclude_single(self, shared_server):
@@ -213,6 +461,143 @@ class TestIncremental:
with open(received_file, "rb") as f:
assert f.read() == b"hello world\n"
def test_size_only_skips_same_size_with_different_mtime(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
received_file = os.path.join(received, "small.txt")
with open(received_file, "wb") as f:
f.write(b"different!!\n")
os.utime(received_file, (time.time() - 3600, time.time() - 3600))
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-M", "--incremental", "--size-only"],
port=shared_server.port,
)
assert result.returncode == 0, f"Size-only sync failed: {result.stderr[:200]}"
with open(received_file, "rb") as f:
assert f.read() == b"different!!\n"
def test_ignore_times_transfers_same_size_and_mtime(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
source_file = os.path.join(SOURCE_DIR, "small.txt")
received_file = os.path.join(received, "small.txt")
source_stat = os.stat(source_file)
with open(received_file, "wb") as f:
f.write(b"stale data!\n")
os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime))
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["-M", "--incremental", "--ignore-times"],
port=shared_server.port)
assert result.returncode == 0, f"Ignore-times sync failed: {result.stderr[:200]}"
with open(received_file, "rb") as f:
assert f.read() == b"hello world\n"
def test_modify_window_allows_subsecond_mtime_difference(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
source_file = os.path.join(SOURCE_DIR, "small.txt")
received_file = os.path.join(received, "small.txt")
source_stat = os.stat(source_file)
with open(received_file, "wb") as f:
f.write(b"modified!!!\n")
os.utime(received_file, ns=(source_stat.st_atime_ns,
source_stat.st_mtime_ns - 1500000000))
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["-M", "--incremental", "--modify-window=2"],
port=shared_server.port)
assert result.returncode == 0, f"Modify-window sync failed: {result.stderr[:200]}"
with open(received_file, "rb") as f:
assert f.read() == b"modified!!!\n"
def test_whole_file_disables_delta_and_keeps_compression(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0
source_file = os.path.join(SOURCE_DIR, "medium.txt")
with open(source_file, "wb") as f:
f.write(b"whole-file replacement\n" * 5000)
result, _ = run_client(
SOURCE_DIR,
DEST_DIR,
flags=["-M", "--incremental", "--delta", "-W", "-c"],
port=shared_server.port,
)
assert result.returncode == 0, f"Whole-file sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
class TestUpdate:
def test_update_skips_older_destination_and_allows_equal_or_newer_source(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
source_file = os.path.join(SOURCE_DIR, "small.txt")
received_file = os.path.join(received, "small.txt")
source_stat = os.stat(source_file)
with open(received_file, "wb") as f:
f.write(b"newer destination\n")
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns + 10_000_000_000))
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
assert result.returncode == 0
with open(received_file, "rb") as f:
assert f.read() == b"newer destination\n"
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns))
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
assert result.returncode == 0
with open(received_file, "rb") as f:
assert f.read() == b"hello world\n"
def test_update_skips_unreadable_newer_destination(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
assert result.returncode == 0
received_file = os.path.join(get_dest_received_dir(DEST_DIR, SOURCE_DIR), "small.txt")
source_stat = os.stat(os.path.join(SOURCE_DIR, "small.txt"))
with open(received_file, "wb") as f:
f.write(b"protected destination\n")
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns + 10_000_000_000))
original_mode = os.stat(received_file).st_mode
try:
os.chmod(received_file, 0)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
assert result.returncode == 0
os.chmod(received_file, original_mode)
with open(received_file, "rb") as f:
assert f.read() == b"protected destination\n"
finally:
os.chmod(received_file, original_mode)
with open(received_file, "wb") as f:
f.write(b"older destination\n")
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns - 10_000_000_000))
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
assert result.returncode == 0
with open(received_file, "rb") as f:
assert f.read() == b"hello world\n"
class TestExisting:
def test_existing_updates_existing_and_skips_new(self, shared_server):
@@ -242,6 +627,60 @@ class TestExisting:
f.write(b"hello world\n")
class TestIgnoreExisting:
def test_ignore_existing_preserves_existing_and_transfers_new(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
existing_file = os.path.join(received, "small.txt")
with open(existing_file, "wb") as f:
f.write(b"destination content\n")
new_source = os.path.join(SOURCE_DIR, "new.txt")
try:
with open(new_source, "wb") as f:
f.write(b"new file\n")
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["--ignore-existing"], port=shared_server.port)
assert result.returncode == 0, f"Sync failed: {(result.stderr or result.stdout)[:200]}"
with open(existing_file, "rb") as f:
assert f.read() == b"destination content\n"
with open(os.path.join(received, "new.txt"), "rb") as f:
assert f.read() == b"new file\n"
finally:
if os.path.lexists(new_source):
os.unlink(new_source)
class TestIgnoreExisting:
def test_ignore_existing_preserves_existing_and_transfers_new(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
existing_file = os.path.join(received, "small.txt")
with open(existing_file, "wb") as f:
f.write(b"destination content\n")
new_source = os.path.join(SOURCE_DIR, "new.txt")
try:
with open(new_source, "wb") as f:
f.write(b"new file\n")
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["--ignore-existing"], port=shared_server.port)
assert result.returncode == 0, f"Sync failed: {(result.stderr or result.stdout)[:200]}"
with open(existing_file, "rb") as f:
assert f.read() == b"destination content\n"
with open(os.path.join(received, "new.txt"), "rb") as f:
assert f.read() == b"new file\n"
finally:
if os.path.lexists(new_source):
os.unlink(new_source)
class TestDelete:
def test_delete_removes_extra_files(self, shared_server):
clean_dir(DEST_DIR)
@@ -291,6 +730,75 @@ class TestProgress:
assert "Sent " in output and "MB" in output, "--progress produced no stable byte marker"
assert "Done." in output, "--progress did not report completion"
def test_human_readable_stats(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-h", "--stats"],
port=shared_server.port,
)
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
assert "Stats:" in result.stderr
assert "KB" in result.stderr
def test_human_readable_progress_multithreaded(self, shared_server):
clean_dir(DEST_DIR)
result, dur = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-m", "-h", "--progress"],
port=shared_server.port,
)
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
output = result.stdout + result.stderr
assert "Sent " in output
assert "KB" in output
assert "Done." in output
class TestInfo:
def test_info_copy_reports_transfers(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--info=copy"],
port=shared_server.port,
)
assert result.returncode == 0, f"Info sync failed: {(result.stderr or result.stdout)[:200]}"
output = result.stdout + result.stderr
assert "[INFO]" in output and "Transferring" in output
def test_info_stats_reports_multithreaded_transfer(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-m", "--info=stats"],
port=shared_server.port,
)
assert result.returncode == 0, f"Info stats sync failed: {(result.stderr or result.stdout)[:200]}"
output = result.stdout + result.stderr
assert "[INFO]" in output and "Transfer summary:" in output
def test_info_rejects_unknown_flag(self):
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["--info=unknown"],
)
assert result.returncode != 0
assert "unsupported --info flag" in result.stderr
@pytest.mark.parametrize("flags", [
["--info=none", "--verbose"],
["--verbose", "--info=none"],
])
def test_info_none_suppresses_verbose_info(self, shared_server, flags):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
assert result.returncode == 0, f"Info sync failed: {(result.stderr or result.stdout)[:200]}"
output = result.stdout + result.stderr
assert "[INFO]" not in output
assert "Transferring" not in output
assert "Transfer summary:" not in output
class TestBandwidthLimit:
def test_bwlimit_runs(self, shared_server):
+5
View File
@@ -71,6 +71,11 @@ class TestTCPFlags:
r = _run_tcp_test("Compression (-c)", shared_server.port, ["-c"])
assert r["status"] == "Success", r["error"]
def test_compression_threads(self, shared_server):
r = _run_tcp_test("Compression threads (-c --compress-threads=2)", shared_server.port,
["-c", "--compress-threads=2"])
assert r["status"] == "Success", r["error"]
def test_chunk_serialization(self, shared_server):
r = _run_tcp_test("Chunk Serialization (-s)", shared_server.port, ["-s"])
assert r["status"] == "Success", r["error"]
+796 -10
View File
@@ -1,8 +1,11 @@
#include "test_client_cli.h"
#include "client_validation.h"
#include "chmod.h"
#include "config.h"
#include "log.h"
#include "test_utils.h"
#include "utils.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
@@ -36,6 +39,15 @@ static void test_validate_config_incompatible_options() {
cfg->use_incremental = true;
cfg->use_chunk_serialization = true;
EXPECT_FALSE(validate_config(cfg));
cfg->use_incremental = false;
cfg->skip_compress_set = true;
EXPECT_FALSE(validate_config(cfg));
cfg->skip_compress_set = false;
cfg->compression_threads = 2;
EXPECT_FALSE(validate_config(cfg));
cfg->use_compression = true;
cfg->use_sendfile = false;
EXPECT_TRUE(validate_config(cfg));
config_delete(cfg);
}
@@ -59,6 +71,13 @@ static void test_validate_config_delta_sendfile_constraints() {
cfg->use_incremental = true;
cfg->use_sendfile = true;
EXPECT_FALSE(validate_config(cfg));
/* Whole-file makes delta selection inactive, so these combinations are valid. */
cfg->whole_file = true;
EXPECT_TRUE(validate_config(cfg));
cfg->use_incremental = false;
EXPECT_TRUE(validate_config(cfg));
config_delete(cfg);
}
@@ -107,6 +126,27 @@ static void test_cli_dry_run() {
config_delete(cfg);
}
static void test_cli_remove_source_files() {
Config* cfg = config_create();
EXPECT_NOT_NULL(cfg);
EXPECT_FALSE(cfg->remove_source_files);
cfg->remove_source_files = true;
EXPECT_TRUE(cfg->remove_source_files);
config_delete(cfg);
}
static void test_parse_args_remove_source_files() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--remove-source-files", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
int ret = parse_args(cfg, 4, argv, positional_args, &positional_count);
EXPECT_EQ_INT(ret, 0);
EXPECT_TRUE(cfg->remove_source_files);
config_delete(cfg);
}
/* Test that --delete sets use_delete */
static void test_cli_delete_flag() {
Config* cfg = config_create();
@@ -181,6 +221,80 @@ static void test_parse_args_valid_port() {
config_delete(cfg);
}
/* Test parse_args with --size-only. */
static void test_parse_args_size_only() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--size-only", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->size_only);
EXPECT_EQ_INT(positional_count, 2);
config_delete(cfg);
}
static void test_parse_args_ignore_existing() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--ignore-existing", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
int ret = parse_args(cfg, 4, argv, positional_args, &positional_count);
EXPECT_EQ_INT(ret, 0);
EXPECT_TRUE(cfg->ignore_existing);
config_delete(cfg);
}
static void test_parse_args_executability() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "-E", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->use_executability);
EXPECT_TRUE(cfg->use_metadata);
config_delete(cfg);
}
static void test_parse_args_chmod() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--chmod=u=rw,go=r", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_EQ_STR(cfg->chmod_spec, "u=rw,go=r");
EXPECT_TRUE(cfg->use_metadata);
mode_t result;
EXPECT_TRUE(chmod_apply(0777, cfg->chmod_spec, &result));
EXPECT_EQ_INT(result, 0644);
config_delete(cfg);
}
static void test_parse_args_numeric_chmod() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--chmod", "7777", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
EXPECT_EQ_STR(cfg->chmod_spec, "7777");
EXPECT_TRUE(cfg->use_metadata);
config_delete(cfg);
}
static void test_parse_args_rejects_invalid_chmod() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--chmod=a+X", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
/* Test parse_args rejects port > 65535 */
static void test_parse_args_invalid_port() {
Config* cfg = config_create();
@@ -247,6 +361,178 @@ static void test_parse_args_valid_compression_level() {
config_delete(cfg);
}
static void test_parse_args_debug_flags() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--debug=io,proto,pack,util", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_ALL);
EXPECT_EQ_INT(get_log_debug_flags(), LOG_DEBUG_ALL);
config_delete(cfg);
}
static void test_parse_args_debug_help() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--debug=help"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 2, argv, positional_args, &positional_count), 1);
config_delete(cfg);
}
static void test_parse_args_debug_flags_validation() {
static const char* const values[] = {"", "io,", ",io", "io,,proto", "acl", "tls", "unknown"};
for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) {
Config* cfg = config_create();
char option[64];
snprintf(option, sizeof(option), "--debug=%s", values[i]);
char* argv[] = {"fastsync", option, "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
}
static void test_parse_args_modify_window() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--modify-window=3", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->modify_window, 3);
config_delete(cfg);
cfg = config_create();
char* short_argv[] = {"fastsync", "-@", "7", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, short_argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->modify_window, 7);
config_delete(cfg);
cfg = config_create();
char* attached_argv[] = {"fastsync", "-@11", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, attached_argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->modify_window, 11);
config_delete(cfg);
}
static void test_parse_args_rejects_invalid_modify_window() {
const char* values[] = {"-1", "not-a-number", ""};
for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--modify-window", (char*)values[i], "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
}
static void test_parse_args_max_alloc_sizes() {
const char* values[] = {"1", "4K", "2m", "3G", "1T", "1P", "1E", "512B"};
const unsigned long long expected[] = {1,
4ULL * 1024,
2ULL * 1024 * 1024,
3ULL * 1024 * 1024 * 1024,
1ULL * 1024 * 1024 * 1024 * 1024,
1ULL * 1024 * 1024 * 1024 * 1024 * 1024,
1ULL * 1024 * 1024 * 1024 * 1024 * 1024 * 1024,
512};
for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--max-alloc", (char*)values[i], "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->max_alloc == expected[i]);
config_delete(cfg);
}
Config* cfg = config_create();
char* argv[] = {"fastsync", "--max-alloc=8M", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->max_alloc == 8ULL * 1024 * 1024);
config_delete(cfg);
}
static void test_parse_args_rejects_invalid_max_alloc() {
const char* values[] = {"0", "-1", "+1", " 1", "1 ",
"1Z", "1K2", "1 K", "1\tK", "18446744073709551615K"};
for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--max-alloc", (char*)values[i], "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
}
static void test_parse_args_skip_compress() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--skip-compress=.ZIP, .GZ", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->skip_compress_set);
EXPECT_EQ_INT(cfg->skip_compress_count, 2);
EXPECT_EQ_STR(cfg->skip_compress_suffixes[0], ".ZIP");
EXPECT_EQ_STR(cfg->skip_compress_suffixes[1], ".GZ");
config_delete(cfg);
}
static void test_parse_args_empty_skip_compress() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--skip-compress=", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->skip_compress_set);
EXPECT_EQ_INT(cfg->skip_compress_count, 0);
config_delete(cfg);
}
static void test_parse_args_compression_threads() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--compress-threads", "4", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->compression_threads, 4);
config_delete(cfg);
cfg = config_create();
char* equals_argv[] = {"fastsync", "--compress-threads=3", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, equals_argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->compression_threads, 3);
config_delete(cfg);
cfg = config_create();
char* invalid_argv[] = {"fastsync", "--compress-threads=0", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, invalid_argv, positional_args, &positional_count), -1);
config_delete(cfg);
cfg = config_create();
char* excessive_argv[] = {"fastsync", "--compress-threads=65", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, excessive_argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
/* Test parse_args unknown option returns error */
static void test_parse_args_unknown_option() {
Config* cfg = config_create();
@@ -260,11 +546,53 @@ static void test_parse_args_unknown_option() {
config_delete(cfg);
}
/* Directory aliases must report the unsupported directory-only behavior clearly. */
static void test_parse_args_rejects_dirs_aliases() {
static const char* const options[] = {"--dirs", "--old-dirs", "--old-d"};
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", (char*)options[i], "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
FILE* log_file = tmpfile();
char log_buffer[256] = {0};
EXPECT_NOT_NULL(log_file);
log_set_file(log_file);
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), -1);
fflush(log_file);
rewind(log_file);
EXPECT_TRUE(fread(log_buffer, 1, sizeof(log_buffer) - 1, log_file) > 0);
EXPECT_TRUE(strstr(log_buffer, options[i]) != NULL);
EXPECT_TRUE(strstr(log_buffer, "directory-only transfer is not implemented") != NULL);
EXPECT_TRUE(strstr(log_buffer, "requires --dirs") == NULL);
log_set_file(NULL);
fclose(log_file);
config_delete(cfg);
}
}
/* --del is recognized as the rsync alias, but its timing mode is not implemented. */
static void test_parse_args_delete_during_alias_unimplemented() {
static const char* const options[] = {"--del", "--delete-during"};
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", (char*)options[i], "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), -1);
EXPECT_FALSE(cfg->use_delete);
config_delete(cfg);
}
}
/* Parsed-but-unimplemented options must fail instead of being silently accepted. */
static void test_parse_args_rejects_unimplemented_options() {
static const char* const options[] = {"-q",
"--quiet",
"--silent",
static const char* const options[] = {"--silent",
"--queue-size",
"-H",
"--hard-links",
@@ -277,13 +605,7 @@ static void test_parse_args_rejects_unimplemented_options() {
"-i",
"--itemize-changes",
"--out-format",
"--info",
"--debug",
"--list-only",
"-h",
"--human-readable",
"-u",
"--update",
"--append",
"--append-verify",
"--delete-excluded",
@@ -310,7 +632,7 @@ static void test_parse_args_rejects_unimplemented_options() {
"--daemon",
"--config",
"--server",
"--compress-choice"};
"--checksum-choice"};
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
Config* cfg = config_create();
@@ -323,6 +645,92 @@ static void test_parse_args_rejects_unimplemented_options() {
}
}
/* Test both rsync-compatible quiet spellings and option ordering. */
static void test_parse_args_quiet() {
static const char* const options[][2] = {
{"-q", "-v"}, {"-v", "-q"}, {"--quiet", "-v"}, {"-v", "--quiet"}};
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", (char*)options[i][0], (char*)options[i][1], "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->quiet);
config_delete(cfg);
}
}
static void test_parse_args_human_readable() {
static const char* const options[] = {"-h", "--human-readable"};
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", (char*)options[i], "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->human_readable);
config_delete(cfg);
}
}
static void test_parse_args_update() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "-u", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->update);
EXPECT_TRUE(cfg->use_metadata);
EXPECT_EQ_INT(positional_count, 2);
config_delete(cfg);
}
static void test_parse_args_info_flags() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--info=copy,skip", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->info_level, LOG_INFO_COPY | LOG_INFO_SKIP);
EXPECT_EQ_INT(get_log_info_flags(), LOG_INFO_COPY | LOG_INFO_SKIP);
config_delete(cfg);
}
static void test_parse_args_info_verbose_order() {
char* argv_info_first[] = {"fastsync", "--info=none", "--verbose", "/src", "/dst"};
char* argv_verbose_first[] = {"fastsync", "--verbose", "--info=none", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
Config* cfg = config_create();
EXPECT_EQ_INT(parse_args(cfg, 5, argv_info_first, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->info_level, 0);
EXPECT_EQ_INT(get_log_info_flags(), 0);
config_delete(cfg);
cfg = config_create();
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv_verbose_first, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->info_level, 0);
EXPECT_EQ_INT(get_log_info_flags(), 0);
config_delete(cfg);
}
static void test_parse_args_rejects_invalid_info_flag() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--info=copy,unknown", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
/* Test parse_args with --archive flag */
static void test_parse_args_archive() {
Config* cfg = config_create();
@@ -339,6 +747,121 @@ static void test_parse_args_archive() {
config_delete(cfg);
}
/* Negations must override archive's implied options in argument order. */
static void test_parse_args_negations() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--archive", "--no-compress", "--no-m",
"--no-preserve", "--no-dry-run", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 8, argv, positional_args, &positional_count), 0);
EXPECT_FALSE(cfg->use_compression);
EXPECT_FALSE(cfg->use_multithreading);
EXPECT_FALSE(cfg->use_metadata);
EXPECT_FALSE(cfg->dry_run);
EXPECT_EQ_INT(positional_count, 2);
config_delete(cfg);
}
static void test_parse_args_negation_order() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--no-z", "-c", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->use_compression);
config_delete(cfg);
}
static void test_parse_args_no_preserve_blocks_implicit_metadata() {
static const char* const options[][3] = {
{"--incremental", "--no-preserve", "/src"},
{"--no-preserve", "--incremental", "/src"},
{"--delta", "--no-preserve", "/src"},
{"--no-preserve", "--delta", "/src"},
};
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", (char*)options[i][0], (char*)options[i][1], (char*)options[i][2],
"/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
EXPECT_FALSE(cfg->use_metadata);
EXPECT_TRUE(cfg->metadata_explicitly_disabled);
config_delete(cfg);
}
}
/* Checksum-choice spellings are recognized and rejected until algorithms are implemented. */
static void test_parse_args_checksum_choice_aliases() {
static const char* const options[] = {"--checksum-choice", "--cc"};
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", (char*)options[i], "xxh64", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
}
static void test_parse_args_rejects_unsafe_negation() {
static const char* const options[] = {"--no-archive", "--no-timeout", "--no-unknown"};
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", (char*)options[i], "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
}
/* Both checksum-choice spellings require a value. */
static void test_parse_args_checksum_choice_requires_value() {
static const char* const options[] = {"--checksum-choice", "--cc"};
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", (char*)options[i]};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 2, argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
}
static void test_parse_args_old_args() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--old-args", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->old_args);
config_delete(cfg);
}
static void test_parse_args_fsync() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--fsync", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->use_fsync);
config_delete(cfg);
}
static void test_parse_args_existing() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--existing", "/src", "/dst"};
@@ -350,6 +873,221 @@ static void test_parse_args_existing() {
config_delete(cfg);
}
static void test_parse_args_ignore_times() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "-I", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->ignore_times);
config_delete(cfg);
cfg = config_create();
char* long_argv[] = {"fastsync", "--ignore-times", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, long_argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->ignore_times);
config_delete(cfg);
}
/* --secluded-args is accepted for compatibility but has no effect. */
static void test_parse_args_secluded_args() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--secluded-args", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_FALSE(cfg->use_chunk_serialization);
config_delete(cfg);
}
static void test_parse_args_short_s_remains_chunk_serialization() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "-s", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->use_chunk_serialization);
config_delete(cfg);
}
static void test_parse_args_8_bit_output() {
Config* cfg = config_create();
char* long_argv[] = {"fastsync", "--8-bit-output", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, long_argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->eight_bit_output);
config_delete(cfg);
cfg = config_create();
char* short_argv[] = {"fastsync", "-8", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, short_argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->eight_bit_output);
config_delete(cfg);
}
static void test_parse_args_stderr_modes() {
static const char* const modes[] = {"errors", "all", "e", "a"};
static const LogStderrMode expected[] = {LOG_STDERR_ERRORS, LOG_STDERR_ALL, LOG_STDERR_ERRORS,
LOG_STDERR_ALL};
for (size_t i = 0; i < sizeof(modes) / sizeof(modes[0]); i++) {
Config* cfg = config_create();
char option[32];
snprintf(option, sizeof(option), "--stderr=%s", modes[i]);
char* argv[] = {"fastsync", option, "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(log_get_stderr_mode(), expected[i]);
config_delete(cfg);
}
log_set_stderr_mode(LOG_STDERR_ERRORS);
}
static void test_parse_args_rejects_unsupported_stderr_modes() {
static const char* const modes[] = {"client", "c", "invalid"};
for (size_t i = 0; i < sizeof(modes) / sizeof(modes[0]); i++) {
Config* cfg = config_create();
char option[32];
snprintf(option, sizeof(option), "--stderr=%s", modes[i]);
char* argv[] = {"fastsync", option, "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
log_set_stderr_mode(LOG_STDERR_ERRORS);
}
/* Test both whole-file spellings and its precedence over delta selection. */
static void test_parse_args_whole_file() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--delta", "--incremental", "-W", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 6, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->whole_file);
EXPECT_TRUE(cfg->use_delta);
config_delete(cfg);
cfg = config_create();
char* long_argv[] = {"fastsync", "--whole-file", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, long_argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->whole_file);
config_delete(cfg);
}
/* Test rsync-compatible compression-choice and compression-level aliases. */
static void test_parse_args_compression_aliases() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--zc", "zstd", "--zl", "10", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
int ret = parse_args(cfg, 7, argv, positional_args, &positional_count);
EXPECT_EQ_INT(ret, 0);
EXPECT_EQ_STR(cfg->compress_choice, "zstd");
EXPECT_EQ_INT(cfg->compression_level, 10);
EXPECT_TRUE(cfg->use_compression);
EXPECT_EQ_INT(positional_count, 2);
config_delete(cfg);
}
/* Test rsync-compatible -P parsing; resumable partial-file retention is not implied. */
static void test_parse_args_partial_progress() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "-P", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
int ret = parse_args(cfg, 4, argv, positional_args, &positional_count);
EXPECT_EQ_INT(ret, 0);
EXPECT_TRUE(cfg->partial);
EXPECT_TRUE(cfg->show_progress);
EXPECT_EQ_INT(positional_count, 2);
config_delete(cfg);
}
static void test_parse_args_compression_equals_and_none() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "-z", "--zc=none", "--zl=7", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
int ret = parse_args(cfg, 6, argv, positional_args, &positional_count);
EXPECT_EQ_INT(ret, 0);
EXPECT_EQ_STR(cfg->compress_choice, "none");
EXPECT_EQ_INT(cfg->compression_level, 7);
EXPECT_FALSE(cfg->use_compression);
config_delete(cfg);
}
static void test_parse_args_compression_canonical_equals() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--compress-choice=zstd", "--compress-level=7", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
EXPECT_EQ_STR(cfg->compress_choice, "zstd");
EXPECT_EQ_INT(cfg->compression_level, 7);
EXPECT_TRUE(cfg->use_compression);
EXPECT_EQ_INT(positional_count, 2);
config_delete(cfg);
}
static void test_parse_args_compression_alias_equals() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--zc=zstd", "--zl=7", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
EXPECT_EQ_STR(cfg->compress_choice, "zstd");
EXPECT_EQ_INT(cfg->compression_level, 7);
EXPECT_TRUE(cfg->use_compression);
EXPECT_EQ_INT(positional_count, 2);
config_delete(cfg);
}
static void test_parse_args_rejects_invalid_compression_level_equals() {
static const char* const values[] = {"0", "23", "invalid"};
for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) {
Config* cfg = config_create();
char option[32];
snprintf(option, sizeof(option), "--compress-level=%s", values[i]);
char* argv[] = {"fastsync", option, "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
}
static void test_parse_args_rejects_invalid_compression_choice() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--compress-choice=bogus", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
void test_client_cli() {
test_validate_config_required_paths();
test_validate_config_incompatible_options();
@@ -358,18 +1096,66 @@ void test_client_cli() {
test_cli_help();
test_cli_archive_flags();
test_cli_dry_run();
test_cli_remove_source_files();
test_parse_args_remove_source_files();
test_cli_delete_flag();
test_cli_exclude_patterns();
test_parse_args_help();
test_parse_args_version();
test_parse_args_valid_port();
test_parse_args_size_only();
test_parse_args_ignore_existing();
test_parse_args_executability();
test_parse_args_chmod();
test_parse_args_numeric_chmod();
test_parse_args_rejects_invalid_chmod();
test_parse_args_invalid_port();
test_parse_args_non_numeric_port();
test_parse_args_invalid_server_port();
test_parse_args_invalid_compression_level();
test_parse_args_valid_compression_level();
test_parse_args_debug_flags();
test_parse_args_debug_help();
test_parse_args_debug_flags_validation();
test_parse_args_modify_window();
test_parse_args_rejects_invalid_modify_window();
test_parse_args_skip_compress();
test_parse_args_empty_skip_compress();
test_parse_args_compression_threads();
test_parse_args_max_alloc_sizes();
test_parse_args_rejects_invalid_max_alloc();
test_parse_args_unknown_option();
test_parse_args_rejects_dirs_aliases();
test_parse_args_delete_during_alias_unimplemented();
test_parse_args_rejects_unimplemented_options();
test_parse_args_quiet();
test_parse_args_human_readable();
test_parse_args_update();
test_parse_args_info_flags();
test_parse_args_info_verbose_order();
test_parse_args_rejects_invalid_info_flag();
test_parse_args_archive();
test_parse_args_negations();
test_parse_args_negation_order();
test_parse_args_no_preserve_blocks_implicit_metadata();
test_parse_args_rejects_unsafe_negation();
test_parse_args_old_args();
test_parse_args_fsync();
test_parse_args_existing();
test_parse_args_ignore_times();
test_parse_args_8_bit_output();
test_parse_args_stderr_modes();
test_parse_args_rejects_unsupported_stderr_modes();
test_parse_args_secluded_args();
test_parse_args_short_s_remains_chunk_serialization();
test_parse_args_whole_file();
test_parse_args_compression_aliases();
test_parse_args_compression_equals_and_none();
test_parse_args_compression_canonical_equals();
test_parse_args_compression_alias_equals();
test_parse_args_rejects_invalid_compression_level_equals();
test_parse_args_rejects_invalid_compression_choice();
test_parse_args_partial_progress();
test_parse_args_checksum_choice_aliases();
test_parse_args_checksum_choice_requires_value();
}
+27
View File
@@ -55,6 +55,31 @@ static void test_data_compress_decompress_large() {
data_destroy(decompressed);
}
static void test_skip_compress_suffix_matching() {
char* suffixes[] = {".ZIP", ".GZ"};
EXPECT_TRUE(compression_should_skip_with_suffixes("archive.zip", suffixes, 2));
EXPECT_TRUE(compression_should_skip_with_suffixes("backup.TAR.GZ", suffixes, 2));
EXPECT_FALSE(compression_should_skip_with_suffixes("notes.txt", suffixes, 2));
EXPECT_FALSE(compression_should_skip_with_suffixes("archive.zip", suffixes, 0));
}
static void test_data_compress_with_threads_roundtrip() {
const size_t size = 8 * 1024 * 1024;
Data* input = data_create_empty(size);
EXPECT_NOT_NULL(input);
for (size_t i = 0; i < size; i++)
((char*)input->data)[i] = (char)((i / 4096) % 7);
Data* compressed = data_compress_with_threads(input, 3, 2);
EXPECT_NOT_NULL(compressed);
Data* decompressed = data_decompress(compressed);
EXPECT_NOT_NULL(decompressed);
EXPECT_EQ_INT((int)decompressed->size, (int)size);
EXPECT_EQ_INT(memcmp(decompressed->data, input->data, size), 0);
data_destroy(input);
data_destroy(compressed);
data_destroy(decompressed);
}
static void test_chunk_compress_decompress_roundtrip() {
char* path1 = "temp_comp_test_1.txt";
char* content1 = "chunk compression test file 1";
@@ -115,5 +140,7 @@ static void test_chunk_compress_decompress_roundtrip() {
void test_compression() {
test_data_compress_decompress_roundtrip();
test_data_compress_decompress_large();
test_skip_compress_suffix_matching();
test_data_compress_with_threads_roundtrip();
test_chunk_compress_decompress_roundtrip();
}
+39 -4
View File
@@ -95,6 +95,7 @@ static void test_pipeline_sender_lifecycle() {
EXPECT_EQ_INT(pcs->queue_loader->capacity, 15);
EXPECT_FALSE(pcs->scanner_done);
EXPECT_FALSE(pcs->loader_done);
EXPECT_EQ_INT((int)pcs->allocation_session.max_alloc, (int)cfg->max_alloc);
pipeline_context_sender_destroy(pcs);
}
@@ -121,12 +122,25 @@ static void test_config_send_receive() {
send_cfg->receive_root_directory = str_dup("/send/dst");
send_cfg->save_to_disk = true;
send_cfg->use_multithreading = true;
send_cfg->use_chunk_serialization = true;
send_cfg->use_chunk_serialization = false;
send_cfg->use_compression = true;
send_cfg->use_metadata = true;
send_cfg->use_executability = true;
send_cfg->use_delta = true;
send_cfg->whole_file = true;
send_cfg->ignore_times = true;
send_cfg->size_only = true;
send_cfg->compression_level = 5;
send_cfg->chunk_size = 1024;
send_cfg->eight_bit_output = true;
send_cfg->modify_window = 4;
send_cfg->existing = true;
send_cfg->ignore_existing = true;
send_cfg->skip_compress_set = true;
send_cfg->skip_compress_count = 1;
send_cfg->skip_compress_suffixes = calloc(1, sizeof(char*));
send_cfg->skip_compress_suffixes[0] = str_dup(".zip");
send_cfg->max_alloc = MAX_SERVER_ALLOC + 1;
/* Use socketpair for bidirectional communication */
int p[2];
@@ -155,14 +169,33 @@ static void test_config_send_receive() {
ok = false;
if (!recv_cfg->use_multithreading)
ok = false;
if (!recv_cfg->use_chunk_serialization)
if (recv_cfg->use_chunk_serialization)
ok = false;
if (recv_cfg->compression_level != 5)
ok = false;
if (recv_cfg->chunk_size != 1024)
ok = false;
if (!recv_cfg->use_executability)
ok = false;
if (!recv_cfg->size_only)
ok = false;
if (!recv_cfg->ignore_times)
ok = false;
if (!recv_cfg->eight_bit_output)
ok = false;
if (recv_cfg->use_delta)
ok = false;
if (recv_cfg->modify_window != 4)
ok = false;
if (!recv_cfg->existing)
ok = false;
if (!recv_cfg->ignore_existing)
ok = false;
if (!recv_cfg->skip_compress_set || recv_cfg->skip_compress_count != 1 ||
strcmp(recv_cfg->skip_compress_suffixes[0], ".zip") != 0)
ok = false;
if (recv_cfg->max_alloc != MAX_SERVER_ALLOC)
ok = false;
}
config_delete(recv_cfg);
close(p[0]);
@@ -188,11 +221,11 @@ static void test_config_send_receive() {
}
static void test_config_send_receive_version_mismatch() {
/* Create a config with a different protocol version */
/* A peer using the previous wire format must be rejected. */
Config* cfg = config_create();
EXPECT_NOT_NULL(cfg);
free(cfg->version);
cfg->version = str_dup("0.0");
cfg->version = str_dup("2.3.0");
cfg->send_directory = str_dup("/src");
cfg->receive_root_directory = str_dup("/dst");
@@ -236,6 +269,8 @@ static void test_config_receive_truncated() {
/* A valid prefix exercises cleanup after allocated wire strings and a
* partially received scalar field. */
EXPECT_TRUE(send_str(p[1], PROTOCOL_VERSION));
unsigned long long max_alloc = DEFAULT_MAX_ALLOC;
EXPECT_TRUE(send_n_data(p[1], &max_alloc, sizeof(max_alloc)));
EXPECT_TRUE(send_str(p[1], "/src"));
EXPECT_TRUE(send_str(p[1], "/dst"));
EXPECT_TRUE(send_int(p[1], 1));
+133
View File
@@ -1,6 +1,7 @@
#include "test_file.h"
#include "file.h"
#include "data.h"
#include "config.h"
#include "utils.h"
#include "protocol.h"
#include "test_utils.h"
@@ -88,6 +89,30 @@ static void test_file_save_to_disk() {
rmdir("test_save_tmp");
}
static void test_file_save_to_disk_with_fsync_config() {
File* f = file_create("saved_file_fsync.txt");
EXPECT_NOT_NULL(f);
const char* content = "Save to disk with fsync";
f->data->data = malloc(strlen(content));
EXPECT_NOT_NULL(f->data->data);
memcpy(f->data->data, content, strlen(content));
f->data->size = strlen(content);
Config* config = config_create();
EXPECT_NOT_NULL(config);
config->use_fsync = true;
EXPECT_TRUE(file_save_to_disk("test_save_fsync_tmp", f, config));
struct stat st;
EXPECT_EQ_INT(stat("test_save_fsync_tmp/saved_file_fsync.txt", &st), 0);
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
file_destroy(f);
config_delete(config);
unlink("test_save_fsync_tmp/saved_file_fsync.txt");
rmdir("test_save_fsync_tmp");
}
static void test_file_save_to_disk_existing() {
const char* root = "test_existing_tmp";
const char* existing_path = "test_existing_tmp/existing.txt";
@@ -129,6 +154,99 @@ static void test_file_save_to_disk_existing() {
config_delete(cfg);
unlink(existing_path);
rmdir("test_existing_tmp");
}
static void test_file_save_to_disk_ignore_existing() {
const char* path = "test_ignore_existing_tmp/existing.txt";
EXPECT_TRUE(file_write_to_disk(path, "old", 3, false, false));
File* file = file_create("existing.txt");
EXPECT_NOT_NULL(file);
file->data->data = malloc(3);
EXPECT_NOT_NULL(file->data->data);
memcpy(file->data->data, "new", 3);
file->data->size = 3;
Config* config = config_create();
EXPECT_NOT_NULL(config);
config->ignore_existing = true;
EXPECT_TRUE(file_save_to_disk("test_ignore_existing_tmp", file, config));
FILE* stream = fopen(path, "rb");
char content[4] = {0};
EXPECT_NOT_NULL(stream);
// cppcheck-suppress knownConditionTrueFalse
if (stream) {
EXPECT_EQ_INT((int)fread(content, 1, 3, stream), 3);
fclose(stream);
}
EXPECT_EQ_STR(content, "old");
file_destroy(file);
config_delete(config);
unlink(path);
rmdir("test_ignore_existing_tmp");
}
static void test_file_save_to_disk_ignore_existing_entry_types() {
const char* root = "test_ignore_existing_entries_tmp";
const char* directory = "test_ignore_existing_entries_tmp/directory";
const char* link = "test_ignore_existing_entries_tmp/link";
const char* target = "test_ignore_existing_entries_tmp/target";
const char* backup = "test_ignore_existing_entries_tmp/backup.txt~";
const char* backup_file = "test_ignore_existing_entries_tmp/backup.txt";
Config* config = config_create();
File* file = file_create("unused");
unlink(link);
unlink(target);
unlink(backup);
unlink(backup_file);
rmdir(directory);
rmdir(root);
EXPECT_NOT_NULL(config);
EXPECT_NOT_NULL(file);
// cppcheck-suppress knownConditionTrueFalse
if (!config || !file)
return;
config->ignore_existing = true;
config->backup = true;
file->data->data = malloc(3);
EXPECT_NOT_NULL(file->data->data);
// cppcheck-suppress knownConditionTrueFalse
if (!file->data->data) {
file_destroy(file);
config_delete(config);
return;
}
memcpy(file->data->data, "new", 3);
file->data->size = 3;
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(directory, 0755), 0);
EXPECT_TRUE(file_write_to_disk(target, "old", 3, false, false));
EXPECT_EQ_INT(symlink("target", link), 0);
free(file->path);
file->path = str_dup("directory");
EXPECT_TRUE(file_save_to_disk(root, file, config));
free(file->path);
file->path = str_dup("link");
EXPECT_TRUE(file_save_to_disk(root, file, config));
free(file->path);
file->path = str_dup("backup.txt");
EXPECT_TRUE(file_write_to_disk(backup_file, "old", 3, false, false));
EXPECT_TRUE(file_save_to_disk(root, file, config));
EXPECT_TRUE(file_path_exists_secure(backup_file));
EXPECT_FALSE(file_path_exists_secure(backup));
file_destroy(file);
config_delete(config);
unlink(link);
unlink(target);
unlink(backup_file);
rmdir(directory);
rmdir(root);
}
@@ -152,6 +270,17 @@ static void test_file_write_to_disk_basic() {
unlink("test_file_write_to_disk_basic.txt");
}
static void test_file_write_to_disk_with_fsync() {
const char* path = "test_file_write_to_disk_fsync.txt";
const char* content = "fsync file content";
EXPECT_TRUE(file_to_disk_secure_with_fsync(path, content, strlen(content), false, false, NULL,
false, true));
struct stat st;
EXPECT_EQ_INT(stat(path, &st), 0);
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
unlink(path);
}
static void test_file_write_to_disk_creates_dirs() {
const char* content = "Nested dir test";
EXPECT_TRUE(file_write_to_disk("test_nested_tmp/nested/file.txt", content, strlen(content), false,
@@ -502,8 +631,12 @@ void test_file() {
test_file_load_data();
test_file_load_data_missing_file();
test_file_save_to_disk();
test_file_save_to_disk_with_fsync_config();
test_file_save_to_disk_existing();
test_file_save_to_disk_ignore_existing();
test_file_save_to_disk_ignore_existing_entry_types();
test_file_write_to_disk_basic();
test_file_write_to_disk_with_fsync();
test_file_write_to_disk_creates_dirs();
test_file_write_to_disk_does_not_follow_symlink();
test_file_content_to_buffer();
+26
View File
@@ -1,6 +1,8 @@
#include "test_log.h"
#include "log.h"
#include "test_utils.h"
#include <string.h>
#include <unistd.h>
/* Test default log level: WARNING and ERROR should print, DEBUG and INFO should not.
* We can't easily capture stderr in unit tests, so we verify the functions don't crash
@@ -90,6 +92,29 @@ static void test_log_filtering() {
EXPECT_TRUE(true);
}
static void test_log_stderr_mode_all() {
int pipe_fds[2];
EXPECT_EQ_INT(pipe(pipe_fds), 0);
int saved_stderr = dup(STDERR_FILENO);
EXPECT_TRUE(saved_stderr >= 0);
EXPECT_TRUE(dup2(pipe_fds[1], STDERR_FILENO) >= 0);
close(pipe_fds[1]);
set_log_level(LOG_LEVEL_WARNING);
log_set_stderr_mode(LOG_STDERR_ALL);
log_message(LOG_LEVEL_WARNING, "warning routed to stderr");
fflush(stderr);
EXPECT_TRUE(dup2(saved_stderr, STDERR_FILENO) >= 0);
close(saved_stderr);
char output[128] = {0};
ssize_t length = read(pipe_fds[0], output, sizeof(output) - 1);
close(pipe_fds[0]);
EXPECT_TRUE(length > 0);
EXPECT_TRUE(strstr(output, "warning routed to stderr") != NULL);
log_set_stderr_mode(LOG_STDERR_ERRORS);
}
/* Test that log_message handles various format strings */
static void test_log_message_formats() {
set_log_level(LOG_LEVEL_DEBUG);
@@ -112,5 +137,6 @@ void test_log() {
test_log_set_level_info();
test_log_set_level_error();
test_log_filtering();
test_log_stderr_mode_all();
test_log_message_formats();
}
+68 -1
View File
@@ -1,4 +1,5 @@
#include "test_metadata.h"
#include "chmod.h"
#include "metadata.h"
#include "protocol.h"
#include "test_utils.h"
@@ -122,6 +123,18 @@ static void test_metadata_rejects_invalid_values() {
close(p[1]);
}
static void test_metadata_mtime_window() {
EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 101, 600000000, 2));
EXPECT_FALSE(metadata_mtime_matches(100, 100000000, 102, 600000000, 2));
EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 102, 100000000, 2));
EXPECT_TRUE(metadata_mtime_matches(100, 900000000, 102, 100000000, 2));
EXPECT_FALSE(metadata_mtime_matches(100, 100000000, 102, 900000000, 2));
EXPECT_TRUE(metadata_mtime_matches(100, 900000000, 102, 900000000, 2));
EXPECT_FALSE(metadata_mtime_matches(100, 900000000, 101, 100000001, 0));
EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 100, 100000001, 0));
EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 100, 100000000, 0));
}
static void test_file_restore_metadata() {
const char* path = "temp_meta_restore_test.txt";
const char* content = "test content";
@@ -134,7 +147,7 @@ static void test_file_restore_metadata() {
m.mtime_sec = 1234567890;
m.mtime_nsec = 0;
file_restore_metadata(path, &m);
file_restore_metadata(path, &m, false);
struct stat st;
EXPECT_EQ_INT(stat(path, &st), 0);
@@ -144,6 +157,56 @@ static void test_file_restore_metadata() {
unlink(path);
}
static void test_file_restore_executability_only() {
const char* path = "temp_exec_restore_test.txt";
EXPECT_TRUE(file_write_to_disk(path, "x", 1, false, false));
EXPECT_EQ_INT(chmod(path, 0644), 0);
FileMetadata m = {
.mode = 0751, .uid = getuid(), .gid = getgid(), .mtime_sec = 0, .mtime_nsec = 0};
file_restore_metadata(path, &m, true);
struct stat st;
EXPECT_EQ_INT(stat(path, &st), 0);
EXPECT_EQ_INT(st.st_mode & 0777, 0755);
unlink(path);
}
static void test_directory_restore_executability_only() {
const char* path = "temp_exec_restore_test_dir";
EXPECT_EQ_INT(mkdir(path, 0700), 0);
FileMetadata m = {
.mode = 0755, .uid = getuid(), .gid = getgid(), .mtime_sec = 0, .mtime_nsec = 0};
file_restore_metadata(path, &m, true);
struct stat st;
EXPECT_EQ_INT(stat(path, &st), 0);
EXPECT_EQ_INT(st.st_mode & 0777, 0711);
rmdir(path);
}
static void test_chmod_changes() {
mode_t result;
EXPECT_TRUE(chmod_apply(0777, "u=rw,go=r", &result));
EXPECT_EQ_INT(result, 0644);
EXPECT_TRUE(chmod_apply(0644, "a+x", &result));
EXPECT_EQ_INT(result, 0755);
result = 0777;
EXPECT_TRUE(chmod_apply(0777, "0000", &result));
EXPECT_EQ_INT(result, 0000);
result = 0777;
EXPECT_TRUE(chmod_apply(0777, "7777", &result));
EXPECT_EQ_INT(result, 07777);
result = 0777;
EXPECT_TRUE(chmod_apply(0777, "755", &result));
EXPECT_EQ_INT(result, 0755);
EXPECT_FALSE(chmod_apply(0777, "888", &result));
EXPECT_FALSE(chmod_apply(0777, "10000", &result));
EXPECT_FALSE(chmod_apply(0777, "a+X", &result));
EXPECT_FALSE(chmod_apply(0777, "a+r,", &result));
}
void test_metadata() {
test_metadata_to_from_buf_roundtrip();
test_metadata_to_buf_null();
@@ -151,5 +214,9 @@ void test_metadata() {
test_metadata_send_receive_roundtrip();
test_metadata_send_null();
test_metadata_rejects_invalid_values();
test_metadata_mtime_window();
test_file_restore_metadata();
test_file_restore_executability_only();
test_directory_restore_executability_only();
test_chmod_changes();
}
+233
View File
@@ -3,6 +3,60 @@
#include <limits.h>
#include <string.h>
#include <unistd.h>
#include <threads.h>
typedef struct {
ProtocolSession* session;
bool allocation_allowed;
} AllocationWorkerArg;
static int allocation_worker(void* arg) {
AllocationWorkerArg* worker = arg;
protocol_session_bind(worker->session);
void* allocation = protocol_alloc(8);
worker->allocation_allowed = allocation != NULL;
free(allocation);
protocol_session_unbind();
return thrd_success;
}
typedef struct {
ProtocolSession* session;
int read_fd;
bool released;
} AccountingWorkerArg;
typedef struct {
ProtocolSession* session;
atomic_int* ready;
atomic_bool* release;
bool received;
} ConcurrentAccountingWorkerArg;
static int accounting_worker(void* arg) {
AccountingWorkerArg* worker = arg;
protocol_session_bind(worker->session);
Data* data = protocol_receive_data_limited(worker->session, 8);
if (data) {
data_destroy(data);
worker->released = atomic_load(&worker->session->total_allocated_bytes) == 0;
}
protocol_session_unbind();
return data ? thrd_success : thrd_error;
}
static int concurrent_accounting_worker(void* arg) {
ConcurrentAccountingWorkerArg* worker = arg;
protocol_session_bind(worker->session);
Data* data = protocol_receive_data_limited(worker->session, 8);
worker->received = data != NULL;
atomic_fetch_add(worker->ready, 1);
while (!atomic_load(worker->release))
thrd_yield();
data_destroy(data);
protocol_session_unbind();
return thrd_success;
}
static void test_send_receive_n_data() {
int p[2];
@@ -187,6 +241,177 @@ static void test_receive_str_truncated() {
close(p[0]);
}
static void test_max_alloc_rejects_single_buffer() {
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
ProtocolSession session;
protocol_session_init(&session, p[0], p[1]);
protocol_session_set_max_alloc(&session, 4);
protocol_session_bind(&session);
char payload[8] = {0};
EXPECT_TRUE(write(p[1], &(size_t){sizeof(payload)}, sizeof(size_t)) == sizeof(size_t));
EXPECT_NULL(protocol_receive_str(&session));
protocol_session_unbind();
close(p[0]);
close(p[1]);
}
static void test_explicit_session_max_alloc_cannot_be_bypassed() {
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
ProtocolSession explicit_session;
ProtocolSession unrelated_session;
protocol_session_init(&explicit_session, p[0], p[1]);
protocol_session_init(&unrelated_session, p[0], p[1]);
protocol_session_set_max_alloc(&explicit_session, 4);
protocol_session_set_max_alloc(&unrelated_session, 64);
protocol_session_bind(&unrelated_session);
unsigned long long size = 8;
EXPECT_EQ_INT((int)write(p[1], &size, sizeof(size)), (int)sizeof(size));
EXPECT_EQ_INT((int)write(p[1], "12345678", 8), 8);
EXPECT_NULL(protocol_receive_data_limited(&explicit_session, 8));
EXPECT_EQ_INT((int)atomic_load(&explicit_session.total_allocated_bytes), 0);
protocol_session_unbind();
close(p[0]);
close(p[1]);
}
static void test_max_alloc_allows_configured_buffer() {
ProtocolSession session;
protocol_session_init(&session, -1, -1);
protocol_session_set_max_alloc(&session, 4);
protocol_session_bind(&session);
void* allowed = protocol_alloc(4);
const void* rejected = protocol_alloc(5);
EXPECT_NOT_NULL(allowed);
EXPECT_NULL(rejected);
free(allowed);
protocol_session_unbind();
}
static void test_max_alloc_is_bound_in_worker_threads() {
enum { WORKER_COUNT = 4 };
ProtocolSession sessions[WORKER_COUNT];
AllocationWorkerArg args[WORKER_COUNT] = {0};
thrd_t threads[WORKER_COUNT];
for (int i = 0; i < WORKER_COUNT; i++) {
protocol_session_init(&sessions[i], -1, -1);
protocol_session_set_max_alloc(&sessions[i], 4);
args[i].session = &sessions[i];
EXPECT_EQ_INT(thrd_create(&threads[i], allocation_worker, &args[i]), thrd_success);
}
for (int i = 0; i < WORKER_COUNT; i++) {
int result;
EXPECT_EQ_INT(thrd_join(threads[i], &result), thrd_success);
EXPECT_EQ_INT(result, thrd_success);
EXPECT_FALSE(args[i].allocation_allowed);
}
}
static void test_protocol_accounting_is_released_in_worker_threads() {
enum { WORKER_COUNT = 4 };
ProtocolSession sessions[WORKER_COUNT];
AccountingWorkerArg args[WORKER_COUNT] = {0};
thrd_t threads[WORKER_COUNT];
for (int i = 0; i < WORKER_COUNT; i++) {
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
protocol_session_init(&sessions[i], p[0], p[1]);
protocol_session_set_max_alloc(&sessions[i], 64);
unsigned long long size = 8;
EXPECT_EQ_INT((int)write(p[1], &size, sizeof(size)), (int)sizeof(size));
EXPECT_EQ_INT((int)write(p[1], "12345678", 8), 8);
close(p[1]);
args[i].session = &sessions[i];
args[i].read_fd = p[0];
EXPECT_EQ_INT(thrd_create(&threads[i], accounting_worker, &args[i]), thrd_success);
}
for (int i = 0; i < WORKER_COUNT; i++) {
int result;
EXPECT_EQ_INT(thrd_join(threads[i], &result), thrd_success);
EXPECT_EQ_INT(result, thrd_success);
EXPECT_TRUE(args[i].released);
EXPECT_EQ_INT((int)atomic_load(&sessions[i].total_allocated_bytes), 0);
close(args[i].read_fd);
}
}
static void test_protocol_accounting_reservation_is_atomic() {
enum { WORKER_COUNT = 8 };
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
ProtocolSession session;
protocol_session_init(&session, p[0], p[1]);
protocol_session_set_max_alloc(&session, 64);
const unsigned long long budget_before = MAX_SERVER_ALLOC - 8;
atomic_store(&session.total_allocated_bytes, budget_before);
for (int i = 0; i < WORKER_COUNT; i++) {
unsigned long long size = 8;
EXPECT_EQ_INT((int)write(p[1], &size, sizeof(size)), (int)sizeof(size));
EXPECT_EQ_INT((int)write(p[1], "12345678", 8), 8);
}
close(p[1]);
atomic_int ready;
atomic_bool release;
atomic_init(&ready, 0);
atomic_init(&release, false);
ConcurrentAccountingWorkerArg args[WORKER_COUNT] = {0};
thrd_t threads[WORKER_COUNT];
for (int i = 0; i < WORKER_COUNT; i++) {
args[i].session = &session;
args[i].ready = &ready;
args[i].release = &release;
EXPECT_EQ_INT(thrd_create(&threads[i], concurrent_accounting_worker, &args[i]), thrd_success);
}
while (atomic_load(&ready) != WORKER_COUNT)
thrd_yield();
bool budget_ok = atomic_load(&session.total_allocated_bytes) == budget_before + 8;
atomic_store(&release, true);
int received = 0;
for (int i = 0; i < WORKER_COUNT; i++) {
int result;
EXPECT_EQ_INT(thrd_join(threads[i], &result), thrd_success);
EXPECT_EQ_INT(result, thrd_success);
received += args[i].received ? 1 : 0;
}
EXPECT_EQ_INT(received, 1);
EXPECT_TRUE(budget_ok);
EXPECT_EQ_INT((int)atomic_load(&session.total_allocated_bytes), (int)budget_before);
close(p[0]);
}
static void test_protocol_string_accounting_is_transient() {
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
ProtocolSession session;
protocol_session_init(&session, p[0], p[1]);
protocol_session_set_max_alloc(&session, 64);
EXPECT_TRUE(protocol_send_str(&session, "temporary"));
char* received = protocol_receive_str(&session);
EXPECT_NOT_NULL(received);
EXPECT_EQ_STR(received, "temporary");
EXPECT_EQ_INT((int)atomic_load(&session.total_allocated_bytes), 0);
free(received);
close(p[0]);
close(p[1]);
}
static void test_protocol_accounting_release_does_not_underflow() {
ProtocolSession session;
protocol_session_init(&session, -1, -1);
atomic_store(&session.total_allocated_bytes, 4);
protocol_session_bind(&session);
protocol_release_memory(8);
EXPECT_EQ_INT((int)atomic_load(&session.total_allocated_bytes), 0);
protocol_release_memory(1);
EXPECT_EQ_INT((int)atomic_load(&session.total_allocated_bytes), 0);
protocol_session_unbind();
}
void test_protocol() {
test_send_receive_n_data();
test_send_receive_n_data_zero();
@@ -198,4 +423,12 @@ void test_protocol() {
test_send_receive_status();
test_receive_n_data_truncated();
test_receive_str_truncated();
test_max_alloc_rejects_single_buffer();
test_explicit_session_max_alloc_cannot_be_bypassed();
test_max_alloc_allows_configured_buffer();
test_max_alloc_is_bound_in_worker_threads();
test_protocol_accounting_is_released_in_worker_threads();
test_protocol_accounting_reservation_is_atomic();
test_protocol_string_accounting_is_transient();
test_protocol_accounting_release_does_not_underflow();
}
+15
View File
@@ -109,6 +109,20 @@ static void test_delta_deserialize_garbage() {
data_destroy(d);
}
static void test_delta_deserialize_respects_max_alloc() {
unsigned char serialized[sizeof(uint64_t) + sizeof(uint32_t)] = {0};
Data data = {.data = serialized, .size = sizeof(serialized)};
ProtocolSession session;
protocol_session_init(&session, -1, -1);
protocol_session_set_max_alloc(&session, sizeof(Delta) - 1);
protocol_session_bind(&session);
const Delta* result = delta_deserialize(&data);
EXPECT_NULL(result);
protocol_session_unbind();
}
static void test_delta_signature_deserialize_truncated() {
char old_data[4096];
for (int i = 0; i < 4096; i++)
@@ -206,6 +220,7 @@ void test_robustness() {
test_delta_deserialize_truncated();
test_delta_deserialize_empty();
test_delta_deserialize_garbage();
test_delta_deserialize_respects_max_alloc();
test_delta_deserialize_truncated_instructions();
test_delta_signature_deserialize_truncated();
test_delta_apply_null();
+1 -1
View File
@@ -396,7 +396,7 @@ static void test_parallel_scanner_root_chunks_without_workers() {
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0,
0, 0, false, false, false, false, false};
ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options);
ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options, NULL);
EXPECT_NOT_NULL(scanner);
int total_files = 0;
+29
View File
@@ -182,11 +182,40 @@ static void test_receive_manifest_rejects_traversal() {
config_delete(cfg);
}
static void test_receive_incremental_check_rejects_invalid_nanoseconds() {
Config* cfg = config_create();
EXPECT_NOT_NULL(cfg);
cfg->receive_root_directory = str_dup("/tmp/dst");
int p[2];
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
io_set_fds(p[0], p[1]);
EXPECT_TRUE(send_str(p[1], "file.txt"));
unsigned long long size = 0;
long long mtime = 100;
long long mtime_nsec = 1000000000LL;
EXPECT_TRUE(send_n_data(p[1], &size, sizeof(size)));
EXPECT_TRUE(send_n_data(p[1], &mtime, sizeof(mtime)));
EXPECT_TRUE(send_n_data(p[1], &mtime_nsec, sizeof(mtime_nsec)));
bool skipped = false;
EXPECT_NULL(receive_incremental_check(p[0], cfg, &skipped));
Status status;
EXPECT_TRUE(receive_status(p[1], &status));
EXPECT_EQ_INT(status, STATUS_ERROR);
EXPECT_FALSE(skipped);
close(p[0]);
close(p[1]);
config_delete(cfg);
}
void test_server() {
if (!is_running_under_valgrind()) {
test_receive_files_finished();
test_receive_files_single_file();
test_receive_files_abort();
test_receive_manifest_rejects_traversal();
test_receive_incremental_check_rejects_invalid_nanoseconds();
}
}
+56
View File
@@ -1,10 +1,66 @@
#include "test_shared_utils.h"
#include "utils.h"
#include "protocol.h"
#include "test_utils.h"
#include <stdlib.h>
#include <string.h>
#include <threads.h>
typedef struct {
bool eight_bit_output;
const char* expected;
int failed;
} EscapeThreadArgs;
static int escape_thread(void* arg) {
EscapeThreadArgs* args = arg;
for (int i = 0; i < 1000; i++) {
char* escaped = output_escape("x\xc3\xa9\n", args->eight_bit_output);
if (!escaped || strcmp(escaped, args->expected) != 0)
args->failed = 1;
free(escaped);
}
return 0;
}
void test_shared_utils() {
char formatted[32];
EXPECT_TRUE(format_human_bytes(0, formatted, sizeof(formatted)));
EXPECT_EQ_STR(formatted, "0 B");
EXPECT_TRUE(format_human_bytes(1024, formatted, sizeof(formatted)));
EXPECT_EQ_STR(formatted, "1.0 KB");
EXPECT_TRUE(format_human_bytes(1536 * 1024, formatted, sizeof(formatted)));
EXPECT_EQ_STR(formatted, "1.5 MB");
EXPECT_FALSE(format_human_bytes(1024, formatted, 4));
char high_bit[] = {'a', (char)0xc3, (char)0xa9, '\n', '\0'};
char* escaped = output_escape(high_bit, false);
EXPECT_EQ_STR(escaped, "a\\#303\\#251\\#012");
free(escaped);
escaped = output_escape(high_bit, true);
EXPECT_EQ_STR(escaped, "a\xc3\xa9\\#012");
free(escaped);
ProtocolSession safe_session;
ProtocolSession eight_bit_session;
protocol_session_init(&safe_session, -1, -1);
protocol_session_init(&eight_bit_session, -1, -1);
protocol_session_set_8_bit_output(&safe_session, false);
protocol_session_set_8_bit_output(&eight_bit_session, true);
EXPECT_FALSE(safe_session.eight_bit_output);
EXPECT_TRUE(eight_bit_session.eight_bit_output);
EscapeThreadArgs safe_args = {false, "x\\#303\\#251\\#012", 0};
EscapeThreadArgs eight_bit_args = {true, "x\xc3\xa9\\#012", 0};
thrd_t safe_thread;
thrd_t eight_bit_thread;
EXPECT_EQ_INT(thrd_create(&safe_thread, escape_thread, &safe_args), thrd_success);
EXPECT_EQ_INT(thrd_create(&eight_bit_thread, escape_thread, &eight_bit_args), thrd_success);
EXPECT_EQ_INT(thrd_join(safe_thread, NULL), thrd_success);
EXPECT_EQ_INT(thrd_join(eight_bit_thread, NULL), thrd_success);
EXPECT_FALSE(safe_args.failed);
EXPECT_FALSE(eight_bit_args.failed);
// Test str_dup
const char* dup_null = str_dup(NULL);
EXPECT_NULL(dup_null);
+33 -13
View File
@@ -4,34 +4,39 @@
static void test_ssh_connect_invalid_dest_no_colon() {
/* cppcheck-suppress constVariablePointer */
Client* client = client_connect_ssh("invalid-destination-no-colon", 22, NULL);
Client* client = client_connect_ssh("invalid-destination-no-colon", 22, NULL, false);
EXPECT_NULL(client);
}
static void test_ssh_connect_invalid_dest_empty() {
/* cppcheck-suppress constVariablePointer */
Client* client = client_connect_ssh("", 22, NULL);
Client* client = client_connect_ssh("", 22, NULL, false);
EXPECT_NULL(client);
}
/* Test client_connect_ssh with malformed destination (just a colon).
* parse_remote_dest succeeds, ssh is exec'd and fails, but the function
* creates a Client that must be cleaned up. */
/* A child that cannot exec ssh must not be returned as a successful client. */
static void test_ssh_connect_malformed() {
Client* client = client_connect_ssh(":", 22, NULL);
/* ssh binary exists, so exec succeeds; the function returns a Client.
* We just verify it doesn't crash and clean up properly. */
if (client != NULL) {
client_disconnect(client);
client_delete(client);
const char* old_path = getenv("PATH");
char* saved_path = old_path ? strdup(old_path) : NULL;
setenv("PATH", "", 1);
/* cppcheck-suppress constVariablePointer */
Client* client = client_connect_ssh(":", 22, NULL, false);
if (saved_path) {
setenv("PATH", saved_path, 1);
free(saved_path);
} else {
unsetenv("PATH");
}
EXPECT_TRUE(true);
EXPECT_NULL(client);
}
/* Test client_connect_ssh with valid format but unreachable host.
* The function launches ssh which will fail to connect, returns a Client. */
static void test_ssh_connect_unreachable() {
Client* client = client_connect_ssh("nonexistent.invalid:/remote/path", 22, NULL);
Client* client = client_connect_ssh("nonexistent.invalid:/remote/path", 22, NULL, false);
if (client != NULL) {
client_disconnect(client);
client_delete(client);
@@ -39,9 +44,24 @@ static void test_ssh_connect_unreachable() {
EXPECT_TRUE(true);
}
static void test_ssh_remote_command_argument_modes() {
char* command = ssh_build_remote_command("fast sync; touch /tmp/pwned", false);
EXPECT_EQ_STR(command, "'fast sync; touch /tmp/pwned' --stdio");
free(command);
command = ssh_build_remote_command("fast'sync", false);
EXPECT_EQ_STR(command, "'fast'\\''sync' --stdio");
free(command);
command = ssh_build_remote_command("fast sync; touch /tmp/pwned", true);
EXPECT_EQ_STR(command, "fast sync; touch /tmp/pwned --stdio");
free(command);
}
void test_transport_ssh() {
test_ssh_connect_invalid_dest_no_colon();
test_ssh_connect_invalid_dest_empty();
test_ssh_connect_malformed();
test_ssh_connect_unreachable();
test_ssh_remote_command_argument_modes();
}