diff --git a/RSYNC_COMPAT.md b/RSYNC_COMPAT.md index d1068b3..a97dd5e 100644 --- a/RSYNC_COMPAT.md +++ b/RSYNC_COMPAT.md @@ -246,7 +246,7 @@ why plain `--append` works on the normal atomic path, not only with `--inplace`. | `--chmod=CHMOD` | Affect file permissions | ✅ Implemented | Supports numeric and symbolic `ugo` `rwx` changes; retains receiver safety masking | | `-A`, `--acls` | Preserve ACLs | ❌ Not Implemented | Removed because it had no effect | | `-X`, `--xattrs` | Preserve extended attributes | ❌ Not Implemented | Removed because it had no effect | -| `-H`, `--hard-links` | Preserve hard links | ❌ Not Implemented | Removed because it had no effect | +| `-H`, `--hard-links` | Preserve hard links | ✅ Implemented | Files on the source that share an inode (`st_dev`+`st_ino`, e.g. a `cp -al` tree) are re-created as hard links to one another on the destination, so duplicate links stay deduplicated and only the first member's data is sent (later members are transmitted as payload-less `STATUS_HARDLINK` frames). The receiver links each sibling to the first member's installed file with an atomic link + rename; on `link()` failure it falls back to a byte-identical local copy of the first member, never a partial/corrupt file. Requires the sequential scan for ordering (the first member is always emitted and installed before any sibling is linked). Works single-threaded and under `-m`, `--inplace`, `--delay-updates` (links staged and published by rename) and `--partial`. Crosses the wire (`preserve_hard_links` bool; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0**, peers must match). Incompatible with `-s` (chunk serialization) and `--append`/`--append-verify`, rejected up front with a distinct error. See the Phase-4 hard-links notes below | | `-D` | Same as --devices --specials | ❌ Not Implemented | Removed because device-file handling is not implemented | | `--devices` | Preserve device files | ❌ Not Implemented | Removed because it had no effect | | `--specials` | Preserve special files | ❌ Not Implemented | | @@ -306,6 +306,41 @@ unlike rsync, plain `-M` never applies ownership and `--usermap`/`--groupmap`/ `--chown` each imply metadata preservation so the source uid/gid actually travel (the flags only take effect where ownership is being preserved/applied). +**Phase-4 hard-links notes:** `-H`/`--hard-links` is real and introduces a +deduplicating wire path for files whose source entries share a filesystem inode. +On the sender, the scanner records each distinct `(st_dev, st_ino)` encounter and +assigns it a stable, run-local link-group id (`HardLinkTable`, mutex-guarded so a +multi-threaded scan could share one instance). The FIRST member of a group is +transferred normally and carries the data; each later (sibling) member is +transmitted as a payload-less `STATUS_HARDLINK` frame carrying its destination +path, the group id, and the first member's destination-relative wire path. +Ordering is guaranteed by forcing the sequential scanner whenever `-H` is on +(even under `-m`), so the first member is always emitted — and, on the receiver's +single write thread, installed — before any of its siblings; the receiver is +therefore always able to link to an already-present first member, including the +"first member already up-to-date/skipped" case (the sibling links to or copies +the existing file). Asymmetric existence policies are handled gracefully: under +`--existing`, if the first member's destination is absent (so it is skipped) but +a sibling's own destination already exists, that existing sibling is left in +place rather than the transfer aborting on the missing first member. The receiver +installs each sibling beneath its confined root +as an atomic hard link (temp link + rename); when `link()` fails (cross-device, +filesystem refuses links) it falls back to a byte-identical local copy of the +first member, never a partial/corrupt file. `--delay-updates` stages each sibling +as a hard link to the first member's STAGED file, so publication's renames +preserve the shared inode; `--inplace` and `--partial` are unaffected (a sibling +is a fresh link/copy). Because a hard link shares an inode, metadata is applied +exactly once on the first member and never re-written through the sibling (whose +members are byte-identical by construction), so all members agree. + +Wire/version: `PROTOCOL_VERSION` was bumped **2.11.0 → 2.12.0** (peers must +match). The config frame already carried the `preserve_hard_links` boolean +(round-trips through `config_send`/`config_receive`); the only new wire element +is the `STATUS_HARDLINK` frame described above. Incompatibilities (rejected up +front with a distinct error on the client, and re-checked on receive): `-H` with +`-s` chunk serialization (the chunk wire has no per-file hard-link info) and `-H` +with `--append`/`--append-verify` (a payload-less sibling cannot be tail-resumed). + ## 9. Symlink Handling | Flag | Rsync Description | FastSync Status | Notes | diff --git a/src/client/client_cli.c b/src/client/client_cli.c index b5216f4..291dbe3 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -475,6 +475,7 @@ static const OptionEntry OPTION_TABLE[] = { {"--copy-links", NULL, OPT_FLAG, offsetof(Config, copy_links)}, {"--safe-links", NULL, OPT_FLAG, offsetof(Config, safe_links)}, {"--copy-unsafe-links", NULL, OPT_FLAG, offsetof(Config, copy_unsafe_links)}, + {"--hard-links", "-H", OPT_FLAG, offsetof(Config, preserve_hard_links)}, {"--sparse", "-S", OPT_FLAG, offsetof(Config, preserve_sparse)}, {"--inplace", NULL, OPT_FLAG, offsetof(Config, inplace)}, {"--preallocate", NULL, OPT_FLAG, offsetof(Config, preallocate)}, @@ -552,6 +553,7 @@ static const NegatableOption NEGATABLE_OPTIONS[] = { {"copy-links", NULL, offsetof(Config, copy_links)}, {"safe-links", NULL, offsetof(Config, safe_links)}, {"copy-unsafe-links", NULL, offsetof(Config, copy_unsafe_links)}, + {"hard-links", "H", offsetof(Config, preserve_hard_links)}, {"sparse", "S", offsetof(Config, preserve_sparse)}, {"inplace", NULL, offsetof(Config, inplace)}, {"preallocate", NULL, offsetof(Config, preallocate)}, diff --git a/src/client/client_send.c b/src/client/client_send.c index 169b119..c92a3cb 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -9,6 +9,7 @@ #include "file.h" #include "file_list.h" #include "filter.h" +#include "hardlink.h" #include "metadata.h" #include "log.h" #include "multiprocessing.h" @@ -44,10 +45,13 @@ static const char* display_bytes(unsigned long long bytes, bool human_readable, /* Compiled scanner inputs that are shared read-only across scanner instances * and, in -m mode, across worker threads. `base_filters` owns the compiled - * command-line + -C rules; the FileListSet allow-set lives in the Config. */ + * command-line + -C rules; the FileListSet allow-set lives in the Config. + * `hardlinks` owns the --hard-links/-H link-group detection table (NULL when + * off) and is shared (mutex-guarded) across every scanner/worker of one scan. */ typedef struct { ScannerOptions options; FilterRuleList* base_filters; /* owned; may be NULL */ + HardLinkTable* hardlinks; /* owned; may be NULL */ } PreparedScanner; /* Build the scanner options for one scan. Returns false and logs on failure. */ @@ -55,6 +59,7 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann if (!out) return false; out->base_filters = NULL; + out->hardlinks = NULL; memset(&out->options, 0, sizeof(out->options)); int rule_count = config->filters ? config->filters->size : 0; @@ -107,6 +112,16 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann options->ignore_missing_args = config->ignore_missing_args || config->delete_missing_args; options->excluded_paths = NULL; options->excluded_mutex = NULL; + options->hardlinks = NULL; + if (config->preserve_hard_links) { + out->hardlinks = hardlink_table_create(); + if (!out->hardlinks) { + filter_rule_list_free(out->base_filters); + out->base_filters = NULL; + return false; + } + options->hardlinks = out->hardlinks; + } return true; } @@ -115,6 +130,8 @@ static void prepared_scanner_destroy(PreparedScanner* prepared) { return; filter_rule_list_free(prepared->base_filters); prepared->base_filters = NULL; + hardlink_table_destroy(prepared->hardlinks); + prepared->hardlinks = NULL; } /* True when some --files-from entry is an ancestor-or-equal directory of @@ -1216,6 +1233,19 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config, change_emit_dir_sent(config, f); continue; } + /* --hard-links/-H sibling: a later member of a hard-link group that has no + data (its payload lives in the first member). Transmit a dedicated + STATUS_HARDLINK frame carrying the first member's destination-relative + wire path so the receiver links this entry to that installed file. */ + if (f->link_group != 0 && !f->link_first && f->hardlink_target != NULL) { + if (!send_status(client->file_descriptor, STATUS_HARDLINK) || + !send_str(client->file_descriptor, file_wire_path(f)) || + !send_int(client->file_descriptor, f->link_group) || + !send_str(client->file_descriptor, f->hardlink_target)) + return -1; + change_emit_file_sent(config, f); + continue; + } bool stream = f->data->data == NULL && f->data->size > 0; bool use_sendfile = (config->use_sendfile && !config->use_compression) || (stream && !config->use_compression); @@ -1385,9 +1415,18 @@ static int scan_directory_multithreaded(void* pipeline_context) { if (!context->early_delete) prepared.options.excluded_paths = context->excluded_paths; bool dirs_mode = prepared.options.dirs; + /* -H also selects the sequential scanner (see the comment at the branch), + * so the loop below must choose the scanner by which object exists, not by + * --dirs alone. */ + bool use_dscanner = dirs_mode || prepared.options.hardlinks; DirectoryScanner* dscanner = NULL; ParallelScanner* scanner = NULL; - if (dirs_mode) { + /* --hard-links/-H forces the sequential scanner even in -m mode: a hard-link + group's first member must be emitted before any of its siblings so the + receiver always links to an already-installed first member. The parallel + scanner hands different subdirectories to different worker threads, which + can reorder a group whose members span directories. */ + if (use_dscanner) { dscanner = directory_scanner_create_with_options(context->config->send_directory, &prepared.options); } else { @@ -1404,12 +1443,12 @@ static int scan_directory_multithreaded(void* pipeline_context) { bool failed = false; Chunk* current_chunk; while (1) { - if (dirs_mode) + if (use_dscanner) current_chunk = directory_scanner_next(dscanner); else current_chunk = parallel_scanner_next(scanner); if (current_chunk == NULL) { - failed = dirs_mode ? directory_scanner_failed(dscanner) : parallel_scanner_failed(scanner); + failed = use_dscanner ? directory_scanner_failed(dscanner) : parallel_scanner_failed(scanner); break; } if (context->config->use_delete && !context->early_delete) { @@ -1434,9 +1473,9 @@ static int scan_directory_multithreaded(void* pipeline_context) { /* Capture the scanner results BEFORE destroying the scanner objects (the io_error flag lives on the scanner, so reading it after destroy would be a use-after-free). */ - bool had_io = - dirs_mode ? directory_scanner_had_io_error(dscanner) : parallel_scanner_had_io_error(scanner); - if (dirs_mode) + bool had_io = use_dscanner ? directory_scanner_had_io_error(dscanner) + : parallel_scanner_had_io_error(scanner); + if (use_dscanner) directory_scanner_destroy(dscanner); else parallel_scanner_destroy(scanner); diff --git a/src/client/client_validation.c b/src/client/client_validation.c index c307b02..cf5d4fe 100644 --- a/src/client/client_validation.c +++ b/src/client/client_validation.c @@ -69,6 +69,21 @@ bool validate_config(const Config* config) { "full transfer)"); return false; } + /* --hard-links/-H transmits each later group member as a dedicated per-file + STATUS_HARDLINK frame, which chunk serialization -s does not support; and a + hard-links sibling carries no payload, so the tail-resume of --append is + meaningless for it. Both combinations are rejected up front rather than + silently degrading. */ + if (config->preserve_hard_links && config->use_chunk_serialization) { + log_message(LOG_LEVEL_ERROR, + "--hard-links/-H cannot be combined with -s (chunk serialization)"); + return false; + } + if (config->preserve_hard_links && (config->append || config->append_verify)) { + log_message(LOG_LEVEL_ERROR, + "--hard-links/-H cannot be combined with --append/--append-verify"); + return false; + } if (config->log_file_format && !config->log_file) { log_message(LOG_LEVEL_ERROR, "--log-file-format requires --log-file"); return false; diff --git a/src/client/scanner.c b/src/client/scanner.c index 87fa020..74c9f96 100644 --- a/src/client/scanner.c +++ b/src/client/scanner.c @@ -165,6 +165,34 @@ static bool entry_passes_selection(const FileListSet* file_list, const FilterRul return true; } +/* Apply --hard-links (-H) detection to one regular File. On a sibling (a + * later member of an already-seen source inode) the File keeps the group id + * and the first member's wire path but carries NO data payload (size 0); the + * first member is left untouched (data present, link_first). Allocation + * failure is fatal: the scanner is marked failed. */ +static void scanner_assign_hardlink(DirectoryScanner* scanner, HardLinkTable* table, File* file, + const struct stat* stats) { + if (!table || !file || !stats) + return; + int gid; + bool is_first; + char* first_path = NULL; + if (!hardlink_table_assign(table, file_wire_path(file), stats->st_dev, stats->st_ino, &gid, + &is_first, &first_path)) { + if (scanner) + scanner->failed = true; + return; + } + file->link_group = gid; + file->link_first = is_first; + if (!is_first) { + file->hardlink_target = first_path; + file->data->size = 0; + } else { + free(first_path); + } +} + /* Append `rel` to the caller's exclusion sink, taking `mtx` when shared across parallel worker threads. Returns false on allocation failure (list left unchanged). */ @@ -356,6 +384,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo scanner->io_error = false; scanner->dirs_mode = options->dirs; scanner->relative_mode = options->relative && options->file_list != NULL; + scanner->hardlinks = options->hardlinks; scanner->prune_empty_dirs = options->prune_empty_dirs; scanner->dirs_root_emitted = false; scanner->list_index = 0; @@ -892,6 +921,8 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) { file->send_path = rel_copy; rel_copy = NULL; } + if (scanner->hardlinks && S_ISREG(stats.st_mode)) + scanner_assign_hardlink(scanner, scanner->hardlinks, file, &stats); if (scanner->use_metadata) file->metadata = file_metadata_create(&stats); if (scanner->use_metadata && !file->metadata) { @@ -1207,6 +1238,24 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo file->send_path = rel; rel = NULL; } + if (options->hardlinks && S_ISREG(st.st_mode)) { + int gid; + bool is_first; + char* first_path = NULL; + if (!hardlink_table_assign((HardLinkTable*)options->hardlinks, file_wire_path(file), st.st_dev, + st.st_ino, &gid, &is_first, &first_path)) { + ps->failed = true; + } else { + file->link_group = gid; + file->link_first = is_first; + if (!is_first) { + file->hardlink_target = first_path; + file->data->size = 0; + } else { + free(first_path); + } + } + } if (options->use_metadata) file->metadata = file_metadata_create(&st); if (options->use_metadata && !file->metadata) { diff --git a/src/client/scanner.h b/src/client/scanner.h index 6c4e5fe..634ac56 100644 --- a/src/client/scanner.h +++ b/src/client/scanner.h @@ -4,6 +4,7 @@ #include "chunk.h" #include "file_list.h" #include "filter.h" +#include "hardlink.h" #include "protocol.h" #include "queue.h" #include @@ -64,6 +65,11 @@ typedef struct { * instead of failing (the --dirs generator is the only scanner path that * observes a listed-but-missing entry). */ bool ignore_missing_args; + /* --hard-links (-H): shared, mutable (mutex-guarded) link-group detection + * table, NULL when -H is off. Owned by the caller (client_send), shared + * read-only here; the parallel scanner passes it unchanged to every worker so + * one table detects every group across all subdirectories. */ + HardLinkTable* hardlinks; } ScannerOptions; /* Internal per-scanner filter state. FilterNode chains represent the ordered @@ -124,6 +130,9 @@ typedef struct { --ignore-errors the scan continues past it and the caller decides what to do; `failed` is reserved for fatal errors that always abort the scan. */ bool io_error; + /* --hard-links (-H): shared link-group detection table (see ScannerOptions). + NULL when -H is off. */ + HardLinkTable* hardlinks; } DirectoryScanner; typedef struct { diff --git a/src/client/usage.c b/src/client/usage.c index 08ab4c2..453e5c1 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -180,6 +180,7 @@ void print_usage(void) { printf(" --copy-links Transform symlinks into referent files\n"); printf(" --safe-links Skip symlinks that point outside transfer tree\n"); printf(" --copy-unsafe-links Only transform unsafe symlinks into referent files\n"); + printf(" -H, --hard-links Preserve hard-link relationships across the transfer\n"); printf(" -S, --sparse Handle sparse files efficiently\n"); printf(" --inplace Update files in-place (no temp+rename)\n"); printf( diff --git a/src/server/receiver.c b/src/server/receiver.c index ea6d29f..548b090 100644 --- a/src/server/receiver.c +++ b/src/server/receiver.c @@ -154,7 +154,7 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver DeleteManifest* deferred_manifest = NULL; while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK || status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH || - status == STATUS_MKDIR || status == STATUS_MANIFEST) { + status == STATUS_MKDIR || status == STATUS_MANIFEST || status == STATUS_HARDLINK) { if (status == STATUS_KEEPALIVE) { if (!send_status(file_descriptor, STATUS_KEEPALIVE)) goto fail; @@ -181,6 +181,10 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver File* dir = file_receive_directory(file_descriptor); if (!dir || !sink->store_file(dir, sink->context)) goto receive_error; + } else if (status == STATUS_HARDLINK) { + File* file = file_receive_hardlink(file_descriptor); + if (!file || !sink->store_file(file, sink->context)) + goto receive_error; } else if (status == STATUS_MANIFEST) { DeleteManifest* manifest = receive_manifest_entries(file_descriptor); if (!manifest) diff --git a/src/shared/config.c b/src/shared/config.c index 11c197e..521f6d2 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -195,6 +195,8 @@ static bool validate_received_config(const Config* config) { which chunk serialization -s disables: reject on the receiver too so a -s sender cannot negotiate an inert append mode. */ !((config->append || config->append_verify) && config->use_chunk_serialization) && + !(config->preserve_hard_links && config->use_chunk_serialization) && + !(config->preserve_hard_links && (config->append || config->append_verify)) && (!config->use_compression || (config->compression_level >= 1 && config->compression_level <= 22)) && config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE && diff --git a/src/shared/config.h b/src/shared/config.h index 841aa83..b5e20c5 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -293,7 +293,7 @@ typedef struct Config { DelayUpdatesContext* delay_context; } Config; -#define PROTOCOL_VERSION "2.11.0" +#define PROTOCOL_VERSION "2.12.0" #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) /* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */ #define MAX_BASIS_DIRS 64 diff --git a/src/shared/file.c b/src/shared/file.c index 7de1bd1..2190adf 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -108,6 +108,9 @@ File* file_create(const char* path) { file->skip = false; file->is_dir = false; file->basis_link = NULL; + file->link_group = 0; + file->link_first = false; + file->hardlink_target = NULL; return file; } @@ -125,6 +128,8 @@ void file_destroy(void* item) { file->send_path = NULL; free(file->basis_link); file->basis_link = NULL; + free(file->hardlink_target); + file->hardlink_target = NULL; free(file); } diff --git a/src/shared/file_receive.c b/src/shared/file_receive.c index 2b10212..129fa41 100644 --- a/src/shared/file_receive.c +++ b/src/shared/file_receive.c @@ -102,6 +102,192 @@ static FileSaveResult file_stage_delayed_update(const char* root_directory, return FILE_SAVE_WRITTEN; } +/* Read the whole content of a confined regular file (used to fall back to a + byte-identical copy when a hard-link sibling's link() fails). Symlink-safe + (parent resolved via file_open_secure_parent + O_NOFOLLOW). A zero-length + file yields *out_size 0 and *out_buf NULL as a SUCCESS. Returns false only + on a real error/read failure, setting *source_absent to true when the reason + was that the path does not exist (ENOENT/ENOTDIR), so the caller can decide + between an abort and a graceful skip. */ +static bool hardlink_read_source(const char* path, void** out_buf, unsigned long long* out_size, + bool* source_absent) { + *out_buf = NULL; + *out_size = 0; + *source_absent = false; + if (!path) + return false; + char* leaf = NULL; + int parent_fd = file_open_secure_parent(path, &leaf, false); + if (parent_fd < 0) { + *source_absent = errno == ENOENT || errno == ENOTDIR; + return false; + } + int fd = openat(parent_fd, leaf, O_RDONLY | O_CLOEXEC | O_NOFOLLOW); + int saved_errno = errno; + free(leaf); + close(parent_fd); + if (fd < 0) { + *source_absent = saved_errno == ENOENT || saved_errno == ENOTDIR; + return false; + } + struct stat st; + if (fstat(fd, &st) != 0 || !S_ISREG(st.st_mode)) { + close(fd); + return false; + } + unsigned long long size = (unsigned long long)st.st_size; + if (size > MAX_RECEIVE_WHOLE_FILE_SIZE || size > SIZE_MAX) { + close(fd); + return false; + } + if (size == 0) { + close(fd); + return true; + } + void* buf = protocol_alloc((size_t)size); + if (!buf) { + close(fd); + return false; + } + size_t got = 0; + while (got < (size_t)size) { + ssize_t n = read(fd, (char*)buf + got, (size_t)size - got); + if (n <= 0) { + free(buf); + close(fd); + return false; + } + got += (size_t)n; + } + close(fd); + *out_buf = buf; + *out_size = size; + return true; +} + +/* The group's first member's installed file is absent, but its destination + path was validated (a sibling is only ever processed after its group's first + member). When the sibling's OWN destination already exists it should be + left alone -- a clean skip -- rather than aborting the whole transfer (the + asymmetric --existing case: the first member was skipped because its + destination was missing, while the sibling already has one). Only when the + sibling's destination is missing too is this a genuine failure to + link/copy, which aborts. */ +static FileSaveResult hardlink_sibling_absent_first(const char* destination_path) { + if (destination_path && file_path_exists_secure(destination_path)) + return FILE_SAVE_SKIPPED; + return FILE_SAVE_ERROR; +} + +/* Install a --hard-links/-H sibling: the destination entry is atomically + replaced (temp + rename) with a hard link to the group's first member. The + first member is guaranteed already installed at `hardlink_target` under the + root because -H relies on the receiver's single-FIFO-writer pipeline (one + receive thread, one write thread, FIFO queue => wire order == write order) + plus the sender's forced sequential scan, so a sibling is always processed + after its group's first member. When link() fails (different filesystem, + filesystem refuses links) a byte-identical copy of the first member is + written instead, so the result is never partial or corrupt. With + --delay-updates the sibling is staged as a hard link to the first member's + STAGED file (publication's renames preserve the shared inode). The final + --existing/--ignore-existing/--update policies are decided against the final + destination like every normal write. */ +static FileSaveResult file_save_hardlink_sibling(const char* root_directory, const File* file, + const Config* config) { + Config* cfg = (Config*)config; + if (!root_directory || !file || !file->path || !file->hardlink_target) + return FILE_SAVE_ERROR; + char* destination_path = path_cat(root_directory, file->path); + if (!destination_path) + return FILE_SAVE_ERROR; + + if (cfg->existing && !file_path_exists_secure(destination_path)) { + free(destination_path); + return FILE_SAVE_SKIPPED; + } + if (cfg->ignore_existing && file_path_exists_secure(destination_path)) { + free(destination_path); + return FILE_SAVE_SKIPPED; + } + if (cfg->update && file_destination_is_newer_secure(destination_path, file->metadata)) { + free(destination_path); + return FILE_SAVE_SKIPPED; + } + + bool preallocate = cfg && cfg->preallocate; + bool preserve_executability = cfg && cfg->use_executability; + bool use_fsync = cfg && cfg->use_fsync; + + if (cfg->delay_updates) { + if (!cfg->delay_context) { + cfg->delay_context = delay_updates_context_create(root_directory); + if (!cfg->delay_context) { + free(destination_path); + return FILE_SAVE_ERROR; + } + } + if (!delay_updates_prepare(cfg->delay_context)) { + free(destination_path); + return FILE_SAVE_ERROR; + } + char* staged_first = path_cat(cfg->delay_context->staging_root, file->hardlink_target); + char* staged_sibling = path_cat(cfg->delay_context->staging_root, file->path); + if (!staged_first || !staged_sibling) { + free(staged_first); + free(staged_sibling); + free(destination_path); + return FILE_SAVE_ERROR; + } + void* content = NULL; + unsigned long long content_size = 0; + bool source_absent = false; + if (!hardlink_read_source(staged_first, &content, &content_size, &source_absent)) { + FileSaveResult absent_result = + source_absent ? hardlink_sibling_absent_first(destination_path) : FILE_SAVE_ERROR; + free(staged_first); + free(staged_sibling); + free(destination_path); + return absent_result; + } + bool ok = + file_to_disk_secure_link(staged_sibling, staged_first, content, content_size, preallocate, + file->metadata, preserve_executability, use_fsync, NULL); + free(content); + if (ok) + ok = delay_updates_record(cfg->delay_context, staged_sibling, destination_path, file->path); + if (!ok) + unlink(staged_sibling); + free(staged_first); + free(staged_sibling); + free(destination_path); + return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR; + } + + char* first_disk = path_cat(root_directory, file->hardlink_target); + if (!first_disk) { + free(destination_path); + return FILE_SAVE_ERROR; + } + void* content = NULL; + unsigned long long content_size = 0; + bool source_absent = false; + if (!hardlink_read_source(first_disk, &content, &content_size, &source_absent)) { + FileSaveResult absent_result = + source_absent ? hardlink_sibling_absent_first(destination_path) : FILE_SAVE_ERROR; + free(first_disk); + free(destination_path); + return absent_result; + } + const char* temp_dir = (cfg && cfg->temp_dir) ? cfg->temp_dir : NULL; + bool ok = + file_to_disk_secure_link(destination_path, first_disk, content, content_size, preallocate, + file->metadata, preserve_executability, use_fsync, temp_dir); + free(content); + free(first_disk); + free(destination_path); + return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR; +} + FileSaveResult file_save_to_disk_full(const char* root_directory, const File* file, const Config* config) { /* Backups are incompatible with ignore-existing: moving the entry first @@ -146,6 +332,14 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR; } + /* --hard-links/-H sibling: a later member of a link group arrives with no + payload and is installed as a hard link to (or, on link() failure, a + byte-identical copy of) the group's first member. Handled entirely here, + before the normal data-write paths (which would create an empty file). */ + if (file->link_group != 0 && !file->link_first && file->hardlink_target != NULL) { + return file_save_hardlink_sibling(root_directory, file, config); + } + /* These options arrive from the client. They are names below the server root, never independent filesystem roots. --temp-dir is confined exactly like --backup-dir/--partial-dir: an absolute or `..`-escaping scratch @@ -1622,6 +1816,58 @@ File* file_receive_directory(int file_descriptor) { return file; } +/* Receive a --hard-links/-H sibling frame (the leading STATUS_HARDLINK code has + already been consumed): the destination path, the run-local link-group id, + and the first (data-carrying) member's destination-relative wire path. The + created File carries no payload; it is installed beneath the receive root as + a hard link to (or, on link failure, a byte-identical copy of) the first + member. All paths are validated like every other received path (non-empty, + relative, no traversal). */ +File* file_receive_hardlink(int file_descriptor) { + char* path = receive_str(file_descriptor); + if (path == NULL) + return NULL; + if (path[0] == '\0' || has_path_traversal(path)) { + char* escaped_path = output_escape(path, log_get_8_bit_output()); + log_message(LOG_LEVEL_ERROR, "Invalid received hard-link path: %s", + escaped_path ? escaped_path : ""); + free(escaped_path); + free(path); + send_status(file_descriptor, STATUS_ERROR); + return NULL; + } + int gid; + if (!receive_int(file_descriptor, &gid) || gid <= 0) { + free(path); + return NULL; + } + char* target = receive_str(file_descriptor); + if (!target) { + free(path); + return NULL; + } + if (target[0] == '\0' || has_path_traversal(target)) { + char* escaped = output_escape(target, log_get_8_bit_output()); + log_message(LOG_LEVEL_ERROR, "Invalid hard-link target path: %s", + escaped ? escaped : ""); + free(escaped); + free(target); + free(path); + send_status(file_descriptor, STATUS_ERROR); + return NULL; + } + File* file = file_create(path); + free(path); + if (file == NULL) { + free(target); + return NULL; + } + file->link_group = gid; + file->link_first = false; + file->hardlink_target = target; + return file; +} + /* Read a delete-manifest frame (the STATUS_MANIFEST leading code has already been consumed): a keep-set entry count followed by that many destination-relative paths, then a protected-prefix count followed by that diff --git a/src/shared/file_receive.h b/src/shared/file_receive.h index 54dae5c..3ecee9c 100644 --- a/src/shared/file_receive.h +++ b/src/shared/file_receive.h @@ -9,6 +9,7 @@ File* file_receive(const Config* config, int file_descriptor); File* file_receive_directory(int file_descriptor); +File* file_receive_hardlink(int file_descriptor); File* receive_incremental_check(int fd, const Config* config, bool* skipped); /* A received delete-manifest frame: the keep-set (`keeps`, destination-relative diff --git a/src/shared/file_types.h b/src/shared/file_types.h index a116916..454d52e 100644 --- a/src/shared/file_types.h +++ b/src/shared/file_types.h @@ -35,6 +35,15 @@ typedef struct { * equals the incoming file, and `data` is kept as the cross-filesystem * fallback (a local copy) if the hard link cannot be created. */ char* basis_link; + /* --hard-links (-H), sender + receiver wire state. link_group is a run-local + * id shared by every member of one source inode (0 = not part of a group). + * The FIRST member (link_first == true) carries its data on the wire and is + * written normally; every sibling (link_first == false) carries NO data and + * hardlink_target holds the first member's wire path so the receiver can link + * to (or copy from) the already-installed first member. */ + int link_group; + bool link_first; + char* hardlink_target; } File; /* The path that should be sent on the wire and used for the receiver-side diff --git a/src/shared/hardlink.c b/src/shared/hardlink.c new file mode 100644 index 0000000..36a1cf5 --- /dev/null +++ b/src/shared/hardlink.c @@ -0,0 +1,127 @@ +#include "hardlink.h" + +#include +#include +#include + +#include "log.h" +#include "utils.h" + +/* ---- Sender-side detection table ---- */ + +HardLinkTable* hardlink_table_create(void) { + HardLinkTable* table = calloc(1, sizeof(HardLinkTable)); + if (!table) + return NULL; + if (mtx_init(&table->mutex, mtx_plain) != thrd_success) { + free(table); + return NULL; + } + table->next_gid = 1; + return table; +} + +static void hardlink_item_destroy(HardLinkItem* item) { + if (!item) + return; + free(item->first_path); + item->first_path = NULL; +} + +void hardlink_table_destroy(HardLinkTable* table) { + if (!table) + return; + for (size_t i = 0; i < table->count; i++) + hardlink_item_destroy(&table->items[i]); + free(table->items); + table->items = NULL; + table->count = 0; + table->capacity = 0; + mtx_destroy(&table->mutex); + free(table); +} + +static HardLinkItem* hardlink_table_find_locked(HardLinkTable* table, dev_t dev, ino_t ino) { + for (size_t i = 0; i < table->count; i++) { + if (table->items[i].dev == dev && table->items[i].ino == ino) + return &table->items[i]; + } + return NULL; +} + +static bool hardlink_table_add_locked(HardLinkTable* table, dev_t dev, ino_t ino, const char* path, + int gid, HardLinkItem** out) { + if (table->count == table->capacity) { + size_t new_capacity = table->capacity == 0 ? 8 : table->capacity * 2; + if (new_capacity < table->capacity) + return false; + HardLinkItem* grown = realloc(table->items, new_capacity * sizeof(HardLinkItem)); + if (!grown) + return false; + table->items = grown; + table->capacity = new_capacity; + } + HardLinkItem* item = &table->items[table->count]; + char* dup = str_dup(path); + if (!dup) + return false; + memset(item, 0, sizeof(*item)); + item->dev = dev; + item->ino = ino; + item->gid = gid; + item->first_path = dup; + table->count++; + *out = item; + return true; +} + +bool hardlink_table_assign(HardLinkTable* table, const char* wire_path, dev_t dev, ino_t ino, + int* gid, bool* is_first, char** first_path_out) { + if (!table || !wire_path || !gid || !is_first || !first_path_out) + return false; + if (mtx_lock(&table->mutex) != thrd_success) + return false; + bool ok = true; + const HardLinkItem* item = hardlink_table_find_locked(table, dev, ino); + int next_gid; + if (item) { + *is_first = false; + char* dup = str_dup(item->first_path); + if (!dup) { + ok = false; + } else { + *gid = item->gid; + *first_path_out = dup; + } + next_gid = -1; + } else { + if (table->next_gid <= 0) { + ok = false; + next_gid = -1; + } else { + next_gid = table->next_gid; + HardLinkItem* created = NULL; + if (!hardlink_table_add_locked(table, dev, ino, wire_path, next_gid, &created)) { + ok = false; + } else { + char* dup = str_dup(wire_path); + if (!dup) { + hardlink_item_destroy(created); + table->count--; + ok = false; + } else { + *is_first = true; + *gid = next_gid; + *first_path_out = dup; + } + } + } + } + if (ok && next_gid > 0) + table->next_gid++; + mtx_unlock(&table->mutex); + if (!ok) { + log_message(LOG_LEVEL_ERROR, "memory allocation failed while detecting hard links"); + } + return ok; +} diff --git a/src/shared/hardlink.h b/src/shared/hardlink.h new file mode 100644 index 0000000..55eb342 --- /dev/null +++ b/src/shared/hardlink.h @@ -0,0 +1,66 @@ +#ifndef HARDLINK_H +#define HARDLINK_H + +#include +#include +#include +#include + +/* + * --hard-links / -H support. + * + * Sender side: a HardLinkTable detects regular files on the source that share + * an (st_dev, st_ino) identity (a `cp -al`-style hard-linked tree) and assigns + * each distinct inode a stable, run-local link-group id. The first member + * encountered carries the file data; every later member is marked as a sibling + * (no data payload) that the receiver creates as a hard link to the first + * member's destination file. Grouping is scoped by st_dev so inode reuse + * across different filesystems is never conflated. The table is mutex-guarded + * so the parallel (multi-threaded) scanner COULD share one instance across its + * worker threads; the first-thread-to-call designates the data-carrying member, + * which is safe because a hard-link group's members are byte-identical. (In + * practice the sender forces the sequential scanner whenever -H is on; the + * mutex guards the shared table for any path that supplies one.) + * + * ORDERING (why there is no receiver-side handshake): the receiver stores every + * file - including a hard-link group's first member - through a SINGLE writer + * thread draining a single FIFO queue driven by a single receive thread, so + * wire order == write order and every sibling is processed AFTER its group's + * first member. The sender additionally forces the sequential scanner with -H + * so the first-member frame always precedes its siblings on the wire. Sibling + * install therefore needs no present/wait registry: it hard-links to the first + * member (or copies it) knowing that path is already installed - or that, if + * the first member was skipped (already up to date), its destination still + * exists. This guarantee is REQUIRED; do not introduce a concurrent + * multi-writer receiver for -H without re-adding an ordering mechanism. + */ + +typedef struct HardLinkItem { + dev_t dev; + ino_t ino; + int gid; + char* first_path; /* wire path of the group's data-carrying first member */ +} HardLinkItem; + +typedef struct HardLinkTable { + mtx_t mutex; + HardLinkItem* items; + size_t count; + size_t capacity; + int next_gid; +} HardLinkTable; + +HardLinkTable* hardlink_table_create(void); +void hardlink_table_destroy(HardLinkTable* table); + +/* Assign a link-group id to the regular file at `wire_path` with (dev, ino). + * On the first encounter the file becomes the group's first (data-carrying) + * member (*is_first = true) and a fresh gid is allocated. On a later member + * *is_first = false and *first_path_out is set to a malloc'd copy of the first + * member's wire path (the caller stores it and owns it; on the first member + * path the returned *first_path_out is a malloc'd copy of its own wire path). + * Returns false on allocation failure (transfer should abort). */ +bool hardlink_table_assign(HardLinkTable* table, const char* wire_path, dev_t dev, ino_t ino, + int* gid, bool* is_first, char** first_path_out); + +#endif diff --git a/src/shared/protocol.c b/src/shared/protocol.c index 9729e9b..2a0df31 100644 --- a/src/shared/protocol.c +++ b/src/shared/protocol.c @@ -405,6 +405,8 @@ static const char* status_to_string(Status status) { return "APPEND_OK"; case STATUS_APPEND_DATA: return "APPEND_DATA"; + case STATUS_HARDLINK: + return "HARDLINK"; default: return "UNKNOWN"; } diff --git a/src/shared/protocol.h b/src/shared/protocol.h index 28c6f9f..92e32f4 100644 --- a/src/shared/protocol.h +++ b/src/shared/protocol.h @@ -84,7 +84,13 @@ enum NET_STATUS { STATUS_APPEND, STATUS_APPEND_SIG, STATUS_APPEND_OK, - STATUS_APPEND_DATA + STATUS_APPEND_DATA, + /* --hard-links/-H: a sibling (later member) of a source hard-link group. + * The sender transmits only the path, the run-local link-group id, and the + * first (data-carrying) member's destination-relative wire path; the receiver + * creates this entry as a hard link to the first member's installed file + * (falling back to a byte-identical copy if link() fails). Protocol 2.12.0. */ + STATUS_HARDLINK }; void io_set_fds(int read_fd, int write_fd); diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index 7df756f..a1a703c 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -3826,3 +3826,115 @@ class TestIdentityMapping: st = os.stat(dst_file) assert st.st_uid == 12345 and st.st_gid == 54321, \ f"--chown not applied: uid={st.st_uid} gid={st.st_gid}" + + +class TestHardLinks: + """-H/--hard-links: source files sharing an inode are re-created as hard + links to one another on the destination (dedup preserved, first copy + transferred once, the rest linked/copied). No root required.""" + + STAGING = ".fastsync-stage" + + def _make_source(self, name): + src = os.path.join(TEST_DATA_DIR, name) + clean_dir(src) + with open(os.path.join(src, "a.txt"), "wb") as fh: + fh.write(b"shared content\n" * 2000) + os.link(os.path.join(src, "a.txt"), os.path.join(src, "b.txt")) + with open(os.path.join(src, "c.txt"), "wb") as fh: + fh.write(b"independent content\n" * 2000) + return src + + @pytest.mark.parametrize("flags", [[], ["-m"], ["--delay-updates"]]) + def test_hard_links_preserved(self, shared_server, flags): + src = self._make_source("hl_src") + dest = os.path.join(TEST_DATA_DIR, "hl_dst") + clean_dir(dest) + result, _ = run_client(src, dest, flags=["-H"] + flags, port=shared_server.port) + assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:300]}" + received = get_dest_received_dir(dest, src) + a = os.path.join(received, "a.txt") + b = os.path.join(received, "b.txt") + c = os.path.join(received, "c.txt") + assert os.path.isfile(a) and os.path.isfile(b) and os.path.isfile(c), \ + "all three destination files exist" + with open(a, "rb") as fa, open(b, "rb") as fb: + assert fa.read() == fb.read(), "hard-linked pair content matches" + assert os.stat(a).st_ino == os.stat(b).st_ino, \ + "source hard links were not preserved on the destination" + assert os.stat(a).st_ino != os.stat(c).st_ino, \ + "independent files were incorrectly hard linked" + with open(a, "rb") as fa, open(c, "rb") as fc: + assert fa.read() != fc.read(), "independent files must differ in content" + assert not os.path.isdir(os.path.join(dest, self.STAGING)), \ + "--delay-updates left a staging tree behind" + + def test_hard_links_rejects_chunk_serialization(self, shared_server): + src = self._make_source("hl_reject_src") + dest = os.path.join(TEST_DATA_DIR, "hl_reject_dst") + clean_dir(dest) + result, _ = run_client(src, dest, flags=["-H", "-s"], port=shared_server.port) + assert result.returncode != 0, "-H with -s was accepted" + + def test_hard_links_rejects_append(self, shared_server): + src = self._make_source("hl_reject_app_src") + dest = os.path.join(TEST_DATA_DIR, "hl_reject_app_dst") + clean_dir(dest) + result, _ = run_client(src, dest, flags=["-H", "--append"], port=shared_server.port) + assert result.returncode != 0, "-H with --append was accepted" + + def test_hard_links_link_to_existing_first_member(self, shared_server): + """A sibling whose first member is already up-to-date at the destination + must still be created as a hard link to that existing file.""" + src = os.path.join(TEST_DATA_DIR, "hl_exist_src") + dest = os.path.join(TEST_DATA_DIR, "hl_exist_dst") + clean_dir(src) + clean_dir(dest) + with open(os.path.join(src, "a.txt"), "wb") as fh: + fh.write(b"seed content\n" * 1500) + result, _ = run_client(src, dest, port=shared_server.port) + assert result.returncode == 0, f"seed failed: {result.stderr[:200]}" + # Introduce a hard-link sibling to the already-transferred first member. + os.link(os.path.join(src, "a.txt"), os.path.join(src, "b.txt")) + result, _ = run_client(src, dest, flags=["-H"], port=shared_server.port) + assert result.returncode == 0, f"-H sync failed: {result.stderr[:300]}" + received = get_dest_received_dir(dest, src) + a = os.path.join(received, "a.txt") + b = os.path.join(received, "b.txt") + assert os.path.isfile(a) and os.path.isfile(b) + assert os.stat(a).st_ino == os.stat(b).st_ino, \ + "new sibling was not linked to the existing first member" + with open(a, "rb") as fa, open(b, "rb") as fb: + assert fa.read() == fb.read() + + def test_hard_links_existing_asymmetric_group(self, shared_server): + """-H --existing with an asymmetric link group must succeed: when the + first member's destination is absent (so it is skipped by --existing) + but a sibling's destination already exists, the existing sibling is left + in place instead of the whole transfer aborting on the absent first + member.""" + src = os.path.join(TEST_DATA_DIR, "hl_existing_src") + dest = os.path.join(TEST_DATA_DIR, "hl_existing_dst") + clean_dir(src) + clean_dir(dest) + with open(os.path.join(src, "a.txt"), "wb") as fh: + fh.write(b"asymmetric group content\n" * 1200) + # b.txt is a hard-link sibling of a.txt on the source. + os.link(os.path.join(src, "a.txt"), os.path.join(src, "b.txt")) + with open(os.path.join(src, "c.txt"), "wb") as fh: + fh.write(b"independent\n" * 1200) + # Pre-seed the destination with ONLY the sibling's file (the first + # member has no destination entry). + received = get_dest_received_dir(dest, src) + os.makedirs(received, exist_ok=True) + with open(os.path.join(received, "b.txt"), "wb") as fh: + fh.write(b"asymmetric group content\n" * 1200) + result, _ = run_client(src, dest, flags=["-H", "--existing"], + port=shared_server.port) + assert result.returncode == 0, \ + f"-H --existing asymmetric group failed: {result.stderr[:300]}" + # The existing sibling was preserved and its content is intact. + with open(os.path.join(received, "b.txt"), "rb") as fh: + assert fh.read() == b"asymmetric group content\n" * 1200 + # Under --existing the absent first member is not created. + assert not os.path.exists(os.path.join(received, "a.txt")) diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index 51950a3..9004b11 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -768,8 +768,6 @@ static void test_parse_args_delete_timing_without_delete_rejected() { static void test_parse_args_rejects_unimplemented_options() { static const char* const options[] = {"--silent", "--queue-size", - "-H", - "--hard-links", "-A", "--acls", "-X", @@ -845,6 +843,52 @@ static void test_parse_args_update() { config_delete(cfg); } +static void test_parse_args_hard_links() { + Config* cfg = config_create(); + char* argv_H[] = {"fastsync", "-H", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv_H, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->preserve_hard_links); + config_delete(cfg); + + cfg = config_create(); + positional_count = 0; + char* argv_long[] = {"fastsync", "--hard-links", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 4, argv_long, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->preserve_hard_links); + config_delete(cfg); + + /* --no-hard-links clears the flag. */ + cfg = config_create(); + positional_count = 0; + char* argv_neg[] = {"fastsync", "--hard-links", "--no-hard-links", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 5, argv_neg, positional_args, &positional_count), 0); + EXPECT_FALSE(cfg->preserve_hard_links); + config_delete(cfg); +} + +/* -H/--hard-links violates the per-file streaming requirement of -s and the + * payload-bearing tail-resume of --append: both combos are rejected up front. */ +static void test_validate_config_hard_links_incompatible_modes() { + Config* cfg = config_create(); + char* argv_s[] = {"fastsync", "-H", "-s", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv_s, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->preserve_hard_links); + EXPECT_FALSE(validate_config(cfg)); + config_delete(cfg); + + cfg = config_create(); + positional_count = 0; + char* argv_append[] = {"fastsync", "-H", "--append", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 5, argv_append, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->preserve_hard_links); + EXPECT_FALSE(validate_config(cfg)); + config_delete(cfg); +} + static void test_parse_args_info_flags() { Config* cfg = config_create(); char* argv[] = {"fastsync", "--info=copy,skip", "/src", "/dst"}; @@ -2314,6 +2358,8 @@ void test_client_cli() { test_parse_args_rejects_unimplemented_options(); test_parse_args_quiet(); test_parse_args_human_readable(); + test_parse_args_hard_links(); + test_validate_config_hard_links_incompatible_modes(); test_parse_args_update(); test_parse_args_info_flags(); test_parse_args_info_verbose_order(); diff --git a/tests/test_config.c b/tests/test_config.c index 0682669..a856374 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -126,6 +126,7 @@ static void test_config_send_receive() { send_cfg->use_compression = true; send_cfg->use_metadata = true; send_cfg->use_executability = true; + send_cfg->preserve_hard_links = true; send_cfg->use_delta = true; send_cfg->whole_file = true; send_cfg->fuzzy = true; @@ -181,6 +182,8 @@ static void test_config_send_receive() { ok = false; if (!recv_cfg->use_executability) ok = false; + if (!recv_cfg->preserve_hard_links) + ok = false; if (!recv_cfg->size_only) ok = false; if (!recv_cfg->ignore_times)