Compare commits
5
Commits
196a27689f
...
d6d502fbb4
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d6d502fbb4 | ||
|
|
e0e0ea6eac | ||
|
|
4799d12e25 | ||
|
|
4bf4da37e5 | ||
|
|
08a5815ca7 |
No files matched your search
+3
-3
@@ -171,9 +171,9 @@ This document maps rsync's full feature set to FastSync's current implementation
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `--checksum` | Skip based on checksum | ✅ Implemented | With `--incremental`, compares xxHash64 content checksums; `-c` remains compression |
|
||||
| `--checksum-choice=STR` | Choose checksum algorithm | ❌ Not Implemented | xxHash used internally |
|
||||
| `--compare-dest=DIR` | Compare dest files relative to DIR | ✅ Implemented | DIR is a receiver-side basis relative to the destination root (confined below it; absolute/`..` rejected). On the receiver's per-file check (implies `--incremental`) an exact match = same size + mtime (unless `--size-only`/`-I`) **and** equal xxHash64 of the sender's file; a match suppresses the data transfer. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. Divergences: when the destination already holds a *different* version rsync deletes it but FastSync instead transfers the data (keeps the mirror complete; never deletes without `--delete`); attribute-only differences on a match are not re-applied (data is skipped so the sender never sends metadata); content is verified by xxHash64, stricter than rsync's default quick check. Requires `--incremental` (implied); needs the per-file check so it is incompatible with `-s` chunk serialization. Wire: new config fields (protocol bumped to 2.8.0) |
|
||||
| `--copy-dest=DIR` | Include copies of unchanged files | ✅ Implemented | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the normal atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Repeatable; command-line order = priority. Content is xxHash64-verified before the copy. Divergences: attributes come from the basis file (sender metadata is not sent on a match); a match that differs only in attributes is copied with the basis attributes rather than rsync's "copy + fix attributes". Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.8.0 |
|
||||
| `--link-dest=DIR` | Hardlink to files when unchanged | ✅ Implemented | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy, never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Content is xxHash64-verified before linking. Divergences: an already up-to-date destination file is not re-linked to a basis file (only files that would otherwise be written are linked); attributes on a link are the basis inode's own (metadata is never written through the shared inode, which would mutate the basis file); basis dirs are excluded from `--delete`; a `--remove-source-files` source matched from a basis dir is retained (treated as skipped). Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.8.0 |
|
||||
| `--compare-dest=DIR` | Compare dest files relative to DIR | ✅ Implemented | DIR is a receiver-side basis relative to the destination root (confined below it; absolute/`..`/`.` rejected, `//` collapsed and trailing `/` dropped). On the receiver's per-file check (implies `--incremental`) an exact match = same size + mtime (unless `--size-only`; `-I` disables matching) **and** equal xxHash64 of the sender's file; a match suppresses the data transfer. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. Divergences: when the destination already holds a *different* version rsync deletes it but FastSync instead transfers the data (keeps the mirror complete; never deletes without `--delete`); attribute-only differences on a match are not re-applied (data is skipped so the sender never sends metadata); content is verified by xxHash64, stricter than rsync's default quick check. Sizing: FastSync's whole-file payload limit is 256 MiB on **every** transfer path (not basis-specific); rsync applies basis dirs to arbitrary sizes, so FastSync refuses a basis run whose source contains a larger file up front with a clear error before any transfer. Wire: a basis-count field is always present on the config frame (protocol bumped to 2.8.0, so clients and servers must both be 2.8.0) |
|
||||
| `--copy-dest=DIR` | Include copies of unchanged files | ✅ Implemented | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the normal atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Repeatable; command-line order = priority. Content is xxHash64-verified before the copy. Divergences: a basis-hit destination keeps the basis file's own mode/uid/gid and mtime (the sender sends no metadata on a skip), so with `--size-only` its mtime can differ from the source and attribute-only differences are copied with the basis attributes rather than rsync's "copy + fix attributes". Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.8.0 |
|
||||
| `--link-dest=DIR` | Hardlink to files when unchanged | ✅ Implemented | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy, never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Content is xxHash64-verified before linking. Divergences and caveats: an already up-to-date destination file is not re-linked to a basis file (only files that would otherwise be written are linked); a link keeps the basis inode's own mode/uid/gid and mtime — metadata is never written through the shared inode (that would mutate the basis file), so a later `--inplace` run that rewrites such a destination path **will mutate the basis snapshot** through the shared inode (use `--copy-dest` when the destination must stay independently writable); with `--size-only` the linked mtime can differ from the source; a `--remove-source-files` source satisfied by a basis dir is treated as skipped and therefore **retained** (never removed); basis dirs are excluded from `--delete`. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.8.0 |
|
||||
| `--fuzzy`, `--no-fuzzy` | Find similar file for basis | ❌ Not Implemented | |
|
||||
|
||||
## 12. Compression
|
||||
|
||||
@@ -218,6 +218,49 @@ static bool files_from_list_valid(const Config* config) {
|
||||
return no_implied_dirs_files_from_valid(config);
|
||||
}
|
||||
|
||||
/* Basis directories are honored by the receiver's per-file incremental check,
|
||||
which (like every whole-file payload path in FastSync) is bounded by
|
||||
MAX_RECEIVE_WHOLE_FILE_SIZE. rsync would apply basis dirs to files of any
|
||||
size; FastSync cannot, so when basis dirs are requested this preflight scan
|
||||
refuses the run up front with a clear diagnostic instead of letting the
|
||||
receiver abort the whole transfer mid-stream with no client explanation.
|
||||
Returns true when the tree can be transferred. */
|
||||
static bool basis_oversize_preflight(const Config* config) {
|
||||
PreparedScanner prepared;
|
||||
if (!prepare_scanner(config, 0, &prepared))
|
||||
return false;
|
||||
DirectoryScanner* scanner =
|
||||
directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
||||
prepared_scanner_destroy(&prepared);
|
||||
if (!scanner)
|
||||
return false;
|
||||
bool ok = true;
|
||||
Chunk* chunk;
|
||||
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
File* f = chunk->items[i];
|
||||
if (f == NULL || f->is_dir || f->data == NULL || f->data->size <= MAX_RECEIVE_WHOLE_FILE_SIZE)
|
||||
continue;
|
||||
char* escaped = output_escape(file_wire_path(f), config->eight_bit_output);
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"%s is %llu bytes, larger than the %llu-byte whole-file transfer limit; "
|
||||
"--compare-dest/--copy-dest/--link-dest cannot sync files above this limit",
|
||||
escaped ? escaped : "<allocation failed>", (unsigned long long)f->data->size,
|
||||
(unsigned long long)MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||
free(escaped);
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
chunk_destroy(chunk);
|
||||
if (!ok)
|
||||
break;
|
||||
}
|
||||
if (directory_scanner_failed(scanner))
|
||||
ok = false;
|
||||
directory_scanner_destroy(scanner);
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Select the configured transport for both transfer execution paths. */
|
||||
static Client* connect_transfer_client(const Config* config) {
|
||||
if (config->transport == TRANSPORT_SSH) {
|
||||
@@ -1159,6 +1202,8 @@ int send_files(Config* config) {
|
||||
return send_dry_run_manifest(config);
|
||||
if (!files_from_list_valid(config))
|
||||
return 1;
|
||||
if (config_has_basis(config) && !basis_oversize_preflight(config))
|
||||
return 1;
|
||||
|
||||
Client* client = connect_transfer_client(config);
|
||||
if (!client) {
|
||||
@@ -1298,6 +1343,8 @@ int send_files_multithreaded(Config** config_ptr) {
|
||||
return send_dry_run_manifest(config);
|
||||
if (!files_from_list_valid(config))
|
||||
return 1;
|
||||
if (config_has_basis(config) && !basis_oversize_preflight(config))
|
||||
return 1;
|
||||
|
||||
long pages = sysconf(_SC_AVPHYS_PAGES);
|
||||
long page_size = sysconf(_SC_PAGE_SIZE);
|
||||
|
||||
+51
-27
@@ -194,52 +194,76 @@ bool config_has_basis(const Config* config) {
|
||||
/* A basis-dir path travels from the client to the receiver and is resolved
|
||||
* below the destination root, so it must be a non-empty relative path with no
|
||||
* "." or ".." component and no traversal: an absolute or escaping path would
|
||||
* make the receiver read or link files outside its authorized root. */
|
||||
bool config_basis_path_valid(const char* path) {
|
||||
* make the receiver read or link files outside its authorized root.
|
||||
*
|
||||
* Returns a malloc'd CANONICAL copy of an accepted path, or NULL when the path
|
||||
* is rejected. Canonicalization collapses interior empty components ("a//b" ->
|
||||
* "a/b"), drops "." components and trailing "/"s, so validation, the delete
|
||||
* walker prefix match and the receiver's basis lookup all agree on one form.
|
||||
* The normalizer is the single source of truth for both config_basis_path_valid
|
||||
* and config_basis_append. */
|
||||
static char* basis_path_normalize(const char* path) {
|
||||
if (!path || path[0] == '\0' || path[0] == '/' || has_path_traversal(path))
|
||||
return false;
|
||||
return NULL;
|
||||
if (strcmp(path, ".") == 0)
|
||||
return false;
|
||||
return NULL;
|
||||
char* dup = str_dup(path);
|
||||
if (!dup)
|
||||
return false;
|
||||
bool ok = true;
|
||||
return NULL;
|
||||
size_t out_len = 0;
|
||||
char* out = malloc(strlen(path) + 1);
|
||||
if (!out) {
|
||||
free(dup);
|
||||
return NULL;
|
||||
}
|
||||
char* saveptr = NULL;
|
||||
bool ok = true;
|
||||
for (char* part = strtok_r(dup, "/", &saveptr); part; part = strtok_r(NULL, "/", &saveptr)) {
|
||||
if (strcmp(part, ".") == 0) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
if (strcmp(part, "..") == 0) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
if (strcmp(part, ".") == 0)
|
||||
continue;
|
||||
if (out_len > 0)
|
||||
out[out_len++] = '/';
|
||||
size_t len = strlen(part);
|
||||
memcpy(out + out_len, part, len);
|
||||
out_len += len;
|
||||
}
|
||||
free(dup);
|
||||
return ok;
|
||||
if (!ok || out_len == 0) {
|
||||
free(out);
|
||||
return NULL;
|
||||
}
|
||||
out[out_len] = '\0';
|
||||
return out;
|
||||
}
|
||||
|
||||
bool config_basis_path_valid(const char* path) {
|
||||
char* normalized = basis_path_normalize(path);
|
||||
if (!normalized)
|
||||
return false;
|
||||
free(normalized);
|
||||
return true;
|
||||
}
|
||||
|
||||
int config_basis_append(Config* config, BasisDestType type, const char* path) {
|
||||
if (!config || !config_basis_path_valid(path) ||
|
||||
if (!config ||
|
||||
(type != BASIS_DEST_COMPARE && type != BASIS_DEST_COPY && type != BASIS_DEST_LINK) ||
|
||||
config->basis_count >= MAX_BASIS_DIRS)
|
||||
return -1;
|
||||
char* normalized = basis_path_normalize(path);
|
||||
if (!normalized)
|
||||
return -1;
|
||||
BasisDest* grown = realloc(config->basis_dirs, (config->basis_count + 1) * sizeof(BasisDest));
|
||||
if (!grown)
|
||||
if (!grown) {
|
||||
free(normalized);
|
||||
return -1;
|
||||
}
|
||||
config->basis_dirs = grown;
|
||||
/* Normalize a user-supplied trailing slash away so the stored path matches
|
||||
the delete-walker prefix form exactly. */
|
||||
size_t len = strlen(path);
|
||||
while (len > 1 && path[len - 1] == '/')
|
||||
len--;
|
||||
char* dup = malloc(len + 1);
|
||||
if (!dup)
|
||||
return -1;
|
||||
memcpy(dup, path, len);
|
||||
dup[len] = '\0';
|
||||
config->basis_dirs[config->basis_count].type = type;
|
||||
config->basis_dirs[config->basis_count].path = dup;
|
||||
config->basis_dirs[config->basis_count].path = normalized;
|
||||
config->basis_count++;
|
||||
return 0;
|
||||
}
|
||||
@@ -563,9 +587,9 @@ static bool receive_basis_options(int fd, Config* c) {
|
||||
char* path = receive_str(fd);
|
||||
if (!path)
|
||||
return false;
|
||||
bool ok = config_basis_path_valid(path);
|
||||
if (ok)
|
||||
ok = config_basis_append(c, (BasisDestType)type, path) == 0;
|
||||
/* config_basis_append validates and canonicalizes the path; a rejected
|
||||
path (absolute / traversal / empty) drops the whole connection. */
|
||||
bool ok = config_basis_append(c, (BasisDestType)type, path) == 0;
|
||||
free(path);
|
||||
if (!ok)
|
||||
return false;
|
||||
|
||||
+24
-13
@@ -849,12 +849,14 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
||||
}
|
||||
if (materialized) {
|
||||
if (!send_status(fd, STATUS_OK)) {
|
||||
basis_match_free(&basis);
|
||||
file_destroy(materialized);
|
||||
close(old_fd);
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
return NULL;
|
||||
}
|
||||
basis_match_free(&basis);
|
||||
free(old_data);
|
||||
close(old_fd);
|
||||
free(full_path);
|
||||
@@ -1069,29 +1071,38 @@ int receive_manifest(int fd, const Config* config, int* next_status) {
|
||||
/* With --delay-updates the staged (not yet published) files live directly
|
||||
under the receive root in the staging directory; the delete walker must
|
||||
not treat them as extras or it would remove every staged file before it
|
||||
can be published. Alternate basis directories (--compare-dest /
|
||||
--copy-dest / --link-dest) are also excluded: they are extra comparison
|
||||
snapshots the user pointed at, not destination content, and deleting them
|
||||
would destroy the very files a --link-dest run just linked into place. */
|
||||
can be published. That staging name is protected only as a DIRECT child
|
||||
of the receive root so a nested destination directory that happens to be
|
||||
named .fastsync-stage is still ordinary content. Alternate basis
|
||||
directories (--compare-dest / --copy-dest / --link-dest) are excluded at
|
||||
any depth: they are extra comparison snapshots the user pointed at, not
|
||||
destination content, and deleting them would destroy the very files a
|
||||
--link-dest run just linked into place. */
|
||||
int skip_count = (config->delay_updates ? 1 : 0) + config->basis_count;
|
||||
const char** skip_prefixes = NULL;
|
||||
DeleteSkipEntry* skips = NULL;
|
||||
bool deletion_ok = false;
|
||||
if (skip_count > 0) {
|
||||
skip_prefixes = calloc((size_t)skip_count, sizeof(char*));
|
||||
if (!skip_prefixes) {
|
||||
skips = calloc((size_t)skip_count, sizeof(DeleteSkipEntry));
|
||||
if (!skips) {
|
||||
array_list_delete(manifest);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
int idx = 0;
|
||||
if (config->delay_updates)
|
||||
skip_prefixes[idx++] = DELAY_UPDATES_STAGING_DIR;
|
||||
for (int i = 0; i < config->basis_count; i++)
|
||||
skip_prefixes[idx++] = config->basis_dirs[i].path;
|
||||
if (config->delay_updates) {
|
||||
skips[idx].prefix = DELAY_UPDATES_STAGING_DIR;
|
||||
skips[idx].top_level_only = true;
|
||||
idx++;
|
||||
}
|
||||
for (int i = 0; i < config->basis_count; i++) {
|
||||
skips[idx].prefix = config->basis_dirs[i].path;
|
||||
skips[idx].top_level_only = false;
|
||||
idx++;
|
||||
}
|
||||
}
|
||||
deletion_ok = delete_extras_limited(config->receive_root_directory, manifest,
|
||||
MAX_SERVER_DELETE_COUNT, skip_prefixes, skip_count);
|
||||
free(skip_prefixes);
|
||||
MAX_SERVER_DELETE_COUNT, skips, skip_count);
|
||||
free(skips);
|
||||
array_list_delete(manifest);
|
||||
if (!deletion_ok)
|
||||
send_status(fd, STATUS_ERROR);
|
||||
|
||||
+25
-19
@@ -204,22 +204,26 @@ static bool is_dir_in_manifest(const char* rel_path, ArrayList* manifest) {
|
||||
return false;
|
||||
}
|
||||
|
||||
/* True when the relative path is, or lies below, one of the protected
|
||||
prefixes. A prefix "a" therefore protects "a" and "a/b/c" but not "ab". */
|
||||
static bool path_under_skip_prefix(const char* rel_path, const char* const* prefixes,
|
||||
int prefix_count) {
|
||||
for (int i = 0; i < prefix_count; i++) {
|
||||
size_t prefix_len = strlen(prefixes[i]);
|
||||
if (strncmp(rel_path, prefixes[i], prefix_len) == 0 &&
|
||||
(rel_path[prefix_len] == '\0' || rel_path[prefix_len] == '/'))
|
||||
/* True when child_rel is, or lies below, a protected entry. A prefix "a"
|
||||
therefore protects "a" and "a/b/c" but not "ab". Entries with top_level_only
|
||||
set only protect DIRECT children of the receive root (at_root); nested
|
||||
directories that share such a name stay ordinary destination content. */
|
||||
static bool path_under_skip_prefix(const char* child_rel, bool at_root,
|
||||
const DeleteSkipEntry* skips, int skip_count) {
|
||||
for (int i = 0; i < skip_count; i++) {
|
||||
if (skips[i].top_level_only && !at_root)
|
||||
continue;
|
||||
size_t prefix_len = strlen(skips[i].prefix);
|
||||
if (strncmp(child_rel, skips[i].prefix, prefix_len) == 0 &&
|
||||
(child_rel[prefix_len] == '\0' || child_rel[prefix_len] == '/'))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifest,
|
||||
size_t max_delete, size_t* deleted_count,
|
||||
const char* const* skip_prefixes, int skip_prefix_count) {
|
||||
size_t max_delete, size_t* deleted_count, const DeleteSkipEntry* skips,
|
||||
int skip_count) {
|
||||
int scanfd = dup(dirfd);
|
||||
if (scanfd < 0)
|
||||
return false;
|
||||
@@ -238,11 +242,14 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifes
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
/* A --delay-updates run keeps its staging directory below the receive
|
||||
root, and basis-dir snapshots live there too. Their contents are not
|
||||
manifest entries, so descending into them would delete every staged /
|
||||
basis file as an "extra". */
|
||||
if (path_under_skip_prefix(child_rel, skip_prefixes, skip_prefix_count)) {
|
||||
/* A --delay-updates run keeps its staging directory as a direct child of
|
||||
the receive root, and basis-dir snapshots live below it too. Their
|
||||
contents are not manifest entries, so descending into them would delete
|
||||
every staged / basis file as an "extra". Only the staging name (a
|
||||
top-level-only prefix) and the basis prefixes are protected: a nested
|
||||
destination directory that happens to be called .fastsync-stage is
|
||||
ordinary content. */
|
||||
if (path_under_skip_prefix(child_rel, rel_path[0] == '\0', skips, skip_count)) {
|
||||
free(child_rel);
|
||||
continue;
|
||||
}
|
||||
@@ -263,7 +270,7 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifes
|
||||
bool child_removed = false;
|
||||
if (childfd >= 0) {
|
||||
child_removed = delete_extras_fd(childfd, child_rel, manifest, max_delete, deleted_count,
|
||||
skip_prefixes, skip_prefix_count);
|
||||
skips, skip_count);
|
||||
if (!child_removed)
|
||||
operation_ok = false;
|
||||
close(childfd);
|
||||
@@ -315,7 +322,7 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifes
|
||||
}
|
||||
|
||||
bool delete_extras_limited(const char* dest_root, ArrayList* manifest, size_t max_delete,
|
||||
const char* const* skip_prefixes, int skip_prefix_count) {
|
||||
const DeleteSkipEntry* skips, int skip_count) {
|
||||
if (!manifest)
|
||||
return false;
|
||||
int rootfd;
|
||||
@@ -332,8 +339,7 @@ bool delete_extras_limited(const char* dest_root, ArrayList* manifest, size_t ma
|
||||
if (rootfd < 0)
|
||||
return false;
|
||||
size_t deleted_count = 0;
|
||||
bool ok = delete_extras_fd(rootfd, "", manifest, max_delete, &deleted_count, skip_prefixes,
|
||||
skip_prefix_count);
|
||||
bool ok = delete_extras_fd(rootfd, "", manifest, max_delete, &deleted_count, skips, skip_count);
|
||||
if (close(rootfd) != 0)
|
||||
ok = false;
|
||||
return ok;
|
||||
|
||||
+14
-7
@@ -10,14 +10,21 @@ char* output_escape(const char* string, bool eight_bit_output);
|
||||
char* path_cat(const char* path1, const char* path2);
|
||||
bool glob_match(const char* pattern, const char* str);
|
||||
bool delete_extras(const char* dest_root, ArrayList* manifest);
|
||||
/* Remove files/dirs under dest_root that are not listed in manifest. The
|
||||
delete walker never descends into (and so never removes) an entry whose
|
||||
relative path equals one of the skip_prefixes or lies below one: used to
|
||||
protect the --delay-updates staging directory (files still to be published)
|
||||
and the --compare-dest/--copy-dest/--link-dest basis trees (snapshots the
|
||||
transfer links from, never destination content). */
|
||||
/* One protected entry for the delete walker. When top_level_only is true the
|
||||
prefix is skipped only as a DIRECT child of dest_root (the --delay-updates
|
||||
staging directory, which must not hide genuine extras inside a nested
|
||||
destination directory that happens to share the staging name); otherwise the
|
||||
prefix is skipped at any depth (the --compare-dest/--copy-dest/--link-dest
|
||||
basis trees, which the transfer links from and are never destination
|
||||
content). */
|
||||
typedef struct {
|
||||
const char* prefix;
|
||||
bool top_level_only;
|
||||
} DeleteSkipEntry;
|
||||
/* Remove files/dirs under dest_root that are not listed in manifest without
|
||||
ever descending into a protected prefix (see DeleteSkipEntry). */
|
||||
bool delete_extras_limited(const char* dest_root, ArrayList* manifest, size_t max_delete,
|
||||
const char* const* skip_prefixes, int skip_prefix_count);
|
||||
const DeleteSkipEntry* skips, int skip_count);
|
||||
bool utils_set_authorized_root(int fd, const char* canonical_path);
|
||||
/* The fd-only compatibility form is fail-closed for path-based operations;
|
||||
* callers should use utils_set_authorized_root with the canonical identity. */
|
||||
|
||||
@@ -1990,7 +1990,11 @@ class TestBasisDestDirs:
|
||||
STAGING = ".fastsync-stage"
|
||||
TS = 1577836800 # 2020-01-01 00:00:00 UTC, used to pin matching mtimes
|
||||
|
||||
# files: rel-path -> (source content, basis content or None, matched?)
|
||||
# fixture files: source and basis share the mtime pin, so a basis "match"
|
||||
# is decided purely by content (xxHash). unchanged.txt is byte-identical;
|
||||
# changed.txt is byte-DIFFERENT but has the SAME SIZE as the source (and
|
||||
# the same pinned mtime), which is what forces the content-hash gate;
|
||||
# added.txt does not exist in the basis at all.
|
||||
UNCHANGED = "unchanged.txt"
|
||||
CHANGED = "changed.txt"
|
||||
ADDED = "added.txt"
|
||||
@@ -2006,21 +2010,21 @@ class TestBasisDestDirs:
|
||||
_pin_mtime(full, self.TS)
|
||||
return src
|
||||
|
||||
def _basis_root(self, dest, source):
|
||||
received = get_dest_received_dir(dest, source)
|
||||
rel = os.path.relpath(received, dest)
|
||||
return os.path.join(dest, rel)
|
||||
|
||||
def _seed_basis(self, dest, source, basis_dir, basis_files):
|
||||
def _seed_basis_file(self, dest, source, basis_dir, rel, content, ts=None):
|
||||
base = os.path.join(dest, basis_dir, os.path.relpath(
|
||||
get_dest_received_dir(dest, source), dest))
|
||||
full = os.path.join(base, rel)
|
||||
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||
with open(full, "wb") as fh:
|
||||
fh.write(content)
|
||||
_pin_mtime(full, self.TS if ts is None else ts)
|
||||
return full
|
||||
|
||||
def _seed_basis(self, dest, source, basis_dir, basis_files):
|
||||
for rel, content in basis_files.items():
|
||||
full = os.path.join(base, rel)
|
||||
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||
with open(full, "wb") as fh:
|
||||
fh.write(content)
|
||||
_pin_mtime(full, self.TS)
|
||||
return base
|
||||
self._seed_basis_file(dest, source, basis_dir, rel, content)
|
||||
return os.path.join(dest, basis_dir, os.path.relpath(
|
||||
get_dest_received_dir(dest, source), dest))
|
||||
|
||||
def _source_tree(self, prefix):
|
||||
return {
|
||||
@@ -2030,13 +2034,38 @@ class TestBasisDestDirs:
|
||||
}
|
||||
|
||||
def _basis_tree(self, prefix):
|
||||
# unchanged.txt matches the source; changed.txt differs in CONTENT but
|
||||
# shares size/mtime pinning; added.txt is missing from the basis.
|
||||
# unchanged.txt is identical to the source; changed.txt has the SAME
|
||||
# byte size and pinned mtime but a different body (equal size forces
|
||||
# the xxHash gate); added.txt is missing from the basis.
|
||||
return {
|
||||
self.UNCHANGED: b"stable content v1\n",
|
||||
self.CHANGED: b"ANCIENT DIFFERENT CONTENT!\n",
|
||||
self.CHANGED: b"CHANGED CONTENT NOW\n",
|
||||
}
|
||||
|
||||
def test_same_size_different_content_is_not_a_basis_match(self, shared_server):
|
||||
# Core safety property: equal size + pinned mtime but different content
|
||||
# must NEVER be hard-linked or copied from the basis -- the xxHash gate
|
||||
# rejects it and the sender's data is transferred instead.
|
||||
for flag, basis_dir in (("--link-dest", "szlb"), ("--copy-dest", "szcp"),
|
||||
("--compare-dest", "szcmp")):
|
||||
source = self._make_source("basis_same_size_src",
|
||||
{self.UNCHANGED: b"same length body\n"})
|
||||
dest = os.path.join(TEST_DATA_DIR, f"basis_same_size_dst_{basis_dir}")
|
||||
clean_dir(dest)
|
||||
basis_file = self._seed_basis_file(dest, source, basis_dir, self.UNCHANGED,
|
||||
b"SAME LENGTH BODY!")
|
||||
result, _ = run_client(source, dest, flags=[f"{flag}={basis_dir}"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"{flag} same-size mismatch failed: {result.stderr[:300]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
dest_file = os.path.join(received, self.UNCHANGED)
|
||||
assert _read_file(dest_file) == b"same length body\n", \
|
||||
f"{flag}: basis content leaked into the destination on a hash mismatch"
|
||||
if flag != "--compare-dest":
|
||||
assert os.stat(dest_file).st_ino != os.stat(basis_file).st_ino, \
|
||||
f"{flag}: linked/copied from a content-mismatched basis file"
|
||||
|
||||
def test_compare_dest_skips_matching_and_transfers_missing(self, shared_server):
|
||||
source = self._make_source("basis_compare_src", self._source_tree("c"))
|
||||
dest = os.path.join(TEST_DATA_DIR, "basis_compare_dst")
|
||||
@@ -2087,7 +2116,7 @@ class TestBasisDestDirs:
|
||||
assert _read_file(unchanged) == b"stable content v1\n", "unchanged file not materialized"
|
||||
# A real local copy, NOT a hard link to the basis file.
|
||||
assert os.stat(unchanged).st_ino != os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
|
||||
# Changed content falls back to a normal transfer of the sender data.
|
||||
# Equal-size/different-content basis file falls back to the sender data.
|
||||
assert _read_file(os.path.join(received, self.CHANGED)) == \
|
||||
self._source_tree("cp")[self.CHANGED]
|
||||
assert _read_file(os.path.join(received, self.ADDED)) == \
|
||||
@@ -2109,7 +2138,8 @@ class TestBasisDestDirs:
|
||||
assert os.stat(unchanged).st_ino == os.stat(basis_file).st_ino, \
|
||||
"link-dest did not produce a hard link"
|
||||
assert os.stat(unchanged).st_nlink >= 2
|
||||
# Content mismatch must fall back to a plain transfer (not a link).
|
||||
# Equal-size/different-content basis file must fall back to a plain
|
||||
# transfer (not a link).
|
||||
changed = os.path.join(received, self.CHANGED)
|
||||
assert _read_file(changed) == self._source_tree("ln")[self.CHANGED]
|
||||
assert os.stat(changed).st_ino != os.stat(os.path.join(basis, self.CHANGED)).st_ino
|
||||
@@ -2181,3 +2211,137 @@ class TestBasisDestDirs:
|
||||
"basis directory was deleted by --delete"
|
||||
assert os.stat(os.path.join(received, self.UNCHANGED)).st_ino == \
|
||||
os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
|
||||
|
||||
def test_delay_delete_keeps_nested_staging_named_dir_as_content(self):
|
||||
# The real --delay-updates staging directory is protected from --delete
|
||||
# only as a DIRECT child of the receive root. A nested destination
|
||||
# directory that merely shares the staging name is ordinary content, so
|
||||
# its extras must still be deleted (regression guard for the walker).
|
||||
source = self._make_source("basis_nested_stage_src",
|
||||
{"top.txt": b"top\n", "sub/real.txt": b"real\n"})
|
||||
dest = os.path.join(TEST_DATA_DIR, "basis_nested_stage_dst")
|
||||
clean_dir(dest)
|
||||
self._seed_basis(dest, source, "nstbasis",
|
||||
{"top.txt": b"top\n", "sub/real.txt": b"real\n"})
|
||||
received = get_dest_received_dir(dest, source)
|
||||
nested = os.path.join(received, "sub", self.STAGING)
|
||||
os.makedirs(nested, exist_ok=True)
|
||||
extra = os.path.join(nested, "extra.txt")
|
||||
with open(extra, "wb") as fh:
|
||||
fh.write(b"nested extra")
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest,
|
||||
flags=["--link-dest=nstbasis", "--delete",
|
||||
"--delay-updates"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"delay-delete nested staging failed: {result.stderr[:300]}"
|
||||
assert not os.path.exists(extra), \
|
||||
"extra inside a nested .fastsync-stage dir was not deleted"
|
||||
assert not os.path.isdir(nested), \
|
||||
"nested .fastsync-stage dir should have been removed after its extra"
|
||||
assert _read_file(os.path.join(received, "sub", "real.txt")) == b"real\n"
|
||||
assert not os.path.isdir(os.path.join(dest, self.STAGING)), \
|
||||
"real delay-updates staging tree was not cleaned up"
|
||||
|
||||
def test_basis_priority_first_match_wins(self, shared_server):
|
||||
# Two link-dest dirs both hold the exact file: the FIRST (command-line
|
||||
# order) basis directory must win and supply the hard link.
|
||||
source = self._make_source("basis_prio_src", {"f.txt": b"content\n"})
|
||||
dest = os.path.join(TEST_DATA_DIR, "basis_prio_dst")
|
||||
clean_dir(dest)
|
||||
first = self._seed_basis_file(dest, source, "b1", "f.txt", b"content\n")
|
||||
self._seed_basis_file(dest, source, "b2", "f.txt", b"content\n")
|
||||
result, _ = run_client(source, dest, flags=["--link-dest=b1", "--link-dest=b2"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, f"link-dest priority failed: {result.stderr[:300]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(first).st_ino, \
|
||||
"first basis dir did not win over the second"
|
||||
|
||||
def test_basis_priority_across_compare_and_link(self, shared_server):
|
||||
# A compare-dest entry listed BEFORE a link-dest entry shadows it (the
|
||||
# exact match is found first and nothing is materialized); reversing the
|
||||
# order lets the link-dest entry win and materialize a hard link.
|
||||
source = self._make_source("basis_prio_mixed_src", {"f.txt": b"content\n"})
|
||||
|
||||
dest = os.path.join(TEST_DATA_DIR, "basis_prio_mixed_dst")
|
||||
clean_dir(dest)
|
||||
self._seed_basis_file(dest, source, "cmpb", "f.txt", b"content\n")
|
||||
self._seed_basis_file(dest, source, "lnb", "f.txt", b"content\n")
|
||||
result, _ = run_client(source, dest,
|
||||
flags=["--compare-dest=cmpb", "--link-dest=lnb"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"mixed priority (compare first) failed: {result.stderr[:300]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert not os.path.exists(os.path.join(received, "f.txt")), \
|
||||
"compare-dest matched first, so the file must stay sparse (no link-dest materialize)"
|
||||
|
||||
dest = os.path.join(TEST_DATA_DIR, "basis_prio_mixed_dst2")
|
||||
clean_dir(dest)
|
||||
self._seed_basis_file(dest, source, "cmpb", "f.txt", b"content\n")
|
||||
linkb2 = self._seed_basis_file(dest, source, "lnb", "f.txt", b"content\n")
|
||||
result, _ = run_client(source, dest,
|
||||
flags=["--link-dest=lnb", "--compare-dest=cmpb"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"mixed priority (link first) failed: {result.stderr[:300]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(linkb2).st_ino, \
|
||||
"link-dest did not materialize when listed before compare-dest"
|
||||
|
||||
def test_link_dest_size_only_ignores_mtime(self, shared_server):
|
||||
# --size-only drops the mtime leg of the quick check: a basis file with
|
||||
# the SAME content but a DIFFERENT mtime is still an exact match.
|
||||
source = self._make_source("basis_sizeonly_src", {"f.txt": b"content\n"})
|
||||
dest = os.path.join(TEST_DATA_DIR, "basis_sizeonly_dst")
|
||||
clean_dir(dest)
|
||||
basis_file = self._seed_basis_file(dest, source, "sob", "f.txt", b"content\n",
|
||||
ts=self.TS + 500)
|
||||
result, _ = run_client(source, dest, flags=["--link-dest=sob", "--size-only"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, f"size-only link-dest failed: {result.stderr[:300]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.stat(os.path.join(received, "f.txt")).st_ino == os.stat(basis_file).st_ino, \
|
||||
"--size-only should link a basis file whose mtime differs"
|
||||
|
||||
def test_link_dest_ignore_times_never_links(self, shared_server):
|
||||
# -I/--ignore-times forces every file to be updated, so a basis dir is
|
||||
# never used to hard-link (rsync parity). The file is transferred and
|
||||
# stored as a fresh inode even though it matches the basis exactly.
|
||||
source = self._make_source("basis_igntimes_src", {"f.txt": b"content\n"})
|
||||
dest = os.path.join(TEST_DATA_DIR, "basis_igntimes_dst")
|
||||
clean_dir(dest)
|
||||
basis_file = self._seed_basis_file(dest, source, "itb", "f.txt", b"content\n")
|
||||
result, _ = run_client(source, dest, flags=["--link-dest=itb", "--ignore-times"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, f"ignore-times link-dest failed: {result.stderr[:300]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
dest_file = os.path.join(received, "f.txt")
|
||||
assert _read_file(dest_file) == b"content\n"
|
||||
assert os.stat(dest_file).st_ino != os.stat(basis_file).st_ino, \
|
||||
"--ignore-times must not hard-link to a basis file"
|
||||
|
||||
def test_basis_refuses_file_above_whole_file_limit(self, shared_server):
|
||||
# Every whole-file payload path in FastSync (basis dirs included) is
|
||||
# bounded by MAX_RECEIVE_WHOLE_FILE_SIZE. rsync supports basis dirs for
|
||||
# arbitrary sizes; FastSync refuses such a run up front with a clear
|
||||
# diagnostic instead of letting the receiver abort the whole transfer
|
||||
# mid-stream with no client-side explanation.
|
||||
source = self._make_source("basis_oversize_src", {"small.txt": b"ok\n"})
|
||||
big = os.path.join(source, "huge.bin")
|
||||
with open(big, "wb") as fh:
|
||||
os.ftruncate(fh.fileno(), 256 * 1024 * 1024 + 4096)
|
||||
dest = os.path.join(TEST_DATA_DIR, "basis_oversize_dst")
|
||||
clean_dir(dest)
|
||||
result, _ = run_client(source, dest, flags=["--link-dest=nope"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode != 0, \
|
||||
"basis run with an over-limit file unexpectedly succeeded"
|
||||
assert "larger than" in result.stderr, \
|
||||
f"no clear over-limit diagnostic: {result.stderr[:300]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert not os.path.exists(received), \
|
||||
"over-limit basis run transferred files before failing"
|
||||
@@ -528,6 +528,29 @@ static void test_config_basis_wire_rejects_escaping() {
|
||||
config_delete(c);
|
||||
}
|
||||
|
||||
/* Basis-dir paths are canonicalized on the way in: trailing slashes and
|
||||
interior empty / "." components are dropped so validation, the delete-walker
|
||||
prefix and the receiver lookup all agree on one stored form. */
|
||||
static void test_config_basis_normalization() {
|
||||
Config* c = config_create();
|
||||
EXPECT_NOT_NULL(c);
|
||||
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "prior/"), 0);
|
||||
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "a//b"), 0);
|
||||
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "./x/./y/"), 0);
|
||||
EXPECT_EQ_INT(c->basis_count, 3);
|
||||
EXPECT_EQ_STR(c->basis_dirs[0].path, "prior");
|
||||
EXPECT_EQ_STR(c->basis_dirs[1].path, "a/b");
|
||||
EXPECT_EQ_STR(c->basis_dirs[2].path, "x/y");
|
||||
|
||||
/* Degenerate values that normalize away to nothing stay rejected. */
|
||||
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "."), -1);
|
||||
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, ".."), -1);
|
||||
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "/abs"), -1);
|
||||
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "a/../b"), -1);
|
||||
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, ""), -1);
|
||||
config_delete(c);
|
||||
}
|
||||
|
||||
static void test_config_is_remote_dest() {
|
||||
/* Valid SSH-style destinations */
|
||||
EXPECT_TRUE(config_is_remote_dest("user@host:/path"));
|
||||
@@ -566,6 +589,7 @@ void test_config() {
|
||||
test_config_delay_updates_reserved_backup_rejected();
|
||||
test_config_basis_roundtrip();
|
||||
test_config_basis_wire_rejects_escaping();
|
||||
test_config_basis_normalization();
|
||||
}
|
||||
test_config_is_remote_dest();
|
||||
}
|
||||
Reference in new issue
Block a user