Compare commits
6
Commits
4cf34026e7
...
196a27689f
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
196a27689f | ||
|
|
0d18b7fc87 | ||
|
|
1fc0cacc32 | ||
|
|
8f846a43b8 | ||
|
|
d38920c972 | ||
|
|
df890f76ea |
No files matched your search
+3
-3
@@ -171,9 +171,9 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `--checksum` | Skip based on checksum | ✅ Implemented | With `--incremental`, compares xxHash64 content checksums; `-c` remains compression |
|
| `--checksum` | Skip based on checksum | ✅ Implemented | With `--incremental`, compares xxHash64 content checksums; `-c` remains compression |
|
||||||
| `--checksum-choice=STR` | Choose checksum algorithm | ❌ Not Implemented | xxHash used internally |
|
| `--checksum-choice=STR` | Choose checksum algorithm | ❌ Not Implemented | xxHash used internally |
|
||||||
| `--compare-dest=DIR` | Compare dest files relative to DIR | ❌ Not Implemented | Removed because it had no effect |
|
| `--compare-dest=DIR` | Compare dest files relative to DIR | ✅ Implemented | DIR is a receiver-side basis relative to the destination root (confined below it; absolute/`..` rejected). On the receiver's per-file check (implies `--incremental`) an exact match = same size + mtime (unless `--size-only`/`-I`) **and** equal xxHash64 of the sender's file; a match suppresses the data transfer. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. Divergences: when the destination already holds a *different* version rsync deletes it but FastSync instead transfers the data (keeps the mirror complete; never deletes without `--delete`); attribute-only differences on a match are not re-applied (data is skipped so the sender never sends metadata); content is verified by xxHash64, stricter than rsync's default quick check. Requires `--incremental` (implied); needs the per-file check so it is incompatible with `-s` chunk serialization. Wire: new config fields (protocol bumped to 2.8.0) |
|
||||||
| `--copy-dest=DIR` | Include copies of unchanged files | ❌ Not Implemented | Removed because it had no effect |
|
| `--copy-dest=DIR` | Include copies of unchanged files | ✅ Implemented | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the normal atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Repeatable; command-line order = priority. Content is xxHash64-verified before the copy. Divergences: attributes come from the basis file (sender metadata is not sent on a match); a match that differs only in attributes is copied with the basis attributes rather than rsync's "copy + fix attributes". Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.8.0 |
|
||||||
| `--link-dest=DIR` | Hardlink to files when unchanged | ❌ Not Implemented | Removed because it had no effect |
|
| `--link-dest=DIR` | Hardlink to files when unchanged | ✅ Implemented | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy, never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Content is xxHash64-verified before linking. Divergences: an already up-to-date destination file is not re-linked to a basis file (only files that would otherwise be written are linked); attributes on a link are the basis inode's own (metadata is never written through the shared inode, which would mutate the basis file); basis dirs are excluded from `--delete`; a `--remove-source-files` source matched from a basis dir is retained (treated as skipped). Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.8.0 |
|
||||||
| `--fuzzy`, `--no-fuzzy` | Find similar file for basis | ❌ Not Implemented | |
|
| `--fuzzy`, `--no-fuzzy` | Find similar file for basis | ❌ Not Implemented | |
|
||||||
|
|
||||||
## 12. Compression
|
## 12. Compression
|
||||||
|
|||||||
@@ -114,6 +114,27 @@ static int set_nonneg_int_option(int* dest, const char* value, const char* optio
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Validate and append one --compare-dest/--copy-dest/--link-dest directory.
|
||||||
|
* The path is interpreted on the receiver relative to the destination root,
|
||||||
|
* so it must be a non-empty relative path with no "." / ".." components (an
|
||||||
|
* absolute or escaping path is rejected up front instead of failing on the
|
||||||
|
* server). Returns 0 on success, -1 on error. */
|
||||||
|
static int set_basis_dest_option(Config* config, BasisDestType type, const char* value,
|
||||||
|
const char* option_name) {
|
||||||
|
if (!value || !value[0]) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", option_name);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (config_basis_append(config, type, value) != 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"%s requires a non-empty relative directory name with no '.', '..', or absolute "
|
||||||
|
"path (resolved below the destination root)",
|
||||||
|
option_name);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
static int set_stderr_mode(const char* value) {
|
static int set_stderr_mode(const char* value) {
|
||||||
if (strcmp(value, "errors") == 0 || strcmp(value, "e") == 0)
|
if (strcmp(value, "errors") == 0 || strcmp(value, "e") == 0)
|
||||||
log_set_stderr_mode(LOG_STDERR_ERRORS);
|
log_set_stderr_mode(LOG_STDERR_ERRORS);
|
||||||
@@ -974,6 +995,36 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
}
|
}
|
||||||
log_message(LOG_LEVEL_ERROR, "%s is not supported yet (xxHash64 is used)", argv[i]);
|
log_message(LOG_LEVEL_ERROR, "%s is not supported yet (xxHash64 is used)", argv[i]);
|
||||||
return -1;
|
return -1;
|
||||||
|
} else if (strncmp(argv[i], "--compare-dest=", 15) == 0) {
|
||||||
|
if (set_basis_dest_option(config, BASIS_DEST_COMPARE, argv[i] + 15, "--compare-dest") != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (opt_is(argv[i], "--compare-dest", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (set_basis_dest_option(config, BASIS_DEST_COMPARE, argv[++i], "--compare-dest") != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (strncmp(argv[i], "--copy-dest=", 12) == 0) {
|
||||||
|
if (set_basis_dest_option(config, BASIS_DEST_COPY, argv[i] + 12, "--copy-dest") != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (opt_is(argv[i], "--copy-dest", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (set_basis_dest_option(config, BASIS_DEST_COPY, argv[++i], "--copy-dest") != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (strncmp(argv[i], "--link-dest=", 12) == 0) {
|
||||||
|
if (set_basis_dest_option(config, BASIS_DEST_LINK, argv[i] + 12, "--link-dest") != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (opt_is(argv[i], "--link-dest", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (set_basis_dest_option(config, BASIS_DEST_LINK, argv[++i], "--link-dest") != 0)
|
||||||
|
return -1;
|
||||||
} else if (argv[i][0] == '-') {
|
} else if (argv[i][0] == '-') {
|
||||||
char* escaped = output_escape(argv[i], false);
|
char* escaped = output_escape(argv[i], false);
|
||||||
fprintf(stderr, "Unknown option: %s\n", escaped ? escaped : "<allocation failed>");
|
fprintf(stderr, "Unknown option: %s\n", escaped ? escaped : "<allocation failed>");
|
||||||
@@ -1010,6 +1061,13 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
config->files_from_set = set;
|
config->files_from_set = set;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* The "unchanged" decision for --compare-dest/--copy-dest/--link-dest must
|
||||||
|
* be made on the receiver against the basis directories, which requires the
|
||||||
|
* per-file STATUS_CHECK handshake: basis-dir options therefore imply
|
||||||
|
* --incremental (and, via the block below, metadata) on the sender. */
|
||||||
|
if (config_has_basis(config))
|
||||||
|
config->use_incremental = true;
|
||||||
|
|
||||||
/* Incremental and delta transfers need metadata unless the user disabled it. */
|
/* Incremental and delta transfers need metadata unless the user disabled it. */
|
||||||
if ((config->use_incremental || config->use_delta) && !config->use_metadata &&
|
if ((config->use_incremental || config->use_delta) && !config->use_metadata &&
|
||||||
!config->metadata_explicitly_disabled) {
|
!config->metadata_explicitly_disabled) {
|
||||||
|
|||||||
@@ -613,7 +613,10 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
|||||||
return -1;
|
return -1;
|
||||||
if (!send_n_data(client->file_descriptor, &mtime_nsec, sizeof(mtime_nsec)))
|
if (!send_n_data(client->file_descriptor, &mtime_nsec, sizeof(mtime_nsec)))
|
||||||
return -1;
|
return -1;
|
||||||
if (config->checksum) {
|
/* With alternate basis directories the receiver must be able to verify the
|
||||||
|
* content of every candidate basis file, so the sender supplies its xxHash64
|
||||||
|
* for every file even when --checksum was not requested. */
|
||||||
|
if (config->checksum || config_has_basis(config)) {
|
||||||
uint64_t checksum;
|
uint64_t checksum;
|
||||||
if (!file_checksum(file, &checksum) ||
|
if (!file_checksum(file, &checksum) ||
|
||||||
!send_n_data(client->file_descriptor, &checksum, sizeof(checksum)))
|
!send_n_data(client->file_descriptor, &checksum, sizeof(checksum)))
|
||||||
|
|||||||
@@ -11,6 +11,12 @@ bool validate_config(const Config* config) {
|
|||||||
print_usage();
|
print_usage();
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
if (config_has_basis(config) && config->use_chunk_serialization) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"--compare-dest/--copy-dest/--link-dest require per-file incremental checks and "
|
||||||
|
"cannot be combined with -s (chunk serialization)");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (config->use_sendfile && (config->use_chunk_serialization || config->use_compression)) {
|
if (config->use_sendfile && (config->use_chunk_serialization || config->use_compression)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "-f/--sendfile cannot be combined with -c (compression) or -s "
|
log_message(LOG_LEVEL_ERROR, "-f/--sendfile cannot be combined with -c (compression) or -s "
|
||||||
"(chunk serialization)");
|
"(chunk serialization)");
|
||||||
|
|||||||
@@ -58,6 +58,13 @@ void print_usage(void) {
|
|||||||
printf(" -@, --modify-window <sec> Modification time tolerance\n");
|
printf(" -@, --modify-window <sec> Modification time tolerance\n");
|
||||||
printf(" -u, --update Skip files newer than the source on receiver\n");
|
printf(" -u, --update Skip files newer than the source on receiver\n");
|
||||||
printf(" --existing Skip files not already present at destination\n");
|
printf(" --existing Skip files not already present at destination\n");
|
||||||
|
printf(" --compare-dest <dir> Treat DIR (relative to destination root) as an extra\n");
|
||||||
|
printf(" comparison basis: unchanged files are not transferred\n");
|
||||||
|
printf(" (requires --incremental, which is implied)\n");
|
||||||
|
printf(" --copy-dest <dir> Like --compare-dest, but copies the unchanged file from DIR\n");
|
||||||
|
printf(" into the destination instead of transferring its data\n");
|
||||||
|
printf(" --link-dest <dir> Like --copy-dest, but hard-links the unchanged file from DIR\n");
|
||||||
|
printf(" into the destination (repeatable; earlier DIRs win)\n");
|
||||||
printf(" --checksum-choice, --cc <alg> Checksum algorithm (not supported yet; xxHash64 is "
|
printf(" --checksum-choice, --cc <alg> Checksum algorithm (not supported yet; xxHash64 is "
|
||||||
"used)\n");
|
"used)\n");
|
||||||
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
|
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
|
||||||
|
|||||||
@@ -214,7 +214,7 @@ static bool receiver_save_file(File* file, void* context_pointer) {
|
|||||||
result = file_save_to_disk_full(context->config->receive_root_directory, file, context->config);
|
result = file_save_to_disk_full(context->config->receive_root_directory, file, context->config);
|
||||||
}
|
}
|
||||||
if (result != FILE_SAVE_ERROR && context->config->remove_source_files && !file->is_dir &&
|
if (result != FILE_SAVE_ERROR && context->config->remove_source_files && !file->is_dir &&
|
||||||
!receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
|
!file->skip && !receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|||||||
+103
-8
@@ -109,9 +109,8 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->rsync_path = NULL;
|
config->rsync_path = NULL;
|
||||||
config->old_args = false;
|
config->old_args = false;
|
||||||
config->temp_dir = NULL;
|
config->temp_dir = NULL;
|
||||||
config->compare_dest = NULL;
|
config->basis_dirs = NULL;
|
||||||
config->copy_dest = NULL;
|
config->basis_count = 0;
|
||||||
config->link_dest = NULL;
|
|
||||||
config->partial_dir = NULL;
|
config->partial_dir = NULL;
|
||||||
config->suffix = NULL;
|
config->suffix = NULL;
|
||||||
config->delete_before = false;
|
config->delete_before = false;
|
||||||
@@ -188,6 +187,63 @@ Config* config_create(void) {
|
|||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool config_has_basis(const Config* config) {
|
||||||
|
return config && config->basis_count > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A basis-dir path travels from the client to the receiver and is resolved
|
||||||
|
* below the destination root, so it must be a non-empty relative path with no
|
||||||
|
* "." or ".." component and no traversal: an absolute or escaping path would
|
||||||
|
* make the receiver read or link files outside its authorized root. */
|
||||||
|
bool config_basis_path_valid(const char* path) {
|
||||||
|
if (!path || path[0] == '\0' || path[0] == '/' || has_path_traversal(path))
|
||||||
|
return false;
|
||||||
|
if (strcmp(path, ".") == 0)
|
||||||
|
return false;
|
||||||
|
char* dup = str_dup(path);
|
||||||
|
if (!dup)
|
||||||
|
return false;
|
||||||
|
bool ok = true;
|
||||||
|
char* saveptr = NULL;
|
||||||
|
for (char* part = strtok_r(dup, "/", &saveptr); part; part = strtok_r(NULL, "/", &saveptr)) {
|
||||||
|
if (strcmp(part, ".") == 0) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (strcmp(part, "..") == 0) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
free(dup);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
int config_basis_append(Config* config, BasisDestType type, const char* path) {
|
||||||
|
if (!config || !config_basis_path_valid(path) ||
|
||||||
|
(type != BASIS_DEST_COMPARE && type != BASIS_DEST_COPY && type != BASIS_DEST_LINK) ||
|
||||||
|
config->basis_count >= MAX_BASIS_DIRS)
|
||||||
|
return -1;
|
||||||
|
BasisDest* grown = realloc(config->basis_dirs, (config->basis_count + 1) * sizeof(BasisDest));
|
||||||
|
if (!grown)
|
||||||
|
return -1;
|
||||||
|
config->basis_dirs = grown;
|
||||||
|
/* Normalize a user-supplied trailing slash away so the stored path matches
|
||||||
|
the delete-walker prefix form exactly. */
|
||||||
|
size_t len = strlen(path);
|
||||||
|
while (len > 1 && path[len - 1] == '/')
|
||||||
|
len--;
|
||||||
|
char* dup = malloc(len + 1);
|
||||||
|
if (!dup)
|
||||||
|
return -1;
|
||||||
|
memcpy(dup, path, len);
|
||||||
|
dup[len] = '\0';
|
||||||
|
config->basis_dirs[config->basis_count].type = type;
|
||||||
|
config->basis_dirs[config->basis_count].path = dup;
|
||||||
|
config->basis_count++;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
bool config_is_remote_dest(const char* s) {
|
bool config_is_remote_dest(const char* s) {
|
||||||
if (s == NULL)
|
if (s == NULL)
|
||||||
return false;
|
return false;
|
||||||
@@ -244,9 +300,13 @@ void config_delete(Config* config) {
|
|||||||
free(config->rsh_command);
|
free(config->rsh_command);
|
||||||
free(config->rsync_path);
|
free(config->rsync_path);
|
||||||
free(config->temp_dir);
|
free(config->temp_dir);
|
||||||
free(config->compare_dest);
|
for (int i = 0; i < config->basis_count; i++) {
|
||||||
free(config->copy_dest);
|
free(config->basis_dirs[i].path);
|
||||||
free(config->link_dest);
|
config->basis_dirs[i].path = NULL;
|
||||||
|
}
|
||||||
|
free(config->basis_dirs);
|
||||||
|
config->basis_dirs = NULL;
|
||||||
|
config->basis_count = 0;
|
||||||
free(config->partial_dir);
|
free(config->partial_dir);
|
||||||
free(config->suffix);
|
free(config->suffix);
|
||||||
free(config->address);
|
free(config->address);
|
||||||
@@ -333,6 +393,17 @@ static bool send_resume_options(int fd, const Config* c) {
|
|||||||
send_str(fd, c->chmod_spec ? c->chmod_spec : "") && send_skip_compress_options(fd, c);
|
send_str(fd, c->chmod_spec ? c->chmod_spec : "") && send_skip_compress_options(fd, c);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool send_basis_options(int fd, const Config* c) {
|
||||||
|
if (!send_int(fd, c->basis_count))
|
||||||
|
return false;
|
||||||
|
for (int i = 0; i < c->basis_count; i++) {
|
||||||
|
if (!send_int(fd, (int)c->basis_dirs[i].type) ||
|
||||||
|
!send_str(fd, c->basis_dirs[i].path ? c->basis_dirs[i].path : ""))
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
static bool receive_core_fields(int fd, Config* c) {
|
static bool receive_core_fields(int fd, Config* c) {
|
||||||
int value;
|
int value;
|
||||||
if (!receive_wire_bool(fd, &c->eight_bit_output))
|
if (!receive_wire_bool(fd, &c->eight_bit_output))
|
||||||
@@ -479,12 +550,35 @@ static bool receive_resume_options(int fd, Config* c) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool receive_basis_options(int fd, Config* c) {
|
||||||
|
int count;
|
||||||
|
if (!receive_int(fd, &count))
|
||||||
|
return false;
|
||||||
|
if (count < 0 || count > MAX_BASIS_DIRS)
|
||||||
|
return false;
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
int type;
|
||||||
|
if (!receive_int(fd, &type) || type <= BASIS_DEST_NONE || type > BASIS_DEST_LINK)
|
||||||
|
return false;
|
||||||
|
char* path = receive_str(fd);
|
||||||
|
if (!path)
|
||||||
|
return false;
|
||||||
|
bool ok = config_basis_path_valid(path);
|
||||||
|
if (ok)
|
||||||
|
ok = config_basis_append(c, (BasisDestType)type, path) == 0;
|
||||||
|
free(path);
|
||||||
|
if (!ok)
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
bool config_send(int file_descriptor, const Config* config) {
|
bool config_send(int file_descriptor, const Config* config) {
|
||||||
protocol_session_set_max_alloc(NULL, config->max_alloc);
|
protocol_session_set_max_alloc(NULL, config->max_alloc);
|
||||||
if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) ||
|
if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) ||
|
||||||
!send_file_options(file_descriptor, config) ||
|
!send_file_options(file_descriptor, config) ||
|
||||||
!send_selection_options(file_descriptor, config) ||
|
!send_selection_options(file_descriptor, config) ||
|
||||||
!send_resume_options(file_descriptor, config))
|
!send_resume_options(file_descriptor, config) || !send_basis_options(file_descriptor, config))
|
||||||
return false;
|
return false;
|
||||||
Status status;
|
Status status;
|
||||||
if (!receive_status(file_descriptor, &status))
|
if (!receive_status(file_descriptor, &status))
|
||||||
@@ -516,7 +610,8 @@ Config* config_receive(int file_descriptor) {
|
|||||||
!receive_delta_fields(file_descriptor, config) ||
|
!receive_delta_fields(file_descriptor, config) ||
|
||||||
!receive_file_options(file_descriptor, config) ||
|
!receive_file_options(file_descriptor, config) ||
|
||||||
!receive_selection_options(file_descriptor, config) ||
|
!receive_selection_options(file_descriptor, config) ||
|
||||||
!receive_resume_options(file_descriptor, config))
|
!receive_resume_options(file_descriptor, config) ||
|
||||||
|
!receive_basis_options(file_descriptor, config))
|
||||||
goto error;
|
goto error;
|
||||||
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
|
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
|
||||||
strcmp(config->compress_choice, "none") != 0) {
|
strcmp(config->compress_choice, "none") != 0) {
|
||||||
|
|||||||
+32
-4
@@ -12,6 +12,22 @@ typedef enum { TRANSPORT_TCP, TRANSPORT_SSH } TransportType;
|
|||||||
Config can carry it; the concrete type lives in delay_updates.h. */
|
Config can carry it; the concrete type lives in delay_updates.h. */
|
||||||
typedef struct DelayUpdatesContext DelayUpdatesContext;
|
typedef struct DelayUpdatesContext DelayUpdatesContext;
|
||||||
|
|
||||||
|
/* Alternate basis-directory modes (--compare-dest / --copy-dest /
|
||||||
|
* --link-dest). Each flag adds one entry to the ordered Config->basis_dirs
|
||||||
|
* list; the receiver consults entries in command-line order and stops at the
|
||||||
|
* first exact match, mirroring rsync's basis-dir priority rules. */
|
||||||
|
typedef enum {
|
||||||
|
BASIS_DEST_NONE = 0,
|
||||||
|
BASIS_DEST_COMPARE, /* compare only: never copies, never materializes */
|
||||||
|
BASIS_DEST_COPY, /* local copy of the matched basis file */
|
||||||
|
BASIS_DEST_LINK /* hard link to the matched basis file */
|
||||||
|
} BasisDestType;
|
||||||
|
|
||||||
|
typedef struct BasisDest {
|
||||||
|
BasisDestType type;
|
||||||
|
char* path; /* relative to the destination root (receiver-confined) */
|
||||||
|
} BasisDest;
|
||||||
|
|
||||||
typedef struct Config {
|
typedef struct Config {
|
||||||
char* version;
|
char* version;
|
||||||
char* send_directory;
|
char* send_directory;
|
||||||
@@ -134,9 +150,12 @@ typedef struct Config {
|
|||||||
char* rsync_path;
|
char* rsync_path;
|
||||||
bool old_args;
|
bool old_args;
|
||||||
char* temp_dir;
|
char* temp_dir;
|
||||||
char* compare_dest;
|
/* Alternate basis directories, ordered by command-line appearance. Each
|
||||||
char* copy_dest;
|
* entry's type selects compare/copy/link behavior on an exact match. These
|
||||||
char* link_dest;
|
* cross the wire so the receiver can consult them; they are interpreted
|
||||||
|
* relative to the destination root and confined there. */
|
||||||
|
BasisDest* basis_dirs;
|
||||||
|
int basis_count;
|
||||||
|
|
||||||
// PR #174: Partial transfer resumption
|
// PR #174: Partial transfer resumption
|
||||||
char* partial_dir;
|
char* partial_dir;
|
||||||
@@ -174,8 +193,10 @@ typedef struct Config {
|
|||||||
DelayUpdatesContext* delay_context;
|
DelayUpdatesContext* delay_context;
|
||||||
} Config;
|
} Config;
|
||||||
|
|
||||||
#define PROTOCOL_VERSION "2.7.0"
|
#define PROTOCOL_VERSION "2.8.0"
|
||||||
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
||||||
|
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
||||||
|
#define MAX_BASIS_DIRS 64
|
||||||
|
|
||||||
Config* config_create(void);
|
Config* config_create(void);
|
||||||
void config_delete(Config* config);
|
void config_delete(Config* config);
|
||||||
@@ -184,4 +205,11 @@ Config* config_receive(int file_descriptor);
|
|||||||
bool config_is_remote_dest(const char* s);
|
bool config_is_remote_dest(const char* s);
|
||||||
void config_parse_ssh_dest(Config* config);
|
void config_parse_ssh_dest(Config* config);
|
||||||
|
|
||||||
|
/* True when at least one --compare-dest/--copy-dest/--link-dest was set. */
|
||||||
|
bool config_has_basis(const Config* config);
|
||||||
|
/* Append one basis-dir entry. Returns 0 on success, -1 on allocation failure. */
|
||||||
|
int config_basis_append(Config* config, BasisDestType type, const char* path);
|
||||||
|
/* Validate a client-provided basis-dir path (relative, confined, non-empty). */
|
||||||
|
bool config_basis_path_valid(const char* path);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
@@ -83,6 +83,7 @@ File* file_create(const char* path) {
|
|||||||
file->metadata = NULL;
|
file->metadata = NULL;
|
||||||
file->skip = false;
|
file->skip = false;
|
||||||
file->is_dir = false;
|
file->is_dir = false;
|
||||||
|
file->basis_link = NULL;
|
||||||
return file;
|
return file;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -98,6 +99,8 @@ void file_destroy(void* item) {
|
|||||||
file->path = NULL;
|
file->path = NULL;
|
||||||
free(file->send_path);
|
free(file->send_path);
|
||||||
file->send_path = NULL;
|
file->send_path = NULL;
|
||||||
|
free(file->basis_link);
|
||||||
|
file->basis_link = NULL;
|
||||||
free(file);
|
free(file);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -630,6 +633,106 @@ bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
|||||||
preserve_executability, false, true, false, temp_dir);
|
preserve_executability, false, true, false, temp_dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Atomic --link-dest install. The destination is replaced (via a temporary
|
||||||
|
* name and a final rename) with a hard link to `basis_path`. When a hard
|
||||||
|
* link cannot be created (the basis lives on a different filesystem, the
|
||||||
|
* filesystem refuses hard links, ...) the install falls back to writing a
|
||||||
|
* local copy from `data`/`data_size`, which the caller has already verified is
|
||||||
|
* byte-identical to the basis file. `metadata` is only applied on that copy
|
||||||
|
* fallback; a successful hard link keeps the basis inode's own attributes
|
||||||
|
* (applying metadata through the shared inode would mutate the basis file).
|
||||||
|
* Returns false only when both the link and the copy fallback fail. */
|
||||||
|
bool file_to_disk_secure_link(const char* path, const char* basis_path, const void* data,
|
||||||
|
unsigned long long data_size, const FileMetadata* metadata,
|
||||||
|
bool preserve_executability, bool use_fsync, const char* temp_dir) {
|
||||||
|
if (!path || !basis_path)
|
||||||
|
return false;
|
||||||
|
char* leaf = NULL;
|
||||||
|
int dirfd = file_open_secure_parent(path, &leaf, true);
|
||||||
|
if (dirfd < 0)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
int scratch_dirfd = -1;
|
||||||
|
if (temp_dir) {
|
||||||
|
scratch_dirfd = file_open_private_dir(temp_dir);
|
||||||
|
if (scratch_dirfd < 0) {
|
||||||
|
int saved_errno = errno;
|
||||||
|
log_message(LOG_LEVEL_ERROR, "could not open --temp-dir scratch directory '%s': %s", temp_dir,
|
||||||
|
strerror(saved_errno));
|
||||||
|
close(dirfd);
|
||||||
|
free(leaf);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
char* basis_leaf = NULL;
|
||||||
|
int basis_dirfd = file_open_secure_parent(basis_path, &basis_leaf, false);
|
||||||
|
bool linked = false;
|
||||||
|
if (basis_dirfd >= 0 && basis_leaf != NULL) {
|
||||||
|
int tmp_size = snprintf(NULL, 0, ".%s.tmp.%ld.%llu", leaf, (long)getpid(), ~0ULL);
|
||||||
|
char* tmp = NULL;
|
||||||
|
if (tmp_size >= 0)
|
||||||
|
tmp = malloc((size_t)tmp_size + 1);
|
||||||
|
if (!tmp) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed while hard-linking basis file");
|
||||||
|
} else {
|
||||||
|
for (unsigned int i = 0; i < 100 && !linked; ++i) {
|
||||||
|
if (scratch_dirfd >= 0)
|
||||||
|
snprintf(tmp, (size_t)tmp_size + 1, ".%s.tmp.%ld.%llu", leaf, (long)getpid(),
|
||||||
|
next_temp_sequence());
|
||||||
|
else
|
||||||
|
snprintf(tmp, (size_t)tmp_size + 1, ".%s.tmp.%ld.%u", leaf, (long)getpid(), i);
|
||||||
|
if (linkat(basis_dirfd, basis_leaf, scratch_dirfd >= 0 ? scratch_dirfd : dirfd, tmp, 0) ==
|
||||||
|
0) {
|
||||||
|
linked = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (errno != EEXIST)
|
||||||
|
break; /* EXDEV / EPERM / ...: give up and fall back to a copy */
|
||||||
|
}
|
||||||
|
if (linked) {
|
||||||
|
int target_dirfd = scratch_dirfd >= 0 ? scratch_dirfd : dirfd;
|
||||||
|
if (use_fsync) {
|
||||||
|
int tfd = openat(target_dirfd, tmp, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
|
if (tfd < 0 || fsync(tfd) != 0) {
|
||||||
|
linked = false;
|
||||||
|
if (tfd >= 0)
|
||||||
|
close(tfd);
|
||||||
|
} else {
|
||||||
|
close(tfd);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (linked && renameat(target_dirfd, tmp, dirfd, leaf) != 0)
|
||||||
|
linked = false;
|
||||||
|
if (!linked)
|
||||||
|
unlinkat(target_dirfd, tmp, 0);
|
||||||
|
}
|
||||||
|
free(tmp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (basis_dirfd >= 0)
|
||||||
|
close(basis_dirfd);
|
||||||
|
free(basis_leaf);
|
||||||
|
basis_leaf = NULL;
|
||||||
|
|
||||||
|
if (!linked) {
|
||||||
|
if (scratch_dirfd >= 0)
|
||||||
|
close(scratch_dirfd);
|
||||||
|
close(dirfd);
|
||||||
|
free(leaf);
|
||||||
|
/* The basis file could not be linked in (missing, cross-device, refused
|
||||||
|
by the filesystem). Write a byte-identical local copy instead. */
|
||||||
|
return file_to_disk_secure_with_fsync(path, data, data_size, false, false, metadata,
|
||||||
|
preserve_executability, use_fsync, temp_dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (scratch_dirfd >= 0)
|
||||||
|
close(scratch_dirfd);
|
||||||
|
close(dirfd);
|
||||||
|
free(leaf);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
|
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse) {
|
bool inplace, bool sparse) {
|
||||||
if (!path || (!data && data_size != 0) || has_path_traversal(path))
|
if (!path || (!data && data_size != 0) || has_path_traversal(path))
|
||||||
|
|||||||
@@ -63,5 +63,12 @@ bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
|||||||
unsigned long long data_size, bool sparse,
|
unsigned long long data_size, bool sparse,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, bool preserve_executability,
|
||||||
const char* temp_dir);
|
const char* temp_dir);
|
||||||
|
/* Atomic --link-dest install: replace `path` with a hard link to `basis_path`
|
||||||
|
(via a temp name + rename); fall back to a byte-identical local copy from
|
||||||
|
`data` when the link is impossible (EXDEV/EPERM/unsupported filesystem).
|
||||||
|
`metadata` is applied only on the copy fallback. */
|
||||||
|
bool file_to_disk_secure_link(const char* path, const char* basis_path, const void* data,
|
||||||
|
unsigned long long data_size, const FileMetadata* metadata,
|
||||||
|
bool preserve_executability, bool use_fsync, const char* temp_dir);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
+257
-12
@@ -75,10 +75,17 @@ static FileSaveResult file_stage_delayed_update(const char* root_directory,
|
|||||||
/* The staged location is brand new (stale leftovers from a prior crash were
|
/* The staged location is brand new (stale leftovers from a prior crash were
|
||||||
wiped by prepare), so the plain atomic temp+rename engine installs the
|
wiped by prepare), so the plain atomic temp+rename engine installs the
|
||||||
complete file there. --temp-dir scratch is deliberately not layered on
|
complete file there. --temp-dir scratch is deliberately not layered on
|
||||||
top of the delay-updates staging tree. */
|
top of the delay-updates staging tree. A --link-dest basis file is hard
|
||||||
bool ok =
|
linked into the staging tree (so publication's rename keeps the link). */
|
||||||
file_to_disk_secure_with_fsync(staged_path, file->data->data, file->data->size, false, sparse,
|
bool ok;
|
||||||
|
if (file->basis_link) {
|
||||||
|
ok = file_to_disk_secure_link(staged_path, file->basis_link, file->data->data, file->data->size,
|
||||||
metadata, preserve_executability, config->use_fsync, NULL);
|
metadata, preserve_executability, config->use_fsync, NULL);
|
||||||
|
} else {
|
||||||
|
ok = file_to_disk_secure_with_fsync(staged_path, file->data->data, file->data->size, false,
|
||||||
|
sparse, metadata, preserve_executability, config->use_fsync,
|
||||||
|
NULL);
|
||||||
|
}
|
||||||
if (!ok) {
|
if (!ok) {
|
||||||
free(staged_path);
|
free(staged_path);
|
||||||
return FILE_SAVE_ERROR;
|
return FILE_SAVE_ERROR;
|
||||||
@@ -272,16 +279,27 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
|
|||||||
while (temp_len > 1 && confined_temp[temp_len - 1] == '/')
|
while (temp_len > 1 && confined_temp[temp_len - 1] == '/')
|
||||||
confined_temp[--temp_len] = '\0';
|
confined_temp[--temp_len] = '\0';
|
||||||
}
|
}
|
||||||
bool ok =
|
/* A --link-dest basis hit installs an atomic hard link (with a byte-copy
|
||||||
config && config->ignore_existing
|
fallback); --inplace and the update/no-replace write variants do not
|
||||||
|
apply to a fresh hard link, whose inode attributes already match. The
|
||||||
|
existing/ignore-existing/update/backup preamble above has already made the
|
||||||
|
policy decision. */
|
||||||
|
bool ok;
|
||||||
|
if (config && file->basis_link) {
|
||||||
|
ok = file_to_disk_secure_link(disk_path, file->basis_link, file->data->data, file->data->size,
|
||||||
|
metadata, preserve_executability, config->use_fsync,
|
||||||
|
confined_temp);
|
||||||
|
} else {
|
||||||
|
ok = config && config->ignore_existing
|
||||||
? file_to_disk_secure_no_replace(disk_path, file->data->data, file->data->size, sparse,
|
? file_to_disk_secure_no_replace(disk_path, file->data->data, file->data->size, sparse,
|
||||||
metadata, preserve_executability, confined_temp)
|
metadata, preserve_executability, confined_temp)
|
||||||
: config && config->update
|
: config && config->update
|
||||||
? file_to_disk_secure_update(disk_path, file->data->data, file->data->size, inplace,
|
? file_to_disk_secure_update(disk_path, file->data->data, file->data->size, inplace,
|
||||||
sparse, metadata, preserve_executability, confined_temp)
|
sparse, metadata, preserve_executability, confined_temp)
|
||||||
: file_to_disk_secure_with_fsync(disk_path, file->data->data, file->data->size, inplace,
|
: file_to_disk_secure_with_fsync(disk_path, file->data->data, file->data->size,
|
||||||
sparse, metadata, preserve_executability,
|
inplace, sparse, metadata, preserve_executability,
|
||||||
config && config->use_fsync, confined_temp);
|
config && config->use_fsync, confined_temp);
|
||||||
|
}
|
||||||
free(confined_temp);
|
free(confined_temp);
|
||||||
confined_temp = NULL;
|
confined_temp = NULL;
|
||||||
if (!ok)
|
if (!ok)
|
||||||
@@ -505,6 +523,143 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ---- Alternate basis directories (--compare-dest / --copy-dest / --link-dest) ----
|
||||||
|
* The receiver consults the ordered basis-dir list only when the destination
|
||||||
|
* entry is NOT already up to date. An "exact match" requires an equal size,
|
||||||
|
* an equal mtime (unless --size-only), and an equal content xxHash64, so a
|
||||||
|
* hard link / local copy is only ever made from byte-identical content. */
|
||||||
|
|
||||||
|
typedef struct BasisMatch {
|
||||||
|
bool hit;
|
||||||
|
BasisDestType type;
|
||||||
|
char* basis_path; /* owned absolute path of the matched basis file */
|
||||||
|
struct stat st; /* fstat() of the matched basis file */
|
||||||
|
Data* content; /* owned basis bytes (or empty Data), NULL when not loaded */
|
||||||
|
} BasisMatch;
|
||||||
|
|
||||||
|
static void basis_match_free(BasisMatch* match) {
|
||||||
|
if (!match)
|
||||||
|
return;
|
||||||
|
free(match->basis_path);
|
||||||
|
match->basis_path = NULL;
|
||||||
|
data_destroy(match->content);
|
||||||
|
match->content = NULL;
|
||||||
|
match->hit = false;
|
||||||
|
match->type = BASIS_DEST_NONE;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Open `path` (via the secure, root-confined primitives) and require it to be
|
||||||
|
a regular file of exactly `expected_size` bytes. Returns an open read-only
|
||||||
|
descriptor and its fstat on success. */
|
||||||
|
static bool basis_open_regular(const char* path, unsigned long long expected_size, int* out_fd,
|
||||||
|
struct stat* out_st) {
|
||||||
|
char* leaf = NULL;
|
||||||
|
int parent_fd = file_open_secure_parent(path, &leaf, false);
|
||||||
|
if (parent_fd < 0)
|
||||||
|
return false;
|
||||||
|
int fd = openat(parent_fd, leaf, O_RDONLY | O_CLOEXEC | O_NOFOLLOW);
|
||||||
|
free(leaf);
|
||||||
|
close(parent_fd);
|
||||||
|
if (fd < 0)
|
||||||
|
return false;
|
||||||
|
struct stat st;
|
||||||
|
if (fstat(fd, &st) != 0 || !S_ISREG(st.st_mode) ||
|
||||||
|
(unsigned long long)st.st_size != expected_size) {
|
||||||
|
close(fd);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
*out_fd = fd;
|
||||||
|
*out_st = st;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Read the whole remaining content of an open descriptor. A zero-length file
|
||||||
|
yields an empty Data (data pointer NULL). */
|
||||||
|
static Data* basis_read_content(int fd, unsigned long long size) {
|
||||||
|
if (size == 0)
|
||||||
|
return data_create_reserve(0);
|
||||||
|
if (size > MAX_RECEIVE_WHOLE_FILE_SIZE || size > SIZE_MAX)
|
||||||
|
return NULL;
|
||||||
|
void* buf = protocol_alloc((size_t)size);
|
||||||
|
if (!buf)
|
||||||
|
return NULL;
|
||||||
|
size_t got = 0;
|
||||||
|
while (got < (size_t)size) {
|
||||||
|
ssize_t n = read(fd, (char*)buf + got, (size_t)size - got);
|
||||||
|
if (n <= 0) {
|
||||||
|
free(buf);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
got += (size_t)n;
|
||||||
|
}
|
||||||
|
return data_create(buf, (size_t)size);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --ignore-times forces every file to be updated, so no basis hit is ever
|
||||||
|
declared (matching rsync, where -I prevents link-dest from linking). */
|
||||||
|
static bool basis_quick_matches(const Config* config, const struct stat* st, time_t check_mtime,
|
||||||
|
long check_mtime_nsec) {
|
||||||
|
if (config->size_only)
|
||||||
|
return true;
|
||||||
|
long mtime_nsec = 0;
|
||||||
|
#ifdef __linux__
|
||||||
|
mtime_nsec = st->st_mtim.tv_nsec;
|
||||||
|
#endif
|
||||||
|
return metadata_mtime_matches(st->st_mtime, mtime_nsec, check_mtime, check_mtime_nsec,
|
||||||
|
config->modify_window);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Search the basis-dir list in command-line order and return the first exact
|
||||||
|
match. When load_content is true the matched bytes are kept in out->content
|
||||||
|
so the caller can materialize the file without re-reading it. */
|
||||||
|
static bool basis_match_find(const Config* config, const char* check_path,
|
||||||
|
unsigned long long check_size, time_t check_mtime,
|
||||||
|
long check_mtime_nsec, uint64_t check_checksum, bool load_content,
|
||||||
|
BasisMatch* out) {
|
||||||
|
memset(out, 0, sizeof(*out));
|
||||||
|
if (!config || !config_has_basis(config) || config->ignore_times)
|
||||||
|
return false;
|
||||||
|
for (int i = 0; i < config->basis_count; i++) {
|
||||||
|
const BasisDest* entry = &config->basis_dirs[i];
|
||||||
|
char* basis_dir = path_cat(config->receive_root_directory, entry->path);
|
||||||
|
if (!basis_dir)
|
||||||
|
continue;
|
||||||
|
char* candidate = path_cat(basis_dir, check_path);
|
||||||
|
free(basis_dir);
|
||||||
|
if (!candidate)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
int fd;
|
||||||
|
struct stat st;
|
||||||
|
if (basis_open_regular(candidate, check_size, &fd, &st)) {
|
||||||
|
if (basis_quick_matches(config, &st, check_mtime, check_mtime_nsec)) {
|
||||||
|
Data* content = basis_read_content(fd, check_size);
|
||||||
|
if (content) {
|
||||||
|
uint64_t basis_hash = check_size == 0 ? delta_xxhash64("", 0)
|
||||||
|
: content->data ? delta_xxhash64(content->data, content->size)
|
||||||
|
: 0;
|
||||||
|
if (basis_hash == check_checksum) {
|
||||||
|
out->hit = true;
|
||||||
|
out->type = entry->type;
|
||||||
|
out->basis_path = candidate;
|
||||||
|
candidate = NULL; /* ownership transferred to out */
|
||||||
|
out->st = st;
|
||||||
|
out->content = load_content ? content : NULL;
|
||||||
|
if (!load_content)
|
||||||
|
data_destroy(content);
|
||||||
|
close(fd);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
data_destroy(content);
|
||||||
|
}
|
||||||
|
close(fd);
|
||||||
|
}
|
||||||
|
free(candidate);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
||||||
if (!config || !skipped) {
|
if (!config || !skipped) {
|
||||||
send_status(fd, STATUS_ERROR);
|
send_status(fd, STATUS_ERROR);
|
||||||
@@ -531,7 +686,8 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
send_status(fd, STATUS_ERROR);
|
send_status(fd, STATUS_ERROR);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
if (config->checksum && !receive_n_data(fd, &check_checksum, sizeof(check_checksum))) {
|
if ((config->checksum || config_has_basis(config)) &&
|
||||||
|
!receive_n_data(fd, &check_checksum, sizeof(check_checksum))) {
|
||||||
free(check_path);
|
free(check_path);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
@@ -642,6 +798,76 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ---- Alternate basis directories ---- */
|
||||||
|
if (config_has_basis(config)) {
|
||||||
|
BasisMatch basis;
|
||||||
|
basis_match_find(config, check_path, check_size, (time_t)check_mtime, (long)check_mtime_nsec,
|
||||||
|
check_checksum, true, &basis);
|
||||||
|
if (basis.hit) {
|
||||||
|
if (basis.type == BASIS_DEST_COMPARE) {
|
||||||
|
/* compare-dest never copies: an exact match only suppresses the data
|
||||||
|
for a file the destination does not already hold (sparse backup).
|
||||||
|
When the destination holds a DIFFERENT version FastSync falls back to
|
||||||
|
a normal transfer rather than deleting the stale entry the way rsync
|
||||||
|
does (see RSYNC_COMPAT.md). */
|
||||||
|
basis_match_free(&basis);
|
||||||
|
if (!has_old_file) {
|
||||||
|
if (!send_status(fd, STATUS_OK)) {
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
free(old_data);
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
*skipped = true;
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
/* copy-dest / link-dest: materialize the unchanged file locally so the
|
||||||
|
sender can skip the data. The store engine re-applies the normal
|
||||||
|
existing/ignore-existing/update/backup/delay-updates policy. */
|
||||||
|
File* materialized = file_create(check_path);
|
||||||
|
if (materialized && basis.content) {
|
||||||
|
materialized->data = basis.content;
|
||||||
|
basis.content = NULL;
|
||||||
|
materialized->metadata = file_metadata_create(&basis.st);
|
||||||
|
materialized->skip = true; /* receiver must not ack this as a data file */
|
||||||
|
if (basis.type == BASIS_DEST_LINK) {
|
||||||
|
materialized->basis_link = basis.basis_path;
|
||||||
|
basis.basis_path = NULL;
|
||||||
|
}
|
||||||
|
if (!materialized->metadata) {
|
||||||
|
file_destroy(materialized);
|
||||||
|
materialized = NULL;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
file_destroy(materialized);
|
||||||
|
materialized = NULL;
|
||||||
|
}
|
||||||
|
if (materialized) {
|
||||||
|
if (!send_status(fd, STATUS_OK)) {
|
||||||
|
file_destroy(materialized);
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
free(old_data);
|
||||||
|
close(old_fd);
|
||||||
|
free(full_path);
|
||||||
|
free(check_path);
|
||||||
|
*skipped = false;
|
||||||
|
return materialized;
|
||||||
|
}
|
||||||
|
/* Materialization setup failed: fall through to the normal transfer. */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
basis_match_free(&basis);
|
||||||
|
}
|
||||||
|
|
||||||
if (try_delta && old_data != NULL) {
|
if (try_delta && old_data != NULL) {
|
||||||
bool delta_failed = false;
|
bool delta_failed = false;
|
||||||
File* delta_file =
|
File* delta_file =
|
||||||
@@ -843,10 +1069,29 @@ int receive_manifest(int fd, const Config* config, int* next_status) {
|
|||||||
/* With --delay-updates the staged (not yet published) files live directly
|
/* With --delay-updates the staged (not yet published) files live directly
|
||||||
under the receive root in the staging directory; the delete walker must
|
under the receive root in the staging directory; the delete walker must
|
||||||
not treat them as extras or it would remove every staged file before it
|
not treat them as extras or it would remove every staged file before it
|
||||||
can be published. */
|
can be published. Alternate basis directories (--compare-dest /
|
||||||
const char* skip_staging = config->delay_updates ? DELAY_UPDATES_STAGING_DIR : NULL;
|
--copy-dest / --link-dest) are also excluded: they are extra comparison
|
||||||
bool deletion_ok = delete_extras_limited(config->receive_root_directory, manifest,
|
snapshots the user pointed at, not destination content, and deleting them
|
||||||
MAX_SERVER_DELETE_COUNT, skip_staging);
|
would destroy the very files a --link-dest run just linked into place. */
|
||||||
|
int skip_count = (config->delay_updates ? 1 : 0) + config->basis_count;
|
||||||
|
const char** skip_prefixes = NULL;
|
||||||
|
bool deletion_ok = false;
|
||||||
|
if (skip_count > 0) {
|
||||||
|
skip_prefixes = calloc((size_t)skip_count, sizeof(char*));
|
||||||
|
if (!skip_prefixes) {
|
||||||
|
array_list_delete(manifest);
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
int idx = 0;
|
||||||
|
if (config->delay_updates)
|
||||||
|
skip_prefixes[idx++] = DELAY_UPDATES_STAGING_DIR;
|
||||||
|
for (int i = 0; i < config->basis_count; i++)
|
||||||
|
skip_prefixes[idx++] = config->basis_dirs[i].path;
|
||||||
|
}
|
||||||
|
deletion_ok = delete_extras_limited(config->receive_root_directory, manifest,
|
||||||
|
MAX_SERVER_DELETE_COUNT, skip_prefixes, skip_count);
|
||||||
|
free(skip_prefixes);
|
||||||
array_list_delete(manifest);
|
array_list_delete(manifest);
|
||||||
if (!deletion_ok)
|
if (!deletion_ok)
|
||||||
send_status(fd, STATUS_ERROR);
|
send_status(fd, STATUS_ERROR);
|
||||||
|
|||||||
@@ -29,6 +29,12 @@ typedef struct {
|
|||||||
/* True when this entry is an explicit directory entry (--dirs mode): the
|
/* True when this entry is an explicit directory entry (--dirs mode): the
|
||||||
* receiver creates the directory instead of writing a regular file. */
|
* receiver creates the directory instead of writing a regular file. */
|
||||||
bool is_dir;
|
bool is_dir;
|
||||||
|
/* Receiver-only, --link-dest: when set, install the destination entry as a
|
||||||
|
* hard link to this absolute (root-confined) path instead of writing
|
||||||
|
* `data`. The matching code has already verified the link target's content
|
||||||
|
* equals the incoming file, and `data` is kept as the cross-filesystem
|
||||||
|
* fallback (a local copy) if the hard link cannot be created. */
|
||||||
|
char* basis_link;
|
||||||
} File;
|
} File;
|
||||||
|
|
||||||
/* The path that should be sent on the wire and used for the receiver-side
|
/* The path that should be sent on the wire and used for the receiver-side
|
||||||
|
|||||||
@@ -297,7 +297,7 @@ int write_thread(void* pipeline_context) {
|
|||||||
/* Record the per-file outcome so a --remove-source-files sender learns
|
/* Record the per-file outcome so a --remove-source-files sender learns
|
||||||
which sources were actually written versus skipped on the receiver.
|
which sources were actually written versus skipped on the receiver.
|
||||||
Explicit directory entries have no source and are never acknowledged. */
|
Explicit directory entries have no source and are never acknowledged. */
|
||||||
if (context->config->remove_source_files && !file->is_dir &&
|
if (context->config->remove_source_files && !file->is_dir && !file->skip &&
|
||||||
!receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
|
!receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||||
|
|||||||
+27
-12
@@ -204,9 +204,22 @@ static bool is_dir_in_manifest(const char* rel_path, ArrayList* manifest) {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* True when the relative path is, or lies below, one of the protected
|
||||||
|
prefixes. A prefix "a" therefore protects "a" and "a/b/c" but not "ab". */
|
||||||
|
static bool path_under_skip_prefix(const char* rel_path, const char* const* prefixes,
|
||||||
|
int prefix_count) {
|
||||||
|
for (int i = 0; i < prefix_count; i++) {
|
||||||
|
size_t prefix_len = strlen(prefixes[i]);
|
||||||
|
if (strncmp(rel_path, prefixes[i], prefix_len) == 0 &&
|
||||||
|
(rel_path[prefix_len] == '\0' || rel_path[prefix_len] == '/'))
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifest,
|
static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifest,
|
||||||
size_t max_delete, size_t* deleted_count,
|
size_t max_delete, size_t* deleted_count,
|
||||||
const char* skip_root_child) {
|
const char* const* skip_prefixes, int skip_prefix_count) {
|
||||||
int scanfd = dup(dirfd);
|
int scanfd = dup(dirfd);
|
||||||
if (scanfd < 0)
|
if (scanfd < 0)
|
||||||
return false;
|
return false;
|
||||||
@@ -220,18 +233,19 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifes
|
|||||||
while ((entry = readdir(dir)) != NULL) {
|
while ((entry = readdir(dir)) != NULL) {
|
||||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||||
continue;
|
continue;
|
||||||
/* A --delay-updates run keeps its staging directory as a direct child of
|
|
||||||
the receive root. Its contents are not manifest entries yet (they are
|
|
||||||
published after deletion), so descending into it would delete every
|
|
||||||
staged file as an "extra". Skip only the top-level staging name; nested
|
|
||||||
directories with the same name are ordinary destination content. */
|
|
||||||
if (rel_path[0] == '\0' && skip_root_child && strcmp(entry->d_name, skip_root_child) == 0)
|
|
||||||
continue;
|
|
||||||
char* child_rel = path_cat((char*)rel_path, entry->d_name);
|
char* child_rel = path_cat((char*)rel_path, entry->d_name);
|
||||||
if (!child_rel) {
|
if (!child_rel) {
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
/* A --delay-updates run keeps its staging directory below the receive
|
||||||
|
root, and basis-dir snapshots live there too. Their contents are not
|
||||||
|
manifest entries, so descending into them would delete every staged /
|
||||||
|
basis file as an "extra". */
|
||||||
|
if (path_under_skip_prefix(child_rel, skip_prefixes, skip_prefix_count)) {
|
||||||
|
free(child_rel);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||||
if (errno != ENOENT)
|
if (errno != ENOENT)
|
||||||
@@ -249,7 +263,7 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifes
|
|||||||
bool child_removed = false;
|
bool child_removed = false;
|
||||||
if (childfd >= 0) {
|
if (childfd >= 0) {
|
||||||
child_removed = delete_extras_fd(childfd, child_rel, manifest, max_delete, deleted_count,
|
child_removed = delete_extras_fd(childfd, child_rel, manifest, max_delete, deleted_count,
|
||||||
skip_root_child);
|
skip_prefixes, skip_prefix_count);
|
||||||
if (!child_removed)
|
if (!child_removed)
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
close(childfd);
|
close(childfd);
|
||||||
@@ -301,7 +315,7 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifes
|
|||||||
}
|
}
|
||||||
|
|
||||||
bool delete_extras_limited(const char* dest_root, ArrayList* manifest, size_t max_delete,
|
bool delete_extras_limited(const char* dest_root, ArrayList* manifest, size_t max_delete,
|
||||||
const char* skip_root_child) {
|
const char* const* skip_prefixes, int skip_prefix_count) {
|
||||||
if (!manifest)
|
if (!manifest)
|
||||||
return false;
|
return false;
|
||||||
int rootfd;
|
int rootfd;
|
||||||
@@ -318,14 +332,15 @@ bool delete_extras_limited(const char* dest_root, ArrayList* manifest, size_t ma
|
|||||||
if (rootfd < 0)
|
if (rootfd < 0)
|
||||||
return false;
|
return false;
|
||||||
size_t deleted_count = 0;
|
size_t deleted_count = 0;
|
||||||
bool ok = delete_extras_fd(rootfd, "", manifest, max_delete, &deleted_count, skip_root_child);
|
bool ok = delete_extras_fd(rootfd, "", manifest, max_delete, &deleted_count, skip_prefixes,
|
||||||
|
skip_prefix_count);
|
||||||
if (close(rootfd) != 0)
|
if (close(rootfd) != 0)
|
||||||
ok = false;
|
ok = false;
|
||||||
return ok;
|
return ok;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool delete_extras(const char* dest_root, ArrayList* manifest) {
|
bool delete_extras(const char* dest_root, ArrayList* manifest) {
|
||||||
return delete_extras_limited(dest_root, manifest, SIZE_MAX, NULL);
|
return delete_extras_limited(dest_root, manifest, SIZE_MAX, NULL, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool has_path_traversal(const char* path) {
|
bool has_path_traversal(const char* path) {
|
||||||
|
|||||||
+7
-5
@@ -10,12 +10,14 @@ char* output_escape(const char* string, bool eight_bit_output);
|
|||||||
char* path_cat(const char* path1, const char* path2);
|
char* path_cat(const char* path1, const char* path2);
|
||||||
bool glob_match(const char* pattern, const char* str);
|
bool glob_match(const char* pattern, const char* str);
|
||||||
bool delete_extras(const char* dest_root, ArrayList* manifest);
|
bool delete_extras(const char* dest_root, ArrayList* manifest);
|
||||||
/* Remove files/dirs under dest_root that are not listed in manifest. When
|
/* Remove files/dirs under dest_root that are not listed in manifest. The
|
||||||
skip_root_child is non-NULL, a direct child of dest_root with that exact
|
delete walker never descends into (and so never removes) an entry whose
|
||||||
name is left untouched (used to protect the --delay-updates staging
|
relative path equals one of the skip_prefixes or lies below one: used to
|
||||||
directory, which holds files that are still to be published). */
|
protect the --delay-updates staging directory (files still to be published)
|
||||||
|
and the --compare-dest/--copy-dest/--link-dest basis trees (snapshots the
|
||||||
|
transfer links from, never destination content). */
|
||||||
bool delete_extras_limited(const char* dest_root, ArrayList* manifest, size_t max_delete,
|
bool delete_extras_limited(const char* dest_root, ArrayList* manifest, size_t max_delete,
|
||||||
const char* skip_root_child);
|
const char* const* skip_prefixes, int skip_prefix_count);
|
||||||
bool utils_set_authorized_root(int fd, const char* canonical_path);
|
bool utils_set_authorized_root(int fd, const char* canonical_path);
|
||||||
/* The fd-only compatibility form is fail-closed for path-based operations;
|
/* The fd-only compatibility form is fail-closed for path-based operations;
|
||||||
* callers should use utils_set_authorized_root with the canonical identity. */
|
* callers should use utils_set_authorized_root with the canonical identity. */
|
||||||
|
|||||||
@@ -1971,3 +1971,213 @@ class TestFilters:
|
|||||||
"""--filter/-C/-F rule layer: excludes prune, ordering is first-match-wins,
|
"""--filter/-C/-F rule layer: excludes prune, ordering is first-match-wins,
|
||||||
the default with no matching rule is include, and legacy --exclude remains
|
the default with no matching rule is include, and legacy --exclude remains
|
||||||
an independent layer."""
|
an independent layer."""
|
||||||
|
|
||||||
|
|
||||||
|
def _pin_mtime(path, ts):
|
||||||
|
os.utime(path, (ts, ts))
|
||||||
|
|
||||||
|
|
||||||
|
class TestBasisDestDirs:
|
||||||
|
"""--compare-dest / --copy-dest / --link-dest alternate basis directories.
|
||||||
|
|
||||||
|
FastSync's basis directories are relative to the destination root and are
|
||||||
|
confined below it. The "unchanged" decision is receiver-side and requires
|
||||||
|
the per-file --incremental handshake (implied by these flags), so the basis
|
||||||
|
snapshot must reproduce the exact destination-relative mirror path of the
|
||||||
|
incoming files.
|
||||||
|
"""
|
||||||
|
|
||||||
|
STAGING = ".fastsync-stage"
|
||||||
|
TS = 1577836800 # 2020-01-01 00:00:00 UTC, used to pin matching mtimes
|
||||||
|
|
||||||
|
# files: rel-path -> (source content, basis content or None, matched?)
|
||||||
|
UNCHANGED = "unchanged.txt"
|
||||||
|
CHANGED = "changed.txt"
|
||||||
|
ADDED = "added.txt"
|
||||||
|
|
||||||
|
def _make_source(self, name, source_files):
|
||||||
|
src = os.path.join(TEST_DATA_DIR, name)
|
||||||
|
clean_dir(src)
|
||||||
|
for rel, content in source_files.items():
|
||||||
|
full = os.path.join(src, rel)
|
||||||
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||||
|
with open(full, "wb") as fh:
|
||||||
|
fh.write(content)
|
||||||
|
_pin_mtime(full, self.TS)
|
||||||
|
return src
|
||||||
|
|
||||||
|
def _basis_root(self, dest, source):
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
rel = os.path.relpath(received, dest)
|
||||||
|
return os.path.join(dest, rel)
|
||||||
|
|
||||||
|
def _seed_basis(self, dest, source, basis_dir, basis_files):
|
||||||
|
base = os.path.join(dest, basis_dir, os.path.relpath(
|
||||||
|
get_dest_received_dir(dest, source), dest))
|
||||||
|
for rel, content in basis_files.items():
|
||||||
|
full = os.path.join(base, rel)
|
||||||
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||||
|
with open(full, "wb") as fh:
|
||||||
|
fh.write(content)
|
||||||
|
_pin_mtime(full, self.TS)
|
||||||
|
return base
|
||||||
|
|
||||||
|
def _source_tree(self, prefix):
|
||||||
|
return {
|
||||||
|
self.UNCHANGED: b"stable content v1\n",
|
||||||
|
self.CHANGED: b"changed content now\n",
|
||||||
|
self.ADDED: b"brand new content\n",
|
||||||
|
}
|
||||||
|
|
||||||
|
def _basis_tree(self, prefix):
|
||||||
|
# unchanged.txt matches the source; changed.txt differs in CONTENT but
|
||||||
|
# shares size/mtime pinning; added.txt is missing from the basis.
|
||||||
|
return {
|
||||||
|
self.UNCHANGED: b"stable content v1\n",
|
||||||
|
self.CHANGED: b"ANCIENT DIFFERENT CONTENT!\n",
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_compare_dest_skips_matching_and_transfers_missing(self, shared_server):
|
||||||
|
source = self._make_source("basis_compare_src", self._source_tree("c"))
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "basis_compare_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
self._seed_basis(dest, source, "cbasis", self._basis_tree("c"))
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["--compare-dest=cbasis"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"compare-dest failed: {result.stderr[:300]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
# compare-dest never copies: an exact basis match is skipped, leaving a
|
||||||
|
# sparse destination (rsync parity).
|
||||||
|
assert not os.path.exists(os.path.join(received, self.UNCHANGED)), \
|
||||||
|
"compare-dest materialized the unchanged file"
|
||||||
|
# Files the destination lacks AND the basis cannot satisfy are still
|
||||||
|
# transferred normally.
|
||||||
|
assert _read_file(os.path.join(received, self.CHANGED)) == \
|
||||||
|
self._source_tree("c")[self.CHANGED], "changed file not transferred"
|
||||||
|
assert _read_file(os.path.join(received, self.ADDED)) == \
|
||||||
|
self._source_tree("c")[self.ADDED], "added file not transferred"
|
||||||
|
|
||||||
|
def test_compare_dest_content_mismatch_forces_transfer(self, shared_server):
|
||||||
|
# The basis holds a file with a DIFFERENT body: even though it shares
|
||||||
|
# the mtime pin, the xxHash check fails and the data must be sent.
|
||||||
|
source = self._make_source("basis_compare_mismatch_src", {self.UNCHANGED: b"real data\n"})
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "basis_compare_mismatch_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
basis = self._seed_basis(dest, source, "cbasis", {self.UNCHANGED: b"stale data!!\n"})
|
||||||
|
result, _ = run_client(source, dest, flags=["--compare-dest=cbasis"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"compare-dest mismatch failed: {result.stderr[:300]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert _read_file(os.path.join(received, self.UNCHANGED)) == b"real data\n", \
|
||||||
|
"content mismatch did not fall back to a normal transfer"
|
||||||
|
assert os.stat(os.path.join(received, self.UNCHANGED)).st_ino != \
|
||||||
|
os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
|
||||||
|
|
||||||
|
def test_copy_dest_copies_unchanged_and_transfers_changed(self, shared_server):
|
||||||
|
source = self._make_source("basis_copy_src", self._source_tree("cp"))
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "basis_copy_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
basis = self._seed_basis(dest, source, "cpbasis", self._basis_tree("cp"))
|
||||||
|
result, _ = run_client(source, dest, flags=["--copy-dest=cpbasis"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"copy-dest failed: {result.stderr[:300]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
unchanged = os.path.join(received, self.UNCHANGED)
|
||||||
|
assert _read_file(unchanged) == b"stable content v1\n", "unchanged file not materialized"
|
||||||
|
# A real local copy, NOT a hard link to the basis file.
|
||||||
|
assert os.stat(unchanged).st_ino != os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
|
||||||
|
# Changed content falls back to a normal transfer of the sender data.
|
||||||
|
assert _read_file(os.path.join(received, self.CHANGED)) == \
|
||||||
|
self._source_tree("cp")[self.CHANGED]
|
||||||
|
assert _read_file(os.path.join(received, self.ADDED)) == \
|
||||||
|
self._source_tree("cp")[self.ADDED]
|
||||||
|
|
||||||
|
def test_link_dest_hardlinks_and_falls_back(self, shared_server):
|
||||||
|
source = self._make_source("basis_link_src", self._source_tree("ln"))
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "basis_link_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
basis = self._seed_basis(dest, source, "lnbasis", self._basis_tree("ln"))
|
||||||
|
result, _ = run_client(source, dest, flags=["--link-dest=lnbasis"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"link-dest failed: {result.stderr[:300]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
unchanged = os.path.join(received, self.UNCHANGED)
|
||||||
|
basis_file = os.path.join(basis, self.UNCHANGED)
|
||||||
|
# Real hard link: same inode as the DIR file, nlink >= 2, no data copy.
|
||||||
|
assert os.path.exists(unchanged)
|
||||||
|
assert os.stat(unchanged).st_ino == os.stat(basis_file).st_ino, \
|
||||||
|
"link-dest did not produce a hard link"
|
||||||
|
assert os.stat(unchanged).st_nlink >= 2
|
||||||
|
# Content mismatch must fall back to a plain transfer (not a link).
|
||||||
|
changed = os.path.join(received, self.CHANGED)
|
||||||
|
assert _read_file(changed) == self._source_tree("ln")[self.CHANGED]
|
||||||
|
assert os.stat(changed).st_ino != os.stat(os.path.join(basis, self.CHANGED)).st_ino
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("flag", ["--compare-dest", "--copy-dest", "--link-dest"])
|
||||||
|
def test_basis_dir_missing_is_a_clean_noop(self, shared_server, flag):
|
||||||
|
# A basis directory that does not exist must simply transfer everything.
|
||||||
|
source = self._make_source("basis_missing_src", {self.UNCHANGED: b"content\n"})
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "basis_missing_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=[f"{flag}=nope"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"{flag} with missing dir failed: {result.stderr[:300]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert _read_file(os.path.join(received, self.UNCHANGED)) == b"content\n"
|
||||||
|
|
||||||
|
def test_link_dest_multithreaded(self, shared_server):
|
||||||
|
source = self._make_source("basis_link_mt_src", self._source_tree("mt"))
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "basis_link_mt_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
basis = self._seed_basis(dest, source, "mtbasis", self._basis_tree("mt"))
|
||||||
|
result, _ = run_client(source, dest, flags=["--link-dest=mtbasis", "-m"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-m link-dest failed: {result.stderr[:300]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert os.stat(os.path.join(received, self.UNCHANGED)).st_ino == \
|
||||||
|
os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
|
||||||
|
assert _read_file(os.path.join(received, self.ADDED)) == \
|
||||||
|
self._source_tree("mt")[self.ADDED]
|
||||||
|
|
||||||
|
def test_link_dest_with_delay_updates_stages_and_publishes_link(self, shared_server):
|
||||||
|
source = self._make_source("basis_link_delay_src", {self.UNCHANGED: b"v1\n"})
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "basis_link_delay_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
basis = self._seed_basis(dest, source, "delaybasis", {self.UNCHANGED: b"v1\n"})
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["--link-dest=delaybasis", "--delay-updates"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"delay-updates link-dest failed: {result.stderr[:300]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
unchanged = os.path.join(received, self.UNCHANGED)
|
||||||
|
assert os.stat(unchanged).st_ino == \
|
||||||
|
os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
|
||||||
|
assert not os.path.isdir(os.path.join(dest, self.STAGING)), \
|
||||||
|
"delay-updates staging tree was not cleaned up"
|
||||||
|
|
||||||
|
def test_delete_does_not_touch_basis_dir(self):
|
||||||
|
"""--delete removes genuine extras but must never treat a basis-dir
|
||||||
|
snapshot (which a --link-dest run just linked from) as destination
|
||||||
|
content."""
|
||||||
|
source = self._make_source("basis_delete_src", {self.UNCHANGED: b"v1\n"})
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "basis_delete_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
basis = self._seed_basis(dest, source, "delbasis", {self.UNCHANGED: b"v1\n"})
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
os.makedirs(received, exist_ok=True)
|
||||||
|
extra = os.path.join(received, "extra.txt")
|
||||||
|
with open(extra, "wb") as fh:
|
||||||
|
fh.write(b"extra")
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
result, _ = run_client(source, dest, flags=["--link-dest=delbasis", "--delete"],
|
||||||
|
port=server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"delete+link-dest failed: {result.stderr[:300]}"
|
||||||
|
assert not os.path.exists(extra), "genuine extra file was not deleted"
|
||||||
|
assert _read_file(os.path.join(received, self.UNCHANGED)) == b"v1\n"
|
||||||
|
assert os.path.exists(os.path.join(basis, self.UNCHANGED)), \
|
||||||
|
"basis directory was deleted by --delete"
|
||||||
|
assert os.stat(os.path.join(received, self.UNCHANGED)).st_ino == \
|
||||||
|
os.stat(os.path.join(basis, self.UNCHANGED)).st_ino
|
||||||
+83
-3
@@ -595,6 +595,86 @@ static void test_parse_args_relative_no_implied_mkpath() {
|
|||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Parse --compare-dest/--copy-dest/--link-dest, including the =value and
|
||||||
|
separate-argument forms, and verify the ordered (repeatable) basis list. */
|
||||||
|
static void test_parse_args_basis_dirs() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
char* argv[] = {"fastsync", "--link-dest=prior", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(config_has_basis(cfg));
|
||||||
|
EXPECT_EQ_INT(cfg->basis_count, 1);
|
||||||
|
EXPECT_EQ_INT(cfg->basis_dirs[0].type, BASIS_DEST_LINK);
|
||||||
|
EXPECT_EQ_STR(cfg->basis_dirs[0].path, "prior");
|
||||||
|
/* Basis dirs are honored by the receiver-side per-file check, so they imply
|
||||||
|
--incremental (and, unless disabled, metadata) on the sender. */
|
||||||
|
EXPECT_TRUE(cfg->use_incremental);
|
||||||
|
EXPECT_TRUE(cfg->use_metadata);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* argv2[] = {"fastsync", "--compare-dest", "cmp", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv2, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_EQ_INT(cfg->basis_count, 1);
|
||||||
|
EXPECT_EQ_INT(cfg->basis_dirs[0].type, BASIS_DEST_COMPARE);
|
||||||
|
EXPECT_EQ_STR(cfg->basis_dirs[0].path, "cmp");
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
/* Repetition is supported: entries keep command-line order and type. */
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* argv3[] = {"fastsync", "--link-dest=a", "--compare-dest=b",
|
||||||
|
"--link-dest=c", "--copy-dest=d", "/src",
|
||||||
|
"/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 7, argv3, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_EQ_INT(cfg->basis_count, 4);
|
||||||
|
EXPECT_EQ_INT(cfg->basis_dirs[0].type, BASIS_DEST_LINK);
|
||||||
|
EXPECT_EQ_STR(cfg->basis_dirs[0].path, "a");
|
||||||
|
EXPECT_EQ_INT(cfg->basis_dirs[1].type, BASIS_DEST_COMPARE);
|
||||||
|
EXPECT_EQ_STR(cfg->basis_dirs[1].path, "b");
|
||||||
|
EXPECT_EQ_INT(cfg->basis_dirs[2].type, BASIS_DEST_LINK);
|
||||||
|
EXPECT_EQ_STR(cfg->basis_dirs[2].path, "c");
|
||||||
|
EXPECT_EQ_INT(cfg->basis_dirs[3].type, BASIS_DEST_COPY);
|
||||||
|
EXPECT_EQ_STR(cfg->basis_dirs[3].path, "d");
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
/* Nested relative basis dirs are allowed (they resolve below the root). */
|
||||||
|
cfg = config_create();
|
||||||
|
positional_count = 0;
|
||||||
|
char* argv4[] = {"fastsync", "--copy-dest=snap/2026-01", "/src", "/dst"};
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv4, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_EQ_INT(cfg->basis_count, 1);
|
||||||
|
EXPECT_EQ_STR(cfg->basis_dirs[0].path, "snap/2026-01");
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Absolute, escaping, or degenerate basis-dir values must be rejected up
|
||||||
|
front: they would resolve outside the destination root on the receiver. */
|
||||||
|
static void test_parse_args_basis_invalid_paths() {
|
||||||
|
static const char* const invalid[] = {"/abs", "..", "a/../b", "."};
|
||||||
|
for (size_t i = 0; i < sizeof(invalid) / sizeof(invalid[0]); i++) {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--link-dest", (char*)invalid[i], "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Basis dirs require the per-file incremental handshake, which -s disables. */
|
||||||
|
static void test_validate_config_basis_rejects_chunk_serialization() {
|
||||||
|
Config* cfg = valid_client_config();
|
||||||
|
EXPECT_EQ_INT(config_basis_append(cfg, BASIS_DEST_LINK, "prior"), 0);
|
||||||
|
cfg->use_chunk_serialization = true;
|
||||||
|
EXPECT_FALSE(validate_config(cfg));
|
||||||
|
cfg->use_chunk_serialization = false;
|
||||||
|
EXPECT_TRUE(validate_config(cfg));
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
/* --del is recognized as the rsync alias, but its timing mode is not implemented. */
|
/* --del is recognized as the rsync alias, but its timing mode is not implemented. */
|
||||||
static void test_parse_args_delete_during_alias_unimplemented() {
|
static void test_parse_args_delete_during_alias_unimplemented() {
|
||||||
static const char* const options[] = {"--del", "--delete-during"};
|
static const char* const options[] = {"--del", "--delete-during"};
|
||||||
@@ -632,9 +712,6 @@ static void test_parse_args_rejects_unimplemented_options() {
|
|||||||
"-e",
|
"-e",
|
||||||
"--rsh",
|
"--rsh",
|
||||||
"--rsync-path",
|
"--rsync-path",
|
||||||
"--compare-dest",
|
|
||||||
"--copy-dest",
|
|
||||||
"--link-dest",
|
|
||||||
"--delete-before",
|
"--delete-before",
|
||||||
"--address",
|
"--address",
|
||||||
"--bind-address",
|
"--bind-address",
|
||||||
@@ -1589,4 +1666,7 @@ void test_client_cli() {
|
|||||||
test_parse_args_files_from();
|
test_parse_args_files_from();
|
||||||
test_parse_args_filter_rules();
|
test_parse_args_filter_rules();
|
||||||
test_parse_args_from0_cvs_filter_file_flags();
|
test_parse_args_from0_cvs_filter_file_flags();
|
||||||
|
test_parse_args_basis_dirs();
|
||||||
|
test_parse_args_basis_invalid_paths();
|
||||||
|
test_validate_config_basis_rejects_chunk_serialization();
|
||||||
}
|
}
|
||||||
@@ -438,6 +438,96 @@ static void test_config_delay_updates_reserved_backup_rejected() {
|
|||||||
config_delete(c);
|
config_delete(c);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Basis-dir lists survive the config wire: each entry's type and path must
|
||||||
|
round-trip unchanged. */
|
||||||
|
static void test_config_basis_roundtrip() {
|
||||||
|
if (is_running_under_valgrind())
|
||||||
|
return;
|
||||||
|
Config* send_cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(send_cfg);
|
||||||
|
send_cfg->send_directory = str_dup("/send/src");
|
||||||
|
send_cfg->receive_root_directory = str_dup("/send/dst");
|
||||||
|
EXPECT_EQ_INT(config_basis_append(send_cfg, BASIS_DEST_LINK, "prior"), 0);
|
||||||
|
EXPECT_EQ_INT(config_basis_append(send_cfg, BASIS_DEST_COMPARE, "snap/2026-01"), 0);
|
||||||
|
EXPECT_EQ_INT(config_basis_append(send_cfg, BASIS_DEST_COPY, "copy"), 0);
|
||||||
|
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
close(p[1]);
|
||||||
|
io_set_fds(p[0], p[0]);
|
||||||
|
Config* recv = config_receive(p[0]);
|
||||||
|
bool ok = recv != NULL && recv->basis_count == 3 && recv->basis_dirs != NULL;
|
||||||
|
if (ok) {
|
||||||
|
ok = recv->basis_dirs[0].type == BASIS_DEST_LINK &&
|
||||||
|
strcmp(recv->basis_dirs[0].path, "prior") == 0;
|
||||||
|
ok = ok && recv->basis_dirs[1].type == BASIS_DEST_COMPARE &&
|
||||||
|
strcmp(recv->basis_dirs[1].path, "snap/2026-01") == 0;
|
||||||
|
ok = ok && recv->basis_dirs[2].type == BASIS_DEST_COPY &&
|
||||||
|
strcmp(recv->basis_dirs[2].path, "copy") == 0;
|
||||||
|
}
|
||||||
|
config_delete(recv);
|
||||||
|
close(p[0]);
|
||||||
|
close(p[1]);
|
||||||
|
_exit(ok ? 0 : 1);
|
||||||
|
} else {
|
||||||
|
close(p[0]);
|
||||||
|
io_set_fds(p[1], p[1]);
|
||||||
|
bool sent = config_send(p[1], send_cfg);
|
||||||
|
int status;
|
||||||
|
waitpid(pid, &status, 0);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(send_cfg);
|
||||||
|
EXPECT_TRUE(sent);
|
||||||
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The receiver must reject a basis-dir path that would escape the destination
|
||||||
|
root. The values are injected directly (bypassing the client-side append
|
||||||
|
validator) so the receiver-side wire validation is what is exercised. */
|
||||||
|
static void test_config_basis_wire_rejects_escaping() {
|
||||||
|
if (is_running_under_valgrind())
|
||||||
|
return;
|
||||||
|
Config* c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
c->send_directory = str_dup("/src");
|
||||||
|
c->receive_root_directory = str_dup("/dst");
|
||||||
|
c->basis_count = 1;
|
||||||
|
c->basis_dirs = calloc(1, sizeof(BasisDest));
|
||||||
|
c->basis_dirs[0].type = BASIS_DEST_LINK;
|
||||||
|
c->basis_dirs[0].path = str_dup("../../etc");
|
||||||
|
EXPECT_FALSE(roundtrip_config_ok(c));
|
||||||
|
config_delete(c);
|
||||||
|
|
||||||
|
c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
c->send_directory = str_dup("/src");
|
||||||
|
c->receive_root_directory = str_dup("/dst");
|
||||||
|
c->basis_count = 1;
|
||||||
|
c->basis_dirs = calloc(1, sizeof(BasisDest));
|
||||||
|
c->basis_dirs[0].type = BASIS_DEST_LINK;
|
||||||
|
c->basis_dirs[0].path = str_dup("/abs");
|
||||||
|
EXPECT_FALSE(roundtrip_config_ok(c));
|
||||||
|
config_delete(c);
|
||||||
|
|
||||||
|
/* A well-formed list still round-trips even with a manually built struct. */
|
||||||
|
c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
c->send_directory = str_dup("/src");
|
||||||
|
c->receive_root_directory = str_dup("/dst");
|
||||||
|
c->basis_count = 1;
|
||||||
|
c->basis_dirs = calloc(1, sizeof(BasisDest));
|
||||||
|
c->basis_dirs[0].type = BASIS_DEST_COPY;
|
||||||
|
c->basis_dirs[0].path = str_dup("safe");
|
||||||
|
EXPECT_TRUE(roundtrip_config_ok(c));
|
||||||
|
config_delete(c);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_config_is_remote_dest() {
|
static void test_config_is_remote_dest() {
|
||||||
/* Valid SSH-style destinations */
|
/* Valid SSH-style destinations */
|
||||||
EXPECT_TRUE(config_is_remote_dest("user@host:/path"));
|
EXPECT_TRUE(config_is_remote_dest("user@host:/path"));
|
||||||
@@ -474,6 +564,8 @@ void test_config() {
|
|||||||
test_config_string_null_vs_empty_roundtrip();
|
test_config_string_null_vs_empty_roundtrip();
|
||||||
test_config_temp_dir_roundtrip();
|
test_config_temp_dir_roundtrip();
|
||||||
test_config_delay_updates_reserved_backup_rejected();
|
test_config_delay_updates_reserved_backup_rejected();
|
||||||
|
test_config_basis_roundtrip();
|
||||||
|
test_config_basis_wire_rejects_escaping();
|
||||||
}
|
}
|
||||||
test_config_is_remote_dest();
|
test_config_is_remote_dest();
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user