feat(stats): receiver-observed created/literal counters (protocol 2.28.0)
- PROTOCOL_VERSION 2.27.0 -> 2.28.0; STATUS_STATS gains literal_bytes and the created reg/dir/link/special counters (golden wire updated) - receiver reports which destination entries it newly created, including implicitly-created parent directories below the logical transfer root, so Number of created files carries rsync's per-type breakdown - Literal data is now exact for a delta transfer (receiver counts the literal fragments it stored) - differential-tested vs rsync 3.4.1 for fresh-create, update and delta
This commit is contained in:
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
cmake_minimum_required(VERSION 3.22)
|
cmake_minimum_required(VERSION 3.22)
|
||||||
|
|
||||||
project(FastFileTransfer VERSION 2.27.0)
|
project(FastFileTransfer VERSION 2.28.0)
|
||||||
|
|
||||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||||
set(CMAKE_C_STANDARD 11)
|
set(CMAKE_C_STANDARD 11)
|
||||||
|
|||||||
+1
-1
@@ -82,7 +82,7 @@ Every one of those has an entry below with its remaining caveats.
|
|||||||
|
|
||||||
| Flag | Rsync Description | FastSync Status | Notes |
|
| Flag | Rsync Description | FastSync Status | Notes |
|
||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `--stats` | Give transfer stats | ⚠️ Caveat | Prints transfer statistics. Protocol 2.25.0 populates the receiver-only counters the sender cannot observe: `Matched data` (a delta basis's reused bytes) and `Number of deleted files` come from the receiver's `STATUS_STATS` report. The sender now tracks the scanned file list per type and only counts regular files the receiver actually stored, so `Number of files` carries rsync's `(reg: X, dir: Y, link: Z, special: W)` breakdown (directories come from the scanner's captured directory list, present for `-a`/`-t`/`-p`), `Number of regular files transferred` excludes symlinks/specials and up-to-date files, `Total file size` includes symlink target lengths, and `Total transferred file size`/`Literal data` count only transferred files — all differential-tested in the sequential and `--threads` paths. **Remaining divergences:** `Number of created files` is the transferred-regular count (FastSync cannot tell which entries the receiver newly created, so on an update where rsync reports 0 created FastSync can report the transferred file) and lacks the type breakdown; a recursive scan that preserves no directory attribute (`-r` without `-t`/`-p`) captures no directory entries, so the `dir:` category is then omitted; `Literal data` is exact for a whole-file transfer but an upper bound for a delta transfer (the sender counts each stored file's whole source size rather than only the literal fragments rsync ships, since it does not measure the delta payload it sends); rsync's per-type breakdown on `Number of deleted files` is not reproduced; and `Total bytes sent`/`received` are FastSync wire bytes framed differently from rsync's, so they are not numerically comparable |
|
| `--stats` | Give transfer stats | ⚠️ Caveat | Prints transfer statistics. Protocol 2.25.0 populates the receiver-only counters the sender cannot observe (`Matched data`, `Number of deleted files`) from the receiver's `STATUS_STATS` report; the sender tracks the scanned file list per type so `Number of files` carries rsync's `(reg: X, dir: Y, link: Z, special: W)` breakdown (directories come from the scanner's captured directory list, present for `-a`/`-t`/`-p`), `Number of regular files transferred` excludes symlinks/specials and up-to-date files, `Total file size` includes symlink target lengths, and `Total transferred file size` counts only transferred files. **Protocol 2.28.0 extends `STATUS_STATS`** with receiver-observed `literal_bytes` and the four `created_*` counters: `Number of created files` now carries rsync's `(reg/dir/link/special)` breakdown (the receiver reports which destination entries it newly created, including implicitly-created parent directories below the transfer root) and `Literal data` is exact for a delta transfer (the receiver counts the literal fragments it stored, not the whole source size) — all differential-tested in the sequential and `--threads` paths against rsync 3.4.1 for fresh-create, update and delta shapes. **Remaining divergences:** a recursive scan that preserves no directory attribute (`-r` without `-t`/`-p`) captures no directory entries, so the `dir:` category is then omitted from `Number of files`; rsync's per-type breakdown on `Number of deleted files` is not reproduced; and `Total bytes sent`/`received` are FastSync wire bytes framed differently from rsync's, so they are not numerically comparable |
|
||||||
| `-h`, `--human-readable` | Human-readable numbers | ✅ Parity | Formats transfer byte and rate counts using rsync's **decimal** (base-1000) units, matching rsync `-h` (e.g. `1.23M`), not binary units. **A lone `-h` with no transfer arguments prints help instead** (protocol 2.26.0), matching the rsync idiom; `-h` alongside a transfer remains human-readable |
|
| `-h`, `--human-readable` | Human-readable numbers | ✅ Parity | Formats transfer byte and rate counts using rsync's **decimal** (base-1000) units, matching rsync `-h` (e.g. `1.23M`), not binary units. **A lone `-h` with no transfer arguments prints help instead** (protocol 2.26.0), matching the rsync idiom; `-h` alongside a transfer remains human-readable |
|
||||||
| `-i`, `--itemize-changes` | Per-file change summary | ✅ Parity | Prints rsync-style `>f+++++++++` lines to stdout only for files actually sent (also under `-j`/`--threads`); unchanged files print nothing, matching single-`-i` behavior |
|
| `-i`, `--itemize-changes` | Per-file change summary | ✅ Parity | Prints rsync-style `>f+++++++++` lines to stdout only for files actually sent (also under `-j`/`--threads`); unchanged files print nothing, matching single-`-i` behavior |
|
||||||
| `--progress` | Show progress | ⚠️ Caveat | Protocol 2.25.0 prints rsync-style per-file progress blocks (percent, transferred/total bytes, rate, elapsed, `(xfr#N, to-chk=M/T)`) fed by the receiver's `STATUS_STATS`, in both the sequential and `--threads` send paths. FastSync now also prints rsync's leading `./` transfer-root line and counts that root entry in the `to-chk` denominator, so a **single-file transfer's name lines and deterministic frames are byte-identical to rsync** (differential test). **Remaining divergences:** for a multi-directory tree rsync prints a per-directory name line and its `to-chk` denominator includes every directory/symlink/special entry; FastSync's streaming scan emits only file-name lines and counts just the root plus transferred files (a full flist pre-count would be needed), and the rate/ETA are wall-clock dependent |
|
| `--progress` | Show progress | ⚠️ Caveat | Protocol 2.25.0 prints rsync-style per-file progress blocks (percent, transferred/total bytes, rate, elapsed, `(xfr#N, to-chk=M/T)`) fed by the receiver's `STATUS_STATS`, in both the sequential and `--threads` send paths. FastSync now also prints rsync's leading `./` transfer-root line and counts that root entry in the `to-chk` denominator, so a **single-file transfer's name lines and deterministic frames are byte-identical to rsync** (differential test). **Remaining divergences:** for a multi-directory tree rsync prints a per-directory name line and its `to-chk` denominator includes every directory/symlink/special entry; FastSync's streaming scan emits only file-name lines and counts just the root plus transferred files (a full flist pre-count would be needed), and the rate/ETA are wall-clock dependent |
|
||||||
|
|||||||
+36
-17
@@ -93,12 +93,11 @@ static const char* stats_bytes(const Config* config, unsigned long long bytes, c
|
|||||||
return buffer;
|
return buffer;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Build rsync's `Number of files` parenthetical: each non-zero category, in
|
/* Build rsync's per-type parenthetical: each non-zero category, in
|
||||||
reg/dir/link/special order. Empty when the flist counted nothing. */
|
reg/dir/link/special order. Empty when every count is zero. */
|
||||||
static void stats_type_breakdown(const TransferStats* stats, char* out, size_t out_size) {
|
static void type_breakdown(unsigned long long reg, unsigned long long dir, unsigned long long link,
|
||||||
unsigned long long total =
|
unsigned long long special, char* out, size_t out_size) {
|
||||||
stats->flist_reg + stats->flist_dir + stats->flist_link + stats->flist_special;
|
if (reg + dir + link + special == 0) {
|
||||||
if (total == 0) {
|
|
||||||
out[0] = '\0';
|
out[0] = '\0';
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -107,10 +106,7 @@ static void stats_type_breakdown(const TransferStats* stats, char* out, size_t o
|
|||||||
const struct {
|
const struct {
|
||||||
const char* name;
|
const char* name;
|
||||||
unsigned long long count;
|
unsigned long long count;
|
||||||
} parts[4] = {{"reg", stats->flist_reg},
|
} parts[4] = {{"reg", reg}, {"dir", dir}, {"link", link}, {"special", special}};
|
||||||
{"dir", stats->flist_dir},
|
|
||||||
{"link", stats->flist_link},
|
|
||||||
{"special", stats->flist_special}};
|
|
||||||
bool first = true;
|
bool first = true;
|
||||||
for (size_t i = 0; i < 4; i++) {
|
for (size_t i = 0; i < 4; i++) {
|
||||||
if (parts[i].count == 0)
|
if (parts[i].count == 0)
|
||||||
@@ -127,6 +123,12 @@ static void stats_type_breakdown(const TransferStats* stats, char* out, size_t o
|
|||||||
out[used] = '\0';
|
out[used] = '\0';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Build rsync's `Number of files` parenthetical from the scan's flist counts. */
|
||||||
|
static void stats_type_breakdown(const TransferStats* stats, char* out, size_t out_size) {
|
||||||
|
type_breakdown(stats->flist_reg, stats->flist_dir, stats->flist_link, stats->flist_special, out,
|
||||||
|
out_size);
|
||||||
|
}
|
||||||
|
|
||||||
/* Print the rsync `--stats` block on stdout. The source-side flist and
|
/* Print the rsync `--stats` block on stdout. The source-side flist and
|
||||||
transferred counters come from `stats` (filled while scanning/sending), the
|
transferred counters come from `stats` (filled while scanning/sending), the
|
||||||
receiver-only counters from the STATUS_STATS frame, and the wire byte totals
|
receiver-only counters from the STATUS_STATS frame, and the wire byte totals
|
||||||
@@ -151,6 +153,7 @@ static void report_transfer_stats(const Config* config, const TransferStats* sta
|
|||||||
char total_buffer[32];
|
char total_buffer[32];
|
||||||
char transferred_buffer[32];
|
char transferred_buffer[32];
|
||||||
char literal_buffer[32];
|
char literal_buffer[32];
|
||||||
|
char matched_buffer[32];
|
||||||
char sent_buffer[32];
|
char sent_buffer[32];
|
||||||
char recv_buffer[32];
|
char recv_buffer[32];
|
||||||
char rate_buffer[32] = {0};
|
char rate_buffer[32] = {0};
|
||||||
@@ -159,8 +162,15 @@ static void report_transfer_stats(const Config* config, const TransferStats* sta
|
|||||||
stats_bytes(config, stats->total_file_size, total_buffer, sizeof(total_buffer));
|
stats_bytes(config, stats->total_file_size, total_buffer, sizeof(total_buffer));
|
||||||
const char* transferred = stats_bytes(config, stats->transferred_file_size, transferred_buffer,
|
const char* transferred = stats_bytes(config, stats->transferred_file_size, transferred_buffer,
|
||||||
sizeof(transferred_buffer));
|
sizeof(transferred_buffer));
|
||||||
const char* literal =
|
/* Protocol 2.28.0: the receiver reports the bytes it literally stored, which
|
||||||
stats_bytes(config, stats->literal_data, literal_buffer, sizeof(literal_buffer));
|
is exact for a delta transfer (the sender's own literal_data counts each
|
||||||
|
stored file's whole source size and is only an upper bound). Fall back to
|
||||||
|
the sender total when the receiver reported no delta/literal accounting
|
||||||
|
(e.g. a local no-server path). */
|
||||||
|
unsigned long long literal_bytes = (recv->literal_bytes != 0 || recv->matched_data != 0)
|
||||||
|
? recv->literal_bytes
|
||||||
|
: stats->literal_data;
|
||||||
|
const char* literal = stats_bytes(config, literal_bytes, literal_buffer, sizeof(literal_buffer));
|
||||||
const char* sent_s = stats_bytes(config, sent, sent_buffer, sizeof(sent_buffer));
|
const char* sent_s = stats_bytes(config, sent, sent_buffer, sizeof(sent_buffer));
|
||||||
const char* recv_s = stats_bytes(config, received, recv_buffer, sizeof(recv_buffer));
|
const char* recv_s = stats_bytes(config, received, recv_buffer, sizeof(recv_buffer));
|
||||||
const char* rate_str = rate_buffer;
|
const char* rate_str = rate_buffer;
|
||||||
@@ -177,21 +187,30 @@ static void report_transfer_stats(const Config* config, const TransferStats* sta
|
|||||||
stats_type_breakdown(stats, breakdown, sizeof(breakdown));
|
stats_type_breakdown(stats, breakdown, sizeof(breakdown));
|
||||||
unsigned long long flist_total =
|
unsigned long long flist_total =
|
||||||
stats->flist_reg + stats->flist_dir + stats->flist_link + stats->flist_special;
|
stats->flist_reg + stats->flist_dir + stats->flist_link + stats->flist_special;
|
||||||
|
char created_breakdown[128];
|
||||||
|
type_breakdown(recv->created_reg, recv->created_dir, recv->created_link, recv->created_special,
|
||||||
|
created_breakdown, sizeof(created_breakdown));
|
||||||
|
unsigned long long created_total =
|
||||||
|
recv->created_reg + recv->created_dir + recv->created_link + recv->created_special;
|
||||||
printf("\n");
|
printf("\n");
|
||||||
if (breakdown[0] != '\0')
|
if (breakdown[0] != '\0')
|
||||||
printf("Number of files: %llu %s\n", flist_total, breakdown);
|
printf("Number of files: %llu %s\n", flist_total, breakdown);
|
||||||
else
|
else
|
||||||
printf("Number of files: %llu\n", flist_total);
|
printf("Number of files: %llu\n", flist_total);
|
||||||
/* FastSync cannot tell which entries the receiver newly created, so it
|
/* Protocol 2.28.0: the receiver reports which destination entries it newly
|
||||||
reports the transferred regular files (which are created on a fresh
|
created, split by type, so this line matches rsync exactly. */
|
||||||
destination). See RSYNC_COMPAT.md for the documented residual. */
|
if (created_breakdown[0] != '\0')
|
||||||
printf("Number of created files: %llu\n", stats->transferred_regular);
|
printf("Number of created files: %llu %s\n", created_total, created_breakdown);
|
||||||
|
else
|
||||||
|
printf("Number of created files: %llu\n", created_total);
|
||||||
printf("Number of deleted files: %llu\n", recv->deleted_files);
|
printf("Number of deleted files: %llu\n", recv->deleted_files);
|
||||||
printf("Number of regular files transferred: %llu\n", stats->transferred_regular);
|
printf("Number of regular files transferred: %llu\n", stats->transferred_regular);
|
||||||
printf("Total file size: %s bytes\n", total);
|
printf("Total file size: %s bytes\n", total);
|
||||||
printf("Total transferred file size: %s bytes\n", transferred);
|
printf("Total transferred file size: %s bytes\n", transferred);
|
||||||
printf("Literal data: %s bytes\n", literal);
|
printf("Literal data: %s bytes\n", literal);
|
||||||
printf("Matched data: %llu bytes\n", recv->matched_data);
|
const char* matched =
|
||||||
|
stats_bytes(config, recv->matched_data, matched_buffer, sizeof(matched_buffer));
|
||||||
|
printf("Matched data: %s bytes\n", matched);
|
||||||
printf("File list size: 0\n");
|
printf("File list size: 0\n");
|
||||||
printf("File list generation time: 0.000 seconds\n");
|
printf("File list generation time: 0.000 seconds\n");
|
||||||
printf("File list transfer time: 0.000 seconds\n");
|
printf("File list transfer time: 0.000 seconds\n");
|
||||||
|
|||||||
@@ -613,6 +613,8 @@ typedef struct {
|
|||||||
static bool receiver_save_file(File* file, void* context_pointer) {
|
static bool receiver_save_file(File* file, void* context_pointer) {
|
||||||
ReceiverSaveContext* context = context_pointer;
|
ReceiverSaveContext* context = context_pointer;
|
||||||
FileSaveResult result = FILE_SAVE_ERROR;
|
FileSaveResult result = FILE_SAVE_ERROR;
|
||||||
|
bool created = false;
|
||||||
|
unsigned created_dirs = 0;
|
||||||
if (context->config->dry_run) {
|
if (context->config->dry_run) {
|
||||||
/* Defense in depth: a dry-run receiver mutates nothing even if a data
|
/* Defense in depth: a dry-run receiver mutates nothing even if a data
|
||||||
frame reaches the sink (the sender is not supposed to send one). */
|
frame reaches the sink (the sender is not supposed to send one). */
|
||||||
@@ -622,12 +624,17 @@ static bool receiver_save_file(File* file, void* context_pointer) {
|
|||||||
--remove-source-files sender keeps its source. */
|
--remove-source-files sender keeps its source. */
|
||||||
result = FILE_SAVE_SKIPPED;
|
result = FILE_SAVE_SKIPPED;
|
||||||
} else {
|
} else {
|
||||||
result = file_save_to_disk_full(context->config->receive_root_directory, file, context->config);
|
result = file_save_to_disk_full_ex(context->config->receive_root_directory, file,
|
||||||
|
context->config, &created, &created_dirs);
|
||||||
}
|
}
|
||||||
/* Wire-stats tally: bytes reconstructed from the basis file (delta matches)
|
/* Wire-stats tally: bytes reconstructed from the basis file (delta matches)
|
||||||
count as matched data in the end-of-transfer report. */
|
count as matched data in the end-of-transfer report. */
|
||||||
if (result != FILE_SAVE_ERROR && file->matched_bytes > 0)
|
if (result != FILE_SAVE_ERROR && file->matched_bytes > 0)
|
||||||
context->stats.matched_data += file->matched_bytes;
|
context->stats.matched_data += file->matched_bytes;
|
||||||
|
/* Protocol 2.28.0: receiver-observed literal bytes and the created-entry
|
||||||
|
breakdown (regular/dir/link/special) for the `--stats` report. */
|
||||||
|
if (result == FILE_SAVE_WRITTEN)
|
||||||
|
receiver_stats_note_saved(&context->stats, file, created, created_dirs);
|
||||||
/* A directory's metadata is deferred, never applied inline: collect it now
|
/* A directory's metadata is deferred, never applied inline: collect it now
|
||||||
and apply it at the end. -O/--omit-dir-times and --preserve_perms/-times
|
and apply it at the end. -O/--omit-dir-times and --preserve_perms/-times
|
||||||
are honored by dir_metadata_list_apply's caller (see
|
are honored by dir_metadata_list_apply's caller (see
|
||||||
|
|||||||
@@ -247,12 +247,23 @@ int write_thread(void* pipeline_context) {
|
|||||||
}
|
}
|
||||||
size_t file_bytes = file->data ? file->data->size : 0;
|
size_t file_bytes = file->data ? file->data->size : 0;
|
||||||
FileSaveResult result = FILE_SAVE_SKIPPED;
|
FileSaveResult result = FILE_SAVE_SKIPPED;
|
||||||
|
bool created = false;
|
||||||
|
unsigned created_dirs = 0;
|
||||||
/* Server-contacting --dry-run: never write. The receiver thread does not
|
/* Server-contacting --dry-run: never write. The receiver thread does not
|
||||||
enqueue anything on the dry-run path, but this keeps the writer thread
|
enqueue anything on the dry-run path, but this keeps the writer thread
|
||||||
provably mutation-free if a data frame ever reached it. */
|
provably mutation-free if a data frame ever reached it. */
|
||||||
bool dry_run = context->config->dry_run;
|
bool dry_run = context->config->dry_run;
|
||||||
if (save_to_disk && !dry_run) {
|
if (save_to_disk && !dry_run) {
|
||||||
result = file_save_to_disk_full(root_directory, file, context->config);
|
result =
|
||||||
|
file_save_to_disk_full_ex(root_directory, file, context->config, &created, &created_dirs);
|
||||||
|
if (result == FILE_SAVE_WRITTEN) {
|
||||||
|
/* Protocol 2.28.0: fold the receiver-observed literal bytes and the
|
||||||
|
created-entry type into the shared stats block under its mutex (the
|
||||||
|
receive thread also writes stats.matched_data). */
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
receiver_stats_note_saved(&context->stats, file, created, created_dirs);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
}
|
||||||
if (result == FILE_SAVE_ERROR) {
|
if (result == FILE_SAVE_ERROR) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||||
|
|||||||
+15
-2
@@ -82,7 +82,7 @@ typedef struct {
|
|||||||
typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode;
|
typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode;
|
||||||
|
|
||||||
/* ===========================================================================
|
/* ===========================================================================
|
||||||
* Config wire-field table (single source of truth for protocol 2.27.0).
|
* Config wire-field table (single source of truth for protocol 2.28.0).
|
||||||
*
|
*
|
||||||
* Every field below crosses the wire. The table is the ONLY place a
|
* Every field below crosses the wire. The table is the ONLY place a
|
||||||
* serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare
|
* serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare
|
||||||
@@ -1006,7 +1006,20 @@ typedef struct Config {
|
|||||||
* the sender can print rsync's `deleting PATH` lines for a real deletion. No
|
* the sender can print rsync's `deleting PATH` lines for a real deletion. No
|
||||||
* change to the fixed STATUS_STATS record itself; only a new trailing config
|
* change to the fixed STATUS_STATS record itself; only a new trailing config
|
||||||
* bool, which still requires the version bump for the strict lockstep. */
|
* bool, which still requires the version bump for the strict lockstep. */
|
||||||
#define PROTOCOL_VERSION "2.27.0"
|
/* (8) --stats receiver-observed counters (protocol 2.28.0): the fixed
|
||||||
|
* STATUS_STATS record grows from three counters to eight. The receiver now
|
||||||
|
* reports the bytes it literally stored (`literal_data`) and the count of
|
||||||
|
* destination entries it newly CREATED, split by type
|
||||||
|
* (reg/dir/link/special), so the sender can print rsync's exact
|
||||||
|
* `Number of created files: N (reg: X, dir: Y, link: Z, special: W)` line and
|
||||||
|
* an exact `Literal data` total even for delta transfers. The config-frame
|
||||||
|
* LAYOUT is unchanged (no new config field), but the STATUS_STATS body grows,
|
||||||
|
* so a 2.27 peer that does not consume the five new fixed-width counters would
|
||||||
|
* desynchronize on the trailing would-delete path list; the strict
|
||||||
|
* same-version handshake (config_receive rejects a mismatched version before
|
||||||
|
* parsing anything else) keeps mixed deployments from ever reaching that
|
||||||
|
* state. */
|
||||||
|
#define PROTOCOL_VERSION "2.28.0"
|
||||||
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
||||||
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
||||||
#define MAX_BASIS_DIRS 64
|
#define MAX_BASIS_DIRS 64
|
||||||
|
|||||||
+156
-21
@@ -187,6 +187,7 @@ File* file_create(const char* path) {
|
|||||||
file->xattrs = NULL;
|
file->xattrs = NULL;
|
||||||
file->dest_state = (OutputDestState){0};
|
file->dest_state = (OutputDestState){0};
|
||||||
file->matched_bytes = 0;
|
file->matched_bytes = 0;
|
||||||
|
file->literal_bytes = 0;
|
||||||
return file;
|
return file;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -614,6 +615,103 @@ static int open_dir_beneath_root(const char* resolved, const char* root) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs) {
|
int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs) {
|
||||||
|
return file_open_secure_parent_counted(path, leaf_out, create_dirs, NULL, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The logical transfer root expressed in the same coordinate as the secure
|
||||||
|
* parent walk's `rel_buf` (relative to the authorized root, with a leading
|
||||||
|
* '/'), used as the floor at or below which a created directory is a real
|
||||||
|
* file-list entry. The on-disk transfer root is the receive root joined to the
|
||||||
|
* wire path; the mirror scaffolding above it (the absolute source path below
|
||||||
|
* the destination root) is not an rsync entry. Returns an allocated string or
|
||||||
|
* NULL (count every created component). */
|
||||||
|
static char* transfer_root_floor(const Config* config) {
|
||||||
|
if (!config || !config->send_directory || config->send_directory[0] == '\0')
|
||||||
|
return NULL;
|
||||||
|
const char* spec = config->send_directory;
|
||||||
|
const char* after = spec;
|
||||||
|
if (spec[0] == '.' && spec[1] == '/') {
|
||||||
|
after = spec + 2;
|
||||||
|
} else {
|
||||||
|
const char* cut = strstr(spec, "/./");
|
||||||
|
if (cut)
|
||||||
|
after = cut + 3;
|
||||||
|
}
|
||||||
|
while (*after == '/')
|
||||||
|
after++;
|
||||||
|
char* wire_root = str_dup(after);
|
||||||
|
if (!wire_root)
|
||||||
|
return NULL;
|
||||||
|
size_t wlen = strlen(wire_root);
|
||||||
|
while (wlen > 0 && wire_root[wlen - 1] == '/')
|
||||||
|
wire_root[--wlen] = '\0';
|
||||||
|
if (wlen == 0) {
|
||||||
|
free(wire_root);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
char* disk_root = config->receive_root_directory
|
||||||
|
? path_cat(config->receive_root_directory, wire_root)
|
||||||
|
: str_dup(wire_root);
|
||||||
|
free(wire_root);
|
||||||
|
if (!disk_root)
|
||||||
|
return NULL;
|
||||||
|
const char* root_path = utils_get_authorized_root_path();
|
||||||
|
const char* floor = disk_root;
|
||||||
|
if (root_path && root_path[0] == '/') {
|
||||||
|
size_t rl = strlen(root_path);
|
||||||
|
while (rl > 0 && root_path[rl - 1] == '/')
|
||||||
|
rl--;
|
||||||
|
if (strncmp(disk_root, root_path, rl) == 0 && (disk_root[rl] == '/' || disk_root[rl] == '\0'))
|
||||||
|
floor = disk_root + rl;
|
||||||
|
}
|
||||||
|
while (*floor == '/')
|
||||||
|
floor++;
|
||||||
|
char* out = str_dup(floor);
|
||||||
|
free(disk_root);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
if (out[0] == '\0') {
|
||||||
|
free(out);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A created parent component counts toward `Number of created files` only when
|
||||||
|
* its receive-root-relative path is at or below the logical transfer root
|
||||||
|
* (`count_floor`). The transfer root itself corresponds to rsync's `.` entry
|
||||||
|
* (created on a fresh destination, pre-existing otherwise); the mirror
|
||||||
|
* scaffolding above it is FastSync's absolute-path layout, not an rsync entry. */
|
||||||
|
static bool created_dir_counts(const char* count_floor, const char* rel_buf,
|
||||||
|
const char* component) {
|
||||||
|
if (!count_floor)
|
||||||
|
return true;
|
||||||
|
char candidate[PATH_MAX];
|
||||||
|
int n = snprintf(candidate, sizeof(candidate), "%s/%s", rel_buf, component);
|
||||||
|
if (n < 0 || (size_t)n >= sizeof(candidate))
|
||||||
|
return false;
|
||||||
|
const char* cand = candidate;
|
||||||
|
while (*cand == '/')
|
||||||
|
cand++;
|
||||||
|
size_t fl = strlen(count_floor);
|
||||||
|
if (strncmp(cand, count_floor, fl) != 0)
|
||||||
|
return false;
|
||||||
|
return cand[fl] == '\0' || cand[fl] == '/';
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Public wrapper for the receiver's created-directory accounting: the logical
|
||||||
|
* transfer root expressed receive-root-relative, or NULL when the wire paths
|
||||||
|
* carry no mirror scaffolding above it (--relative and --files-from, whose
|
||||||
|
* paths are already relative to the transfer root). The caller frees a
|
||||||
|
* non-NULL result. */
|
||||||
|
char* file_transfer_root_floor(const Config* config) {
|
||||||
|
if (!config || config->relative || config->files_from_set != NULL)
|
||||||
|
return NULL;
|
||||||
|
return transfer_root_floor(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
int file_open_secure_parent_counted(const char* path, char** leaf_out, bool create_dirs,
|
||||||
|
unsigned* dirs_created, const char* count_floor) {
|
||||||
char* copy = str_dup(path);
|
char* copy = str_dup(path);
|
||||||
if (!copy)
|
if (!copy)
|
||||||
return -1;
|
return -1;
|
||||||
@@ -674,6 +772,14 @@ int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs)
|
|||||||
if (next < 0 && create_dirs && errno == ENOENT) {
|
if (next < 0 && create_dirs && errno == ENOENT) {
|
||||||
bool created = mkdirat(fd, component, (mode_t)(0777 & ~(mode_t)file_process_umask())) == 0;
|
bool created = mkdirat(fd, component, (mode_t)(0777 & ~(mode_t)file_process_umask())) == 0;
|
||||||
if (created || errno == EEXIST) {
|
if (created || errno == EEXIST) {
|
||||||
|
/* Protocol 2.28.0: only directories the logical file list would
|
||||||
|
create count toward `Number of created files`; the mirror
|
||||||
|
scaffolding above the transfer root (e.g. the absolute source path
|
||||||
|
under the destination root) is not an rsync entry. `count_floor`
|
||||||
|
is a receive-root-relative prefix that must be reached before a
|
||||||
|
created component is counted. */
|
||||||
|
if (created && dirs_created && created_dir_counts(count_floor, rel_buf, component))
|
||||||
|
(*dirs_created)++;
|
||||||
/* P7 Wave E: --copy-as owns EVERY entry, including the intermediate
|
/* P7 Wave E: --copy-as owns EVERY entry, including the intermediate
|
||||||
directories this walk creates implicitly. Its target ids are a
|
directories this walk creates implicitly. Its target ids are a
|
||||||
global policy, so they are available here without per-entry source
|
global policy, so they are available here without per-entry source
|
||||||
@@ -1026,15 +1132,14 @@ static void restore_extra_fd(int fd, const FileMetadata* metadata, const FileXat
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool file_to_disk_secure_impl(const char* path, const void* data,
|
static bool
|
||||||
unsigned long long data_size, bool inplace, bool sparse,
|
file_to_disk_secure_impl(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool preallocate, const FileMetadata* metadata,
|
bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata,
|
||||||
FileAttrPolicy policy, bool update, bool no_replace,
|
FileAttrPolicy policy, bool update, bool no_replace, bool use_fsync,
|
||||||
bool use_fsync, const char* temp_dir,
|
const char* temp_dir, const FileXattrList* xattrs, bool fake_super,
|
||||||
const FileXattrList* xattrs, bool fake_super,
|
bool keep_partial, unsigned* dirs_created, const char* count_floor) {
|
||||||
bool keep_partial) {
|
|
||||||
char* leaf = NULL;
|
char* leaf = NULL;
|
||||||
int dirfd = file_open_secure_parent(path, &leaf, true);
|
int dirfd = file_open_secure_parent_counted(path, &leaf, true, dirs_created, count_floor);
|
||||||
if (dirfd < 0)
|
if (dirfd < 0)
|
||||||
return false;
|
return false;
|
||||||
int fd = -1;
|
int fd = -1;
|
||||||
@@ -1325,7 +1430,7 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
|||||||
"non-atomic copy into the destination directory");
|
"non-atomic copy into the destination directory");
|
||||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||||
policy, update, no_replace, use_fsync, NULL, xattrs, fake_super,
|
policy, update, no_replace, use_fsync, NULL, xattrs, fake_super,
|
||||||
keep_partial);
|
keep_partial, dirs_created, count_floor);
|
||||||
}
|
}
|
||||||
return ok;
|
return ok;
|
||||||
}
|
}
|
||||||
@@ -1334,7 +1439,8 @@ bool file_to_disk_secure(const char* path, const void* data, unsigned long long
|
|||||||
bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata,
|
bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata,
|
||||||
FileAttrPolicy policy, const char* temp_dir) {
|
FileAttrPolicy policy, const char* temp_dir) {
|
||||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||||
policy, false, false, false, temp_dir, NULL, false, false);
|
policy, false, false, false, temp_dir, NULL, false, false, NULL,
|
||||||
|
NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
|
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
|
||||||
@@ -1342,7 +1448,8 @@ bool file_to_disk_secure_update(const char* path, const void* data, unsigned lon
|
|||||||
const FileMetadata* metadata, FileAttrPolicy policy,
|
const FileMetadata* metadata, FileAttrPolicy policy,
|
||||||
const char* temp_dir) {
|
const char* temp_dir) {
|
||||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||||
policy, true, false, false, temp_dir, NULL, false, false);
|
policy, true, false, false, temp_dir, NULL, false, false, NULL,
|
||||||
|
NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
||||||
@@ -1350,7 +1457,8 @@ bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
|||||||
bool preallocate, const FileMetadata* metadata,
|
bool preallocate, const FileMetadata* metadata,
|
||||||
FileAttrPolicy policy, bool use_fsync, const char* temp_dir) {
|
FileAttrPolicy policy, bool use_fsync, const char* temp_dir) {
|
||||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||||
policy, false, false, use_fsync, temp_dir, NULL, false, false);
|
policy, false, false, use_fsync, temp_dir, NULL, false, false,
|
||||||
|
NULL, NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
||||||
@@ -1358,7 +1466,8 @@ bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
|||||||
const FileMetadata* metadata, FileAttrPolicy policy,
|
const FileMetadata* metadata, FileAttrPolicy policy,
|
||||||
const char* temp_dir) {
|
const char* temp_dir) {
|
||||||
return file_to_disk_secure_impl(path, data, data_size, false, sparse, preallocate, metadata,
|
return file_to_disk_secure_impl(path, data, data_size, false, sparse, preallocate, metadata,
|
||||||
policy, false, true, false, temp_dir, NULL, false, false);
|
policy, false, true, false, temp_dir, NULL, false, false, NULL,
|
||||||
|
NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Receiver write-path variant that also applies the per-file xattrs (-X/-A)
|
/* Receiver write-path variant that also applies the per-file xattrs (-X/-A)
|
||||||
@@ -1371,9 +1480,21 @@ bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long
|
|||||||
const FileMetadata* metadata, FileAttrPolicy policy, bool update,
|
const FileMetadata* metadata, FileAttrPolicy policy, bool update,
|
||||||
bool no_replace, bool use_fsync, const FileXattrList* xattrs,
|
bool no_replace, bool use_fsync, const FileXattrList* xattrs,
|
||||||
bool fake_super, bool keep_partial, const char* temp_dir) {
|
bool fake_super, bool keep_partial, const char* temp_dir) {
|
||||||
|
return file_to_disk_secure_attrs_counted(path, data, data_size, inplace, sparse, preallocate,
|
||||||
|
metadata, policy, update, no_replace, use_fsync, xattrs,
|
||||||
|
fake_super, keep_partial, temp_dir, NULL, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool file_to_disk_secure_attrs_counted(const char* path, const void* data,
|
||||||
|
unsigned long long data_size, bool inplace, bool sparse,
|
||||||
|
bool preallocate, const FileMetadata* metadata,
|
||||||
|
FileAttrPolicy policy, bool update, bool no_replace,
|
||||||
|
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
||||||
|
bool keep_partial, const char* temp_dir,
|
||||||
|
unsigned* dirs_created, const char* count_floor) {
|
||||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||||
policy, update, no_replace, use_fsync, temp_dir, xattrs,
|
policy, update, no_replace, use_fsync, temp_dir, xattrs,
|
||||||
fake_super, keep_partial);
|
fake_super, keep_partial, dirs_created, count_floor);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Atomic --link-dest install. The destination is replaced (via a temporary
|
/* Atomic --link-dest install. The destination is replaced (via a temporary
|
||||||
@@ -1396,11 +1517,12 @@ static bool file_to_disk_secure_link_impl(const char* path, const char* basis_pa
|
|||||||
bool preallocate, const FileMetadata* metadata,
|
bool preallocate, const FileMetadata* metadata,
|
||||||
FileAttrPolicy policy, bool use_fsync,
|
FileAttrPolicy policy, bool use_fsync,
|
||||||
const FileXattrList* xattrs, bool fake_super,
|
const FileXattrList* xattrs, bool fake_super,
|
||||||
const char* temp_dir) {
|
const char* temp_dir, unsigned* dirs_created,
|
||||||
|
const char* count_floor) {
|
||||||
if (!path || !basis_path)
|
if (!path || !basis_path)
|
||||||
return false;
|
return false;
|
||||||
char* leaf = NULL;
|
char* leaf = NULL;
|
||||||
int dirfd = file_open_secure_parent(path, &leaf, true);
|
int dirfd = file_open_secure_parent_counted(path, &leaf, true, dirs_created, count_floor);
|
||||||
if (dirfd < 0)
|
if (dirfd < 0)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
@@ -1483,9 +1605,9 @@ static bool file_to_disk_secure_link_impl(const char* path, const char* basis_pa
|
|||||||
free(leaf);
|
free(leaf);
|
||||||
/* The basis file could not be linked in (missing, cross-device, refused
|
/* The basis file could not be linked in (missing, cross-device, refused
|
||||||
by the filesystem). Write a byte-identical local copy instead. */
|
by the filesystem). Write a byte-identical local copy instead. */
|
||||||
return file_to_disk_secure_attrs(path, data, data_size, false, false, preallocate, metadata,
|
return file_to_disk_secure_attrs_counted(
|
||||||
policy, false, false, use_fsync, xattrs, fake_super, false,
|
path, data, data_size, false, false, preallocate, metadata, policy, false, false, use_fsync,
|
||||||
temp_dir);
|
xattrs, fake_super, false, temp_dir, dirs_created, count_floor);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (scratch_dirfd >= 0)
|
if (scratch_dirfd >= 0)
|
||||||
@@ -1500,7 +1622,7 @@ bool file_to_disk_secure_link(const char* path, const char* basis_path, const vo
|
|||||||
const FileMetadata* metadata, FileAttrPolicy policy, bool use_fsync,
|
const FileMetadata* metadata, FileAttrPolicy policy, bool use_fsync,
|
||||||
const char* temp_dir) {
|
const char* temp_dir) {
|
||||||
return file_to_disk_secure_link_impl(path, basis_path, data, data_size, preallocate, metadata,
|
return file_to_disk_secure_link_impl(path, basis_path, data, data_size, preallocate, metadata,
|
||||||
policy, use_fsync, NULL, false, temp_dir);
|
policy, use_fsync, NULL, false, temp_dir, NULL, NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, const void* data,
|
bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, const void* data,
|
||||||
@@ -1508,8 +1630,21 @@ bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, co
|
|||||||
const FileMetadata* metadata, FileAttrPolicy policy,
|
const FileMetadata* metadata, FileAttrPolicy policy,
|
||||||
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
||||||
const char* temp_dir) {
|
const char* temp_dir) {
|
||||||
|
return file_to_disk_secure_link_attrs_counted(path, basis_path, data, data_size, preallocate,
|
||||||
|
metadata, policy, use_fsync, xattrs, fake_super,
|
||||||
|
temp_dir, NULL, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool file_to_disk_secure_link_attrs_counted(const char* path, const char* basis_path,
|
||||||
|
const void* data, unsigned long long data_size,
|
||||||
|
bool preallocate, const FileMetadata* metadata,
|
||||||
|
FileAttrPolicy policy, bool use_fsync,
|
||||||
|
const FileXattrList* xattrs, bool fake_super,
|
||||||
|
const char* temp_dir, unsigned* dirs_created,
|
||||||
|
const char* count_floor) {
|
||||||
return file_to_disk_secure_link_impl(path, basis_path, data, data_size, preallocate, metadata,
|
return file_to_disk_secure_link_impl(path, basis_path, data, data_size, preallocate, metadata,
|
||||||
policy, use_fsync, xattrs, fake_super, temp_dir);
|
policy, use_fsync, xattrs, fake_super, temp_dir,
|
||||||
|
dirs_created, count_floor);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
|
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
|
||||||
|
|||||||
@@ -87,6 +87,11 @@ bool file_path_exists_secure(const char* path);
|
|||||||
bool file_stat_secure(const char* path, struct stat* st);
|
bool file_stat_secure(const char* path, struct stat* st);
|
||||||
bool file_destination_is_newer_secure(const char* path, const FileMetadata* metadata);
|
bool file_destination_is_newer_secure(const char* path, const FileMetadata* metadata);
|
||||||
int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs);
|
int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs);
|
||||||
|
/* Protocol 2.28.0 variant: also increments *dirs_created for every missing
|
||||||
|
* parent directory this walk creates that lies strictly below `count_floor`
|
||||||
|
* (a receive-root-relative path, or NULL to count all of them). */
|
||||||
|
int file_open_secure_parent_counted(const char* path, char** leaf_out, bool create_dirs,
|
||||||
|
unsigned* dirs_created, const char* count_floor);
|
||||||
bool file_ensure_directory_secure(const char* path);
|
bool file_ensure_directory_secure(const char* path);
|
||||||
bool file_directory_exists_secure(const char* path);
|
bool file_directory_exists_secure(const char* path);
|
||||||
bool file_rename_secure(const char* old_path, const char* new_path);
|
bool file_rename_secure(const char* old_path, const char* new_path);
|
||||||
@@ -158,5 +163,27 @@ bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, co
|
|||||||
const FileMetadata* metadata, FileAttrPolicy policy,
|
const FileMetadata* metadata, FileAttrPolicy policy,
|
||||||
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
||||||
const char* temp_dir);
|
const char* temp_dir);
|
||||||
|
/* Protocol 2.28.0 receiver-stat variants: like the two above but additionally
|
||||||
|
* report through `dirs_created` (when non-NULL) how many parent directories the
|
||||||
|
* confined secure walk had to create that lie strictly below `count_floor` (a
|
||||||
|
* receive-root-relative prefix, or NULL for all). Used to reproduce rsync's
|
||||||
|
* `Number of created files` directory count on a fresh destination. */
|
||||||
|
bool file_to_disk_secure_attrs_counted(const char* path, const void* data,
|
||||||
|
unsigned long long data_size, bool inplace, bool sparse,
|
||||||
|
bool preallocate, const FileMetadata* metadata,
|
||||||
|
FileAttrPolicy policy, bool update, bool no_replace,
|
||||||
|
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
||||||
|
bool keep_partial, const char* temp_dir,
|
||||||
|
unsigned* dirs_created, const char* count_floor);
|
||||||
|
bool file_to_disk_secure_link_attrs_counted(const char* path, const char* basis_path,
|
||||||
|
const void* data, unsigned long long data_size,
|
||||||
|
bool preallocate, const FileMetadata* metadata,
|
||||||
|
FileAttrPolicy policy, bool use_fsync,
|
||||||
|
const FileXattrList* xattrs, bool fake_super,
|
||||||
|
const char* temp_dir, unsigned* dirs_created,
|
||||||
|
const char* count_floor);
|
||||||
|
/* The logical transfer root expressed receive-root-relative, or NULL when the
|
||||||
|
* wire paths carry no mirror scaffolding above it. Caller frees non-NULL. */
|
||||||
|
char* file_transfer_root_floor(const Config* config);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+77
-12
@@ -37,7 +37,12 @@
|
|||||||
#define MANIFEST_ENTRY_OVERHEAD (sizeof(char*) + 16)
|
#define MANIFEST_ENTRY_OVERHEAD (sizeof(char*) + 16)
|
||||||
|
|
||||||
bool file_save_to_disk(const char* root_directory, const File* file, const Config* config) {
|
bool file_save_to_disk(const char* root_directory, const File* file, const Config* config) {
|
||||||
return file_save_to_disk_full(root_directory, file, config) != FILE_SAVE_ERROR;
|
return file_save_to_disk_full_ex(root_directory, file, config, NULL, NULL) != FILE_SAVE_ERROR;
|
||||||
|
}
|
||||||
|
|
||||||
|
FileSaveResult file_save_to_disk_full(const char* root_directory, const File* file,
|
||||||
|
const Config* config) {
|
||||||
|
return file_save_to_disk_full_ex(root_directory, file, config, NULL, NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* --delay-updates receiver path: write the file into a private staging tree
|
/* --delay-updates receiver path: write the file into a private staging tree
|
||||||
@@ -208,13 +213,14 @@ static FileSaveResult hardlink_sibling_absent_first(const char* destination_path
|
|||||||
--existing/--ignore-existing/--update policies are decided against the final
|
--existing/--ignore-existing/--update policies are decided against the final
|
||||||
destination like every normal write. */
|
destination like every normal write. */
|
||||||
static FileSaveResult file_save_hardlink_sibling(const char* root_directory, const File* file,
|
static FileSaveResult file_save_hardlink_sibling(const char* root_directory, const File* file,
|
||||||
const Config* config) {
|
const Config* config, bool* created) {
|
||||||
Config* cfg = (Config*)config;
|
Config* cfg = (Config*)config;
|
||||||
if (!root_directory || !file || !file->path || !file->hardlink_target)
|
if (!root_directory || !file || !file->path || !file->hardlink_target)
|
||||||
return FILE_SAVE_ERROR;
|
return FILE_SAVE_ERROR;
|
||||||
char* destination_path = path_cat(root_directory, file->path);
|
char* destination_path = path_cat(root_directory, file->path);
|
||||||
if (!destination_path)
|
if (!destination_path)
|
||||||
return FILE_SAVE_ERROR;
|
return FILE_SAVE_ERROR;
|
||||||
|
bool existed = file_path_exists_secure(destination_path);
|
||||||
|
|
||||||
if (cfg->existing && !file_path_exists_secure(destination_path)) {
|
if (cfg->existing && !file_path_exists_secure(destination_path)) {
|
||||||
free(destination_path);
|
free(destination_path);
|
||||||
@@ -278,6 +284,8 @@ static FileSaveResult file_save_hardlink_sibling(const char* root_directory, con
|
|||||||
free(staged_first);
|
free(staged_first);
|
||||||
free(staged_sibling);
|
free(staged_sibling);
|
||||||
free(destination_path);
|
free(destination_path);
|
||||||
|
if (ok && created && !existed)
|
||||||
|
*created = true;
|
||||||
return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR;
|
return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -324,6 +332,8 @@ static FileSaveResult file_save_hardlink_sibling(const char* root_directory, con
|
|||||||
free(content);
|
free(content);
|
||||||
free(first_disk);
|
free(first_disk);
|
||||||
free(destination_path);
|
free(destination_path);
|
||||||
|
if (ok && created && !existed)
|
||||||
|
*created = true;
|
||||||
return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR;
|
return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -360,7 +370,7 @@ bool file_special_rdev_valid(int32_t major, int32_t minor, mode_t mode) {
|
|||||||
* non-device entry must carry an empty rdev.
|
* non-device entry must carry an empty rdev.
|
||||||
*/
|
*/
|
||||||
static FileSaveResult file_save_special_to_disk(const char* root_directory, const File* file,
|
static FileSaveResult file_save_special_to_disk(const char* root_directory, const File* file,
|
||||||
const Config* config) {
|
const Config* config, bool* created) {
|
||||||
/* The empty-path and structural checks stay unconditional; the redundant
|
/* The empty-path and structural checks stay unconditional; the redundant
|
||||||
".." list-path re-check is skipped under --trust-sender exactly like the
|
".." list-path re-check is skipped under --trust-sender exactly like the
|
||||||
receive layer (confinement is deferred to the secure parent walk below,
|
receive layer (confinement is deferred to the secure parent walk below,
|
||||||
@@ -412,6 +422,7 @@ static FileSaveResult file_save_special_to_disk(const char* root_directory, cons
|
|||||||
char* destination = path_cat(root_directory, file->path);
|
char* destination = path_cat(root_directory, file->path);
|
||||||
if (!destination)
|
if (!destination)
|
||||||
return FILE_SAVE_ERROR;
|
return FILE_SAVE_ERROR;
|
||||||
|
bool existed = file_path_exists_secure(destination);
|
||||||
char* leaf = NULL;
|
char* leaf = NULL;
|
||||||
int parent_fd = file_open_secure_parent(destination, &leaf, true);
|
int parent_fd = file_open_secure_parent(destination, &leaf, true);
|
||||||
if (parent_fd < 0) {
|
if (parent_fd < 0) {
|
||||||
@@ -532,6 +543,8 @@ static FileSaveResult file_save_special_to_disk(const char* root_directory, cons
|
|||||||
free(destination);
|
free(destination);
|
||||||
/* A failed required --copy-as ownership marks the node as failed; every other
|
/* A failed required --copy-as ownership marks the node as failed; every other
|
||||||
* identity policy stays best-effort. */
|
* identity policy stays best-effort. */
|
||||||
|
if (owner_ok && created && !existed)
|
||||||
|
*created = true;
|
||||||
return owner_ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR;
|
return owner_ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -610,8 +623,13 @@ static FileSaveResult file_save_write_device(const char* root_directory, const F
|
|||||||
return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_SKIPPED;
|
return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_SKIPPED;
|
||||||
}
|
}
|
||||||
|
|
||||||
FileSaveResult file_save_to_disk_full(const char* root_directory, const File* file,
|
FileSaveResult file_save_to_disk_full_ex(const char* root_directory, const File* file,
|
||||||
const Config* config) {
|
const Config* config, bool* created,
|
||||||
|
unsigned* created_dirs) {
|
||||||
|
if (created)
|
||||||
|
*created = false;
|
||||||
|
if (created_dirs)
|
||||||
|
*created_dirs = 0;
|
||||||
/* Central no-mutation guard: a server-contacting --dry-run (or a local batch
|
/* Central no-mutation guard: a server-contacting --dry-run (or a local batch
|
||||||
apply that somehow carries dry_run) must never touch the destination, no
|
apply that somehow carries dry_run) must never touch the destination, no
|
||||||
matter which caller reached this primitive. The per-caller guards remain,
|
matter which caller reached this primitive. The per-caller guards remain,
|
||||||
@@ -658,7 +676,7 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
|
|||||||
/* Device/special node (--devices/--specials): recreate the node instead of
|
/* Device/special node (--devices/--specials): recreate the node instead of
|
||||||
writing content (privilege-gated, confined, rdev-validated). */
|
writing content (privilege-gated, confined, rdev-validated). */
|
||||||
if (file->is_special)
|
if (file->is_special)
|
||||||
return file_save_special_to_disk(root_directory, file, config);
|
return file_save_special_to_disk(root_directory, file, config, created);
|
||||||
/* --write-devices: write straight into an existing device node. Writing
|
/* --write-devices: write straight into an existing device node. Writing
|
||||||
into a device is a super-user activity, so --no-super must suppress it just
|
into a device is a super-user activity, so --no-super must suppress it just
|
||||||
like device-node creation; the default AUTO/--super attempt it (the wide
|
like device-node creation; the default AUTO/--super attempt it (the wide
|
||||||
@@ -689,6 +707,7 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
|
|||||||
char* dir_path = path_cat(root_directory, file->path);
|
char* dir_path = path_cat(root_directory, file->path);
|
||||||
if (!dir_path)
|
if (!dir_path)
|
||||||
return FILE_SAVE_ERROR;
|
return FILE_SAVE_ERROR;
|
||||||
|
bool dir_existed = file_path_exists_secure(dir_path);
|
||||||
bool ok = file_ensure_directory_secure(dir_path);
|
bool ok = file_ensure_directory_secure(dir_path);
|
||||||
/* P7 Wave E: apply the negotiated ownership to the directory ITSELF (not
|
/* P7 Wave E: apply the negotiated ownership to the directory ITSELF (not
|
||||||
just the files inside it). --copy-as and every explicit identity policy
|
just the files inside it). --copy-as and every explicit identity policy
|
||||||
@@ -712,6 +731,8 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
|
|||||||
free(leaf);
|
free(leaf);
|
||||||
}
|
}
|
||||||
free(dir_path);
|
free(dir_path);
|
||||||
|
if (ok && created && !dir_existed)
|
||||||
|
*created = true;
|
||||||
return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR;
|
return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -728,6 +749,7 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
|
|||||||
char* link_path = path_cat(root_directory, file->path);
|
char* link_path = path_cat(root_directory, file->path);
|
||||||
if (!link_path)
|
if (!link_path)
|
||||||
return FILE_SAVE_ERROR;
|
return FILE_SAVE_ERROR;
|
||||||
|
bool link_existed = file_path_exists_secure(link_path);
|
||||||
/* The link value is stored verbatim (rsync -l parity: absolute and
|
/* The link value is stored verbatim (rsync -l parity: absolute and
|
||||||
".."-bearing targets are preserved; the scanner's --safe-links /
|
".."-bearing targets are preserved; the scanner's --safe-links /
|
||||||
--copy-unsafe-links decide which links are sent at all). --munge-links
|
--copy-unsafe-links decide which links are sent at all). --munge-links
|
||||||
@@ -768,6 +790,8 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
|
|||||||
ok = file_restore_symlink_metadata(link_path, file->metadata, link_policy,
|
ok = file_restore_symlink_metadata(link_path, file->metadata, link_policy,
|
||||||
config->omit_link_times);
|
config->omit_link_times);
|
||||||
}
|
}
|
||||||
|
if (ok && created && !link_existed)
|
||||||
|
*created = true;
|
||||||
free(link_path);
|
free(link_path);
|
||||||
return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR;
|
return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR;
|
||||||
}
|
}
|
||||||
@@ -777,7 +801,7 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
|
|||||||
byte-identical copy of) the group's first member. Handled entirely here,
|
byte-identical copy of) the group's first member. Handled entirely here,
|
||||||
before the normal data-write paths (which would create an empty file). */
|
before the normal data-write paths (which would create an empty file). */
|
||||||
if (file->link_group != 0 && !file->link_first && file->hardlink_target != NULL) {
|
if (file->link_group != 0 && !file->link_first && file->hardlink_target != NULL) {
|
||||||
return file_save_hardlink_sibling(root_directory, file, config);
|
return file_save_hardlink_sibling(root_directory, file, config, created);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* These options arrive from the client. --backup-dir, --partial-dir and
|
/* These options arrive from the client. --backup-dir, --partial-dir and
|
||||||
@@ -808,12 +832,18 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
|
|||||||
free(disk_path);
|
free(disk_path);
|
||||||
return FILE_SAVE_ERROR;
|
return FILE_SAVE_ERROR;
|
||||||
}
|
}
|
||||||
|
/* Snapshot the final destination's existence BEFORE any backup/force/partial
|
||||||
|
step can move or remove it, so the receiver can report rsync's
|
||||||
|
`Number of created files` (protocol 2.28.0). */
|
||||||
|
bool dest_existed = file_path_exists_secure(destination_path);
|
||||||
|
|
||||||
/* --delay-updates diverts the whole write into the staging tree; the rest of
|
/* --delay-updates diverts the whole write into the staging tree; the rest of
|
||||||
this function is the immediate-install path. */
|
this function is the immediate-install path. */
|
||||||
if (config && config->delay_updates) {
|
if (config && config->delay_updates) {
|
||||||
FileSaveResult result =
|
FileSaveResult result =
|
||||||
file_stage_delayed_update(root_directory, destination_path, file, (Config*)config);
|
file_stage_delayed_update(root_directory, destination_path, file, (Config*)config);
|
||||||
|
if (result == FILE_SAVE_WRITTEN && created && !dest_existed)
|
||||||
|
*created = true;
|
||||||
free(confined_backup);
|
free(confined_backup);
|
||||||
free(confined_partial);
|
free(confined_partial);
|
||||||
free(destination_path);
|
free(destination_path);
|
||||||
@@ -939,19 +969,23 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
|
|||||||
existing/ignore-existing/update/backup preamble above has already made the
|
existing/ignore-existing/update/backup preamble above has already made the
|
||||||
policy decision. */
|
policy decision. */
|
||||||
bool ok;
|
bool ok;
|
||||||
|
char* count_floor = file_transfer_root_floor(config);
|
||||||
if (config && file->basis_link) {
|
if (config && file->basis_link) {
|
||||||
ok = file_to_disk_secure_link_attrs(
|
ok = file_to_disk_secure_link_attrs_counted(
|
||||||
disk_path, file->basis_link, file->data->data, file->data->size, config->preallocate,
|
disk_path, file->basis_link, file->data->data, file->data->size, config->preallocate,
|
||||||
metadata, policy, config->use_fsync, file->xattrs, config->fake_super, confined_temp);
|
metadata, policy, config->use_fsync, file->xattrs, config->fake_super, confined_temp,
|
||||||
|
created_dirs, count_floor);
|
||||||
} else {
|
} else {
|
||||||
/* The plain no-replace / update / with-fsync engines, plus per-file xattr
|
/* The plain no-replace / update / with-fsync engines, plus per-file xattr
|
||||||
(-X/-A) and --fake-super application on the written fd. */
|
(-X/-A) and --fake-super application on the written fd. */
|
||||||
ok = file_to_disk_secure_attrs(
|
ok = file_to_disk_secure_attrs_counted(
|
||||||
disk_path, file->data->data, file->data->size, inplace, sparse,
|
disk_path, file->data->data, file->data->size, inplace, sparse,
|
||||||
config && config->preallocate, metadata, policy, config && config->update,
|
config && config->preallocate, metadata, policy, config && config->update,
|
||||||
config && config->ignore_existing, config && config->use_fsync, file->xattrs,
|
config && config->ignore_existing, config && config->use_fsync, file->xattrs,
|
||||||
config ? config->fake_super : false, config ? config->partial : false, confined_temp);
|
config ? config->fake_super : false, config ? config->partial : false, confined_temp,
|
||||||
|
created_dirs, count_floor);
|
||||||
}
|
}
|
||||||
|
free(count_floor);
|
||||||
free(confined_temp);
|
free(confined_temp);
|
||||||
confined_temp = NULL;
|
confined_temp = NULL;
|
||||||
if (!ok)
|
if (!ok)
|
||||||
@@ -972,6 +1006,8 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi
|
|||||||
free(confined_partial);
|
free(confined_partial);
|
||||||
free(destination_path);
|
free(destination_path);
|
||||||
free(disk_path);
|
free(disk_path);
|
||||||
|
if (created && !dest_existed)
|
||||||
|
*created = true;
|
||||||
return FILE_SAVE_WRITTEN;
|
return FILE_SAVE_WRITTEN;
|
||||||
|
|
||||||
fail:
|
fail:
|
||||||
@@ -984,6 +1020,30 @@ fail:
|
|||||||
return FILE_SAVE_ERROR;
|
return FILE_SAVE_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void receiver_stats_note_saved(ReceiverStats* stats, const File* file, bool created,
|
||||||
|
unsigned created_dirs) {
|
||||||
|
if (!stats || !file)
|
||||||
|
return;
|
||||||
|
bool is_sibling = file->link_group != 0 && !file->link_first;
|
||||||
|
if (!file->is_dir && !file->is_symlink && !file->is_special && !is_sibling) {
|
||||||
|
unsigned long long literal = file->literal_bytes;
|
||||||
|
if (literal == 0 && file->matched_bytes == 0)
|
||||||
|
literal = file->data ? file->data->size : 0;
|
||||||
|
stats->literal_bytes += literal;
|
||||||
|
}
|
||||||
|
stats->created_dir += created_dirs;
|
||||||
|
if (!created)
|
||||||
|
return;
|
||||||
|
if (file->is_dir)
|
||||||
|
stats->created_dir++;
|
||||||
|
else if (file->is_symlink)
|
||||||
|
stats->created_link++;
|
||||||
|
else if (file->is_special)
|
||||||
|
stats->created_special++;
|
||||||
|
else
|
||||||
|
stats->created_reg++;
|
||||||
|
}
|
||||||
|
|
||||||
/* Receive a file's xattr block (when the config enables xattr transport) and
|
/* Receive a file's xattr block (when the config enables xattr transport) and
|
||||||
* attach it to `file`. Returns false on a malformed/oversized frame. */
|
* attach it to `file`. Returns false on a malformed/oversized frame. */
|
||||||
static bool receive_file_xattrs(File* file, int fd, const Config* config) {
|
static bool receive_file_xattrs(File* file, int fd, const Config* config) {
|
||||||
@@ -1094,11 +1154,15 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
/* Wire-stats tally: bytes taken straight from the basis file (matched
|
/* Wire-stats tally: bytes taken straight from the basis file (matched
|
||||||
delta blocks). Computed before the delta is destroyed. */
|
delta blocks) and bytes shipped literally (protocol 2.28.0). Computed
|
||||||
|
before the delta is destroyed. */
|
||||||
unsigned long long matched = 0;
|
unsigned long long matched = 0;
|
||||||
|
unsigned long long literal = 0;
|
||||||
for (uint32_t k = 0; k < delta->instruction_count; k++) {
|
for (uint32_t k = 0; k < delta->instruction_count; k++) {
|
||||||
if (delta->instructions[k].type == DELTA_INSTR_BLOCK_MATCH)
|
if (delta->instructions[k].type == DELTA_INSTR_BLOCK_MATCH)
|
||||||
matched += delta->instructions[k].match.length;
|
matched += delta->instructions[k].match.length;
|
||||||
|
else if (delta->instructions[k].type == DELTA_INSTR_LITERAL)
|
||||||
|
literal += delta->instructions[k].literal.length;
|
||||||
}
|
}
|
||||||
void* new_data = delta_apply(old_data, old_size, delta, config->delta_block_size);
|
void* new_data = delta_apply(old_data, old_size, delta, config->delta_block_size);
|
||||||
delta_destroy(delta);
|
delta_destroy(delta);
|
||||||
@@ -1119,6 +1183,7 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
file->matched_bytes = matched;
|
file->matched_bytes = matched;
|
||||||
|
file->literal_bytes = literal;
|
||||||
|
|
||||||
if (config->use_metadata) {
|
if (config->use_metadata) {
|
||||||
int meta_ok = 1;
|
int meta_ok = 1;
|
||||||
|
|||||||
@@ -179,6 +179,22 @@ typedef enum { FILE_SAVE_ERROR = 0, FILE_SAVE_WRITTEN = 1, FILE_SAVE_SKIPPED = 2
|
|||||||
|
|
||||||
FileSaveResult file_save_to_disk_full(const char* root_directory, const File* file,
|
FileSaveResult file_save_to_disk_full(const char* root_directory, const File* file,
|
||||||
const Config* config);
|
const Config* config);
|
||||||
|
/* Protocol 2.28.0 variant: also reports through `created` (when non-NULL)
|
||||||
|
* whether the destination entry did not exist before this save, and through
|
||||||
|
* `created_dirs` how many parent directories the confined walk created, so the
|
||||||
|
* receiver can build rsync's `Number of created files` breakdown. The plain
|
||||||
|
* file_save_to_disk_full() is this with both out-params NULL. */
|
||||||
|
FileSaveResult file_save_to_disk_full_ex(const char* root_directory, const File* file,
|
||||||
|
const Config* config, bool* created,
|
||||||
|
unsigned* created_dirs);
|
||||||
bool file_save_to_disk(const char* root_directory, const File* file, const Config* config);
|
bool file_save_to_disk(const char* root_directory, const File* file, const Config* config);
|
||||||
|
|
||||||
|
/* Protocol 2.28.0 receiver counter accumulator: fold one successfully saved
|
||||||
|
* entry into `stats`, adding its receiver-observed literal bytes and, when
|
||||||
|
* `created`, the matching created-by-type counter (regular file / symlink /
|
||||||
|
* special) plus `created_dirs` implicitly-created parent directories.
|
||||||
|
* Non-first hardlink siblings contribute no literal bytes. */
|
||||||
|
void receiver_stats_note_saved(ReceiverStats* stats, const File* file, bool created,
|
||||||
|
unsigned created_dirs);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -97,6 +97,11 @@ typedef struct {
|
|||||||
* basis file (matched delta blocks) for this entry. 0 when the file was sent
|
* basis file (matched delta blocks) for this entry. 0 when the file was sent
|
||||||
* whole. Accumulated into ReceiverStats.matched_data by the receiver sink. */
|
* whole. Accumulated into ReceiverStats.matched_data by the receiver sink. */
|
||||||
unsigned long long matched_bytes;
|
unsigned long long matched_bytes;
|
||||||
|
/* Receiver-only (protocol 2.28.0) wire-stats tally: the literal delta fragment
|
||||||
|
* bytes this entry carried (DELTA_INSTR_LITERAL). 0 when the file was sent
|
||||||
|
* whole; the sink then falls back to the whole payload size. Accumulated
|
||||||
|
* into ReceiverStats.literal_bytes. */
|
||||||
|
unsigned long long literal_bytes;
|
||||||
} File;
|
} File;
|
||||||
|
|
||||||
/* The path that should be sent on the wire and used for the receiver-side
|
/* The path that should be sent on the wire and used for the receiver-side
|
||||||
|
|||||||
+15
-13
@@ -105,25 +105,27 @@ bool format_dest_state_receive(int fd, OutputDestState* state) {
|
|||||||
bool format_stats_send(int fd, const ReceiverStats* stats) {
|
bool format_stats_send(int fd, const ReceiverStats* stats) {
|
||||||
if (!stats)
|
if (!stats)
|
||||||
return false;
|
return false;
|
||||||
unsigned long long matched = stats->matched_data;
|
unsigned long long fields[8] = {
|
||||||
unsigned long long deleted = stats->deleted_files;
|
stats->matched_data, stats->deleted_files, stats->would_delete_count, stats->literal_bytes,
|
||||||
unsigned long long would = stats->would_delete_count;
|
stats->created_reg, stats->created_dir, stats->created_link, stats->created_special,
|
||||||
return send_n_data(fd, &matched, sizeof(matched)) && send_n_data(fd, &deleted, sizeof(deleted)) &&
|
};
|
||||||
send_n_data(fd, &would, sizeof(would));
|
return send_n_data(fd, fields, sizeof(fields));
|
||||||
}
|
}
|
||||||
|
|
||||||
bool format_stats_receive(int fd, ReceiverStats* stats) {
|
bool format_stats_receive(int fd, ReceiverStats* stats) {
|
||||||
if (!stats)
|
if (!stats)
|
||||||
return false;
|
return false;
|
||||||
unsigned long long matched = 0;
|
unsigned long long fields[8] = {0};
|
||||||
unsigned long long deleted = 0;
|
if (!receive_n_data(fd, fields, sizeof(fields)))
|
||||||
unsigned long long would = 0;
|
|
||||||
if (!receive_n_data(fd, &matched, sizeof(matched)) ||
|
|
||||||
!receive_n_data(fd, &deleted, sizeof(deleted)) || !receive_n_data(fd, &would, sizeof(would)))
|
|
||||||
return false;
|
return false;
|
||||||
memset(stats, 0, sizeof(*stats));
|
memset(stats, 0, sizeof(*stats));
|
||||||
stats->matched_data = matched;
|
stats->matched_data = fields[0];
|
||||||
stats->deleted_files = deleted;
|
stats->deleted_files = fields[1];
|
||||||
stats->would_delete_count = would;
|
stats->would_delete_count = fields[2];
|
||||||
|
stats->literal_bytes = fields[3];
|
||||||
|
stats->created_reg = fields[4];
|
||||||
|
stats->created_dir = fields[5];
|
||||||
|
stats->created_link = fields[6];
|
||||||
|
stats->created_special = fields[7];
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|||||||
+16
-4
@@ -57,14 +57,26 @@ bool format_dest_state_send(int fd, const OutputDestState* state);
|
|||||||
bool format_dest_state_receive(int fd, OutputDestState* state);
|
bool format_dest_state_receive(int fd, OutputDestState* state);
|
||||||
|
|
||||||
/* End-of-transfer receiver counters reported through STATUS_STATS (protocol
|
/* End-of-transfer receiver counters reported through STATUS_STATS (protocol
|
||||||
* 2.25.0) when the wire config carries report_stats. `would_delete_count` is
|
* 2.25.0, extended in 2.28.0) when the wire config carries report_stats.
|
||||||
* the number of destination-relative paths the receiver would have deleted in a
|
* `would_delete_count` is the number of destination-relative paths the receiver
|
||||||
* -n/--dry-run --delete run; that many wire strings immediately follow the
|
* would have deleted in a -n/--dry-run --delete run; that many wire strings
|
||||||
* fixed record (sent/read by the caller). */
|
* immediately follow the fixed record (sent/read by the caller).
|
||||||
|
*
|
||||||
|
* Protocol 2.28.0 adds the receiver-observed counters the sender cannot see:
|
||||||
|
* `literal_bytes` is the file data the receiver actually stored literally
|
||||||
|
* (whole files plus the literal fragments of a delta) and the four `created_*`
|
||||||
|
* counters split the destination entries the receiver newly created by type,
|
||||||
|
* reproducing rsync's `Number of created files` breakdown and an exact
|
||||||
|
* `Literal data` for a delta run. */
|
||||||
typedef struct {
|
typedef struct {
|
||||||
unsigned long long matched_data;
|
unsigned long long matched_data;
|
||||||
unsigned long long deleted_files;
|
unsigned long long deleted_files;
|
||||||
unsigned long long would_delete_count;
|
unsigned long long would_delete_count;
|
||||||
|
unsigned long long literal_bytes;
|
||||||
|
unsigned long long created_reg;
|
||||||
|
unsigned long long created_dir;
|
||||||
|
unsigned long long created_link;
|
||||||
|
unsigned long long created_special;
|
||||||
} ReceiverStats;
|
} ReceiverStats;
|
||||||
|
|
||||||
/* Fixed-width STATUS_STATS counter record. The status frame and the optional
|
/* Fixed-width STATUS_STATS counter record. The status frame and the optional
|
||||||
|
|||||||
@@ -56,10 +56,13 @@ STDOUT_OUTFMT = "outfmt"
|
|||||||
STDOUT_STATS = "stats"
|
STDOUT_STATS = "stats"
|
||||||
|
|
||||||
# rsync --stats lines that are protocol-independent and must match exactly.
|
# rsync --stats lines that are protocol-independent and must match exactly.
|
||||||
# Deliberately excluded: the per-type "Number of files"/"Number of created
|
# `Number of files` and `Number of created files` carry rsync's per-type
|
||||||
# files" breakdown and Total bytes sent/received (documented residual, see the
|
# breakdown; protocol 2.28.0 reports the receiver-created split over
|
||||||
# `--stats` row in RSYNC_COMPAT.md).
|
# STATUS_STATS. Deliberately excluded: Total bytes sent/received (protocol
|
||||||
|
# framing differs, see the `--stats` row in RSYNC_COMPAT.md).
|
||||||
STATS_KEYS = (
|
STATS_KEYS = (
|
||||||
|
"Number of files",
|
||||||
|
"Number of created files",
|
||||||
"Number of deleted files",
|
"Number of deleted files",
|
||||||
"Number of regular files transferred",
|
"Number of regular files transferred",
|
||||||
"Total file size",
|
"Total file size",
|
||||||
@@ -443,6 +446,11 @@ def run_differential( # noqa: PLR0913 (explicit scenario parameters)
|
|||||||
rroot, froot = os.path.join(rdst, rel), get_dest_received_dir(fdst, src)
|
rroot, froot = os.path.join(rdst, rel), get_dest_received_dir(fdst, src)
|
||||||
else:
|
else:
|
||||||
rroot, froot = rdst, get_dest_received_dir(fdst, src)
|
rroot, froot = rdst, get_dest_received_dir(fdst, src)
|
||||||
|
# rsync's destination root always exists (clean_dir created it). FastSync's
|
||||||
|
# logical transfer root is the mirror path below the destination argument,
|
||||||
|
# so pre-create it too: `Number of created files` counts the root only when
|
||||||
|
# it is genuinely absent, and the two tools must start from the same state.
|
||||||
|
os.makedirs(froot, exist_ok=True)
|
||||||
if seed:
|
if seed:
|
||||||
seed(src, rroot, froot)
|
seed(src, rroot, froot)
|
||||||
|
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ from common import ( # noqa: E402
|
|||||||
verify_transfer,
|
verify_transfer,
|
||||||
)
|
)
|
||||||
|
|
||||||
PROTOCOL_VERSION = b"2.27.0"
|
PROTOCOL_VERSION = b"2.28.0"
|
||||||
STATUS_MANIFEST = 5
|
STATUS_MANIFEST = 5
|
||||||
STATUS_OK = 0
|
STATUS_OK = 0
|
||||||
|
|
||||||
|
|||||||
@@ -289,6 +289,15 @@ def _make_one_file(root, name="f.bin", size=100):
|
|||||||
fh.write(bytes((i * 7 + 3) & 0xFF for i in range(size)))
|
fh.write(bytes((i * 7 + 3) & 0xFF for i in range(size)))
|
||||||
|
|
||||||
|
|
||||||
|
def _pick_stats(text, keys):
|
||||||
|
out = {}
|
||||||
|
for line in text.splitlines():
|
||||||
|
for key in keys:
|
||||||
|
if line.startswith(key + ":"):
|
||||||
|
out[key] = line
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
class TestWireStatsParity:
|
class TestWireStatsParity:
|
||||||
"""Wire-counter output parity: --out-format %b/%c/%C, --progress and
|
"""Wire-counter output parity: --out-format %b/%c/%C, --progress and
|
||||||
--stats versus real rsync 3.4.1."""
|
--stats versus real rsync 3.4.1."""
|
||||||
@@ -496,6 +505,9 @@ class TestWireStatsParity:
|
|||||||
_make_one_file(source, "f.bin", 6000)
|
_make_one_file(source, "f.bin", 6000)
|
||||||
clean_dir(dest)
|
clean_dir(dest)
|
||||||
clean_dir(rdst)
|
clean_dir(rdst)
|
||||||
|
# Start both tools from the same state: rsync's destination root exists,
|
||||||
|
# so pre-create FastSync's mirrored logical root as well.
|
||||||
|
os.makedirs(get_dest_received_dir(dest, source), exist_ok=True)
|
||||||
rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"])
|
rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"])
|
||||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
flags = ["-a", "--stats"] + (["--threads"] if mt else [])
|
flags = ["-a", "--stats"] + (["--threads"] if mt else [])
|
||||||
@@ -526,17 +538,18 @@ class TestWireStatsParity:
|
|||||||
@requires_rsync
|
@requires_rsync
|
||||||
@pytest.mark.ci
|
@pytest.mark.ci
|
||||||
def test_stats_file_count_breakdown_matches_rsync(self, shared_server):
|
def test_stats_file_count_breakdown_matches_rsync(self, shared_server):
|
||||||
"""`Number of files` now carries rsync's per-type breakdown: the scanner
|
"""`Number of files` and `Number of created files` both carry rsync's
|
||||||
accounts directory entries (captured for -a/-t/-p) plus reg/link/special
|
per-type breakdown (protocol 2.28.0 reports the receiver-created
|
||||||
from the transfer list. `Number of created files` still lacks the type
|
reg/dir/link/special split over STATUS_STATS)."""
|
||||||
breakdown (FastSync cannot tell which entries the receiver newly
|
|
||||||
created), so that residual is pinned separately."""
|
|
||||||
source = os.path.join(TEST_DATA_DIR, "wire_stc_src")
|
source = os.path.join(TEST_DATA_DIR, "wire_stc_src")
|
||||||
dest = os.path.join(TEST_DATA_DIR, "wire_stc_dst")
|
dest = os.path.join(TEST_DATA_DIR, "wire_stc_dst")
|
||||||
rdst = os.path.join(TEST_DATA_DIR, "wire_stc_rdst")
|
rdst = os.path.join(TEST_DATA_DIR, "wire_stc_rdst")
|
||||||
_make_one_file(source, "f.bin", 6000)
|
_make_one_file(source, "f.bin", 6000)
|
||||||
clean_dir(dest)
|
clean_dir(dest)
|
||||||
clean_dir(rdst)
|
clean_dir(rdst)
|
||||||
|
# Start both tools from the same state: rsync's destination root exists,
|
||||||
|
# so pre-create FastSync's mirrored logical root as well.
|
||||||
|
os.makedirs(get_dest_received_dir(dest, source), exist_ok=True)
|
||||||
rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"])
|
rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"])
|
||||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
result, _ = run_client(source, dest, flags=["-a", "--stats"],
|
result, _ = run_client(source, dest, flags=["-a", "--stats"],
|
||||||
@@ -556,10 +569,71 @@ class TestWireStatsParity:
|
|||||||
|
|
||||||
assert re.match(r"Number of files: 2 \(reg: 1, dir: 1\)$", r_files), r_files
|
assert re.match(r"Number of files: 2 \(reg: 1, dir: 1\)$", r_files), r_files
|
||||||
assert r_files == f_files, (r_files, f_files)
|
assert r_files == f_files, (r_files, f_files)
|
||||||
# rsync always carries the created type breakdown; FastSync prints the
|
|
||||||
# bare transferred-regular count (documented residual).
|
|
||||||
assert re.match(r"Number of created files: 1 \(reg: 1\)$", r_created), r_created
|
assert re.match(r"Number of created files: 1 \(reg: 1\)$", r_created), r_created
|
||||||
assert re.fullmatch(r"Number of created files: 1", f_created), f_created
|
assert f_created == r_created, (r_created, f_created)
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_stats_created_and_literal_fresh_update_delta(self, shared_server, mt):
|
||||||
|
"""The receiver-observed counters must match rsync for the three
|
||||||
|
transfer shapes: a fresh create (created breakdown + whole-file literal),
|
||||||
|
an update (created == 0, whole-file literal), and a delta update (only
|
||||||
|
the literal delta fragments are counted, not the whole file)."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "wire_stcd_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "wire_stcd_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "wire_stcd_rdst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
os.makedirs(source, exist_ok=True)
|
||||||
|
os.makedirs(get_dest_received_dir(dest, source), exist_ok=True)
|
||||||
|
with open(os.path.join(source, "big.bin"), "wb") as fh:
|
||||||
|
fh.write(bytes(range(256)) * 4096) # 1 MiB
|
||||||
|
mt_flag = ["--threads"] if mt else []
|
||||||
|
|
||||||
|
def compare(tag):
|
||||||
|
# Pin the delta block size on both ends: rsync's adaptive block size
|
||||||
|
# would otherwise make the literal/matched split non-comparable.
|
||||||
|
rsync_result = _rsync(["-a", "--stats", "--no-whole-file", "-B8192",
|
||||||
|
source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
result, _ = run_client(
|
||||||
|
source, dest,
|
||||||
|
flags=["-a", "--stats", "--incremental", "--delta", "-B8192"] + mt_flag,
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
keys = ("Number of created files", "Literal data", "Matched data",
|
||||||
|
"Total transferred file size")
|
||||||
|
r = _pick_stats(rsync_result.stdout, keys)
|
||||||
|
f = _pick_stats(result.stdout, keys)
|
||||||
|
assert r == f, f"{tag}: rsync={r} fastsync={f}"
|
||||||
|
return r
|
||||||
|
|
||||||
|
fresh = compare("fresh")
|
||||||
|
assert re.match(r"Number of created files: 1 \(reg: 1\)$",
|
||||||
|
fresh["Number of created files"]), fresh
|
||||||
|
|
||||||
|
# Update the source and re-run: the destination already exists.
|
||||||
|
sleep_mtime = os.path.getmtime(os.path.join(source, "big.bin")) + 2
|
||||||
|
with open(os.path.join(source, "big.bin"), "r+b") as fh:
|
||||||
|
fh.seek(100)
|
||||||
|
fh.write(b"XXXXXXXXXX")
|
||||||
|
os.utime(os.path.join(source, "big.bin"), (sleep_mtime, sleep_mtime))
|
||||||
|
update = compare("update")
|
||||||
|
assert update["Number of created files"] == "Number of created files: 0", update
|
||||||
|
|
||||||
|
# Second delta update: change bytes far apart, so rsync ships only the
|
||||||
|
# literal fragments and FastSync must report the same Literal data.
|
||||||
|
sleep_mtime = os.path.getmtime(os.path.join(source, "big.bin")) + 2
|
||||||
|
with open(os.path.join(source, "big.bin"), "r+b") as fh:
|
||||||
|
fh.seek(500000)
|
||||||
|
fh.write(b"YYYYYYYYYY")
|
||||||
|
os.utime(os.path.join(source, "big.bin"), (sleep_mtime, sleep_mtime))
|
||||||
|
delta = compare("delta")
|
||||||
|
assert delta["Number of created files"] == "Number of created files: 0", delta
|
||||||
|
lit = int(delta["Literal data"].split(":", 1)[1].strip().split()[0].replace(",", ""))
|
||||||
|
assert 0 < lit < 1024 * 1024, delta
|
||||||
|
|
||||||
@requires_rsync
|
@requires_rsync
|
||||||
@pytest.mark.ci
|
@pytest.mark.ci
|
||||||
|
|||||||
@@ -94,14 +94,14 @@ def _seed_protocol_source(source):
|
|||||||
class TestProtocol:
|
class TestProtocol:
|
||||||
@pytest.mark.ci
|
@pytest.mark.ci
|
||||||
def test_protocol_current_version_accepted(self, shared_server):
|
def test_protocol_current_version_accepted(self, shared_server):
|
||||||
"""--protocol=2.27.0 (the current PROTOCOL_VERSION) is accepted and the
|
"""--protocol=2.28.0 (the current PROTOCOL_VERSION) is accepted and the
|
||||||
transfer completes normally."""
|
transfer completes normally."""
|
||||||
source = os.path.join(TEST_DATA_DIR, "proto_ok_src")
|
source = os.path.join(TEST_DATA_DIR, "proto_ok_src")
|
||||||
dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst")
|
dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst")
|
||||||
shutil.rmtree(dest, ignore_errors=True)
|
shutil.rmtree(dest, ignore_errors=True)
|
||||||
os.makedirs(dest)
|
os.makedirs(dest)
|
||||||
_seed_protocol_source(source)
|
_seed_protocol_source(source)
|
||||||
result, _ = run_client(source, dest, flags=["--protocol=2.27.0"],
|
result, _ = run_client(source, dest, flags=["--protocol=2.28.0"],
|
||||||
port=shared_server.port)
|
port=shared_server.port)
|
||||||
assert result.returncode == 0, \
|
assert result.returncode == 0, \
|
||||||
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
|
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
|
||||||
|
|||||||
@@ -318,7 +318,7 @@ static void test_parse_args_protocol_accept_current() {
|
|||||||
Config* cfg = valid_client_config();
|
Config* cfg = valid_client_config();
|
||||||
EXPECT_NOT_NULL(cfg);
|
EXPECT_NOT_NULL(cfg);
|
||||||
char* argv_equals[] = {"fastsync", "--source-dir", "/src",
|
char* argv_equals[] = {"fastsync", "--source-dir", "/src",
|
||||||
"--dest-dir", "/dst", "--protocol=2.27.0"};
|
"--dest-dir", "/dst", "--protocol=2.28.0"};
|
||||||
int positional_args[2];
|
int positional_args[2];
|
||||||
int positional_count = 0;
|
int positional_count = 0;
|
||||||
EXPECT_EQ_INT(parse_args(cfg, 6, argv_equals, positional_args, &positional_count), 0);
|
EXPECT_EQ_INT(parse_args(cfg, 6, argv_equals, positional_args, &positional_count), 0);
|
||||||
@@ -328,7 +328,7 @@ static void test_parse_args_protocol_accept_current() {
|
|||||||
cfg = valid_client_config();
|
cfg = valid_client_config();
|
||||||
EXPECT_NOT_NULL(cfg);
|
EXPECT_NOT_NULL(cfg);
|
||||||
char* argv_space[] = {"fastsync", "--source-dir", "/src", "--dest-dir",
|
char* argv_space[] = {"fastsync", "--source-dir", "/src", "--dest-dir",
|
||||||
"/dst", "--protocol", "2.27.0"};
|
"/dst", "--protocol", "2.28.0"};
|
||||||
positional_count = 0;
|
positional_count = 0;
|
||||||
EXPECT_EQ_INT(parse_args(cfg, 7, argv_space, positional_args, &positional_count), 0);
|
EXPECT_EQ_INT(parse_args(cfg, 7, argv_space, positional_args, &positional_count), 0);
|
||||||
EXPECT_EQ_STR(cfg->version, PROTOCOL_VERSION);
|
EXPECT_EQ_STR(cfg->version, PROTOCOL_VERSION);
|
||||||
|
|||||||
+8
-6
@@ -2831,15 +2831,17 @@ static void golden_config_populate(Config* c) {
|
|||||||
c->copy_as_gid = 222;
|
c->copy_as_gid = 222;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* The pinned golden frame (protocol 2.27.0). The values below are the only
|
/* The pinned golden frame (protocol 2.28.0). The values below are the only
|
||||||
* thing that ties the generated table to the historical wire format; update
|
* thing that ties the generated table to the historical wire format; update
|
||||||
* them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.24.0
|
* them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.24.0
|
||||||
* delete-plan wave changed only the version string; 2.25.0 appended the
|
* delete-plan wave changed only the version string; 2.25.0 appended the
|
||||||
* report_stats bool, 2.26.0 appended the compression_algo int, and 2.27.0
|
* report_stats bool, 2.26.0 appended the compression_algo int, 2.27.0 appended
|
||||||
* appended the report_deletes bool. The byte-exact values are recomputed for
|
* the report_deletes bool, and 2.28.0 changed only the version string (the
|
||||||
* the merged layout. */
|
* STATUS_STATS body grew, but the config frame layout is unchanged, so the
|
||||||
|
* frame length is identical). The byte-exact values are recomputed for the
|
||||||
|
* merged layout. */
|
||||||
#define GOLDEN_WIRE_LEN 709
|
#define GOLDEN_WIRE_LEN 709
|
||||||
#define GOLDEN_WIRE_HASH 14423869696887880000ULL
|
#define GOLDEN_WIRE_HASH 417335736347473203ULL
|
||||||
|
|
||||||
static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) {
|
static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) {
|
||||||
unsigned long long h = 1469598103934665603ULL;
|
unsigned long long h = 1469598103934665603ULL;
|
||||||
@@ -2921,7 +2923,7 @@ static unsigned long long capture_wire_hash(const Config* cfg, size_t* out_len)
|
|||||||
return h;
|
return h;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Byte-for-byte wire compatibility guard (protocol 2.27.0). The expected hash
|
/* Byte-for-byte wire compatibility guard (protocol 2.28.0). The expected hash
|
||||||
* pins the pre-X-macro byte stream; the refactor MUST NOT change it. */
|
* pins the pre-X-macro byte stream; the refactor MUST NOT change it. */
|
||||||
static void test_config_wire_golden() {
|
static void test_config_wire_golden() {
|
||||||
if (is_running_under_valgrind())
|
if (is_running_under_valgrind())
|
||||||
|
|||||||
@@ -86,9 +86,43 @@ static void test_dest_state_roundtrip() {
|
|||||||
close(fds[1]);
|
close(fds[1]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_stats_roundtrip() {
|
||||||
|
/* STATUS_STATS grew from three counters (2.25.0) to eight (2.28.0); the codec
|
||||||
|
* must carry every field, including the receiver-observed literal/created
|
||||||
|
* counters, across the wire in order. */
|
||||||
|
int fds[2];
|
||||||
|
if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) != 0)
|
||||||
|
return;
|
||||||
|
ReceiverStats out;
|
||||||
|
memset(&out, 0, sizeof(out));
|
||||||
|
out.matched_data = 111111111ULL;
|
||||||
|
out.deleted_files = 7;
|
||||||
|
out.would_delete_count = 3;
|
||||||
|
out.literal_bytes = 222222222ULL;
|
||||||
|
out.created_reg = 5;
|
||||||
|
out.created_dir = 4;
|
||||||
|
out.created_link = 2;
|
||||||
|
out.created_special = 1;
|
||||||
|
ReceiverStats in;
|
||||||
|
memset(&in, 0, sizeof(in));
|
||||||
|
EXPECT_TRUE(format_stats_send(fds[0], &out));
|
||||||
|
EXPECT_TRUE(format_stats_receive(fds[1], &in));
|
||||||
|
EXPECT_TRUE(in.matched_data == out.matched_data);
|
||||||
|
EXPECT_TRUE(in.deleted_files == out.deleted_files);
|
||||||
|
EXPECT_TRUE(in.would_delete_count == out.would_delete_count);
|
||||||
|
EXPECT_TRUE(in.literal_bytes == out.literal_bytes);
|
||||||
|
EXPECT_TRUE(in.created_reg == out.created_reg);
|
||||||
|
EXPECT_TRUE(in.created_dir == out.created_dir);
|
||||||
|
EXPECT_TRUE(in.created_link == out.created_link);
|
||||||
|
EXPECT_TRUE(in.created_special == out.created_special);
|
||||||
|
close(fds[0]);
|
||||||
|
close(fds[1]);
|
||||||
|
}
|
||||||
|
|
||||||
void test_format(void) {
|
void test_format(void) {
|
||||||
test_human_size_decimal();
|
test_human_size_decimal();
|
||||||
test_big_num_grouping();
|
test_big_num_grouping();
|
||||||
test_datetime_format();
|
test_datetime_format();
|
||||||
test_dest_state_roundtrip();
|
test_dest_state_roundtrip();
|
||||||
|
test_stats_roundtrip();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,8 @@
|
|||||||
#include "config.h"
|
#include "config.h"
|
||||||
#include "delta.h"
|
#include "delta.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
|
#include "file_receive.h"
|
||||||
|
#include "format.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "test_utils.h"
|
#include "test_utils.h"
|
||||||
@@ -134,6 +136,124 @@ static void test_receive_files_single_file() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Protocol 2.28.0: a fresh single-file transfer over the wire reports the
|
||||||
|
* receiver-observed literal bytes and the created-regular counter through the
|
||||||
|
* terminal STATUS_STATS frame, and an update reports created_reg == 0. */
|
||||||
|
static void test_receive_stats_frame_created_and_literal() {
|
||||||
|
const char* content = "stats frame content";
|
||||||
|
size_t len = strlen(content);
|
||||||
|
char root_template[] = "/tmp/fastsync_stats_XXXXXX";
|
||||||
|
char* root = mkdtemp(root_template);
|
||||||
|
EXPECT_NOT_NULL(root);
|
||||||
|
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
free(cfg->version);
|
||||||
|
cfg->version = str_dup(PROTOCOL_VERSION);
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup(root);
|
||||||
|
cfg->save_to_disk = true;
|
||||||
|
cfg->report_stats = true;
|
||||||
|
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
close(p[1]);
|
||||||
|
io_set_fds(p[0], p[0]);
|
||||||
|
int ret = receiver_receive_files(cfg, p[0]);
|
||||||
|
close(p[0]);
|
||||||
|
config_delete(cfg);
|
||||||
|
_exit(ret == 0 ? 0 : 1);
|
||||||
|
}
|
||||||
|
close(p[0]);
|
||||||
|
io_set_fds(p[1], p[1]);
|
||||||
|
send_status(p[1], STATUS_NEXT);
|
||||||
|
File* file = file_create("created.bin");
|
||||||
|
EXPECT_NOT_NULL(file);
|
||||||
|
file->data->data = malloc(len);
|
||||||
|
EXPECT_NOT_NULL(file->data->data);
|
||||||
|
memcpy(file->data->data, content, len);
|
||||||
|
file->data->size = len;
|
||||||
|
send_str(p[1], file->path);
|
||||||
|
send_data(p[1], file->data);
|
||||||
|
file_destroy(file);
|
||||||
|
send_status(p[1], STATUS_FINISHED);
|
||||||
|
|
||||||
|
Status status;
|
||||||
|
EXPECT_TRUE(receive_status(p[1], &status));
|
||||||
|
EXPECT_EQ_INT(status, STATUS_STATS);
|
||||||
|
ReceiverStats stats;
|
||||||
|
EXPECT_TRUE(format_stats_receive(p[1], &stats));
|
||||||
|
int would = 0;
|
||||||
|
EXPECT_TRUE(receive_int(p[1], &would));
|
||||||
|
EXPECT_EQ_INT(would, 0);
|
||||||
|
EXPECT_TRUE(stats.created_reg == 1);
|
||||||
|
EXPECT_TRUE(stats.created_dir == 0);
|
||||||
|
EXPECT_TRUE(stats.created_link == 0);
|
||||||
|
EXPECT_TRUE(stats.created_special == 0);
|
||||||
|
EXPECT_TRUE(stats.literal_bytes == (unsigned long long)len);
|
||||||
|
EXPECT_TRUE(stats.matched_data == 0);
|
||||||
|
|
||||||
|
Status final;
|
||||||
|
EXPECT_TRUE(receive_status(p[1], &final));
|
||||||
|
EXPECT_EQ_INT(final, STATUS_OK);
|
||||||
|
int wstatus;
|
||||||
|
waitpid(pid, &wstatus, 0);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(cfg);
|
||||||
|
EXPECT_TRUE(WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0);
|
||||||
|
|
||||||
|
/* Second run against the now-existing destination: no created file. */
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
free(cfg->version);
|
||||||
|
cfg->version = str_dup(PROTOCOL_VERSION);
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup(root);
|
||||||
|
cfg->save_to_disk = true;
|
||||||
|
cfg->report_stats = true;
|
||||||
|
pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
close(p[1]);
|
||||||
|
io_set_fds(p[0], p[0]);
|
||||||
|
int ret = receiver_receive_files(cfg, p[0]);
|
||||||
|
close(p[0]);
|
||||||
|
config_delete(cfg);
|
||||||
|
_exit(ret == 0 ? 0 : 1);
|
||||||
|
}
|
||||||
|
close(p[0]);
|
||||||
|
io_set_fds(p[1], p[1]);
|
||||||
|
send_status(p[1], STATUS_NEXT);
|
||||||
|
file = file_create("created.bin");
|
||||||
|
EXPECT_NOT_NULL(file);
|
||||||
|
file->data->data = malloc(len);
|
||||||
|
EXPECT_NOT_NULL(file->data->data);
|
||||||
|
memcpy(file->data->data, content, len);
|
||||||
|
file->data->size = len;
|
||||||
|
send_str(p[1], file->path);
|
||||||
|
send_data(p[1], file->data);
|
||||||
|
file_destroy(file);
|
||||||
|
send_status(p[1], STATUS_FINISHED);
|
||||||
|
EXPECT_TRUE(receive_status(p[1], &status));
|
||||||
|
EXPECT_EQ_INT(status, STATUS_STATS);
|
||||||
|
EXPECT_TRUE(format_stats_receive(p[1], &stats));
|
||||||
|
EXPECT_TRUE(receive_int(p[1], &would));
|
||||||
|
EXPECT_TRUE(stats.created_reg == 0);
|
||||||
|
EXPECT_TRUE(stats.literal_bytes == (unsigned long long)len);
|
||||||
|
EXPECT_TRUE(receive_status(p[1], &final));
|
||||||
|
waitpid(pid, &wstatus, 0);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(cfg);
|
||||||
|
EXPECT_TRUE(WIFEXITED(wstatus) && WEXITSTATUS(wstatus) == 0);
|
||||||
|
}
|
||||||
|
|
||||||
/* Test receive_files with STATUS_ABORT */
|
/* Test receive_files with STATUS_ABORT */
|
||||||
static void test_receive_files_abort() {
|
static void test_receive_files_abort() {
|
||||||
Config* cfg = config_create();
|
Config* cfg = config_create();
|
||||||
@@ -1183,6 +1303,7 @@ void test_server() {
|
|||||||
if (!is_running_under_valgrind()) {
|
if (!is_running_under_valgrind()) {
|
||||||
test_receive_files_finished();
|
test_receive_files_finished();
|
||||||
test_receive_files_single_file();
|
test_receive_files_single_file();
|
||||||
|
test_receive_stats_frame_created_and_literal();
|
||||||
test_receive_files_abort();
|
test_receive_files_abort();
|
||||||
test_receive_manifest_rejects_traversal();
|
test_receive_manifest_rejects_traversal();
|
||||||
test_receive_incremental_check_rejects_invalid_nanoseconds();
|
test_receive_incremental_check_rejects_invalid_nanoseconds();
|
||||||
|
|||||||
Reference in New Issue
Block a user