fix(parity): actual-removal delete-delay counts, rsync-accurate stats/progress/%C
- --delete-delay: count/track only entries actually removed; a directory refilled before commit (ENOTEMPTY) no longer inflates Number of deleted files or the --max-delete budget (unit + integration + rsync differential). - --stats: per-type Number of files breakdown; only stored regular files count as transferred; transferred/literal byte totals and Total file size (symlink target lengths) now match rsync for whole-file transfers. - --progress: print the leading ./ root line and include the root entry in the to-chk denominator (single-file output byte-identical to rsync). - --out-format %C: use the selected transfer checksum and render every algorithm exactly like rsync; checksum_digest_file gains md4/sha1/none. Reclassify --out-format to Divergent (protocol-specific %b/delta-%c). - Docs: RSYNC_COMPAT tally 107/25/24, HANDOFF update. No wire change.
This commit is contained in:
+21
-18
@@ -194,25 +194,18 @@ char* change_render_itemize(const Config* config, const ChangeEvent* event) {
|
||||
|
||||
/* ---- --out-format / --log-file-format ---- */
|
||||
|
||||
/* rsync 3.4.1's `%C` uses the negotiated transfer checksum; with the default
|
||||
* "auto" choice on both ends that is xxh128. FastSync's internal XXH64 default
|
||||
* is not an rsync algorithm, so map it to xxh128 for parity. */
|
||||
/* rsync 3.4.1's `%C` uses the negotiated TRANSFER checksum (the first name of a
|
||||
* two-name "transfer,pre-transfer" --checksum-choice), not the pre-transfer
|
||||
* whole-file digest FastSync compares against on the wire. The default "auto"
|
||||
* resolves to xxh128, so an explicit selection and the default both render the
|
||||
* selected algorithm's digest. */
|
||||
static ChecksumAlgo out_format_checksum_algo(const Config* config) {
|
||||
switch ((ChecksumAlgo)config->checksum_algo) {
|
||||
case CHECKSUM_ALGO_MD5:
|
||||
return CHECKSUM_ALGO_MD5;
|
||||
case CHECKSUM_ALGO_XXH3:
|
||||
return CHECKSUM_ALGO_XXH3;
|
||||
case CHECKSUM_ALGO_XXH128:
|
||||
return CHECKSUM_ALGO_XXH128;
|
||||
case CHECKSUM_ALGO_XXH64:
|
||||
default:
|
||||
return CHECKSUM_ALGO_XXH128;
|
||||
}
|
||||
return (ChecksumAlgo)config->checksum_transfer_algo;
|
||||
}
|
||||
|
||||
/* Render a digest as rsync's sum_as_hex: for xxh128 the HIGH 64-bit half is
|
||||
* printed before the low half; every other algorithm prints its bytes in order. */
|
||||
/* Render a digest as rsync's sum_as_hex: xxh128 prints the HIGH 64-bit half
|
||||
* before the low half, and xxh64/xxh3 print their 64-bit value big-endian; every
|
||||
* other algorithm prints its bytes in order. */
|
||||
static void digest_to_hex(ChecksumAlgo algo, const uint8_t* digest, size_t len, char* out) {
|
||||
if (algo == CHECKSUM_ALGO_XXH128 && len == 16) {
|
||||
uint64_t low = 0;
|
||||
@@ -222,6 +215,12 @@ static void digest_to_hex(ChecksumAlgo algo, const uint8_t* digest, size_t len,
|
||||
snprintf(out, len * 2 + 1, "%016llx%016llx", (unsigned long long)high, (unsigned long long)low);
|
||||
return;
|
||||
}
|
||||
if ((algo == CHECKSUM_ALGO_XXH64 || algo == CHECKSUM_ALGO_XXH3) && len == 8) {
|
||||
uint64_t value = 0;
|
||||
memcpy(&value, digest, sizeof(value));
|
||||
snprintf(out, len * 2 + 1, "%016llx", (unsigned long long)value);
|
||||
return;
|
||||
}
|
||||
static const char hex[] = "0123456789abcdef";
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
out[i * 2] = hex[(digest[i] >> 4) & 0xf];
|
||||
@@ -260,6 +259,9 @@ static void fill_event_checksum(const Config* config, const File* file, ChangeEv
|
||||
if (file->path == NULL)
|
||||
return;
|
||||
ChecksumAlgo algo = out_format_checksum_algo(config);
|
||||
/* rsync renders `--checksum-choice=none` as a blank 2-character column. */
|
||||
if (algo == CHECKSUM_ALGO_NONE)
|
||||
return;
|
||||
uint8_t digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||
size_t len = 0;
|
||||
/* rsync's %C is the transfer checksum, which is always seeded with 0 (it is
|
||||
@@ -329,9 +331,10 @@ char* change_render_format(const char* format, const Config* config, const Chang
|
||||
if (event->checksum_known) {
|
||||
ok = strbuf_append(&line, event->checksum);
|
||||
} else {
|
||||
/* rsync pads a non-regular / untransferred entry with spaces. */
|
||||
/* rsync pads a non-regular / untransferred / `none` entry with spaces;
|
||||
`none` renders as a blank 2-character column. */
|
||||
ChecksumAlgo algo = out_format_checksum_algo(config);
|
||||
int width = checksum_digest_len(algo) * 2;
|
||||
int width = algo == CHECKSUM_ALGO_NONE ? 2 : checksum_digest_len(algo) * 2;
|
||||
for (int i = 0; i < width && ok; i++)
|
||||
ok = strbuf_append_char(&line, ' ');
|
||||
}
|
||||
|
||||
+154
-36
@@ -83,15 +83,53 @@ static const char* stats_bytes(const Config* config, unsigned long long bytes, c
|
||||
return buffer;
|
||||
}
|
||||
|
||||
/* Print the rsync `--stats` block on stdout. Byte totals use the process-wide
|
||||
wire counters and the receiver-only counters come from the STATUS_STATS frame;
|
||||
the labels, layout and rate/speedup formulas match rsync 3.4.1. Shared by the
|
||||
single-threaded and multithreaded send paths. */
|
||||
static void report_transfer_stats(const Config* config, int total_files,
|
||||
unsigned long long total_bytes, time_t start,
|
||||
/* Build rsync's `Number of files` parenthetical: each non-zero category, in
|
||||
reg/dir/link/special order. Empty when the flist counted nothing. */
|
||||
static void stats_type_breakdown(const TransferStats* stats, char* out, size_t out_size) {
|
||||
unsigned long long total =
|
||||
stats->flist_reg + stats->flist_dir + stats->flist_link + stats->flist_special;
|
||||
if (total == 0) {
|
||||
out[0] = '\0';
|
||||
return;
|
||||
}
|
||||
out[0] = '\0';
|
||||
size_t used = 0;
|
||||
const struct {
|
||||
const char* name;
|
||||
unsigned long long count;
|
||||
} parts[4] = {{"reg", stats->flist_reg},
|
||||
{"dir", stats->flist_dir},
|
||||
{"link", stats->flist_link},
|
||||
{"special", stats->flist_special}};
|
||||
bool first = true;
|
||||
for (size_t i = 0; i < 4; i++) {
|
||||
if (parts[i].count == 0)
|
||||
continue;
|
||||
int written = snprintf(out + used, out_size - used, "%s%s: %llu", first ? "(" : ", ",
|
||||
parts[i].name, parts[i].count);
|
||||
if (written < 0 || (size_t)written >= out_size - used)
|
||||
break;
|
||||
used += (size_t)written;
|
||||
first = false;
|
||||
}
|
||||
if (!first && used + 1 < out_size)
|
||||
out[used++] = ')';
|
||||
out[used] = '\0';
|
||||
}
|
||||
|
||||
/* Print the rsync `--stats` block on stdout. The source-side flist and
|
||||
transferred counters come from `stats` (filled while scanning/sending), the
|
||||
receiver-only counters from the STATUS_STATS frame, and the wire byte totals
|
||||
from the process-wide protocol counters. The labels, layout and
|
||||
rate/speedup formulas match rsync 3.4.1. Shared by the single-threaded and
|
||||
multithreaded send paths. */
|
||||
static void report_transfer_stats(const Config* config, const TransferStats* stats, time_t start,
|
||||
const ReceiverStats* recv) {
|
||||
if (!config->stats || config->quiet)
|
||||
return;
|
||||
TransferStats empty = {0};
|
||||
if (stats == NULL)
|
||||
stats = ∅
|
||||
ReceiverStats none = {0};
|
||||
if (recv == NULL)
|
||||
recv = &none;
|
||||
@@ -101,11 +139,18 @@ static void report_transfer_stats(const Config* config, int total_files,
|
||||
double elapsed = difftime(time(NULL), start);
|
||||
double rate = (double)(sent + received) / (0.5 + elapsed);
|
||||
char total_buffer[32];
|
||||
char transferred_buffer[32];
|
||||
char literal_buffer[32];
|
||||
char sent_buffer[32];
|
||||
char recv_buffer[32];
|
||||
char rate_buffer[32] = {0};
|
||||
char human_rate[32] = {0};
|
||||
const char* total = stats_bytes(config, total_bytes, total_buffer, sizeof(total_buffer));
|
||||
const char* total =
|
||||
stats_bytes(config, stats->total_file_size, total_buffer, sizeof(total_buffer));
|
||||
const char* transferred = stats_bytes(config, stats->transferred_file_size, transferred_buffer,
|
||||
sizeof(transferred_buffer));
|
||||
const char* literal =
|
||||
stats_bytes(config, stats->literal_data, literal_buffer, sizeof(literal_buffer));
|
||||
const char* sent_s = stats_bytes(config, sent, sent_buffer, sizeof(sent_buffer));
|
||||
const char* recv_s = stats_bytes(config, received, recv_buffer, sizeof(recv_buffer));
|
||||
const char* rate_str = rate_buffer;
|
||||
@@ -116,15 +161,26 @@ static void report_transfer_stats(const Config* config, int total_files,
|
||||
} else {
|
||||
snprintf(rate_buffer, sizeof(rate_buffer), "%.2f", rate);
|
||||
}
|
||||
double speedup = (sent + received) > 0 ? (double)total_bytes / (double)(sent + received) : 0.0;
|
||||
double speedup =
|
||||
(sent + received) > 0 ? (double)stats->total_file_size / (double)(sent + received) : 0.0;
|
||||
char breakdown[128];
|
||||
stats_type_breakdown(stats, breakdown, sizeof(breakdown));
|
||||
unsigned long long flist_total =
|
||||
stats->flist_reg + stats->flist_dir + stats->flist_link + stats->flist_special;
|
||||
printf("\n");
|
||||
printf("Number of files: %d\n", total_files);
|
||||
printf("Number of created files: %d\n", total_files);
|
||||
if (breakdown[0] != '\0')
|
||||
printf("Number of files: %llu %s\n", flist_total, breakdown);
|
||||
else
|
||||
printf("Number of files: %llu\n", flist_total);
|
||||
/* FastSync cannot tell which entries the receiver newly created, so it
|
||||
reports the transferred regular files (which are created on a fresh
|
||||
destination). See RSYNC_COMPAT.md for the documented residual. */
|
||||
printf("Number of created files: %llu\n", stats->transferred_regular);
|
||||
printf("Number of deleted files: %llu\n", recv->deleted_files);
|
||||
printf("Number of regular files transferred: %d\n", total_files);
|
||||
printf("Number of regular files transferred: %llu\n", stats->transferred_regular);
|
||||
printf("Total file size: %s bytes\n", total);
|
||||
printf("Total transferred file size: %s bytes\n", total);
|
||||
printf("Literal data: %s bytes\n", total);
|
||||
printf("Total transferred file size: %s bytes\n", transferred);
|
||||
printf("Literal data: %s bytes\n", literal);
|
||||
printf("Matched data: %llu bytes\n", recv->matched_data);
|
||||
printf("File list size: 0\n");
|
||||
printf("File list generation time: 0.000 seconds\n");
|
||||
@@ -138,6 +194,45 @@ static void report_transfer_stats(const Config* config, int total_files,
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
/* Classify one scanned source entry into the rsync flist counters. Called for
|
||||
every entry the sender walks, transferred or skipped. Directory entries are
|
||||
counted here only for the explicit -d/--dirs generator; a recursive scan's
|
||||
directories are accounted from the scanner's dir_entries list at report time. */
|
||||
static void transfer_stats_note_entry(TransferStats* stats, const File* file) {
|
||||
if (stats == NULL || file == NULL)
|
||||
return;
|
||||
if (file->is_dir) {
|
||||
stats->flist_dir++;
|
||||
return;
|
||||
}
|
||||
if (file->is_symlink) {
|
||||
stats->flist_link++;
|
||||
stats->total_file_size += file->symlink_target ? strlen(file->symlink_target) : 0;
|
||||
return;
|
||||
}
|
||||
if (file->is_special) {
|
||||
stats->flist_special++;
|
||||
return;
|
||||
}
|
||||
stats->flist_reg++;
|
||||
stats->total_file_size += file->data ? file->data->size : 0;
|
||||
}
|
||||
|
||||
/* Account for a regular file (or a whole-file append) the receiver actually
|
||||
stored: rsync's transferred-file count and transferred/literal byte totals. */
|
||||
static void transfer_stats_note_transferred(TransferStats* stats, const File* file) {
|
||||
if (stats == NULL || file == NULL)
|
||||
return;
|
||||
if (file->is_dir || file->is_symlink || file->is_special)
|
||||
return;
|
||||
if (file->link_group != 0 && !file->link_first)
|
||||
return;
|
||||
unsigned long long size = file->data ? file->data->size : 0;
|
||||
stats->transferred_regular++;
|
||||
stats->transferred_file_size += size;
|
||||
stats->literal_data += size;
|
||||
}
|
||||
|
||||
/* ---- rsync-style per-file --progress ------------------------------------
|
||||
* rsync prints, for each transferred regular file, the file name followed by a
|
||||
* two-frame progress line: the first at the initial 32 KiB read window (always
|
||||
@@ -188,10 +283,14 @@ static void progress_final_frame(unsigned long long size, char* out, size_t out_
|
||||
unsigned long long remain = (unsigned long long)(diff_ms / 1000);
|
||||
snprintf(rembuf, sizeof(rembuf), "%4u:%02u:%02u", (unsigned)(remain / 3600),
|
||||
(unsigned)((remain / 60) % 60), (unsigned)(remain % 60));
|
||||
/* rsync's `to-chk` denominator is the whole file list, which includes the
|
||||
transfer-root directory FastSync never emits as a transfer entry. Count
|
||||
that root entry so a single-file transfer matches rsync exactly. */
|
||||
unsigned long long total = g_progress_seen + 1;
|
||||
unsigned long long to_chk =
|
||||
g_progress_seen > g_progress_xferred ? g_progress_seen - g_progress_xferred : 0;
|
||||
snprintf(out, out_size, "\r%15s %3d%% %7.2f%s %s (xfr#%llu, to-chk=%llu/%llu)\n", ofs_buf, 100,
|
||||
rate, units, rembuf, g_progress_xferred, to_chk, g_progress_seen);
|
||||
rate, units, rembuf, g_progress_xferred, to_chk, total);
|
||||
}
|
||||
|
||||
static void client_progress_begin(const Config* config) {
|
||||
@@ -201,6 +300,10 @@ static void client_progress_begin(const Config* config) {
|
||||
if (!g_progress_active)
|
||||
return;
|
||||
printf("sending incremental file list\n");
|
||||
/* rsync prints the transfer-root directory's name before the first file when
|
||||
that directory is created; FastSync mirrors the source root below the
|
||||
receive root and creates it on a fresh destination, so emit it here. */
|
||||
printf("./\n");
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
@@ -1986,7 +2089,16 @@ static int send_dry_run_remote(Config* config) {
|
||||
else
|
||||
printf("Total: %d files, %.1f MB\n", file_count, (double)total_bytes / (double)BYTES_PER_MIB);
|
||||
}
|
||||
report_transfer_stats(config, file_count, total_bytes, dry_start, &dry_stats);
|
||||
{
|
||||
TransferStats dry_transfer;
|
||||
memset(&dry_transfer, 0, sizeof(dry_transfer));
|
||||
dry_transfer.flist_reg = (unsigned long long)file_count;
|
||||
dry_transfer.total_file_size = total_bytes;
|
||||
dry_transfer.transferred_regular = (unsigned long long)file_count;
|
||||
dry_transfer.transferred_file_size = total_bytes;
|
||||
dry_transfer.literal_data = total_bytes;
|
||||
report_transfer_stats(config, &dry_transfer, dry_start, &dry_stats);
|
||||
}
|
||||
ret = io_error ? 1 : 0;
|
||||
|
||||
dry_fail:
|
||||
@@ -2236,7 +2348,7 @@ static bool source_is_regular_file(const File* file) {
|
||||
}
|
||||
|
||||
static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
ArrayList* remove_sources) {
|
||||
ArrayList* remove_sources, TransferStats* stats) {
|
||||
if (config->use_chunk_serialization) {
|
||||
if (remove_sources) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
@@ -2263,10 +2375,13 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
if (chunk->items[i] == NULL)
|
||||
continue;
|
||||
transfer_stats_note_entry(stats, chunk->items[i]);
|
||||
if (chunk->items[i]->is_dir)
|
||||
change_emit_dir_sent(config, chunk->items[i]);
|
||||
else
|
||||
change_emit_file_sent(config, chunk->items[i]);
|
||||
if (!chunk->items[i]->is_dir)
|
||||
transfer_stats_note_transferred(stats, chunk->items[i]);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -2275,6 +2390,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
File* f = chunk->items[i];
|
||||
if (f == NULL)
|
||||
continue;
|
||||
transfer_stats_note_entry(stats, f);
|
||||
if (f->is_dir) {
|
||||
/* Explicit directory entry (--dirs): a MKDIR frame carrying the
|
||||
destination path (and metadata when negotiated). Directories have no
|
||||
@@ -2328,6 +2444,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
source_file_destroy(source);
|
||||
return -1;
|
||||
}
|
||||
transfer_stats_note_transferred(stats, f);
|
||||
change_emit_file_sent_bytes(config, f, protocol_bytes_written() - bytes_before,
|
||||
protocol_bytes_read() - read_before);
|
||||
client_progress_file(config, f);
|
||||
@@ -2436,7 +2553,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
return thrd_error;
|
||||
}
|
||||
if (send_chunk_with_removal(client, current_chunk, context->config,
|
||||
context->remove_source_files) != 0) {
|
||||
context->remove_source_files, &context->stats) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "unexpected error while sending chunk");
|
||||
chunk_destroy(current_chunk);
|
||||
pipeline_cancel(context);
|
||||
@@ -2533,13 +2650,12 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
"server reported a deletion failure (--delete); see the server log for the reason");
|
||||
if (ok)
|
||||
remove_transferred_sources(context->config, context->remove_source_files);
|
||||
mtx_lock(&context->mutex_progress);
|
||||
int total_files = context->total_files;
|
||||
unsigned long long total_bytes = context->total_bytes;
|
||||
mtx_unlock(&context->mutex_progress);
|
||||
report_transfer_stats(context->config, total_files, total_bytes, start, &recv_stats);
|
||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
|
||||
(double)total_bytes / (double)BYTES_PER_MIB);
|
||||
if (context->dir_entries)
|
||||
context->stats.flist_dir += (unsigned long long)context->dir_entries->size;
|
||||
report_transfer_stats(context->config, &context->stats, start, &recv_stats);
|
||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %llu files, %.1f MB",
|
||||
context->stats.transferred_regular,
|
||||
(double)context->stats.transferred_file_size / (double)BYTES_PER_MIB);
|
||||
disconnect_transfer_client(client);
|
||||
mark_sender_done(context);
|
||||
protocol_session_unbind();
|
||||
@@ -3003,8 +3119,8 @@ int send_files(Config* config) {
|
||||
goto send_fail;
|
||||
|
||||
Chunk* current_chunk;
|
||||
unsigned long long total_bytes = 0;
|
||||
int total_files = 0;
|
||||
TransferStats transfer_stats;
|
||||
memset(&transfer_stats, 0, sizeof(transfer_stats));
|
||||
time_t start = time(NULL);
|
||||
client_progress_begin(config);
|
||||
/* True when the stop deadline cut the scan short so the keep-set manifest is
|
||||
@@ -3030,11 +3146,6 @@ int send_files(Config* config) {
|
||||
scan_stopped_early = true;
|
||||
break;
|
||||
}
|
||||
unsigned long long chunk_bytes = 0;
|
||||
for (int i = 0; i < current_chunk->element_count; i++) {
|
||||
chunk_bytes += current_chunk->items[i]->data->size;
|
||||
total_files++;
|
||||
}
|
||||
if (manifest && !add_chunk_to_manifest(manifest, current_chunk)) {
|
||||
chunk_destroy(current_chunk);
|
||||
goto send_fail;
|
||||
@@ -3061,13 +3172,13 @@ int send_files(Config* config) {
|
||||
send_failed = true;
|
||||
break;
|
||||
}
|
||||
if (send_chunk_with_removal(client, current_chunk, config, remove_sources) != 0) {
|
||||
if (send_chunk_with_removal(client, current_chunk, config, remove_sources, &transfer_stats) !=
|
||||
0) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
|
||||
chunk_destroy(current_chunk);
|
||||
send_failed = true;
|
||||
break;
|
||||
}
|
||||
total_bytes += chunk_bytes;
|
||||
chunk_destroy(current_chunk);
|
||||
}
|
||||
if (send_failed) {
|
||||
@@ -3152,9 +3263,16 @@ int send_files(Config* config) {
|
||||
"server reported a deletion failure (--delete); see the server log for the reason");
|
||||
if (ok)
|
||||
remove_transferred_sources(config, remove_sources);
|
||||
report_transfer_stats(config, total_files, total_bytes, start, &recv_stats);
|
||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
|
||||
(double)total_bytes / (double)BYTES_PER_MIB);
|
||||
/* A recursive -a scan has no directory entries in its chunks; account them
|
||||
from the scanner's captured directory list (present whenever a directory
|
||||
attribute is preserved, e.g. -a/-t/-p). The -d generator counts its
|
||||
explicit directory entries inline instead. */
|
||||
if (dir_entries)
|
||||
transfer_stats.flist_dir += (unsigned long long)dir_entries->size;
|
||||
report_transfer_stats(config, &transfer_stats, start, &recv_stats);
|
||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %llu files, %.1f MB",
|
||||
transfer_stats.transferred_regular,
|
||||
(double)transfer_stats.transferred_file_size / (double)BYTES_PER_MIB);
|
||||
/* A skipped source entry (--ignore-errors past an unreadable directory, or a
|
||||
dereferenced symlink with no referent) makes rsync report a partial
|
||||
transfer (exit 23) even though the rest of the run succeeded. A
|
||||
|
||||
+27
-2
@@ -223,17 +223,25 @@ bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, ui
|
||||
if (fd < 0)
|
||||
return false;
|
||||
|
||||
if (algo == CHECKSUM_ALGO_NONE) {
|
||||
/* No checksum requested: nothing to read; an empty digest succeeds. */
|
||||
close(fd);
|
||||
*out_len = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
uint8_t buffer[64 * 1024];
|
||||
bool ok = false;
|
||||
|
||||
if (algo == CHECKSUM_ALGO_MD5) {
|
||||
if (algo == CHECKSUM_ALGO_MD5 || algo == CHECKSUM_ALGO_SHA1) {
|
||||
const EVP_MD* md = algo == CHECKSUM_ALGO_MD5 ? EVP_md5() : EVP_sha1();
|
||||
EVP_MD_CTX* ctx = EVP_MD_CTX_new();
|
||||
if (!ctx) {
|
||||
close(fd);
|
||||
return false;
|
||||
}
|
||||
unsigned int digest_len = 0;
|
||||
if (EVP_DigestInit_ex(ctx, EVP_md5(), NULL) == 1) {
|
||||
if (EVP_DigestInit_ex(ctx, md, NULL) == 1) {
|
||||
ok = true;
|
||||
ssize_t got;
|
||||
while ((got = read(fd, buffer, sizeof(buffer))) > 0) {
|
||||
@@ -254,6 +262,23 @@ bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, ui
|
||||
return ok;
|
||||
}
|
||||
|
||||
if (algo == CHECKSUM_ALGO_MD4) {
|
||||
Md4Ctx ctx;
|
||||
md4_init(&ctx);
|
||||
ok = true;
|
||||
ssize_t got;
|
||||
while ((got = read(fd, buffer, sizeof(buffer))) > 0)
|
||||
md4_update(&ctx, buffer, (size_t)got);
|
||||
if (got < 0)
|
||||
ok = false;
|
||||
if (ok) {
|
||||
md4_final(&ctx, out);
|
||||
*out_len = 16;
|
||||
}
|
||||
close(fd);
|
||||
return ok;
|
||||
}
|
||||
|
||||
XXH64_state_t xxh64;
|
||||
XXH3_state_t* xxh3 = NULL;
|
||||
if (algo == CHECKSUM_ALGO_XXH64) {
|
||||
|
||||
@@ -412,6 +412,11 @@ struct DeletePlanSession {
|
||||
bool dry_run;
|
||||
size_t max_delete;
|
||||
size_t deleted;
|
||||
/* Removals charged against --max-delete. In --delete-delay mode a path is
|
||||
planned (and the budget consumed) while scanning, but `deleted` advances
|
||||
only when the commit actually unlinks it, so an entry that survives the
|
||||
commit (a directory refilled mid-transfer -> ENOTEMPTY) is not reported. */
|
||||
size_t planned;
|
||||
size_t skipped;
|
||||
bool limit_hit;
|
||||
bool limit_logged;
|
||||
@@ -565,7 +570,7 @@ static bool build_plan_skips(const Config* config, const DeletePlanSession* sess
|
||||
}
|
||||
|
||||
static bool budget_available(const DeletePlanSession* session) {
|
||||
return session->deleted < session->max_delete;
|
||||
return session->planned < session->max_delete;
|
||||
}
|
||||
|
||||
static void note_skipped(DeletePlanSession* session) {
|
||||
@@ -579,7 +584,9 @@ static void log_deleted(const char* rel) {
|
||||
free(escaped);
|
||||
}
|
||||
|
||||
/* Append a snapshot path for --delete-delay. */
|
||||
/* Append a snapshot path for --delete-delay. The budget is charged here, but
|
||||
* `deleted` is not: the path counts only once apply_deferred_path truly
|
||||
* unlinks it. */
|
||||
static bool defer_add(DeletePlanSession* session, const char* rel) {
|
||||
char* copy = str_dup(rel);
|
||||
if (!copy)
|
||||
@@ -588,7 +595,7 @@ static bool defer_add(DeletePlanSession* session, const char* rel) {
|
||||
free(copy);
|
||||
return false;
|
||||
}
|
||||
session->deleted++;
|
||||
session->planned++;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -633,6 +640,7 @@ static bool process_extra_dir(int dirfd, const char* name, const char* child_rel
|
||||
}
|
||||
if (unlinkat(dirfd, name, AT_REMOVEDIR) == 0) {
|
||||
session->deleted++;
|
||||
session->planned++;
|
||||
log_deleted(child_rel);
|
||||
*removed = true;
|
||||
return true;
|
||||
@@ -657,6 +665,7 @@ static bool process_extra_file(int dirfd, const char* name, const char* child_re
|
||||
}
|
||||
if (unlinkat(dirfd, name, 0) == 0) {
|
||||
session->deleted++;
|
||||
session->planned++;
|
||||
log_deleted(child_rel);
|
||||
} else if (errno != ENOENT) {
|
||||
return false;
|
||||
@@ -768,13 +777,14 @@ static bool apply_missing(DeletePlanSession* session, const Config* config) {
|
||||
return true;
|
||||
DeleteManifest manifest = {
|
||||
.keeps = NULL, .protected = NULL, .missing = session->missing, .dirs = NULL};
|
||||
size_t remaining = budget_available(session) ? session->max_delete - session->deleted : 0;
|
||||
size_t remaining = budget_available(session) ? session->max_delete - session->planned : 0;
|
||||
size_t deleted = 0;
|
||||
size_t skipped = 0;
|
||||
bool limit = false;
|
||||
bool ok = manifest_delete_missing_args_limited(config, &manifest, remaining, &deleted, &skipped,
|
||||
&limit);
|
||||
session->deleted += deleted;
|
||||
session->planned += deleted;
|
||||
session->skipped += skipped;
|
||||
if (limit)
|
||||
session->limit_hit = true;
|
||||
@@ -839,7 +849,6 @@ int delete_plan_session_receive(DeletePlanSession* session, const Config* config
|
||||
/* Apply one snapshotted --delete-delay path (post-order: children precede their
|
||||
* parent directory). */
|
||||
static bool apply_deferred_path(DeletePlanSession* session, const Config* config, const char* rel) {
|
||||
(void)session;
|
||||
char* full = path_cat(config->receive_root_directory, rel);
|
||||
if (!full)
|
||||
return false;
|
||||
@@ -863,8 +872,10 @@ static bool apply_deferred_path(DeletePlanSession* session, const Config* config
|
||||
else
|
||||
rc = unlinkat(parent_fd, leaf, 0);
|
||||
bool ok = rc == 0 || errno == ENOENT || errno == ENOTEMPTY || errno == EEXIST;
|
||||
if (rc == 0)
|
||||
if (rc == 0) {
|
||||
session->deleted++;
|
||||
log_deleted(rel);
|
||||
}
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
return ok;
|
||||
|
||||
@@ -73,4 +73,22 @@ typedef struct {
|
||||
bool format_stats_send(int fd, const ReceiverStats* stats);
|
||||
bool format_stats_receive(int fd, ReceiverStats* stats);
|
||||
|
||||
/* Sender-side file-list accounting for rsync's `--stats` block. Filled while
|
||||
* the scan/send loops walk each entry: the flist counters describe every
|
||||
* scanned source entry (transferred or skipped), while the transferred/literal
|
||||
* counters describe only the regular files the receiver actually stored. The
|
||||
* type split lets the client print rsync's `Number of files` breakdown; the
|
||||
* receiver-only counters (matched data, deleted, created) come from
|
||||
* STATUS_STATS. */
|
||||
typedef struct {
|
||||
unsigned long long flist_reg;
|
||||
unsigned long long flist_dir;
|
||||
unsigned long long flist_link;
|
||||
unsigned long long flist_special;
|
||||
unsigned long long total_file_size; /* sum of entry sizes (link target len) */
|
||||
unsigned long long transferred_regular; /* regular files actually stored */
|
||||
unsigned long long transferred_file_size; /* source size of those files */
|
||||
unsigned long long literal_data; /* literal bytes sent for them */
|
||||
} TransferStats;
|
||||
|
||||
#endif
|
||||
|
||||
@@ -42,6 +42,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
||||
context->total_files = 0;
|
||||
context->progress_bytes = 0;
|
||||
context->total_bytes = 0;
|
||||
memset(&context->stats, 0, sizeof(context->stats));
|
||||
context->sender_done = false;
|
||||
atomic_init(&context->cancelled, false);
|
||||
protocol_session_init(&context->allocation_session, -1, -1);
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
#include "config.h"
|
||||
#include "delete_plan.h"
|
||||
#include "file.h"
|
||||
#include "format.h"
|
||||
#include "protocol.h"
|
||||
#include "queue.h"
|
||||
#include "stop_condition.h"
|
||||
@@ -86,6 +87,9 @@ typedef struct {
|
||||
int total_files;
|
||||
unsigned long long progress_bytes;
|
||||
unsigned long long total_bytes;
|
||||
/* Per-type flist / transferred accounting for the rsync --stats breakdown and
|
||||
the progress `to-chk` denominator. Guarded by mutex_progress. */
|
||||
TransferStats stats;
|
||||
bool sender_done;
|
||||
atomic_bool cancelled;
|
||||
ProtocolSession allocation_session;
|
||||
|
||||
Reference in New Issue
Block a user