feat(stats): populate receiver wire counters on both receive paths
The single-threaded and -m receivers never populated ReceiverStats.matched_data or .deleted_files, so --stats always printed 0 for both even when rsync reported nonzero. Track the bytes reconstructed from the basis file while applying a delta, and tally the delete-commit counts (manifest and per-directory sessions) into the receiver stats. The -m pipeline now carries its own stats/would-delete fields and emits the STATUS_STATS frame before the terminal success, so --threads finally reports the counters and renders -n --delete lines. Also normalize the -n --delete would-delete enumeration's absolute basis prefixes exactly like the real commit path (fixing an over-report) and fix the basis_delete_relative off-by-one when the receive root is '/'. Unit tests cover the root mapping and the basis protection; integration tests cover matched/deleted stats for both receivers and the --threads dry-run delete lines.
This commit is contained in:
+22
-4
@@ -83,6 +83,13 @@ bool receiver_send_stats_frame(int fd, const Config* config, const ReceiverStats
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Add a delete commit's tally to the sink's end-of-transfer wire counters (when
|
||||||
|
the sink reports them). Runs on the receiving thread, so no locking. */
|
||||||
|
static void receiver_tally_deleted(const ReceiverSink* sink, size_t deleted) {
|
||||||
|
if (sink && sink->stats && deleted > 0)
|
||||||
|
sink->stats->deleted_files += deleted;
|
||||||
|
}
|
||||||
|
|
||||||
static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) {
|
static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) {
|
||||||
if (!chunk || !sink || !sink->store_file)
|
if (!chunk || !sink || !sink->store_file)
|
||||||
return false;
|
return false;
|
||||||
@@ -390,9 +397,12 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
A later transfer failure does not restore these deletions. A
|
A later transfer failure does not restore these deletions. A
|
||||||
--max-delete-capped commit still succeeds and the transfer proceeds;
|
--max-delete-capped commit still succeeds and the transfer proceeds;
|
||||||
the terminal success frame reports the cap. */
|
the terminal success frame reports the cap. */
|
||||||
DeleteCommitResult deletion = (config->use_delete || config->delete_missing_args)
|
size_t deleted = 0;
|
||||||
? manifest_delete_all(config, manifest)
|
DeleteCommitResult deletion =
|
||||||
: DELETE_COMMIT_OK;
|
(config->use_delete || config->delete_missing_args)
|
||||||
|
? manifest_delete_all_counted(config, manifest, &deleted)
|
||||||
|
: DELETE_COMMIT_OK;
|
||||||
|
receiver_tally_deleted(sink, deleted);
|
||||||
delete_manifest_free(manifest);
|
delete_manifest_free(manifest);
|
||||||
if (deletion == DELETE_COMMIT_ERROR) {
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
send_status(file_descriptor, STATUS_ERROR);
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
@@ -469,7 +479,10 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
*pending_manifest = deferred_manifest;
|
*pending_manifest = deferred_manifest;
|
||||||
deferred_manifest = NULL;
|
deferred_manifest = NULL;
|
||||||
} else {
|
} else {
|
||||||
DeleteCommitResult deletion = manifest_delete_all(config, deferred_manifest);
|
size_t deleted = 0;
|
||||||
|
DeleteCommitResult deletion =
|
||||||
|
manifest_delete_all_counted(config, deferred_manifest, &deleted);
|
||||||
|
receiver_tally_deleted(sink, deleted);
|
||||||
delete_manifest_free(deferred_manifest);
|
delete_manifest_free(deferred_manifest);
|
||||||
deferred_manifest = NULL;
|
deferred_manifest = NULL;
|
||||||
if (deletion == DELETE_COMMIT_ERROR) {
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
@@ -496,6 +509,7 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
} else {
|
} else {
|
||||||
DeleteCommitResult deletion = delete_plan_session_commit(plan_session, config);
|
DeleteCommitResult deletion = delete_plan_session_commit(plan_session, config);
|
||||||
bool limit = delete_plan_session_limit_reached(plan_session);
|
bool limit = delete_plan_session_limit_reached(plan_session);
|
||||||
|
receiver_tally_deleted(sink, delete_plan_session_deleted(plan_session));
|
||||||
delete_plan_session_destroy(plan_session);
|
delete_plan_session_destroy(plan_session);
|
||||||
plan_session = NULL;
|
plan_session = NULL;
|
||||||
if (deletion == DELETE_COMMIT_ERROR) {
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
@@ -572,6 +586,10 @@ static bool receiver_save_file(File* file, void* context_pointer) {
|
|||||||
} else {
|
} else {
|
||||||
result = file_save_to_disk_full(context->config->receive_root_directory, file, context->config);
|
result = file_save_to_disk_full(context->config->receive_root_directory, file, context->config);
|
||||||
}
|
}
|
||||||
|
/* Wire-stats tally: bytes reconstructed from the basis file (delta matches)
|
||||||
|
count as matched data in the end-of-transfer report. */
|
||||||
|
if (result != FILE_SAVE_ERROR && file->matched_bytes > 0)
|
||||||
|
context->stats.matched_data += file->matched_bytes;
|
||||||
/* A directory's metadata is deferred, never applied inline: collect it now
|
/* A directory's metadata is deferred, never applied inline: collect it now
|
||||||
and apply it at the end. -O/--omit-dir-times and --preserve_perms/-times
|
and apply it at the end. -O/--omit-dir-times and --preserve_perms/-times
|
||||||
are honored by dir_metadata_list_apply's caller (see
|
are honored by dir_metadata_list_apply's caller (see
|
||||||
|
|||||||
@@ -29,6 +29,8 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
|
|||||||
context->deferred_manifest = NULL;
|
context->deferred_manifest = NULL;
|
||||||
context->deferred_plans = NULL;
|
context->deferred_plans = NULL;
|
||||||
context->delete_limit_reached = false;
|
context->delete_limit_reached = false;
|
||||||
|
memset(&context->stats, 0, sizeof(context->stats));
|
||||||
|
context->would_delete = NULL;
|
||||||
atomic_init(&context->cancelled, false);
|
atomic_init(&context->cancelled, false);
|
||||||
int init = 0;
|
int init = 0;
|
||||||
if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
|
if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
|
||||||
@@ -41,6 +43,9 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
|
|||||||
goto fail;
|
goto fail;
|
||||||
// cppcheck-suppress unreadVariable
|
// cppcheck-suppress unreadVariable
|
||||||
init++;
|
init++;
|
||||||
|
context->would_delete = array_list_create(free);
|
||||||
|
if (!context->would_delete)
|
||||||
|
goto fail;
|
||||||
return context;
|
return context;
|
||||||
|
|
||||||
fail:
|
fail:
|
||||||
@@ -64,6 +69,8 @@ void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
|
|||||||
queue_destroy(context->queue);
|
queue_destroy(context->queue);
|
||||||
receiver_outcomes_destroy(&context->outcomes);
|
receiver_outcomes_destroy(&context->outcomes);
|
||||||
dir_time_list_free(&context->dir_times);
|
dir_time_list_free(&context->dir_times);
|
||||||
|
if (context->would_delete)
|
||||||
|
array_list_delete(context->would_delete);
|
||||||
mtx_destroy(&context->mutex);
|
mtx_destroy(&context->mutex);
|
||||||
cnd_destroy(&context->condition_not_full);
|
cnd_destroy(&context->condition_not_full);
|
||||||
cnd_destroy(&context->condition_not_empty);
|
cnd_destroy(&context->condition_not_empty);
|
||||||
@@ -136,6 +143,11 @@ bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, Fi
|
|||||||
|
|
||||||
static bool receiver_enqueue_file(File* file, void* context_pointer) {
|
static bool receiver_enqueue_file(File* file, void* context_pointer) {
|
||||||
PipelineContextReceiver* context = (PipelineContextReceiver*)context_pointer;
|
PipelineContextReceiver* context = (PipelineContextReceiver*)context_pointer;
|
||||||
|
if (file && file->matched_bytes > 0) {
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
context->stats.matched_data += file->matched_bytes;
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
}
|
||||||
return pipeline_context_receiver_enqueue_file(context, file);
|
return pipeline_context_receiver_enqueue_file(context, file);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -171,8 +183,8 @@ int receive_thread(void* pipeline_context) {
|
|||||||
false,
|
false,
|
||||||
NULL,
|
NULL,
|
||||||
receiver_pipeline_note_delete_limit,
|
receiver_pipeline_note_delete_limit,
|
||||||
NULL,
|
&context->stats,
|
||||||
NULL};
|
context->would_delete};
|
||||||
if (receiver_process_pending((Config*)config, file_descriptor, &sink, &context->deferred_manifest,
|
if (receiver_process_pending((Config*)config, file_descriptor, &sink, &context->deferred_manifest,
|
||||||
&context->deferred_plans) != 0) {
|
&context->deferred_plans) != 0) {
|
||||||
receiver_thread_fail(context);
|
receiver_thread_fail(context);
|
||||||
|
|||||||
@@ -54,6 +54,13 @@ typedef struct PipelineContextReceiver {
|
|||||||
directory entries. Only write_thread mutates it (before it joins); the
|
directory entries. Only write_thread mutates it (before it joins); the
|
||||||
caller (server.c) applies it after the delete/delay-updates phase. */
|
caller (server.c) applies it after the delete/delay-updates phase. */
|
||||||
DirTimeList dir_times;
|
DirTimeList dir_times;
|
||||||
|
/* End-of-transfer wire counters (protocol 2.25.0). receive_thread accumulates
|
||||||
|
matched_data under `mutex`; server.c adds the delete-commit tallies after
|
||||||
|
both threads join and emits the STATUS_STATS frame. */
|
||||||
|
ReceiverStats stats;
|
||||||
|
/* -n/--dry-run --delete would-delete path list, collected by receive_thread
|
||||||
|
and reported in the STATUS_STATS frame. */
|
||||||
|
struct ArrayList* would_delete;
|
||||||
} PipelineContextReceiver;
|
} PipelineContextReceiver;
|
||||||
|
|
||||||
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue_receiver,
|
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue_receiver,
|
||||||
|
|||||||
+10
-2
@@ -955,7 +955,10 @@ void handler(int file_descriptor) {
|
|||||||
--delay-updates run; the walker skips the staging directory. A
|
--delay-updates run; the walker skips the staging directory. A
|
||||||
server-contacting --dry-run deletes nothing (no manifest is sent). */
|
server-contacting --dry-run deletes nothing (no manifest is sent). */
|
||||||
if (context->deferred_manifest) {
|
if (context->deferred_manifest) {
|
||||||
DeleteCommitResult deletion = manifest_delete_all(config, context->deferred_manifest);
|
size_t deleted = 0;
|
||||||
|
DeleteCommitResult deletion =
|
||||||
|
manifest_delete_all_counted(config, context->deferred_manifest, &deleted);
|
||||||
|
context->stats.deleted_files += deleted;
|
||||||
if (deletion == DELETE_COMMIT_ERROR) {
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
transfer_ok = false;
|
transfer_ok = false;
|
||||||
} else if (deletion == DELETE_COMMIT_LIMIT_REACHED) {
|
} else if (deletion == DELETE_COMMIT_LIMIT_REACHED) {
|
||||||
@@ -975,6 +978,7 @@ void handler(int file_descriptor) {
|
|||||||
DeleteCommitResult deletion = config->dry_run ? DELETE_COMMIT_OK
|
DeleteCommitResult deletion = config->dry_run ? DELETE_COMMIT_OK
|
||||||
: delete_plan_session_commit(
|
: delete_plan_session_commit(
|
||||||
context->deferred_plans, config);
|
context->deferred_plans, config);
|
||||||
|
context->stats.deleted_files += delete_plan_session_deleted(context->deferred_plans);
|
||||||
if (deletion == DELETE_COMMIT_ERROR) {
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
transfer_ok = false;
|
transfer_ok = false;
|
||||||
} else if (deletion == DELETE_COMMIT_LIMIT_REACHED) {
|
} else if (deletion == DELETE_COMMIT_LIMIT_REACHED) {
|
||||||
@@ -1003,7 +1007,11 @@ void handler(int file_descriptor) {
|
|||||||
}
|
}
|
||||||
if (transfer_ok) {
|
if (transfer_ok) {
|
||||||
Status final_status = context->delete_limit_reached ? STATUS_DELETE_LIMIT : STATUS_OK;
|
Status final_status = context->delete_limit_reached ? STATUS_DELETE_LIMIT : STATUS_OK;
|
||||||
if (!receiver_send_final_success(file_descriptor, config, &context->outcomes, final_status))
|
/* Emit the optional wire-stats record first (protocol 2.25.0), then the
|
||||||
|
success/outcome frame, exactly like the single-threaded receiver. */
|
||||||
|
if (!receiver_send_stats_frame(file_descriptor, config, &context->stats,
|
||||||
|
context->would_delete) ||
|
||||||
|
!receiver_send_final_success(file_descriptor, config, &context->outcomes, final_status))
|
||||||
transfer_ok = false;
|
transfer_ok = false;
|
||||||
} else {
|
} else {
|
||||||
send_error_detail(file_descriptor, "transfer failed on receiver");
|
send_error_detail(file_descriptor, "transfer failed on receiver");
|
||||||
|
|||||||
@@ -444,6 +444,10 @@ bool delete_plan_session_limit_reached(const DeletePlanSession* session) {
|
|||||||
return session && session->limit_hit;
|
return session && session->limit_hit;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t delete_plan_session_deleted(const DeletePlanSession* session) {
|
||||||
|
return session ? session->deleted : 0;
|
||||||
|
}
|
||||||
|
|
||||||
/* True for a destination-relative path section entry (non-empty, relative,
|
/* True for a destination-relative path section entry (non-empty, relative,
|
||||||
* traversal-free). */
|
* traversal-free). */
|
||||||
static bool valid_rel_path(const char* value) {
|
static bool valid_rel_path(const char* value) {
|
||||||
|
|||||||
@@ -70,5 +70,8 @@ int delete_plan_session_receive(DeletePlanSession* session, const Config* config
|
|||||||
DeleteCommitResult delete_plan_session_commit(DeletePlanSession* session, const Config* config);
|
DeleteCommitResult delete_plan_session_commit(DeletePlanSession* session, const Config* config);
|
||||||
/* True once the shared --max-delete budget stopped part of a deletion. */
|
/* True once the shared --max-delete budget stopped part of a deletion. */
|
||||||
bool delete_plan_session_limit_reached(const DeletePlanSession* session);
|
bool delete_plan_session_limit_reached(const DeletePlanSession* session);
|
||||||
|
/* Number of destination entries the session's plans removed (or, for
|
||||||
|
--delete-delay, snapshotted for removal), for the end-of-transfer stats. */
|
||||||
|
size_t delete_plan_session_deleted(const DeletePlanSession* session);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -1093,6 +1093,13 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
|||||||
*failed = true;
|
*failed = true;
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
/* Wire-stats tally: bytes taken straight from the basis file (matched
|
||||||
|
delta blocks). Computed before the delta is destroyed. */
|
||||||
|
unsigned long long matched = 0;
|
||||||
|
for (uint32_t k = 0; k < delta->instruction_count; k++) {
|
||||||
|
if (delta->instructions[k].type == DELTA_INSTR_BLOCK_MATCH)
|
||||||
|
matched += delta->instructions[k].match.length;
|
||||||
|
}
|
||||||
void* new_data = delta_apply(old_data, old_size, delta, config->delta_block_size);
|
void* new_data = delta_apply(old_data, old_size, delta, config->delta_block_size);
|
||||||
delta_destroy(delta);
|
delta_destroy(delta);
|
||||||
|
|
||||||
@@ -1111,6 +1118,7 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
|||||||
*failed = true;
|
*failed = true;
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
file->matched_bytes = matched;
|
||||||
|
|
||||||
if (config->use_metadata) {
|
if (config->use_metadata) {
|
||||||
int meta_ok = 1;
|
int meta_ok = 1;
|
||||||
@@ -3149,8 +3157,9 @@ typedef struct {
|
|||||||
compares paths relative to the receive root, so a relative entry is already
|
compares paths relative to the receive root, so a relative entry is already
|
||||||
in the right form; an absolute entry that lies below the root is converted to
|
in the right form; an absolute entry that lies below the root is converted to
|
||||||
its root-relative form, and one outside the root returns NULL (the walk
|
its root-relative form, and one outside the root returns NULL (the walk
|
||||||
cannot reach it, and it is not protected data beneath the root). */
|
cannot reach it, and it is not protected data beneath the root). Exposed so
|
||||||
static char* basis_delete_relative(const Config* config, const char* path) {
|
tests can exercise the root-of-"/" child mapping directly. */
|
||||||
|
char* file_receive_basis_delete_relative(const Config* config, const char* path) {
|
||||||
if (!path)
|
if (!path)
|
||||||
return NULL;
|
return NULL;
|
||||||
if (path[0] != '/')
|
if (path[0] != '/')
|
||||||
@@ -3163,6 +3172,13 @@ static char* basis_delete_relative(const Config* config, const char* path) {
|
|||||||
root_len--;
|
root_len--;
|
||||||
if (strncmp(path, root, root_len) != 0)
|
if (strncmp(path, root, root_len) != 0)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
if (root_len == 1 && root[0] == '/') {
|
||||||
|
/* The receive root is "/": every absolute path is below it, and the child
|
||||||
|
relative form is everything after the leading '/'. */
|
||||||
|
if (path[1] == '\0')
|
||||||
|
return NULL; /* identical to the root, not a child */
|
||||||
|
return str_dup(path + 1);
|
||||||
|
}
|
||||||
if (path[root_len] != '/')
|
if (path[root_len] != '/')
|
||||||
return NULL; /* identical or a sibling sharing a name prefix */
|
return NULL; /* identical or a sibling sharing a name prefix */
|
||||||
return str_dup(path + root_len + 1);
|
return str_dup(path + root_len + 1);
|
||||||
@@ -3217,7 +3233,7 @@ static bool delete_extras_budgeted(const Config* config, DeleteManifest* manifes
|
|||||||
for (int i = 0; i < config->basis_count; i++) {
|
for (int i = 0; i < config->basis_count; i++) {
|
||||||
/* An absolute basis outside the receive root is unreachable by this walk,
|
/* An absolute basis outside the receive root is unreachable by this walk,
|
||||||
so it contributes no protection prefix (and no slot). */
|
so it contributes no protection prefix (and no slot). */
|
||||||
char* prefix = basis_delete_relative(config, config->basis_dirs[i].path);
|
char* prefix = file_receive_basis_delete_relative(config, config->basis_dirs[i].path);
|
||||||
if (!prefix)
|
if (!prefix)
|
||||||
continue;
|
continue;
|
||||||
owned_prefixes[i] = prefix;
|
owned_prefixes[i] = prefix;
|
||||||
@@ -3304,7 +3320,7 @@ static bool delete_missing_args_budgeted(const Config* config, DeleteManifest* m
|
|||||||
idx++;
|
idx++;
|
||||||
}
|
}
|
||||||
for (int i = 0; i < config->basis_count; i++) {
|
for (int i = 0; i < config->basis_count; i++) {
|
||||||
char* prefix = basis_delete_relative(config, config->basis_dirs[i].path);
|
char* prefix = file_receive_basis_delete_relative(config, config->basis_dirs[i].path);
|
||||||
if (!prefix)
|
if (!prefix)
|
||||||
continue;
|
continue;
|
||||||
owned_prefixes[i] = prefix;
|
owned_prefixes[i] = prefix;
|
||||||
@@ -3469,10 +3485,16 @@ bool manifest_would_delete_list(const Config* config, DeleteManifest* manifest,
|
|||||||
int skip_count = (config->delay_updates ? 1 : 0) + config->basis_count +
|
int skip_count = (config->delay_updates ? 1 : 0) + config->basis_count +
|
||||||
(manifest->protected ? manifest->protected->size : 0);
|
(manifest->protected ? manifest->protected->size : 0);
|
||||||
DeleteSkipEntry* skips = NULL;
|
DeleteSkipEntry* skips = NULL;
|
||||||
|
char** owned_prefixes = NULL;
|
||||||
|
int used = 0;
|
||||||
if (skip_count > 0) {
|
if (skip_count > 0) {
|
||||||
skips = calloc((size_t)skip_count, sizeof(DeleteSkipEntry));
|
skips = calloc((size_t)skip_count, sizeof(DeleteSkipEntry));
|
||||||
if (!skips)
|
owned_prefixes = calloc((size_t)config->basis_count, sizeof(char*));
|
||||||
|
if (!skips || (config->basis_count > 0 && !owned_prefixes)) {
|
||||||
|
free(skips);
|
||||||
|
free(owned_prefixes);
|
||||||
return false;
|
return false;
|
||||||
|
}
|
||||||
int idx = 0;
|
int idx = 0;
|
||||||
if (config->delay_updates) {
|
if (config->delay_updates) {
|
||||||
skips[idx].prefix = DELAY_UPDATES_STAGING_DIR;
|
skips[idx].prefix = DELAY_UPDATES_STAGING_DIR;
|
||||||
@@ -3480,7 +3502,14 @@ bool manifest_would_delete_list(const Config* config, DeleteManifest* manifest,
|
|||||||
idx++;
|
idx++;
|
||||||
}
|
}
|
||||||
for (int i = 0; i < config->basis_count; i++) {
|
for (int i = 0; i < config->basis_count; i++) {
|
||||||
skips[idx].prefix = config->basis_dirs[i].path;
|
/* Normalize exactly like the real commit path: a relative entry is
|
||||||
|
already root-relative, an absolute one inside the receive root is
|
||||||
|
converted, and one outside contributes no protection prefix. */
|
||||||
|
char* prefix = file_receive_basis_delete_relative(config, config->basis_dirs[i].path);
|
||||||
|
if (!prefix)
|
||||||
|
continue;
|
||||||
|
owned_prefixes[i] = prefix;
|
||||||
|
skips[idx].prefix = prefix;
|
||||||
skips[idx].top_level_only = false;
|
skips[idx].top_level_only = false;
|
||||||
idx++;
|
idx++;
|
||||||
}
|
}
|
||||||
@@ -3489,9 +3518,15 @@ bool manifest_would_delete_list(const Config* config, DeleteManifest* manifest,
|
|||||||
skips[idx].top_level_only = false;
|
skips[idx].top_level_only = false;
|
||||||
idx++;
|
idx++;
|
||||||
}
|
}
|
||||||
|
used = idx;
|
||||||
}
|
}
|
||||||
bool ok = delete_extras_list(config->receive_root_directory, manifest->keeps, manifest->dirs,
|
bool ok = delete_extras_list(config->receive_root_directory, manifest->keeps, manifest->dirs,
|
||||||
skips, skip_count, out, count_out);
|
skips, used, out, count_out);
|
||||||
|
if (owned_prefixes) {
|
||||||
|
for (int i = 0; i < config->basis_count; i++)
|
||||||
|
free(owned_prefixes[i]);
|
||||||
|
}
|
||||||
|
free(owned_prefixes);
|
||||||
free(skips);
|
free(skips);
|
||||||
return ok;
|
return ok;
|
||||||
}
|
}
|
||||||
@@ -3531,6 +3566,13 @@ bool manifest_delete_missing_args_limited(const Config* config, DeleteManifest*
|
|||||||
Both draw from one --max-delete budget; the result reports a cap-stopped
|
Both draw from one --max-delete budget; the result reports a cap-stopped
|
||||||
(partial) commit distinctly so the client can exit 25 like rsync. */
|
(partial) commit distinctly so the client can exit 25 like rsync. */
|
||||||
DeleteCommitResult manifest_delete_all(const Config* config, DeleteManifest* manifest) {
|
DeleteCommitResult manifest_delete_all(const Config* config, DeleteManifest* manifest) {
|
||||||
|
return manifest_delete_all_counted(config, manifest, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
DeleteCommitResult manifest_delete_all_counted(const Config* config, DeleteManifest* manifest,
|
||||||
|
size_t* deleted) {
|
||||||
|
if (deleted)
|
||||||
|
*deleted = 0;
|
||||||
if (!config || !manifest)
|
if (!config || !manifest)
|
||||||
return DELETE_COMMIT_ERROR;
|
return DELETE_COMMIT_ERROR;
|
||||||
/* Central no-mutation guard: a dry-run never deletes. No manifest is sent on
|
/* Central no-mutation guard: a dry-run never deletes. No manifest is sent on
|
||||||
@@ -3551,6 +3593,8 @@ DeleteCommitResult manifest_delete_all(const Config* config, DeleteManifest* man
|
|||||||
return DELETE_COMMIT_ERROR;
|
return DELETE_COMMIT_ERROR;
|
||||||
if (config->use_delete && !delete_extras_budgeted(config, manifest, &budget))
|
if (config->use_delete && !delete_extras_budgeted(config, manifest, &budget))
|
||||||
return DELETE_COMMIT_ERROR;
|
return DELETE_COMMIT_ERROR;
|
||||||
|
if (deleted)
|
||||||
|
*deleted = budget.deleted;
|
||||||
if (budget.limit_hit) {
|
if (budget.limit_hit) {
|
||||||
if (user_limited) {
|
if (user_limited) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Deletions stopped due to --max-delete limit (%zu skipped)",
|
log_message(LOG_LEVEL_ERROR, "Deletions stopped due to --max-delete limit (%zu skipped)",
|
||||||
|
|||||||
@@ -142,6 +142,10 @@ typedef enum {
|
|||||||
do or everything committed, DELETE_COMMIT_LIMIT_REACHED when the budget
|
do or everything committed, DELETE_COMMIT_LIMIT_REACHED when the budget
|
||||||
stopped part of the work, or DELETE_COMMIT_ERROR on a genuine failure. */
|
stopped part of the work, or DELETE_COMMIT_ERROR on a genuine failure. */
|
||||||
DeleteCommitResult manifest_delete_all(const Config* config, DeleteManifest* manifest);
|
DeleteCommitResult manifest_delete_all(const Config* config, DeleteManifest* manifest);
|
||||||
|
/* Like manifest_delete_all, but reports how many destination entries the commit
|
||||||
|
removed (for the end-of-transfer wire stats). `deleted` may be NULL. */
|
||||||
|
DeleteCommitResult manifest_delete_all_counted(const Config* config, DeleteManifest* manifest,
|
||||||
|
size_t* deleted);
|
||||||
|
|
||||||
/* -n/--dry-run --delete would-delete reporting: walk the destination exactly as
|
/* -n/--dry-run --delete would-delete reporting: walk the destination exactly as
|
||||||
the delete pass would and append (strdup'd) destination-relative paths that
|
the delete pass would and append (strdup'd) destination-relative paths that
|
||||||
@@ -150,6 +154,10 @@ DeleteCommitResult manifest_delete_all(const Config* config, DeleteManifest* man
|
|||||||
clean walk; `*count_out` receives the number of paths appended. */
|
clean walk; `*count_out` receives the number of paths appended. */
|
||||||
bool manifest_would_delete_list(const Config* config, DeleteManifest* manifest, ArrayList* out,
|
bool manifest_would_delete_list(const Config* config, DeleteManifest* manifest, ArrayList* out,
|
||||||
size_t* count_out);
|
size_t* count_out);
|
||||||
|
/* Convert one basis-directory path to the receive-root-relative protection
|
||||||
|
prefix the delete walker uses (NULL when it lies outside the root). Exposed
|
||||||
|
for unit tests of the root-of-"/" and normalization edge cases. */
|
||||||
|
char* file_receive_basis_delete_relative(const Config* config, const char* path);
|
||||||
|
|
||||||
/* Outcome of a single file_save_to_disk operation. The receiver needs to
|
/* Outcome of a single file_save_to_disk operation. The receiver needs to
|
||||||
distinguish "written" from "skipped" so --remove-source-files can be told
|
distinguish "written" from "skipped" so --remove-source-files can be told
|
||||||
|
|||||||
@@ -93,6 +93,10 @@ typedef struct {
|
|||||||
* no report was requested/received, in which case -i/--out-format treats the
|
* no report was requested/received, in which case -i/--out-format treats the
|
||||||
* entry conservatively as newly created. */
|
* entry conservatively as newly created. */
|
||||||
OutputDestState dest_state;
|
OutputDestState dest_state;
|
||||||
|
/* Receiver-only wire-stats tally: the number of bytes reconstructed from the
|
||||||
|
* basis file (matched delta blocks) for this entry. 0 when the file was sent
|
||||||
|
* whole. Accumulated into ReceiverStats.matched_data by the receiver sink. */
|
||||||
|
unsigned long long matched_bytes;
|
||||||
} File;
|
} File;
|
||||||
|
|
||||||
/* The path that should be sent on the wire and used for the receiver-side
|
/* The path that should be sent on the wire and used for the receiver-side
|
||||||
|
|||||||
@@ -80,3 +80,89 @@ class TestRelativePerDirDeleteScope:
|
|||||||
assert not os.path.exists(os.path.join(dest, "foo", "extra.txt"))
|
assert not os.path.exists(os.path.join(dest, "foo", "extra.txt"))
|
||||||
assert not os.path.exists(os.path.join(rdst, "foo", "extra.txt"))
|
assert not os.path.exists(os.path.join(rdst, "foo", "extra.txt"))
|
||||||
assert _tree(dest) == _tree(rdst)
|
assert _tree(dest) == _tree(rdst)
|
||||||
|
|
||||||
|
|
||||||
|
def _stats_value(text, label):
|
||||||
|
for line in text.splitlines():
|
||||||
|
if line.startswith(label + ":"):
|
||||||
|
return int(line.split(":", 1)[1].strip().split()[0].replace(",", ""))
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_delta_pair(tag):
|
||||||
|
"""Source file plus a same-size/basis destination file whose mtime differs,
|
||||||
|
and an extra destination file to be deleted."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"stats_{tag}_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, f"stats_{tag}_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, f"stats_{tag}_rdst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
payload = (b"0123456789abcdef" * 16384)[:200000]
|
||||||
|
_write(os.path.join(source, "f.bin"), payload)
|
||||||
|
# Destination basis: same length, one byte changed, deliberately older.
|
||||||
|
basis = bytearray(payload)
|
||||||
|
basis[100000] ^= 0xFF
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
for root in (rdst, received):
|
||||||
|
_write(os.path.join(root, "f.bin"), bytes(basis))
|
||||||
|
_write(os.path.join(root, "extra.txt"), b"delete me\n")
|
||||||
|
old = 1000000
|
||||||
|
os.utime(os.path.join(root, "f.bin"), (old, old))
|
||||||
|
return source, dest, rdst
|
||||||
|
|
||||||
|
|
||||||
|
class TestReceiverWireStats:
|
||||||
|
"""Blocker #3/#4: the receiver must populate the STATUS_STATS counters
|
||||||
|
(matched data, deleted files) on both the single-threaded and -m paths."""
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("threads", [False, True])
|
||||||
|
def test_stats_reports_matched_and_deleted(self, threads):
|
||||||
|
source, dest, rdst = _seed_delta_pair(f"mt{int(threads)}")
|
||||||
|
rsync_result = _rsync(["-a", "--stats", "--delete", "--no-whole-file", source + "/",
|
||||||
|
rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
assert _stats_value(rsync_result.stdout, "Matched data") > 0
|
||||||
|
assert _stats_value(rsync_result.stdout, "Number of deleted files") == 1
|
||||||
|
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
flags = ["-a", "--stats", "--delete", "--delta", "--incremental"]
|
||||||
|
if threads:
|
||||||
|
flags.append("--threads")
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert _stats_value(result.stdout, "Matched data") > 0, result.stdout
|
||||||
|
assert _stats_value(result.stdout, "Number of deleted files") == 1, result.stdout
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_threads_dry_run_delete_lines_match_rsync(self):
|
||||||
|
"""-n --delete --threads must emit transfer-relative `*deleting` lines."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "stats_drydel_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "stats_drydel_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "stats_drydel_rdst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
_write(os.path.join(source, "a.txt"), b"a\n")
|
||||||
|
for root in (rdst, get_dest_received_dir(dest, source)):
|
||||||
|
_write(os.path.join(root, "extra.txt"), b"x\n")
|
||||||
|
_write(os.path.join(root, "sub", "y.txt"), b"y\n")
|
||||||
|
rsync_result = _rsync(["-a", "-n", "--delete", "-i", source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
rsync_del = sorted(
|
||||||
|
line for line in rsync_result.stdout.splitlines() if line.startswith("*deleting")
|
||||||
|
)
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["-a", "-n", "--delete", "-i", "--threads"],
|
||||||
|
port=server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
fast_del = sorted(
|
||||||
|
line for line in result.stdout.splitlines() if line.startswith("*deleting")
|
||||||
|
)
|
||||||
|
assert fast_del and fast_del == rsync_del, f"rsync={rsync_del}\nfastsync={fast_del}"
|
||||||
|
|||||||
@@ -2002,6 +2002,100 @@ static void test_manifest_delete_missing_dir_budget_double_count() {
|
|||||||
rmdir(root);
|
rmdir(root);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Blocker #7: when the receive root is "/", every absolute basis path is below
|
||||||
|
it and its child relative form must drop only the single leading slash. */
|
||||||
|
static void test_basis_delete_relative_root_slash() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->receive_root_directory = str_dup("/");
|
||||||
|
|
||||||
|
char* rel = file_receive_basis_delete_relative(cfg, "/a");
|
||||||
|
EXPECT_NOT_NULL(rel);
|
||||||
|
EXPECT_EQ_STR(rel, "a");
|
||||||
|
free(rel);
|
||||||
|
rel = file_receive_basis_delete_relative(cfg, "/a/b");
|
||||||
|
EXPECT_NOT_NULL(rel);
|
||||||
|
EXPECT_EQ_STR(rel, "a/b");
|
||||||
|
free(rel);
|
||||||
|
/* The root itself is not a child. */
|
||||||
|
EXPECT_NULL(file_receive_basis_delete_relative(cfg, "/"));
|
||||||
|
/* A relative entry is already root-relative. */
|
||||||
|
rel = file_receive_basis_delete_relative(cfg, "x/y");
|
||||||
|
EXPECT_NOT_NULL(rel);
|
||||||
|
EXPECT_EQ_STR(rel, "x/y");
|
||||||
|
free(rel);
|
||||||
|
/* An absolute path outside a non-"/" root is unreachable. */
|
||||||
|
free(cfg->receive_root_directory);
|
||||||
|
cfg->receive_root_directory = str_dup("/root");
|
||||||
|
EXPECT_NULL(file_receive_basis_delete_relative(cfg, "/other/a"));
|
||||||
|
rel = file_receive_basis_delete_relative(cfg, "/root/a");
|
||||||
|
EXPECT_NOT_NULL(rel);
|
||||||
|
EXPECT_EQ_STR(rel, "a");
|
||||||
|
free(rel);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Blocker #6: -n --delete would-delete enumeration must normalize an absolute
|
||||||
|
basis directory under the receive root exactly like the real commit path, so
|
||||||
|
the basis snapshot is protected rather than reported as a deletable extra. */
|
||||||
|
static void test_manifest_would_delete_protects_absolute_basis() {
|
||||||
|
char root[PATH_MAX];
|
||||||
|
snprintf(root, sizeof(root), "/tmp/fastsync_wdbasis_%d", (int)getpid());
|
||||||
|
char* basis = path_cat(root, "basis");
|
||||||
|
char* basis_file = path_cat(basis, "snapshot.bin");
|
||||||
|
char* extra = path_cat(root, "extra.txt");
|
||||||
|
EXPECT_NOT_NULL(basis);
|
||||||
|
EXPECT_NOT_NULL(basis_file);
|
||||||
|
EXPECT_NOT_NULL(extra);
|
||||||
|
mkdir(root, 0755);
|
||||||
|
mkdir(basis, 0755);
|
||||||
|
EXPECT_TRUE(file_write_to_disk(basis_file, "x", 1, false, false));
|
||||||
|
EXPECT_TRUE(file_write_to_disk(extra, "e", 1, false, false));
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->receive_root_directory = str_dup(root);
|
||||||
|
cfg->use_delete = true;
|
||||||
|
EXPECT_EQ_INT(config_basis_append(cfg, BASIS_DEST_COMPARE, basis), 0);
|
||||||
|
|
||||||
|
const char* synced[] = {"."};
|
||||||
|
DeleteManifest manifest = {0};
|
||||||
|
manifest.keeps = make_manifest_string_list(NULL, 0);
|
||||||
|
manifest.protected = make_manifest_string_list(NULL, 0);
|
||||||
|
manifest.dirs = make_manifest_string_list(synced, 1);
|
||||||
|
EXPECT_NOT_NULL(manifest.keeps);
|
||||||
|
EXPECT_NOT_NULL(manifest.protected);
|
||||||
|
EXPECT_NOT_NULL(manifest.dirs);
|
||||||
|
ArrayList* out = array_list_create(free);
|
||||||
|
EXPECT_NOT_NULL(out);
|
||||||
|
size_t count = 0;
|
||||||
|
EXPECT_TRUE(manifest_would_delete_list(cfg, &manifest, out, &count));
|
||||||
|
bool saw_basis = false;
|
||||||
|
bool saw_extra = false;
|
||||||
|
for (int i = 0; i < out->size; i++) {
|
||||||
|
const char* p = (const char*)out->items[i];
|
||||||
|
if (strcmp(p, "basis") == 0 || strncmp(p, "basis/", 6) == 0)
|
||||||
|
saw_basis = true;
|
||||||
|
if (strcmp(p, "extra.txt") == 0)
|
||||||
|
saw_extra = true;
|
||||||
|
}
|
||||||
|
EXPECT_FALSE(saw_basis);
|
||||||
|
EXPECT_TRUE(saw_extra);
|
||||||
|
|
||||||
|
array_list_delete(out);
|
||||||
|
array_list_delete(manifest.keeps);
|
||||||
|
array_list_delete(manifest.protected);
|
||||||
|
array_list_delete(manifest.dirs);
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink(basis_file);
|
||||||
|
rmdir(basis);
|
||||||
|
unlink(extra);
|
||||||
|
rmdir(root);
|
||||||
|
free(basis);
|
||||||
|
free(basis_file);
|
||||||
|
free(extra);
|
||||||
|
}
|
||||||
|
|
||||||
void test_file() {
|
void test_file() {
|
||||||
test_file_create();
|
test_file_create();
|
||||||
test_file_special_rdev_valid();
|
test_file_special_rdev_valid();
|
||||||
@@ -2057,4 +2151,6 @@ void test_file() {
|
|||||||
test_inplace_refuses_fifo_destination();
|
test_inplace_refuses_fifo_destination();
|
||||||
test_inplace_refuses_device_destination();
|
test_inplace_refuses_device_destination();
|
||||||
test_manifest_delete_missing_dir_budget_double_count();
|
test_manifest_delete_missing_dir_budget_double_count();
|
||||||
|
test_basis_delete_relative_root_slash();
|
||||||
|
test_manifest_would_delete_protects_absolute_basis();
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user