feat(delete): per-directory delete plans for --delete-during/--delete-delay (protocol 2.24.0)

Stream one delete plan per source directory from sender to receiver instead of
a single whole-tree keep-set manifest:

- --delete-during applies each directory's extras as its plan arrives, before
  that directory's data (rsync's generator-order deletion).
- --delete-delay snapshots each directory's extras while the plan arrives and
  commits the removals only after a fully-successful transfer, so files created
  after the scan survive (matching rsync's delete-delay, not delete-after).
- Type conflicts (a destination file blocking a source directory, or vice
  versa) are cleared immediately in both modes, so the nested write succeeds.

The plan carries the destination-relative directory, its kept child directory
names and its kept child file names; the first frame also carries the global
protected prefixes, size-skipped prefixes and --delete-missing-args paths.
--delete-before keeps the existing whole-tree early manifest; plain --delete and
--delete-after keep the end-of-transfer manifest commit.

Preserves the existing safety surface: protected/size-skipped prefixes and the
--delay-updates/basis skips are honored at any depth, deletion is scoped to the
synchronized directories (--files-from), MAX_SERVER_DELETE_COUNT and
--max-delete (partial + exit 25) are shared across plans, symlinks are never
followed, and paths are confined to the receive root.
This commit is contained in:
2026-09-16 22:45:26 +02:00
parent 478f80be9f
commit 448edc0432
25 changed files with 1279 additions and 84 deletions
+7 -1
View File
@@ -228,7 +228,13 @@ Config* config_create(void) {
bool config_delete_timing_early(const Config* config) {
if (!config)
return false;
return config->delete_before || config->delete_during;
return config->delete_before;
}
bool config_delete_timing_per_dir(const Config* config) {
if (!config)
return false;
return config->delete_during || config->delete_delay;
}
/* A delete-timing flag is only meaningful together with --delete. At most one
+10 -6
View File
@@ -912,7 +912,7 @@ typedef struct Config {
* carries the new report_dest_info bool appended after the --copy-as block.
* This is both a config-frame layout change (one trailing bool) and a frame
* sequence change (the new status). */
#define PROTOCOL_VERSION "2.23.0"
#define PROTOCOL_VERSION "2.24.0"
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
#define MAX_BASIS_DIRS 64
@@ -1006,13 +1006,17 @@ int config_parse_daemon_dest(Config* config);
* 0. */
int config_parse_transport_dest(Config* config);
/* True when the negotiated delete timing performs the extra-file deletion
* BEFORE the transfer data (--delete-before / --delete-during). The flag is
* a pure function of the config and is used identically on the sender (to pick
/* True for the whole-tree delete-before timing: a complete keep-set manifest is
* transmitted before any data and committed (with an ack) before the first data
* byte. Pure function of the config, used identically on the sender (to pick
* the manifest-first frame order) and the receiver (to delete when the early
* manifest arrives). When false the deletion is committed only after the whole
* transfer succeeded (--delete / --delete-after / --delete-delay). */
* manifest arrives). */
bool config_delete_timing_early(const Config* config);
/* True for the per-directory timings (--delete-during / --delete-delay). The
* sender streams a delete plan per source directory in directory order; the
* receiver applies each plan on arrival (during) or snapshots its extras and
* commits them only after a fully-successful transfer (delay). */
bool config_delete_timing_per_dir(const Config* config);
/* Delete-timing sanity: with deletion enabled at most one timing flag may be
* set (none = the default delete-after commit timing); without deletion no
* timing flag may be set (each timing flag implies --delete). */
+838
View File
@@ -0,0 +1,838 @@
#include "delete_plan.h"
#include "charset.h"
#include "delay_updates.h"
#include "file.h"
#include "log.h"
#include "utils.h"
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
/* Mirrors MAX_SERVER_DELETE_COUNT in file_receive.c: the server's hard bound on
* the number of entries one deletion commit may remove. A client
* --max-delete=NUM smaller than this replaces it for the run. */
#define DELETE_PLAN_SERVER_LIMIT 100000U
/* Per-frame entry cap for the name sections (the dir/file child lists). */
#define DELETE_PLAN_MAX_NAMES MAX_MANIFEST_ENTRIES
/* ------------------------------------------------------------------ */
/* Sender: plan builder */
/* ------------------------------------------------------------------ */
typedef struct PlanNode {
char* dir;
ArrayList* files; /* basenames kept directly in dir */
ArrayList* dirs; /* basenames of kept child directories */
bool sent;
struct PlanNode* hash_next;
} PlanNode;
struct DeletePlanSender {
PlanNode** buckets;
size_t capacity;
size_t count;
bool config_sent;
bool all_synced;
const ArrayList* synced_dirs;
const ArrayList* protected_prefixes;
const ArrayList* size_skipped;
const ArrayList* missing_args;
size_t entries;
};
static size_t plan_hash(const char* key) {
size_t h = 5381;
for (const unsigned char* p = (const unsigned char*)key; *p; p++)
h = ((h << 5) + h) + *p;
return h;
}
static bool list_contains_str(const ArrayList* list, const char* value) {
if (!list)
return false;
for (int i = 0; i < list->size; i++) {
if (strcmp((const char*)list->items[i], value) == 0)
return true;
}
return false;
}
static bool list_add_str_unique(ArrayList* list, const char* value) {
if (!list || !value)
return false;
if (list_contains_str(list, value))
return true;
char* copy = str_dup(value);
if (!copy)
return false;
if (!array_list_add(list, copy)) {
free(copy);
return false;
}
return true;
}
DeletePlanSender* delete_plan_sender_create(void) {
DeletePlanSender* sender = calloc(1, sizeof(DeletePlanSender));
if (!sender)
return NULL;
sender->capacity = 64;
sender->buckets = calloc(sender->capacity, sizeof(PlanNode*));
if (!sender->buckets) {
free(sender);
return NULL;
}
sender->all_synced = true;
return sender;
}
static void plan_node_destroy(PlanNode* node) {
if (!node)
return;
free(node->dir);
array_list_delete(node->files);
array_list_delete(node->dirs);
free(node);
}
void delete_plan_sender_destroy(DeletePlanSender* sender) {
if (!sender)
return;
for (size_t i = 0; i < sender->capacity; i++) {
PlanNode* node = sender->buckets[i];
while (node) {
PlanNode* next = node->hash_next;
plan_node_destroy(node);
node = next;
}
}
free(sender->buckets);
free(sender);
}
static PlanNode* plan_find(const DeletePlanSender* sender, const char* dir) {
size_t index = plan_hash(dir) & (sender->capacity - 1);
for (PlanNode* node = sender->buckets[index]; node; node = node->hash_next) {
if (strcmp(node->dir, dir) == 0)
return node;
}
return NULL;
}
static bool plan_grow(DeletePlanSender* sender) {
size_t new_capacity = sender->capacity * 2;
PlanNode** buckets = calloc(new_capacity, sizeof(PlanNode*));
if (!buckets)
return false;
for (size_t i = 0; i < sender->capacity; i++) {
PlanNode* node = sender->buckets[i];
while (node) {
PlanNode* next = node->hash_next;
size_t index = plan_hash(node->dir) & (new_capacity - 1);
node->hash_next = buckets[index];
buckets[index] = node;
node = next;
}
}
free(sender->buckets);
sender->buckets = buckets;
sender->capacity = new_capacity;
return true;
}
static PlanNode* plan_ensure(DeletePlanSender* sender, const char* dir) {
PlanNode* node = plan_find(sender, dir);
if (node)
return node;
if (sender->count + 1 > sender->capacity * 3 / 4 && !plan_grow(sender))
return NULL;
node = calloc(1, sizeof(PlanNode));
if (!node)
return NULL;
node->dir = str_dup(dir);
node->files = array_list_create(free);
node->dirs = array_list_create(free);
if (!node->dir || !node->files || !node->dirs) {
plan_node_destroy(node);
return NULL;
}
size_t index = plan_hash(dir) & (sender->capacity - 1);
node->hash_next = sender->buckets[index];
sender->buckets[index] = node;
sender->count++;
return node;
}
static char* path_parent_dir(const char* path) {
const char* slash = strrchr(path, '/');
if (!slash)
return str_dup(".");
if (slash == path)
return str_dup(".");
size_t len = (size_t)(slash - path);
char* parent = malloc(len + 1);
if (!parent)
return NULL;
memcpy(parent, path, len);
parent[len] = '\0';
return parent;
}
static char* path_base_name(const char* path) {
const char* slash = strrchr(path, '/');
return str_dup(slash ? slash + 1 : path);
}
/* Copy `path`, stripping a leading '/' and any trailing '/'. */
static char* plan_clean_path(const char* path) {
while (*path == '/')
path++;
size_t len = strlen(path);
while (len > 0 && path[len - 1] == '/')
len--;
char* clean = malloc(len + 1);
if (!clean)
return NULL;
memcpy(clean, path, len);
clean[len] = '\0';
return clean;
}
static bool plan_ensure_ancestors(DeletePlanSender* sender, const char* dir) {
char* current = str_dup(dir);
if (!current)
return false;
bool ok = true;
while (strcmp(current, ".") != 0) {
char* parent = path_parent_dir(current);
char* base = path_base_name(current);
PlanNode* parent_node = parent ? plan_ensure(sender, parent) : NULL;
if (!parent || !base || !parent_node || !list_add_str_unique(parent_node->dirs, base)) {
ok = false;
free(parent);
free(base);
break;
}
free(base);
free(current);
current = parent;
}
free(current);
return ok;
}
bool delete_plan_sender_add(DeletePlanSender* sender, const char* path, bool is_dir) {
if (!sender || !path)
return false;
char* clean = plan_clean_path(path);
if (!clean)
return false;
if (*clean == '\0') {
free(clean);
return true;
}
char* parent = path_parent_dir(clean);
char* base = path_base_name(clean);
PlanNode* parent_node = parent ? plan_ensure(sender, parent) : NULL;
bool ok = parent && base && parent_node;
if (ok) {
if (is_dir) {
ok = list_add_str_unique(parent_node->dirs, base) && plan_ensure(sender, clean) != NULL;
} else {
ok = list_add_str_unique(parent_node->files, base);
}
}
if (ok)
ok = plan_ensure_ancestors(sender, parent);
if (ok)
sender->entries++;
free(clean);
free(parent);
free(base);
return ok;
}
void delete_plan_sender_finalize(DeletePlanSender* sender, const ArrayList* synced_dirs) {
if (!sender)
return;
sender->synced_dirs = synced_dirs;
sender->all_synced = synced_dirs == NULL;
}
bool delete_plan_sender_empty(const DeletePlanSender* sender) {
return !sender || sender->entries == 0;
}
void delete_plan_sender_set_config(DeletePlanSender* sender, const ArrayList* protected_prefixes,
const ArrayList* size_skipped, const ArrayList* missing_args) {
if (!sender)
return;
sender->protected_prefixes = protected_prefixes;
sender->size_skipped = size_skipped;
sender->missing_args = missing_args;
}
static bool plan_is_allowed(const DeletePlanSender* sender, const char* dir) {
if (sender->all_synced)
return true;
return list_contains_str(sender->synced_dirs, dir);
}
static int send_str_section(int fd, const ArrayList* list) {
int count = list ? list->size : 0;
if (!send_int(fd, count))
return -1;
for (int i = 0; i < count; i++) {
if (!send_wire_str(fd, (const char*)list->items[i]))
return -1;
}
return 0;
}
static int send_plan_node(int fd, DeletePlanSender* sender, PlanNode* node) {
if (!send_status(fd, STATUS_DELETE_PLAN))
return -1;
if (!send_int(fd, sender->config_sent ? 0 : 1))
return -1;
if (!sender->config_sent) {
if (send_str_section(fd, sender->protected_prefixes) != 0 ||
send_str_section(fd, sender->size_skipped) != 0 ||
send_str_section(fd, sender->missing_args) != 0)
return -1;
sender->config_sent = true;
}
if (!send_wire_str(fd, node->dir))
return -1;
if (send_str_section(fd, node->dirs) != 0 || send_str_section(fd, node->files) != 0)
return -1;
node->sent = true;
return 0;
}
static int send_prefix_plan(int fd, DeletePlanSender* sender, const char* dir) {
PlanNode* node = plan_find(sender, dir);
if (!node || node->sent)
return 0;
if (!plan_is_allowed(sender, dir))
return 0;
return send_plan_node(fd, sender, node);
}
int delete_plan_send_root(int fd, DeletePlanSender* sender) {
if (!sender)
return -1;
if (!plan_ensure(sender, "."))
return -1;
return send_prefix_plan(fd, sender, ".");
}
int delete_plan_send_for_path(int fd, DeletePlanSender* sender, const char* path, bool is_dir) {
if (!sender || !path)
return -1;
char* clean = plan_clean_path(path);
if (!clean)
return -1;
int rc = send_prefix_plan(fd, sender, ".");
if (rc == 0 && *clean != '\0') {
size_t len = strlen(clean);
size_t end = len;
if (!is_dir) {
const char* slash = strrchr(clean, '/');
end = slash ? (size_t)(slash - clean) : 0;
}
for (size_t i = 1; i <= end && rc == 0; i++) {
if (i == end || clean[i] == '/') {
char* prefix = malloc(i + 1);
if (!prefix) {
rc = -1;
break;
}
memcpy(prefix, clean, i);
prefix[i] = '\0';
rc = send_prefix_plan(fd, sender, prefix);
free(prefix);
}
}
}
free(clean);
return rc;
}
/* ------------------------------------------------------------------ */
/* Receiver: delete session */
/* ------------------------------------------------------------------ */
struct DeletePlanSession {
bool defer;
bool dry_run;
size_t max_delete;
size_t deleted;
size_t skipped;
bool limit_hit;
bool config_seen;
bool missing_applied;
ArrayList* protected_prefixes;
ArrayList* size_skipped;
ArrayList* missing;
ArrayList* deferred;
};
DeletePlanSession* delete_plan_session_create(const Config* config) {
if (!config)
return NULL;
DeletePlanSession* session = calloc(1, sizeof(DeletePlanSession));
if (!session)
return NULL;
session->defer = config->delete_delay;
session->dry_run = config->dry_run;
bool user_limited =
config->max_delete >= 0 && (size_t)config->max_delete < DELETE_PLAN_SERVER_LIMIT;
session->max_delete =
user_limited ? (size_t)config->max_delete : (size_t)DELETE_PLAN_SERVER_LIMIT;
session->protected_prefixes = array_list_create(free);
session->size_skipped = array_list_create(free);
session->missing = array_list_create(free);
session->deferred = array_list_create(free);
if (!session->protected_prefixes || !session->size_skipped || !session->missing ||
!session->deferred) {
delete_plan_session_destroy(session);
return NULL;
}
return session;
}
void delete_plan_session_destroy(DeletePlanSession* session) {
if (!session)
return;
array_list_delete(session->protected_prefixes);
array_list_delete(session->size_skipped);
array_list_delete(session->missing);
array_list_delete(session->deferred);
free(session);
}
bool delete_plan_session_limit_reached(const DeletePlanSession* session) {
return session && session->limit_hit;
}
/* True for a destination-relative path section entry (non-empty, relative,
* traversal-free). */
static bool valid_rel_path(const char* value) {
return value && value[0] != '\0' && value[0] != '/' && !has_path_traversal(value);
}
/* True for a single child name (non-empty, no slash, not "."/".."). */
static bool valid_name(const char* value) {
return value && value[0] != '\0' && strcmp(value, ".") != 0 && strcmp(value, "..") != 0 &&
strchr(value, '/') == NULL;
}
static bool read_section(int fd, ArrayList* list, bool rel_path) {
int count;
if (!receive_int(fd, &count) || count < 0 || count > MAX_MANIFEST_ENTRIES)
return false;
size_t bytes = 0;
for (int i = 0; i < count; i++) {
char* value = receive_wire_str(fd);
bool ok = value && (rel_path ? valid_rel_path(value) : valid_name(value));
if (ok) {
size_t entry_size = strlen(value) + sizeof(char*) + 16;
if (entry_size > MAX_MANIFEST_BYTES - bytes) {
ok = false;
} else {
bytes += entry_size;
ok = array_list_add(list, value);
}
}
if (!ok) {
free(value);
return false;
}
}
return true;
}
static int open_plan_dir(const Config* config, const char* dir) {
char* full = (strcmp(dir, ".") == 0) ? str_dup(config->receive_root_directory)
: path_cat(config->receive_root_directory, dir);
if (!full)
return -1;
int root_fd = utils_get_authorized_root_fd();
int fd = -1;
if (root_fd >= 0) {
if (utils_get_authorized_root_path())
fd = utils_open_authorized_destination(full);
else if (strcmp(dir, ".") == 0)
fd = dup(root_fd);
} else {
fd = open(full, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
}
free(full);
return fd;
}
typedef struct PlanSkips {
DeleteSkipEntry* entries;
int count;
} PlanSkips;
static bool build_plan_skips(const Config* config, const DeletePlanSession* session, PlanSkips* out) {
out->entries = NULL;
out->count = 0;
int count = (config->delay_updates ? 1 : 0) + config->basis_count +
session->protected_prefixes->size + session->size_skipped->size;
if (count == 0)
return true;
out->entries = calloc((size_t)count, sizeof(DeleteSkipEntry));
if (!out->entries)
return false;
int idx = 0;
if (config->delay_updates) {
out->entries[idx].prefix = DELAY_UPDATES_STAGING_DIR;
out->entries[idx].top_level_only = true;
idx++;
}
for (int i = 0; i < config->basis_count; i++) {
out->entries[idx].prefix = config->basis_dirs[i].path;
out->entries[idx].top_level_only = false;
idx++;
}
for (int i = 0; i < session->protected_prefixes->size; i++) {
out->entries[idx].prefix = (const char*)session->protected_prefixes->items[i];
out->entries[idx].top_level_only = false;
idx++;
}
for (int i = 0; i < session->size_skipped->size; i++) {
out->entries[idx].prefix = (const char*)session->size_skipped->items[i];
out->entries[idx].top_level_only = false;
idx++;
}
out->count = idx;
return true;
}
static bool budget_available(const DeletePlanSession* session) {
return session->deleted < session->max_delete;
}
static void note_skipped(DeletePlanSession* session) {
session->limit_hit = true;
session->skipped++;
}
static void log_deleted(const char* rel) {
char* escaped = output_escape(rel, log_get_8_bit_output());
fprintf(stderr, " Deleted: %s\n", escaped ? escaped : "<allocation failed>");
free(escaped);
}
/* Append a snapshot path for --delete-delay. */
static bool defer_add(DeletePlanSession* session, const char* rel) {
char* copy = str_dup(rel);
if (!copy)
return false;
if (!array_list_add(session->deferred, copy)) {
free(copy);
return false;
}
session->deleted++;
return true;
}
/* Process the direct children of one directory. `keep_dirs`/`keep_files`
* (basenames) are the source entries that must be kept; NULL means every child
* is an extra (the forced path used inside a removed extra directory tree).
* `survives` reports that at least one child remains (kept, protected, or
* skipped by the budget). `force_now` removes even in --delete-delay mode
* (type conflicts must clear before the incoming data). */
static bool process_children(int dirfd, const char* dir_rel, const ArrayList* keep_dirs,
const ArrayList* keep_files, bool at_root, bool force_now,
const PlanSkips* skips, DeletePlanSession* session, bool* survives);
static bool process_extra_dir(int dirfd, const char* name, const char* child_rel, bool force_now,
const PlanSkips* skips, DeletePlanSession* session, bool* removed) {
*removed = false;
int childfd = openat(dirfd, name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (childfd < 0) {
if (errno == ENOENT) {
*removed = true;
return true;
}
return false;
}
bool survives = false;
bool ok = process_children(childfd, child_rel, NULL, NULL, false, force_now, skips, session,
&survives);
close(childfd);
if (!ok)
return false;
if (survives)
return true;
if (!budget_available(session)) {
note_skipped(session);
return true;
}
if (session->defer && !force_now) {
if (!defer_add(session, child_rel))
return false;
*removed = true;
return true;
}
if (unlinkat(dirfd, name, AT_REMOVEDIR) == 0) {
session->deleted++;
log_deleted(child_rel);
*removed = true;
return true;
}
if (errno == ENOENT) {
*removed = true;
return true;
}
/* ENOTEMPTY/EEXIST: a protected entry the walker leaves behind survived, so
the directory stays; any other errno is a genuine failure. */
return errno == ENOTEMPTY || errno == EEXIST;
}
static bool process_extra_file(int dirfd, const char* name, const char* child_rel, bool force_now,
DeletePlanSession* session) {
if (!budget_available(session)) {
note_skipped(session);
return true;
}
if (session->defer && !force_now) {
return defer_add(session, child_rel);
}
if (unlinkat(dirfd, name, 0) == 0) {
session->deleted++;
log_deleted(child_rel);
} else if (errno != ENOENT) {
return false;
}
return true;
}
static bool process_children(int dirfd, const char* dir_rel, const ArrayList* keep_dirs,
const ArrayList* keep_files, bool at_root, bool force_now,
const PlanSkips* skips, DeletePlanSession* session, bool* survives) {
*survives = false;
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (scanfd < 0)
return false;
DIR* dir = fdopendir(scanfd);
if (!dir) {
close(scanfd);
return false;
}
bool operation_ok = true;
bool local_survives = false;
const struct dirent* entry;
while ((entry = readdir(dir)) != NULL) {
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
continue;
char* child_rel = (strcmp(dir_rel, ".") == 0) ? str_dup(entry->d_name)
: path_cat(dir_rel, entry->d_name);
if (!child_rel) {
operation_ok = false;
continue;
}
if (path_under_skip_prefix(child_rel, at_root, skips->entries, skips->count)) {
local_survives = true;
free(child_rel);
continue;
}
struct stat st;
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
if (errno != ENOENT)
operation_ok = false;
free(child_rel);
continue;
}
bool is_dir = S_ISDIR(st.st_mode);
bool in_keep_dirs = is_dir && list_contains_str(keep_dirs, entry->d_name);
bool in_keep_files = !is_dir && list_contains_str(keep_files, entry->d_name);
if (in_keep_dirs) {
local_survives = true;
} else if (keep_dirs && !is_dir && list_contains_str(keep_dirs, entry->d_name)) {
/* Destination file blocks a source directory: clear it now, whatever the
delete timing, so the directory can be created. */
if (!process_extra_file(dirfd, entry->d_name, child_rel, true, session))
operation_ok = false;
} else if (in_keep_files) {
local_survives = true;
} else if (keep_files && is_dir && list_contains_str(keep_files, entry->d_name)) {
/* Destination directory blocks a source file: remove it now. */
bool removed = false;
if (!process_extra_dir(dirfd, entry->d_name, child_rel, true, skips, session, &removed))
operation_ok = false;
else if (!removed)
local_survives = true;
} else if (is_dir) {
bool removed = false;
if (!process_extra_dir(dirfd, entry->d_name, child_rel, force_now, skips, session, &removed))
operation_ok = false;
else if (!removed)
local_survives = true;
} else {
if (!process_extra_file(dirfd, entry->d_name, child_rel, force_now, session))
operation_ok = false;
}
free(child_rel);
}
closedir(dir);
*survives = local_survives;
return operation_ok;
}
static bool apply_plan_dir(DeletePlanSession* session, const Config* config, const char* dir,
const ArrayList* dirs, const ArrayList* files) {
int dirfd = open_plan_dir(config, dir);
if (dirfd < 0) {
/* An absent destination directory has nothing to delete. */
return errno == ENOENT || errno == ENOTDIR;
}
PlanSkips skips;
if (!build_plan_skips(config, session, &skips)) {
close(dirfd);
return false;
}
bool survives = false;
bool ok = process_children(dirfd, dir, dirs, files, strcmp(dir, ".") == 0, false, &skips,
session, &survives);
free(skips.entries);
close(dirfd);
if (!ok)
log_message(LOG_LEVEL_ERROR, "deletion failed while removing extraneous files");
return ok;
}
static bool apply_missing(DeletePlanSession* session, const Config* config) {
if (session->missing_applied)
return true;
session->missing_applied = true;
if (session->missing->size == 0)
return true;
DeleteManifest manifest = {.keeps = NULL, .protected = NULL, .missing = session->missing,
.dirs = NULL};
size_t remaining = budget_available(session) ? session->max_delete - session->deleted : 0;
size_t deleted = 0;
size_t skipped = 0;
bool limit = false;
bool ok = manifest_delete_missing_args_limited(config, &manifest, remaining, &deleted, &skipped,
&limit);
session->deleted += deleted;
session->skipped += skipped;
if (limit)
session->limit_hit = true;
return ok;
}
int delete_plan_session_receive(DeletePlanSession* session, const Config* config, int fd) {
if (!session || !config) {
send_status(fd, STATUS_ERROR);
return -1;
}
int has_config;
if (!receive_int(fd, &has_config) || (has_config != 0 && has_config != 1)) {
send_status(fd, STATUS_ERROR);
return -1;
}
if (has_config) {
if (session->config_seen || !read_section(fd, session->protected_prefixes, true) ||
!read_section(fd, session->size_skipped, true) ||
!read_section(fd, session->missing, true)) {
send_status(fd, STATUS_ERROR);
return -1;
}
session->config_seen = true;
}
char* dir = receive_wire_str(fd);
ArrayList* dirs = array_list_create(free);
ArrayList* files = array_list_create(free);
bool parsed = dir && (strcmp(dir, ".") == 0 || valid_rel_path(dir)) && dirs && files &&
read_section(fd, dirs, false) && read_section(fd, files, false);
if (!parsed) {
free(dir);
array_list_delete(dirs);
array_list_delete(files);
send_status(fd, STATUS_ERROR);
return -1;
}
bool enabled = config->use_delete || config->delete_missing_args;
bool ok = true;
if (!session->dry_run && enabled) {
if (!session->defer && !apply_missing(session, config))
ok = false;
if (ok && !apply_plan_dir(session, config, dir, dirs, files))
ok = false;
}
free(dir);
array_list_delete(dirs);
array_list_delete(files);
if (!ok) {
send_status(fd, STATUS_ERROR);
return -1;
}
if (session->limit_hit)
log_message(LOG_LEVEL_WARNING, "Deletions stopped due to the delete limit (%zu skipped)",
session->skipped);
return 0;
}
/* Apply one snapshotted --delete-delay path (post-order: children precede their
* parent directory). */
static bool apply_deferred_path(DeletePlanSession* session, const Config* config, const char* rel) {
(void)session;
char* full = path_cat(config->receive_root_directory, rel);
if (!full)
return false;
char* leaf = NULL;
int parent_fd = file_open_secure_parent(full, &leaf, false);
free(full);
if (parent_fd < 0) {
free(leaf);
return errno == ENOENT || errno == ENOTDIR;
}
struct stat st;
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0) {
bool absent = errno == ENOENT;
close(parent_fd);
free(leaf);
return absent;
}
int rc;
if (S_ISDIR(st.st_mode))
rc = unlinkat(parent_fd, leaf, AT_REMOVEDIR);
else
rc = unlinkat(parent_fd, leaf, 0);
bool ok = rc == 0 || errno == ENOENT || errno == ENOTEMPTY || errno == EEXIST;
if (rc == 0)
log_deleted(rel);
close(parent_fd);
free(leaf);
return ok;
}
DeleteCommitResult delete_plan_session_commit(DeletePlanSession* session, const Config* config) {
if (!session || !config)
return DELETE_COMMIT_ERROR;
bool ok = true;
if (session->defer) {
for (int i = 0; i < session->deferred->size && ok; i++)
ok = apply_deferred_path(session, config, (const char*)session->deferred->items[i]);
}
if (ok)
ok = apply_missing(session, config);
if (!ok)
return DELETE_COMMIT_ERROR;
if (session->limit_hit)
return DELETE_COMMIT_LIMIT_REACHED;
return DELETE_COMMIT_OK;
}
+67
View File
@@ -0,0 +1,67 @@
#ifndef DELETE_PLAN_H
#define DELETE_PLAN_H
#include "array_list.h"
#include "config.h"
#include "file_receive.h"
#include "protocol.h"
#include <stdbool.h>
/* Per-directory delete plans (protocol 2.24.0).
*
* rsync's --delete-during removes a directory's extras while the generator
* processes that directory, and --delete-delay records the deletion list during
* the scan but applies it only after a fully-successful transfer. FastSync has
* no per-directory generator pass; instead the sender streams one plan per
* source directory, in directory order, and the receiver applies it when it
* arrives (during) or snapshots its extras and commits them at the end (delay).
*
* The sender side builds a plan set from the path-only pre-scan (it needs every
* directory's complete direct-child list before the first data byte of that
* directory). The receiver side is a session that carries the global protected
* prefixes (filter-excluded and size-skipped source mirrors), the
* --delete-missing-args exact deletions, the shared --max-delete budget and,
* for --delete-delay, the snapshotted extras. */
/* ---- Sender: plan builder ---- */
typedef struct DeletePlanSender DeletePlanSender;
DeletePlanSender* delete_plan_sender_create(void);
void delete_plan_sender_destroy(DeletePlanSender* sender);
/* Record one transmitted entry. `path` is the destination-relative wire path;
* is_dir marks an explicit directory entry (--dirs, a -x mount point). */
bool delete_plan_sender_add(DeletePlanSender* sender, const char* path, bool is_dir);
/* Drop plans for directories outside `synced_dirs` (the --files-from
* synchronization scope; pass NULL when a full recursive transfer synchronized
* every directory). The receive root is the "." sentinel. */
void delete_plan_sender_finalize(DeletePlanSender* sender, const ArrayList* synced_dirs);
/* True when no transmitted entry was recorded (an ambiguous empty scan). */
bool delete_plan_sender_empty(const DeletePlanSender* sender);
/* Attach the global config sections advertised on the first plan frame. */
void delete_plan_sender_set_config(DeletePlanSender* sender, const ArrayList* protected_prefixes,
const ArrayList* size_skipped, const ArrayList* missing_args);
/* Send the root plan (even before any data, so root extras are handled like
* rsync's first generator directory). Returns -1 on I/O error. */
int delete_plan_send_root(int fd, DeletePlanSender* sender);
/* Send the plans for every ancestor of `path` (root-first) and, when is_dir,
* for `path` itself; already-sent plans are skipped. */
int delete_plan_send_for_path(int fd, DeletePlanSender* sender, const char* path, bool is_dir);
/* ---- Receiver: delete session ---- */
typedef struct DeletePlanSession DeletePlanSession;
DeletePlanSession* delete_plan_session_create(const Config* config);
void delete_plan_session_destroy(DeletePlanSession* session);
/* Read one STATUS_DELETE_PLAN frame (the leading status already consumed) and
* act on it. Returns 0 on success (including a dry-run/disabled no-op) and -1
* after signalling STATUS_ERROR on a malformed frame or a deletion failure. */
int delete_plan_session_receive(DeletePlanSession* session, const Config* config, int fd);
/* Apply the deferred snapshot (--delete-delay) and the missing-args deletions.
* Safe to call once; returns the commit outcome. */
DeleteCommitResult delete_plan_session_commit(DeletePlanSession* session, const Config* config);
/* True once the shared --max-delete budget stopped part of a deletion. */
bool delete_plan_session_limit_reached(const DeletePlanSession* session);
#endif
+15
View File
@@ -3282,6 +3282,21 @@ bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest
return delete_missing_args_budgeted(config, manifest, &budget);
}
bool manifest_delete_missing_args_limited(const Config* config, DeleteManifest* manifest,
size_t max_delete, size_t* deleted, size_t* skipped,
bool* limit_hit) {
DeleteBudgetState budget = {
.max_delete = max_delete, .deleted = 0, .skipped = 0, .limit_hit = false};
bool ok = delete_missing_args_budgeted(config, manifest, &budget);
if (deleted)
*deleted = budget.deleted;
if (skipped)
*skipped = budget.skipped;
if (limit_hit)
*limit_hit = budget.limit_hit;
return ok;
}
/* Commit every deletion family the manifest carries. The --delete-missing-args
exact-path deletions run FIRST: they are explicit user requests and must not
be blocked by the extras walker's filter-exclusion protection (a protected
+8
View File
@@ -118,6 +118,14 @@ bool manifest_delete_extras(const Config* config, DeleteManifest* manifest);
confinement or I/O error (the run then fails); tolerated per-path cases are
reported and skipped. */
bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest);
/* Budgeted form of manifest_delete_missing_args for the per-directory delete
session: each removed mirror draws from `max_delete` (SIZE_MAX = unlimited)
and the tallies are accumulated into `*deleted`/`*skipped`. `*limit_hit` is set
when the budget stopped the pass with entries left over. Returns false only
on a genuine deletion error. */
bool manifest_delete_missing_args_limited(const Config* config, DeleteManifest* manifest,
size_t max_delete, size_t* deleted, size_t* skipped,
bool* limit_hit);
/* Outcome of committing a delete manifest. LIMIT_REACHED reports rsync's
partial --max-delete result: the budget allowed some deletions and the rest
were skipped (the run still stores all file data but the client exits 25). */
+3
View File
@@ -36,6 +36,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
context->scan_had_io_error = false;
context->remove_source_files = NULL;
context->early_delete = false;
context->delete_plans = NULL;
context->scan_stopped_early = false;
context->total_files = 0;
context->progress_bytes = 0;
@@ -187,6 +188,8 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) {
if (context->manifest) {
array_list_delete(context->manifest);
}
if (context->delete_plans)
delete_plan_sender_destroy(context->delete_plans);
if (context->excluded_paths)
array_list_delete(context->excluded_paths);
if (context->size_skipped_paths)
+10 -4
View File
@@ -7,6 +7,7 @@
#include "array_list.h"
#include "chunk.h"
#include "config.h"
#include "delete_plan.h"
#include "file.h"
#include "protocol.h"
#include "queue.h"
@@ -66,11 +67,16 @@ typedef struct {
--ignore-errors kept the run going. */
bool scan_had_io_error;
ArrayList* remove_source_files;
/* True when --delete-before/--delete-during require the keep-set manifest to
be transmitted before any file data: context->manifest is then prebuilt by
a path-only pre-scan on the calling thread and the pipeline scanner must
not append to it. Set once before the worker threads start. */
/* True when --delete-before requires the whole-tree keep-set manifest to be
transmitted before any file data: context->manifest is then prebuilt by a
path-only pre-scan on the calling thread and the pipeline scanner must not
append to it. Set once before the worker threads start. */
bool early_delete;
/* Non-NULL for --delete-during/--delete-delay: the per-directory plan set
prebuilt by the path-only pre-scan on the calling thread. The sender
thread transmits the root plan before any data and the remaining plans
alongside the chunks. Set once before the worker threads start. */
DeletePlanSender* delete_plans;
mtx_t mutex_progress;
int total_files;
unsigned long long progress_bytes;
+15 -1
View File
@@ -181,7 +181,21 @@ enum NET_STATUS {
* (new vs modified, and which of size/time/perms/owner/group differ) without
* changing the transfer decision itself. Appended after
* STATUS_DELETE_LIMIT so no existing status is renumbered. */
STATUS_DEST_INFO
STATUS_DEST_INFO,
/* Per-directory delete plan (protocol 2.24.0). The sender of a
* --delete-during/--delete-delay transfer streams one frame per source
* directory in directory order instead of a single whole-tree keep-set
* manifest. The receiver applies the plan when it arrives
* (--delete-during removes that directory's extras immediately) or records
* the extras and applies them only after the whole transfer succeeded
* (--delete-delay). Payload: an int32 has_config flag (1 on the first plan
* of the run, 0 afterwards); when set, the three global config sections
* (protected-prefix count+paths, size-skipped count+paths, missing-args
* count+paths); then the destination-relative directory path wire string
* ("." for the receive root); then the child-directory count + names and the
* child-file count + names that must be kept. Appended after
* STATUS_DEST_INFO so no existing status is renumbered. */
STATUS_DELETE_PLAN
};
void io_set_fds(int read_fd, int write_fd);
+2 -2
View File
@@ -60,7 +60,7 @@ bool path_is_within_root(const char* root, const char* path) {
* two differ in create-vs-no-create, in what path component they stop at, and
* in the extra receiver policies they apply, so they are intentionally kept
* separate. Both rely on the shared lexical path_is_within_root check. */
static int open_authorized_destination(const char* dest_root) {
int utils_open_authorized_destination(const char* dest_root) {
int root_fd = utils_get_authorized_root_fd();
const char* root_path = utils_get_authorized_root_path();
if (root_fd < 0 || !root_path || !dest_root || !path_is_within_root(root_path, dest_root))
@@ -752,7 +752,7 @@ DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* m
int root_fd = utils_get_authorized_root_fd();
if (root_fd >= 0) {
if (utils_get_authorized_root_path())
rootfd = open_authorized_destination(dest_root);
rootfd = utils_open_authorized_destination(dest_root);
else if (dest_root == NULL)
rootfd = dup(root_fd);
else
+5
View File
@@ -139,6 +139,11 @@ DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* m
const DeleteSkipEntry* skips, int skip_count,
size_t* deleted_out, size_t* skipped_out);
bool delete_extras(const char* dest_root, const ArrayList* manifest);
/* Open the existing destination directory at `dest_root`, confined to the
authorized root with an O_NOFOLLOW component walk (the same confinement the
deletion walker uses for its root). Returns a new fd the caller owns, or -1
on error (including a destination that does not exist). */
int utils_open_authorized_destination(const char* dest_root);
bool utils_set_authorized_root(int fd, const char* canonical_path);
/* The fd-only compatibility form is fail-closed for path-based operations;
* callers should use utils_set_authorized_root with the canonical identity. */