fix: close valid residuals of issues #286-#297 (triage cycle) #308
@@ -141,6 +141,10 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co
|
|||||||
update = 'h';
|
update = 'h';
|
||||||
else if (created)
|
else if (created)
|
||||||
update = (event->is_directory || event->is_symlink || event->is_special) ? 'c' : '>';
|
update = (event->is_directory || event->is_symlink || event->is_special) ? 'c' : '>';
|
||||||
|
else if (event->is_directory)
|
||||||
|
/* rsync: an existing directory that only has attribute changes carries no
|
||||||
|
transfer, so the update column is `.` rather than `>`. */
|
||||||
|
update = '.';
|
||||||
else
|
else
|
||||||
update = '>';
|
update = '>';
|
||||||
code[0] = update;
|
code[0] = update;
|
||||||
@@ -168,12 +172,15 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co
|
|||||||
code[11] = '\0';
|
code[11] = '\0';
|
||||||
}
|
}
|
||||||
|
|
||||||
/* rsync %n: the transfer-relative name, with a trailing slash for directories. */
|
/* rsync %n: the transfer-relative name, with a trailing slash for directories.
|
||||||
|
* The transfer root is `.` (so `%n` renders `./`), matching rsync's root entry. */
|
||||||
static bool append_name(StrBuf* buf, const ChangeEvent* event) {
|
static bool append_name(StrBuf* buf, const ChangeEvent* event) {
|
||||||
if (!strbuf_append(buf, event->name != NULL ? event->name : ""))
|
const char* name = event->name != NULL ? event->name : "";
|
||||||
|
if (event->is_directory && name[0] == '\0')
|
||||||
|
return strbuf_append(buf, "./");
|
||||||
|
if (!strbuf_append(buf, name))
|
||||||
return false;
|
return false;
|
||||||
if (event->is_directory && (event->name == NULL || event->name[0] == '\0' ||
|
if (event->is_directory && name[strlen(name) - 1] != '/')
|
||||||
event->name[strlen(event->name) - 1] != '/'))
|
|
||||||
return strbuf_append_char(buf, '/');
|
return strbuf_append_char(buf, '/');
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|||||||
+207
-21
@@ -243,17 +243,31 @@ void transfer_stats_note_transferred(TransferStats* stats, const File* file) {
|
|||||||
#define RSYNC_PROGRESS_IO_WINDOW (32ULL * 1024ULL)
|
#define RSYNC_PROGRESS_IO_WINDOW (32ULL * 1024ULL)
|
||||||
|
|
||||||
/* Paths-only pre-count of the source file list, built once at transfer start
|
/* Paths-only pre-count of the source file list, built once at transfer start
|
||||||
* when progress output is requested. rsync's `to-chk` denominator is the whole
|
* when progress output or -i/--out-format needs it. rsync's `to-chk`
|
||||||
* file list -- every regular file, directory, symlink and special plus the
|
* denominator is the whole file list -- every regular file, directory, symlink
|
||||||
* transfer root -- while the streaming scan never emits directories. A
|
* and special plus the transfer root -- while the streaming scan only emits
|
||||||
* metadata-only walk (no file reads, no hashing) supplies that total and the
|
* empty directories. A metadata-only walk (no file reads, no hashing) supplies
|
||||||
* directory names, so the opt-in pass leaves non-progress runs untouched. */
|
* that total and a metadata-bearing File for every directory, so --progress can
|
||||||
|
* name them and -i/--out-format can itemize them without a second full scan. */
|
||||||
|
/* One directory in the pre-count, keyed by its transfer-relative display name
|
||||||
|
* ("" is the transfer root). `file` is owned by ProgressPrecount.dir_files and
|
||||||
|
* carries the source metadata needed by -i/--out-format (%M/%B/%U/%G). */
|
||||||
|
typedef struct {
|
||||||
|
char* name; /* owned */
|
||||||
|
File* file;
|
||||||
|
} DirRef;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
unsigned long long total;
|
unsigned long long total;
|
||||||
ArrayList* dir_paths; /* owned char* in transfer-relative display form */
|
ArrayList* dir_paths; /* owned char* in transfer-relative display form */
|
||||||
|
ArrayList* dir_files; /* owned File* captured during the metadata walk */
|
||||||
|
ArrayList* dir_refs; /* owned DirRef*, sorted by name for prefix lookup */
|
||||||
} ProgressPrecount;
|
} ProgressPrecount;
|
||||||
|
|
||||||
static bool g_progress_active;
|
static bool g_progress_active;
|
||||||
|
/* True when -i/--out-format need the pre-counted directory entries fed into the
|
||||||
|
* change-event stream (independent of --progress). */
|
||||||
|
static bool g_change_dirs_active;
|
||||||
static unsigned long long g_progress_xferred;
|
static unsigned long long g_progress_xferred;
|
||||||
static unsigned long long g_progress_index;
|
static unsigned long long g_progress_index;
|
||||||
static unsigned long long g_progress_total;
|
static unsigned long long g_progress_total;
|
||||||
@@ -270,14 +284,101 @@ bool progress_requested(const Config* config) {
|
|||||||
(config->show_progress || (config->info_level & LOG_INFO_PROGRESS) != 0);
|
(config->show_progress || (config->info_level & LOG_INFO_PROGRESS) != 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void dir_ref_destroy(void* item) {
|
||||||
|
DirRef* ref = (DirRef*)item;
|
||||||
|
if (ref == NULL)
|
||||||
|
return;
|
||||||
|
free(ref->name);
|
||||||
|
free(ref);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Sort DirRef pointers by their transfer-relative name for binary search. */
|
||||||
|
static int dir_ref_compare(const void* left, const void* right) {
|
||||||
|
const DirRef* a = *(const DirRef* const*)left;
|
||||||
|
const DirRef* b = *(const DirRef* const*)right;
|
||||||
|
return strcmp(a->name, b->name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Look up the pre-counted directory File for a transfer-relative name ("" is
|
||||||
|
* the transfer root). Returns NULL when no pre-count was built or the name is
|
||||||
|
* not a known directory. */
|
||||||
|
static File* progress_dir_lookup(const char* name) {
|
||||||
|
if (name == NULL || g_progress_precount.dir_refs == NULL)
|
||||||
|
return NULL;
|
||||||
|
ArrayList* refs = g_progress_precount.dir_refs;
|
||||||
|
size_t lo = 0;
|
||||||
|
size_t hi = (size_t)refs->size;
|
||||||
|
while (lo < hi) {
|
||||||
|
size_t mid = lo + (hi - lo) / 2;
|
||||||
|
DirRef* ref = (DirRef*)refs->items[mid];
|
||||||
|
int cmp = strcmp(ref->name, name);
|
||||||
|
if (cmp < 0)
|
||||||
|
lo = mid + 1;
|
||||||
|
else if (cmp > 0)
|
||||||
|
hi = mid;
|
||||||
|
else
|
||||||
|
return ref->file;
|
||||||
|
}
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
static void progress_precount_dispose(ProgressPrecount* p) {
|
static void progress_precount_dispose(ProgressPrecount* p) {
|
||||||
if (p->dir_paths != NULL) {
|
if (p->dir_paths != NULL) {
|
||||||
array_list_delete(p->dir_paths);
|
array_list_delete(p->dir_paths);
|
||||||
p->dir_paths = NULL;
|
p->dir_paths = NULL;
|
||||||
}
|
}
|
||||||
|
if (p->dir_files != NULL) {
|
||||||
|
array_list_delete(p->dir_files);
|
||||||
|
p->dir_files = NULL;
|
||||||
|
}
|
||||||
|
if (p->dir_refs != NULL) {
|
||||||
|
array_list_delete(p->dir_refs);
|
||||||
|
p->dir_refs = NULL;
|
||||||
|
}
|
||||||
p->total = 0;
|
p->total = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Record the transfer root's pre-transfer state for -i/--out-format. The
|
||||||
|
* receive root always exists, so rsync never marks it `cd`; its only observable
|
||||||
|
* change is its timestamp, which FastSync cannot observe remotely. Force a time
|
||||||
|
* mismatch so the root renders rsync's `.d..t...... ./` rather than the `cd`
|
||||||
|
* a zeroed destination state would produce. */
|
||||||
|
static void progress_precount_mark_root(File* root) {
|
||||||
|
if (root == NULL)
|
||||||
|
return;
|
||||||
|
root->dest_state.known = true;
|
||||||
|
root->dest_state.existed = true;
|
||||||
|
root->dest_state.mode = root->metadata != NULL ? root->metadata->mode : 0;
|
||||||
|
root->dest_state.uid = root->metadata != NULL ? root->metadata->uid : 0;
|
||||||
|
root->dest_state.gid = root->metadata != NULL ? root->metadata->gid : 0;
|
||||||
|
root->dest_state.size = 0;
|
||||||
|
root->dest_state.mtime_sec = (root->metadata != NULL ? root->metadata->mtime_sec : 0) - 3600;
|
||||||
|
root->dest_state.mtime_nsec = root->metadata != NULL ? root->metadata->mtime_nsec : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Append one DirRef (name -> file) to the pre-count, marking the transfer
|
||||||
|
* root's destination state. Returns false on allocation failure. */
|
||||||
|
static bool progress_precount_add_ref(ProgressPrecount* p, const Config* config, File* file) {
|
||||||
|
const char* rel = delete_display_path(config, file_wire_path(file));
|
||||||
|
char* name = rel != NULL ? str_dup(rel) : NULL;
|
||||||
|
if (name == NULL)
|
||||||
|
return false;
|
||||||
|
DirRef* ref = malloc(sizeof(*ref));
|
||||||
|
if (ref == NULL) {
|
||||||
|
free(name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
ref->name = name;
|
||||||
|
ref->file = file;
|
||||||
|
if (name[0] == '\0')
|
||||||
|
progress_precount_mark_root(file);
|
||||||
|
if (!array_list_add(p->dir_refs, ref)) {
|
||||||
|
dir_ref_destroy(ref);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
void client_progress_cleanup(void) {
|
void client_progress_cleanup(void) {
|
||||||
if (g_progress_dir_index_valid) {
|
if (g_progress_dir_index_valid) {
|
||||||
path_index_free(&g_progress_dir_index);
|
path_index_free(&g_progress_dir_index);
|
||||||
@@ -293,6 +394,7 @@ void client_progress_cleanup(void) {
|
|||||||
}
|
}
|
||||||
progress_precount_dispose(&g_progress_precount);
|
progress_precount_dispose(&g_progress_precount);
|
||||||
g_progress_active = false;
|
g_progress_active = false;
|
||||||
|
g_change_dirs_active = false;
|
||||||
g_progress_total = 0;
|
g_progress_total = 0;
|
||||||
g_progress_index = 0;
|
g_progress_index = 0;
|
||||||
g_progress_xferred = 0;
|
g_progress_xferred = 0;
|
||||||
@@ -395,6 +497,11 @@ void print_delete_reports(const Config* config, const ArrayList* paths) {
|
|||||||
fflush(stdout);
|
fflush(stdout);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Emit every not-yet-seen ancestor directory of `rel`, outermost first, in the
|
||||||
|
* order rsync's depth-first flist walk visits them. With -i/--out-format each
|
||||||
|
* ancestor becomes a real change line (`cd+++++++++ sub/`, `.d..t...... ./`)
|
||||||
|
* rendered by the shared itemize code; otherwise it is the `--info=name` /
|
||||||
|
* --progress directory name line. */
|
||||||
static void client_progress_emit_ancestors(const Config* config, const char* rel) {
|
static void client_progress_emit_ancestors(const Config* config, const char* rel) {
|
||||||
if (!g_progress_dir_index_valid || !g_progress_emitted_valid || g_progress_emitted_keys == NULL ||
|
if (!g_progress_dir_index_valid || !g_progress_emitted_valid || g_progress_emitted_keys == NULL ||
|
||||||
rel == NULL)
|
rel == NULL)
|
||||||
@@ -413,9 +520,15 @@ static void client_progress_emit_ancestors(const Config* config, const char* rel
|
|||||||
char* key = str_dup(prefix);
|
char* key = str_dup(prefix);
|
||||||
if (key != NULL && array_list_add(g_progress_emitted_keys, key)) {
|
if (key != NULL && array_list_add(g_progress_emitted_keys, key)) {
|
||||||
str_hash_set_insert_ref(&g_progress_emitted, key);
|
str_hash_set_insert_ref(&g_progress_emitted, key);
|
||||||
char* escaped = output_escape(prefix, config->eight_bit_output);
|
if (g_change_dirs_active) {
|
||||||
printf("%s/\n", escaped ? escaped : prefix);
|
File* dir = progress_dir_lookup(prefix);
|
||||||
free(escaped);
|
if (dir != NULL)
|
||||||
|
change_emit_dir_sent(config, dir);
|
||||||
|
} else {
|
||||||
|
char* escaped = output_escape(prefix, config->eight_bit_output);
|
||||||
|
printf("%s/\n", escaped ? escaped : prefix);
|
||||||
|
free(escaped);
|
||||||
|
}
|
||||||
g_progress_index++;
|
g_progress_index++;
|
||||||
} else {
|
} else {
|
||||||
free(key);
|
free(key);
|
||||||
@@ -425,6 +538,20 @@ static void client_progress_emit_ancestors(const Config* config, const char* rel
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Feed a transferred entry's ancestor directories into the change-event stream
|
||||||
|
* before the entry's own line, so -i/--out-format and --progress report
|
||||||
|
* directories in rsync's depth-first order. Every directory is an ancestor of
|
||||||
|
* some emitted entry (a file, symlink, special, hard link or the empty-directory
|
||||||
|
* entry the scanner emits for a leaf), so this covers the whole tree. */
|
||||||
|
void client_change_emit_ancestors(const Config* config, const File* file) {
|
||||||
|
if (config == NULL || file == NULL)
|
||||||
|
return;
|
||||||
|
if (!g_progress_active && !g_change_dirs_active)
|
||||||
|
return;
|
||||||
|
const char* rel = delete_display_path(config, file_wire_path(file));
|
||||||
|
client_progress_emit_ancestors(config, rel);
|
||||||
|
}
|
||||||
|
|
||||||
/* rsync's --info=name/progress line for one entry: transfer-relative name (a
|
/* rsync's --info=name/progress line for one entry: transfer-relative name (a
|
||||||
* trailing slash for directories) plus the ` -> target` symlink suffix. */
|
* trailing slash for directories) plus the ` -> target` symlink suffix. */
|
||||||
static char* progress_entry_line(const File* file, const char* rel) {
|
static char* progress_entry_line(const File* file, const char* rel) {
|
||||||
@@ -462,7 +589,7 @@ void client_progress_begin(const Config* config) {
|
|||||||
g_progress_active = progress_requested(config);
|
g_progress_active = progress_requested(config);
|
||||||
g_progress_xferred = 0;
|
g_progress_xferred = 0;
|
||||||
g_progress_index = 1; /* the transfer root is file-list entry #0 */
|
g_progress_index = 1; /* the transfer root is file-list entry #0 */
|
||||||
if (!g_progress_active) {
|
if (!g_progress_active && !g_change_dirs_active) {
|
||||||
/* `--info=flist` prints rsync's file-list header even without progress. */
|
/* `--info=flist` prints rsync's file-list header even without progress. */
|
||||||
if (!config->quiet && info_flag_enabled(config, LOG_INFO_FLIST)) {
|
if (!config->quiet && info_flag_enabled(config, LOG_INFO_FLIST)) {
|
||||||
printf("sending incremental file list\n");
|
printf("sending incremental file list\n");
|
||||||
@@ -470,11 +597,18 @@ void client_progress_begin(const Config* config) {
|
|||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
printf("sending incremental file list\n");
|
if (g_progress_active)
|
||||||
/* rsync prints the transfer-root directory's name before the first file when
|
printf("sending incremental file list\n");
|
||||||
that directory is created; FastSync mirrors the source root below the
|
/* rsync prints the transfer-root directory before the first entry. Under
|
||||||
receive root and creates it on a fresh destination, so emit it here. */
|
-i/--out-format it is the root change line (`.d..t...... ./`); otherwise it
|
||||||
printf("./\n");
|
is the plain --info=name / --progress name line. */
|
||||||
|
if (g_change_dirs_active) {
|
||||||
|
File* root = progress_dir_lookup("");
|
||||||
|
if (root != NULL)
|
||||||
|
change_emit_dir_sent(config, root);
|
||||||
|
} else {
|
||||||
|
printf("./\n");
|
||||||
|
}
|
||||||
fflush(stdout);
|
fflush(stdout);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -487,7 +621,6 @@ void client_progress_file(const Config* config, const File* file) {
|
|||||||
unsigned long long size = file->data->size;
|
unsigned long long size = file->data->size;
|
||||||
if (!config->itemize_changes && config->out_format == NULL) {
|
if (!config->itemize_changes && config->out_format == NULL) {
|
||||||
const char* rel = delete_display_path(config, file_wire_path(file));
|
const char* rel = delete_display_path(config, file_wire_path(file));
|
||||||
client_progress_emit_ancestors(config, rel);
|
|
||||||
char* escaped = output_escape(rel, config->eight_bit_output);
|
char* escaped = output_escape(rel, config->eight_bit_output);
|
||||||
printf("%s\n", escaped ? escaped : (rel ? rel : ""));
|
printf("%s\n", escaped ? escaped : (rel ? rel : ""));
|
||||||
free(escaped);
|
free(escaped);
|
||||||
@@ -510,7 +643,6 @@ void client_progress_name(const Config* config, const File* file) {
|
|||||||
return;
|
return;
|
||||||
const char* rel = delete_display_path(config, file_wire_path(file));
|
const char* rel = delete_display_path(config, file_wire_path(file));
|
||||||
if (!config->itemize_changes && config->out_format == NULL) {
|
if (!config->itemize_changes && config->out_format == NULL) {
|
||||||
client_progress_emit_ancestors(config, rel);
|
|
||||||
char* line = progress_entry_line(file, rel ? rel : "");
|
char* line = progress_entry_line(file, rel ? rel : "");
|
||||||
if (line != NULL) {
|
if (line != NULL) {
|
||||||
char* escaped = output_escape(line, config->eight_bit_output);
|
char* escaped = output_escape(line, config->eight_bit_output);
|
||||||
@@ -550,8 +682,12 @@ static bool progress_precount_add_dir(ProgressPrecount* p, const char* path) {
|
|||||||
* so the data pass's link-group state is never perturbed. */
|
* so the data pass's link-group state is never perturbed. */
|
||||||
static bool progress_precount_scan(const Config* config, ProgressPrecount* out) {
|
static bool progress_precount_scan(const Config* config, ProgressPrecount* out) {
|
||||||
out->dir_paths = array_list_create(free);
|
out->dir_paths = array_list_create(free);
|
||||||
if (out->dir_paths == NULL)
|
out->dir_files = array_list_create(file_destroy);
|
||||||
|
out->dir_refs = array_list_create(dir_ref_destroy);
|
||||||
|
if (out->dir_paths == NULL || out->dir_files == NULL || out->dir_refs == NULL) {
|
||||||
|
progress_precount_dispose(out);
|
||||||
return false;
|
return false;
|
||||||
|
}
|
||||||
out->total = 0;
|
out->total = 0;
|
||||||
PreparedScanner prepared;
|
PreparedScanner prepared;
|
||||||
memset(&prepared, 0, sizeof(prepared));
|
memset(&prepared, 0, sizeof(prepared));
|
||||||
@@ -568,12 +704,14 @@ static bool progress_precount_scan(const Config* config, ProgressPrecount* out)
|
|||||||
local.preserve_xattrs = false;
|
local.preserve_xattrs = false;
|
||||||
local.preserve_acls = false;
|
local.preserve_acls = false;
|
||||||
local.checksum = false;
|
local.checksum = false;
|
||||||
local.capture_dir_times = false;
|
/* Capture one metadata-bearing File per traversed directory (including the
|
||||||
|
transfer root) so -i/--out-format can render %M/%B/%U/%G for directories. */
|
||||||
|
local.capture_dir_times = true;
|
||||||
local.excluded_paths = NULL;
|
local.excluded_paths = NULL;
|
||||||
local.size_skipped_paths = NULL;
|
local.size_skipped_paths = NULL;
|
||||||
local.synced_dirs = NULL;
|
local.synced_dirs = NULL;
|
||||||
local.plan_dirs = NULL;
|
local.plan_dirs = NULL;
|
||||||
local.dir_entries = NULL;
|
local.dir_entries = out->dir_files;
|
||||||
local.dir_entries_mutex = NULL;
|
local.dir_entries_mutex = NULL;
|
||||||
local.hardlinks = NULL;
|
local.hardlinks = NULL;
|
||||||
DirectoryScanner* scanner = directory_scanner_create_with_options(config->send_directory, &local);
|
DirectoryScanner* scanner = directory_scanner_create_with_options(config->send_directory, &local);
|
||||||
@@ -598,6 +736,18 @@ static bool progress_precount_scan(const Config* config, ProgressPrecount* out)
|
|||||||
progress_precount_dispose(out);
|
progress_precount_dispose(out);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
/* Build the name -> File lookup from the captured directory Files. */
|
||||||
|
for (int i = 0; i < out->dir_files->size; i++) {
|
||||||
|
File* f = (File*)out->dir_files->items[i];
|
||||||
|
if (f == NULL)
|
||||||
|
continue;
|
||||||
|
if (!progress_precount_add_ref(out, config, f)) {
|
||||||
|
progress_precount_dispose(out);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (out->dir_refs->size > 1)
|
||||||
|
qsort(out->dir_refs->items, (size_t)out->dir_refs->size, sizeof(DirRef*), dir_ref_compare);
|
||||||
out->total += 1; /* the transfer root "." */
|
out->total += 1; /* the transfer root "." */
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -609,9 +759,26 @@ static bool progress_precount_from_plan_dirs(const Config* config, const ArrayLi
|
|||||||
unsigned long long non_dir_count,
|
unsigned long long non_dir_count,
|
||||||
ProgressPrecount* out) {
|
ProgressPrecount* out) {
|
||||||
out->dir_paths = array_list_create(free);
|
out->dir_paths = array_list_create(free);
|
||||||
if (out->dir_paths == NULL)
|
out->dir_files = array_list_create(file_destroy);
|
||||||
|
out->dir_refs = array_list_create(dir_ref_destroy);
|
||||||
|
if (out->dir_paths == NULL || out->dir_files == NULL || out->dir_refs == NULL) {
|
||||||
|
progress_precount_dispose(out);
|
||||||
return false;
|
return false;
|
||||||
|
}
|
||||||
out->total = non_dir_count + 1;
|
out->total = non_dir_count + 1;
|
||||||
|
/* The delete pre-scan's plan list omits the transfer root, so synthesize its
|
||||||
|
entry here; it is only used for the root change line. */
|
||||||
|
File* root = file_create("");
|
||||||
|
if (root == NULL || !array_list_add(out->dir_files, root)) {
|
||||||
|
file_destroy(root);
|
||||||
|
progress_precount_dispose(out);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
root->is_dir = true;
|
||||||
|
if (!progress_precount_add_ref(out, config, root)) {
|
||||||
|
progress_precount_dispose(out);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
for (int i = 0; i < plan_dirs->size; i++) {
|
for (int i = 0; i < plan_dirs->size; i++) {
|
||||||
const char* path = (const char*)plan_dirs->items[i];
|
const char* path = (const char*)plan_dirs->items[i];
|
||||||
const char* rel = config->send_directory != NULL
|
const char* rel = config->send_directory != NULL
|
||||||
@@ -621,7 +788,25 @@ static bool progress_precount_from_plan_dirs(const Config* config, const ArrayLi
|
|||||||
progress_precount_dispose(out);
|
progress_precount_dispose(out);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
File* dir = file_create("");
|
||||||
|
if (dir == NULL) {
|
||||||
|
progress_precount_dispose(out);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
dir->is_dir = true;
|
||||||
|
dir->send_path = str_dup(rel != NULL ? rel : "");
|
||||||
|
if (dir->send_path == NULL || !array_list_add(out->dir_files, dir)) {
|
||||||
|
file_destroy(dir);
|
||||||
|
progress_precount_dispose(out);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!progress_precount_add_ref(out, config, dir)) {
|
||||||
|
progress_precount_dispose(out);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
if (out->dir_refs->size > 1)
|
||||||
|
qsort(out->dir_refs->items, (size_t)out->dir_refs->size, sizeof(DirRef*), dir_ref_compare);
|
||||||
out->total += (unsigned long long)out->dir_paths->size;
|
out->total += (unsigned long long)out->dir_paths->size;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -633,7 +818,8 @@ void client_progress_prepare(const Config* config, const ArrayList* plan_dirs,
|
|||||||
unsigned long long plan_non_dir_count) {
|
unsigned long long plan_non_dir_count) {
|
||||||
client_progress_cleanup();
|
client_progress_cleanup();
|
||||||
g_progress_active = progress_requested(config);
|
g_progress_active = progress_requested(config);
|
||||||
if (!g_progress_active)
|
g_change_dirs_active = config->itemize_changes || config->out_format != NULL;
|
||||||
|
if (!g_progress_active && !g_change_dirs_active)
|
||||||
return;
|
return;
|
||||||
bool ok = plan_dirs != NULL ? progress_precount_from_plan_dirs(
|
bool ok = plan_dirs != NULL ? progress_precount_from_plan_dirs(
|
||||||
config, plan_dirs, plan_non_dir_count, &g_progress_precount)
|
config, plan_dirs, plan_non_dir_count, &g_progress_precount)
|
||||||
|
|||||||
@@ -838,6 +838,10 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
|||||||
if (chunk->items[i] == NULL)
|
if (chunk->items[i] == NULL)
|
||||||
continue;
|
continue;
|
||||||
transfer_stats_note_entry(stats, chunk->items[i]);
|
transfer_stats_note_entry(stats, chunk->items[i]);
|
||||||
|
/* The chunk-serialization path emits no --progress name lines, so only
|
||||||
|
feed -i/--out-format its ancestor directory lines here. */
|
||||||
|
if (config->itemize_changes || config->out_format != NULL)
|
||||||
|
client_change_emit_ancestors(config, chunk->items[i]);
|
||||||
if (chunk->items[i]->is_dir)
|
if (chunk->items[i]->is_dir)
|
||||||
change_emit_dir_sent(config, chunk->items[i]);
|
change_emit_dir_sent(config, chunk->items[i]);
|
||||||
else
|
else
|
||||||
@@ -859,6 +863,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
|||||||
source to remove and no incremental check. */
|
source to remove and no incremental check. */
|
||||||
if (!send_directory_entry(client, f, config))
|
if (!send_directory_entry(client, f, config))
|
||||||
return -1;
|
return -1;
|
||||||
|
client_change_emit_ancestors(config, f);
|
||||||
change_emit_dir_sent(config, f);
|
change_emit_dir_sent(config, f);
|
||||||
client_progress_name(config, f);
|
client_progress_name(config, f);
|
||||||
continue;
|
continue;
|
||||||
@@ -873,6 +878,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
|||||||
!send_int(client->file_descriptor, f->link_group) ||
|
!send_int(client->file_descriptor, f->link_group) ||
|
||||||
!send_wire_str(client->file_descriptor, f->hardlink_target))
|
!send_wire_str(client->file_descriptor, f->hardlink_target))
|
||||||
return -1;
|
return -1;
|
||||||
|
client_change_emit_ancestors(config, f);
|
||||||
change_emit_file_sent(config, f);
|
change_emit_file_sent(config, f);
|
||||||
client_progress_name(config, f);
|
client_progress_name(config, f);
|
||||||
continue;
|
continue;
|
||||||
@@ -881,6 +887,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
|||||||
if (f->is_symlink) {
|
if (f->is_symlink) {
|
||||||
if (!send_symlink_entry(client, f, config))
|
if (!send_symlink_entry(client, f, config))
|
||||||
return -1;
|
return -1;
|
||||||
|
client_change_emit_ancestors(config, f);
|
||||||
change_emit_file_sent(config, f);
|
change_emit_file_sent(config, f);
|
||||||
client_progress_name(config, f);
|
client_progress_name(config, f);
|
||||||
continue;
|
continue;
|
||||||
@@ -890,6 +897,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
|||||||
if (f->is_special) {
|
if (f->is_special) {
|
||||||
if (!file_send_special(f, client->file_descriptor, config->use_metadata))
|
if (!file_send_special(f, client->file_descriptor, config->use_metadata))
|
||||||
return -1;
|
return -1;
|
||||||
|
client_change_emit_ancestors(config, f);
|
||||||
change_emit_file_sent(config, f);
|
change_emit_file_sent(config, f);
|
||||||
client_progress_name(config, f);
|
client_progress_name(config, f);
|
||||||
continue;
|
continue;
|
||||||
@@ -913,6 +921,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
transfer_stats_note_transferred(stats, f);
|
transfer_stats_note_transferred(stats, f);
|
||||||
|
client_change_emit_ancestors(config, f);
|
||||||
change_emit_file_sent_bytes(config, f, protocol_bytes_written() - bytes_before,
|
change_emit_file_sent_bytes(config, f, protocol_bytes_written() - bytes_before,
|
||||||
protocol_bytes_read() - read_before);
|
protocol_bytes_read() - read_before);
|
||||||
client_progress_file(config, f);
|
client_progress_file(config, f);
|
||||||
@@ -1567,7 +1576,10 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
|
|||||||
* runs the shared cleanup). */
|
* runs the shared cleanup). */
|
||||||
static bool send_files_run(Config* config, SendFilesState* state) {
|
static bool send_files_run(Config* config, SendFilesState* state) {
|
||||||
Client* client = state->client;
|
Client* client = state->client;
|
||||||
if (progress_requested(config))
|
/* --progress needs the file-list total; -i/--out-format needs the directory
|
||||||
|
entries. Either way one paths-only pre-count supplies both, and a
|
||||||
|
--delete-during/--delete-delay pre-scan is reused when present. */
|
||||||
|
if (progress_requested(config) || config->itemize_changes || config->out_format != NULL)
|
||||||
client_progress_prepare(config, state->plan_dirs, state->per_dir_non_dir_count);
|
client_progress_prepare(config, state->plan_dirs, state->per_dir_non_dir_count);
|
||||||
/* Phase 6: compute the client-only stop deadline once at transfer start. The
|
/* Phase 6: compute the client-only stop deadline once at transfer start. The
|
||||||
early-delete pre-scan above deliberately ignores it so the keep-set (and
|
early-delete pre-scan above deliberately ignores it so the keep-set (and
|
||||||
@@ -2045,9 +2057,10 @@ int send_files_multithreaded(Config* config) {
|
|||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
/* --progress/--info=progress: pre-count the file list for rsync's to-chk
|
/* --progress/--info=progress: pre-count the file list for rsync's to-chk
|
||||||
denominator, reusing a --delete-during/--delete-delay pre-scan when one
|
denominator; -i/--out-format: pre-count the directory entries. One pass
|
||||||
|
supplies both, reusing a --delete-during/--delete-delay pre-scan when one
|
||||||
already ran. */
|
already ran. */
|
||||||
if (progress_requested(config))
|
if (progress_requested(config) || config->itemize_changes || config->out_format != NULL)
|
||||||
client_progress_prepare(config, context->plan_dirs, pre_scan_non_dir);
|
client_progress_prepare(config, context->plan_dirs, pre_scan_non_dir);
|
||||||
|
|
||||||
thrd_t scanner, loader, sender;
|
thrd_t scanner, loader, sender;
|
||||||
|
|||||||
@@ -64,6 +64,9 @@ void client_progress_cleanup(void);
|
|||||||
void client_progress_begin(const Config* config);
|
void client_progress_begin(const Config* config);
|
||||||
void client_progress_file(const Config* config, const File* file);
|
void client_progress_file(const Config* config, const File* file);
|
||||||
void client_progress_name(const Config* config, const File* file);
|
void client_progress_name(const Config* config, const File* file);
|
||||||
|
/* Emit a transferred entry's ancestor directories (as -i/--out-format change
|
||||||
|
* lines or --progress name lines) before the entry's own line. */
|
||||||
|
void client_change_emit_ancestors(const Config* config, const File* file);
|
||||||
void client_progress_uptodate(const Config* config, const File* file);
|
void client_progress_uptodate(const Config* config, const File* file);
|
||||||
void client_progress_prepare(const Config* config, const ArrayList* plan_dirs,
|
void client_progress_prepare(const Config* config, const ArrayList* plan_dirs,
|
||||||
unsigned long long plan_non_dir_count);
|
unsigned long long plan_non_dir_count);
|
||||||
|
|||||||
@@ -2786,7 +2786,12 @@ class TestItemizeChanges:
|
|||||||
flags=["--preserve", "-i", "--incremental"],
|
flags=["--preserve", "-i", "--incremental"],
|
||||||
port=shared_server.port)
|
port=shared_server.port)
|
||||||
assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}"
|
assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}"
|
||||||
itemized = [line for line in result.stdout.splitlines() if line and line[0] in ">.<c"]
|
# -i also emits the transfer-root and directory lines; only FILE entries
|
||||||
|
# matter here, so drop any line whose name has a trailing '/'.
|
||||||
|
itemized = [
|
||||||
|
line for line in result.stdout.splitlines()
|
||||||
|
if line and line[0] in ">.<c" and not line.rsplit(" ", 1)[-1].endswith("/")
|
||||||
|
]
|
||||||
assert itemized == [], f"unchanged files were itemized: {itemized[:5]}"
|
assert itemized == [], f"unchanged files were itemized: {itemized[:5]}"
|
||||||
|
|
||||||
def test_multithreaded_emits_same_itemize_lines(self, shared_server):
|
def test_multithreaded_emits_same_itemize_lines(self, shared_server):
|
||||||
|
|||||||
@@ -26,6 +26,17 @@ def _rsync(args):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _file_entry_line(text):
|
||||||
|
"""The file entry line for a single-file transfer.
|
||||||
|
|
||||||
|
-i/--out-format emit the transfer-root (and directory) lines too, so the
|
||||||
|
file entry is not necessarily the first line; for the one-file corpora used
|
||||||
|
by the wire-counter tests it is the last non-empty line.
|
||||||
|
"""
|
||||||
|
lines = [line for line in text.splitlines() if line.strip()]
|
||||||
|
return lines[-1] if lines else ""
|
||||||
|
|
||||||
|
|
||||||
def _make_selection_tree(root):
|
def _make_selection_tree(root):
|
||||||
clean_dir(root)
|
clean_dir(root)
|
||||||
os.makedirs(os.path.join(root, "sub"))
|
os.makedirs(os.path.join(root, "sub"))
|
||||||
@@ -184,6 +195,51 @@ class TestItemizeParity:
|
|||||||
)
|
)
|
||||||
assert fast_lines == rsync_lines, f"rsync={rsync_lines} fastsync={fast_lines}"
|
assert fast_lines == rsync_lines, f"rsync={rsync_lines} fastsync={fast_lines}"
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_itemize_directory_lines_match_rsync(self, shared_server):
|
||||||
|
"""#292: -i/--out-format emit rsync's directory lines (including the
|
||||||
|
transfer root) in rsync's depth-first order."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "out_itemdir_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "out_itemdir_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "out_itemdir_rdst")
|
||||||
|
clean_dir(source)
|
||||||
|
os.makedirs(os.path.join(source, "sub", "deep"))
|
||||||
|
os.makedirs(os.path.join(source, "emptydir"))
|
||||||
|
with open(os.path.join(source, "a.txt"), "wb") as fh:
|
||||||
|
fh.write(b"hello\n")
|
||||||
|
with open(os.path.join(source, "sub", "b.txt"), "wb") as fh:
|
||||||
|
fh.write(b"world\n")
|
||||||
|
with open(os.path.join(source, "sub", "deep", "d.txt"), "wb") as fh:
|
||||||
|
fh.write(b"deep\n")
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
|
||||||
|
def dir_lines(text):
|
||||||
|
# Any line whose name ends with '/' is a directory entry.
|
||||||
|
return sorted(
|
||||||
|
line for line in text.splitlines()
|
||||||
|
if line.rsplit(" ", 1)[-1].endswith("/")
|
||||||
|
)
|
||||||
|
|
||||||
|
for fmt in (None, "%i %n%L"):
|
||||||
|
rsync_flags = ["-a", "-i"] if fmt is None else ["-a", "--out-format=" + fmt]
|
||||||
|
fast_flags = rsync_flags
|
||||||
|
clean_dir(rdst)
|
||||||
|
clean_dir(dest)
|
||||||
|
rsync_result = _rsync(rsync_flags + [source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
result, _ = run_client(source, dest, flags=fast_flags,
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
expected = [l for l in dir_lines(rsync_result.stdout)
|
||||||
|
if not l.rsplit(" ", 1)[-1] == "./"]
|
||||||
|
fast = dir_lines(result.stdout)
|
||||||
|
assert [l for l in fast if not l.rsplit(" ", 1)[-1] == "./"] == expected, (
|
||||||
|
f"fmt={fmt} rsync={rsync_result.stdout!r} fastsync={result.stdout!r}"
|
||||||
|
)
|
||||||
|
assert ".d..t...... ./" in fast, f"missing root line: {result.stdout!r}"
|
||||||
|
|
||||||
@requires_rsync
|
@requires_rsync
|
||||||
@pytest.mark.ci
|
@pytest.mark.ci
|
||||||
def test_itemize_modified_file_matches_rsync(self, shared_server):
|
def test_itemize_modified_file_matches_rsync(self, shared_server):
|
||||||
@@ -387,8 +443,8 @@ class TestWireStatsParity:
|
|||||||
result, _ = run_client(source, dest, flags=["-a", "--out-format=" + fmt],
|
result, _ = run_client(source, dest, flags=["-a", "--out-format=" + fmt],
|
||||||
port=shared_server.port)
|
port=shared_server.port)
|
||||||
assert result.returncode == 0, result.stderr[:300]
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
rb, rl = (int(x) for x in rsync_result.stdout.split()[:2])
|
rb, rl = (int(x) for x in _file_entry_line(rsync_result.stdout).split()[:2])
|
||||||
fb, fl = (int(x) for x in result.stdout.split()[:2])
|
fb, fl = (int(x) for x in _file_entry_line(result.stdout).split()[:2])
|
||||||
assert rl == fl == 5000, (rsync_result.stdout, result.stdout)
|
assert rl == fl == 5000, (rsync_result.stdout, result.stdout)
|
||||||
assert rb > rl, f"rsync %b must include framing: {rsync_result.stdout!r}"
|
assert rb > rl, f"rsync %b must include framing: {rsync_result.stdout!r}"
|
||||||
assert fb > fl, f"fastsync %b must include framing: {result.stdout!r}"
|
assert fb > fl, f"fastsync %b must include framing: {result.stdout!r}"
|
||||||
@@ -421,7 +477,9 @@ class TestWireStatsParity:
|
|||||||
assert file_lines(result.stdout) == file_lines(rsync_result.stdout), (
|
assert file_lines(result.stdout) == file_lines(rsync_result.stdout), (
|
||||||
f"rsync={rsync_result.stdout!r} fastsync={result.stdout!r}"
|
f"rsync={rsync_result.stdout!r} fastsync={result.stdout!r}"
|
||||||
)
|
)
|
||||||
assert result.stdout.split()[0] == rsync_result.stdout.split()[0] == "16", (
|
fs_c = _file_entry_line(result.stdout).split()[0]
|
||||||
|
rs_c = _file_entry_line(rsync_result.stdout).split()[0]
|
||||||
|
assert fs_c == rs_c == "16", (
|
||||||
f"%c must be rsync's 16-byte sum header: {result.stdout!r}"
|
f"%c must be rsync's 16-byte sum header: {result.stdout!r}"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -450,8 +508,8 @@ class TestWireStatsParity:
|
|||||||
"--out-format=" + fmt],
|
"--out-format=" + fmt],
|
||||||
port=shared_server.port)
|
port=shared_server.port)
|
||||||
assert result.returncode == 0, result.stderr[:300]
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
rs_c = int(rsync_result.stdout.split()[0])
|
rs_c = int(_file_entry_line(rsync_result.stdout).split()[0])
|
||||||
fs_c = int(result.stdout.split()[0])
|
fs_c = int(_file_entry_line(result.stdout).split()[0])
|
||||||
# No basis exists, so rsync still reports only its sum header.
|
# No basis exists, so rsync still reports only its sum header.
|
||||||
assert rs_c == 16, rsync_result.stdout
|
assert rs_c == 16, rsync_result.stdout
|
||||||
# FastSync reports its own handshake bytes and is not aligned.
|
# FastSync reports its own handshake bytes and is not aligned.
|
||||||
|
|||||||
Reference in New Issue
Block a user