diff --git a/RSYNC_COMPAT.md b/RSYNC_COMPAT.md index b298894..68aa106 100644 --- a/RSYNC_COMPAT.md +++ b/RSYNC_COMPAT.md @@ -6,11 +6,11 @@ This document maps rsync's full feature set to FastSync's current implementation | Status | Count | Description | |--------|-------|-------------| -| ✅ Implemented | 63 | Feature works end-to-end | +| ✅ Implemented | 67 | Feature works end-to-end | | 🔀 Alt Arg | 3 | Functionality exists but under different flag/semantics | | ⚠️ Partial | 5 | Flag parsed/stored but behavior incomplete | | 🔄 Compatibility No-op | 1 | Flag is accepted for CLI compatibility but has no effect | -| ❌ Not Implemented | 75 | Flag not recognized or no behavior | +| ❌ Not Implemented | 71 | Flag not recognized or no behavior | | **Total** | **147** | | --- @@ -72,10 +72,10 @@ This document maps rsync's full feature set to FastSync's current implementation | Flag | Rsync Description | FastSync Status | Notes | |------|-------------------|-----------------|-------| | `-r`, `--recursive` | Recurse into directories | ✅ Implemented | Default behavior | -| `-R`, `--relative` | Use relative path names | ❌ Not Implemented | Removed because it had no effect | -| `--no-implied-dirs` | Don't send implied dirs with -R | ❌ Not Implemented | | -| `-d`, `--dirs`, `--old-dirs`, `--old-d` | Transfer dirs without recursing | ❌ Not Implemented | The aliases are recognized and rejected explicitly; they depend on the unimplemented `--dirs` behavior | -| `--mkpath` | Create missing path components | ❌ Not Implemented | | +| `-R`, `--relative` | Use relative path names | ✅ Implemented | Meaningful together with `--files-from` (FastSync's default full-tree scan always mirrors the full source argument path below the destination root, so -R does not change it). With `-R` + `--files-from` each listed entry is transmitted under its bare relative destination path: an entry `sub/x.txt` lands at `/sub/x.txt` (its leading components preserved) instead of under the `/` mirror. Only the path sent on the wire changes; the client still reads the absolute source path, and the delete manifest derives from the sent (relative) paths so `--delete` and `--remove-source-files` stay consistent in both layouts. Works single-threaded and under `-m` (including chunk serialization) | +| `--no-implied-dirs` | Don't send implied dirs with -R | ✅ Implemented | Client-side, meaningful only with `-R` + `--files-from`. rsync would normally create the ancestor directories implied by a listed file so it can be written; with `--no-implied-dirs` a listed file whose parent directory is not itself (or via an ancestor) explicitly listed cannot be placed, and FastSync fails the whole run up front with a clear error (`--no-implied-dirs: cannot place file '...': parent directory '...' is not explicitly listed`). Listing the directory (or an ancestor of it, or the whole tree `.`) permits the file. In every other mode the option has no effect. FastSync has no per-entry skip channel, so the rsync "omit the file" case is surfaced as a hard pre-transfer error | +| `-d`, `--dirs`, `--old-dirs`, `--old-d` | Transfer dirs without recursing | ✅ Implemented | `-d ` transmits an explicit directory entry for the source-root directory, so the destination mirror is created empty and nothing is descended into. With `--files-from` exactly the listed items are transferred: a listed directory is created empty (no descent) and a listed file is transferred with its content; the dest layout follows the same -R rules as plain files. A new wire frame (`STATUS_MKDIR`) carries each directory entry (path only); the receiver creates it with the same confined mkdir-parent semantics as regular writes, in single-threaded and `-m` receivers (chunk serialization carries a per-entry type marker). Directory entries appear in the delete manifest so `--delete` prunes correctly. FastSync divergences: directory mtimes/modes are not transmitted, filter/`--exclude` rules are not re-applied to the listed dirs mode (there is no descent during which they would apply), and `-d` never creates the intermediate directories between the destination root and a listed file beyond the usual on-demand parent creation | +| `--mkpath` | Create missing path components | ✅ Implemented | Wire option (client → server). At connection start the server creates the client's destination root directory (and any missing leading components below its own authorized root) when `--mkpath` is set, failing the connection cleanly if it cannot. Without `--mkpath` a destination root that does not exist yet is rejected up front (rsync semantics), so the flag is the only way to transfer into a not-yet-created destination directory. Creation is confined by the same secure mkdir walk as file writes (`O_NOFOLLOW`, no `..`) | ## 5. Transfer Modifications diff --git a/src/client/change_list.c b/src/client/change_list.c index 2467035..5cddaf0 100644 --- a/src/client/change_list.c +++ b/src/client/change_list.c @@ -289,7 +289,10 @@ void change_emit_file_sent(const Config* config, const File* file) { return; ChangeEvent event; memset(&event, 0, sizeof(event)); - event.path = file->path; + /* The displayed path is the one transmitted (with -R + --files-from this is + the bare relative destination path); the metadata fallback below still + stats the local absolute path. */ + event.path = file_wire_path(file); event.decision = CHANGE_SENT; event.is_directory = false; event.size = file->data != NULL ? file->data->size : 0; @@ -308,3 +311,19 @@ void change_emit_file_sent(const Config* config, const File* file) { } change_emit(config, &event); } + +/* Build and emit a CHANGE_SENT event for an explicit directory entry (-d). */ +void change_emit_dir_sent(const Config* config, const File* file) { + if (file == NULL || !change_list_enabled(config)) + return; + ChangeEvent event; + memset(&event, 0, sizeof(event)); + event.path = file_wire_path(file); + event.decision = CHANGE_SENT; + event.is_directory = true; + event.size = 0; + event.bytes_sent = 0; + if (file->metadata != NULL) + event.mtime_sec = file->metadata->mtime_sec; + change_emit(config, &event); +} diff --git a/src/client/change_list.h b/src/client/change_list.h index 499bcaf..9434026 100644 --- a/src/client/change_list.h +++ b/src/client/change_list.h @@ -72,4 +72,7 @@ void change_emit(const Config* config, const ChangeEvent* event); /* Build and emit a CHANGE_SENT event for a file the client just sent. */ void change_emit_file_sent(const Config* config, const File* file); +/* Build and emit a CHANGE_SENT event for an explicit directory entry (-d). */ +void change_emit_dir_sent(const Config* config, const File* file); + #endif diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 7e251ff..fe08ea9 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -425,8 +425,12 @@ static const OptionEntry OPTION_TABLE[] = { {"--ignore-existing", NULL, OPT_FLAG, offsetof(Config, ignore_existing)}, {"--delay-updates", NULL, OPT_FLAG, offsetof(Config, delay_updates)}, {"--chmod", NULL, OPT_STRING, offsetof(Config, chmod_spec)}, - {"--dirs", "--old-dirs", OPT_UNSUPPORTED, 0}, - {"--old-d", NULL, OPT_UNSUPPORTED, 0}, + {"--dirs", "-d", OPT_FLAG, offsetof(Config, dirs)}, + {"--old-dirs", NULL, OPT_FLAG, offsetof(Config, dirs)}, + {"--old-d", NULL, OPT_FLAG, offsetof(Config, dirs)}, + {"--relative", "-R", OPT_FLAG, offsetof(Config, relative)}, + {"--no-implied-dirs", NULL, OPT_FLAG, offsetof(Config, no_implied_dirs)}, + {"--mkpath", NULL, OPT_FLAG, offsetof(Config, mkpath)}, {"--delete-during", "--del", OPT_UNSUPPORTED, 0}, {"--source-dir", NULL, OPT_STRING, offsetof(Config, send_directory)}, @@ -576,9 +580,7 @@ static int apply_table_option(Config* config, const OptionEntry* entry, const ch return 0; } case OPT_UNSUPPORTED: { - const char* reason = "directory-only transfer is not implemented"; - if (strcmp(entry->name, "--delete-during") == 0) - reason = "delete-during is not implemented"; + const char* reason = "delete-during is not implemented"; log_message(LOG_LEVEL_ERROR, "%s: %s; refusing to ignore option", option_name, reason); return -1; } @@ -611,6 +613,13 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args, config->show_progress = true; continue; } + /* "--no-implied-dirs" is a real rsync option name, not a negation of + * "--implied-dirs", so it must be handled before the generic --no-* + * negation branch. */ + if (strcmp(argv[i], "--no-implied-dirs") == 0) { + config->no_implied_dirs = true; + continue; + } if (strncmp(argv[i], "--no-", strlen("--no-")) == 0) { if (apply_negation(config, argv[i]) != 0) return -1; diff --git a/src/client/client_send.c b/src/client/client_send.c index ab13476..842b673 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -100,6 +100,8 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann options->file_list = (const FileListSet*)config->files_from_set; options->base_filters = out->base_filters; options->per_dir_filters = config->per_dir_filter; + options->dirs = config->dirs; + options->relative = config->relative; return true; } @@ -110,6 +112,72 @@ static void prepared_scanner_destroy(PreparedScanner* prepared) { prepared->base_filters = NULL; } +/* True when some --files-from entry is an ancestor-or-equal directory of + * `rel` (an empty entry -- the whole tree "." -- counts as the root). */ +static bool file_list_ancestor_listed(const FileListSet* set, const char* rel) { + if (!set) + return true; + for (int i = 0; i < set->count; i++) { + const char* listed = set->entries[i]; + if (listed[0] == '\0') + return true; + size_t n = strlen(listed); + if (strncmp(rel, listed, n) == 0 && (rel[n] == '/' || rel[n] == '\0')) + return true; + } + return false; +} + +/* --no-implied-dirs (meaningful only with -R + --files-from): a listed file + * may only be placed when its parent directory (or one of its ancestors) is + * itself an explicitly listed entry. rsync omits a file whose implied parent + * directory is suppressed, and an explicitly listed file that cannot be placed + * fails the transfer; FastSync fails the whole run up front with a clear error + * (it has no per-entry skip channel). Without -R or --files-from the option + * has no effect. */ +static bool no_implied_dirs_files_from_valid(const Config* config) { + if (!config->no_implied_dirs || !config->relative) + return true; + const FileListSet* set = (const FileListSet*)config->files_from_set; + if (!set) + return true; + for (int i = 0; i < set->count; i++) { + const char* entry = set->entries[i]; + if (entry[0] == '\0') + continue; + char* full = path_cat(config->send_directory, entry); + if (!full) + return false; + struct stat st; + bool is_file = lstat(full, &st) == 0 && S_ISREG(st.st_mode); + free(full); + if (!is_file) + continue; + const char* slash = strrchr(entry, '/'); + if (!slash) + continue; /* top-level file: its parent is the receive root */ + size_t parent_len = (size_t)(slash - entry); + if (parent_len == 0) + continue; + char* parent = malloc(parent_len + 1); + if (!parent) + return false; + memcpy(parent, entry, parent_len); + parent[parent_len] = '\0'; + bool listed = file_list_ancestor_listed(set, parent); + if (!listed) { + log_message(LOG_LEVEL_ERROR, + "--no-implied-dirs: cannot place file '%s': parent directory '%s' is not " + "explicitly listed (list the directory or drop --no-implied-dirs)", + entry, parent); + } + free(parent); + if (!listed) + return false; + } + return true; +} + /* --files-from semantics: every listed entry must resolve under the source * root, otherwise rsync reports a hard error instead of silently transferring * nothing. An empty list is also an error. An entry of "." (the whole tree) @@ -147,7 +215,7 @@ static bool files_from_list_valid(const Config* config) { } free(full); } - return true; + return no_implied_dirs_files_from_valid(config); } /* Select the configured transport for both transfer execution paths. */ @@ -194,7 +262,7 @@ static bool add_chunk_to_manifest(ArrayList* manifest, const Chunk* chunk) { if (!manifest) return true; for (int i = 0; i < chunk->element_count; i++) { - const char* path = chunk->items[i]->path; + const char* path = file_wire_path(chunk->items[i]); if (*path == '/') path++; char* entry = str_dup(path); @@ -533,7 +601,7 @@ static int incremental_check(Client* client, File* file, const Config* config, *out_sig = NULL; if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1; - if (!send_str(client->file_descriptor, file->path)) + if (!send_str(client->file_descriptor, file_wire_path(file))) return -1; unsigned long long fsize = file->data->size; long long mtime = file->metadata ? file->metadata->mtime_sec : 0; @@ -633,6 +701,17 @@ static bool send_file_direct(File* file, int fd, bool use_metadata, int compress config->compression_threads); } +/* Transmit one explicit directory entry (--dirs): a STATUS_MKDIR frame whose + payload is only the destination path. The receiver validates the path and + creates the directory under the receive root. */ +static bool send_directory_entry(Client* client, File* file) { + if (!file || !file_wire_path(file)) + return false; + if (!send_status(client->file_descriptor, STATUS_MKDIR)) + return false; + return send_str(client->file_descriptor, file_wire_path(file)); +} + // Send a single file directly via sendfile (non-incremental path). static bool send_file_direct_sendfile(File* file, int fd, bool use_metadata, const Config* config) { if (!send_status(fd, STATUS_NEXT)) @@ -755,7 +834,11 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config, } data_destroy(data); for (int i = 0; i < chunk->element_count; i++) { - if (chunk->items[i] != NULL) + if (chunk->items[i] == NULL) + continue; + if (chunk->items[i]->is_dir) + change_emit_dir_sent(config, chunk->items[i]); + else change_emit_file_sent(config, chunk->items[i]); } return 0; @@ -765,6 +848,15 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config, File* f = chunk->items[i]; if (f == NULL) continue; + if (f->is_dir) { + /* Explicit directory entry (--dirs): a MKDIR frame carrying only the + destination path. Directories have no source to remove and no + incremental check. */ + if (!send_directory_entry(client, f)) + return -1; + change_emit_dir_sent(config, f); + continue; + } bool stream = f->data->data == NULL && f->data->size > 0; bool use_sendfile = (config->use_sendfile && !config->use_compression) || (stream && !config->use_compression); @@ -880,6 +972,9 @@ static int send_chunks_multithreaded(void* pipeline_context) { } } +/* Scan thread of the -m pipeline. --dirs disables recursive traversal (the + transfer is a small set of explicit directory/file entries), so it uses the + sequential scanner rather than spawning worker threads. */ static int scan_directory_multithreaded(void* pipeline_context) { PipelineContextSender* context = (PipelineContextSender*)pipeline_context; protocol_session_bind(&context->allocation_session); @@ -889,29 +984,42 @@ static int scan_directory_multithreaded(void* pipeline_context) { protocol_session_unbind(); return thrd_error; } - ParallelScanner* scanner = parallel_scanner_create_with_options( - context->config->send_directory, &prepared.options, &context->allocation_session); - - Chunk* current_chunk; - if (scanner == NULL) { - log_message(LOG_LEVEL_ERROR, "Failed to create parallel scanner"); + bool dirs_mode = prepared.options.dirs; + DirectoryScanner* dscanner = NULL; + ParallelScanner* scanner = NULL; + if (dirs_mode) { + dscanner = + directory_scanner_create_with_options(context->config->send_directory, &prepared.options); + } else { + scanner = parallel_scanner_create_with_options(context->config->send_directory, + &prepared.options, &context->allocation_session); + } + if (dscanner == NULL && scanner == NULL) { + log_message(LOG_LEVEL_ERROR, "Failed to create scanner"); pipeline_cancel(context); prepared_scanner_destroy(&prepared); protocol_session_unbind(); return thrd_error; } - while ((current_chunk = parallel_scanner_next(scanner)) != NULL) { + bool failed = false; + Chunk* current_chunk; + while (1) { + if (dirs_mode) + current_chunk = directory_scanner_next(dscanner); + else + current_chunk = parallel_scanner_next(scanner); + if (current_chunk == NULL) { + failed = dirs_mode ? directory_scanner_failed(dscanner) : parallel_scanner_failed(scanner); + break; + } if (context->config->use_delete) { mtx_lock(&context->mutex_scanner); bool manifest_ok = add_chunk_to_manifest(context->manifest, current_chunk); mtx_unlock(&context->mutex_scanner); if (!manifest_ok) { - pipeline_cancel(context); + failed = true; chunk_destroy(current_chunk); - parallel_scanner_destroy(scanner); - prepared_scanner_destroy(&prepared); - protocol_session_unbind(); - return thrd_error; + break; } } if (!queue_enqueue_multithreaded_cancel( @@ -919,15 +1027,15 @@ static int scan_directory_multithreaded(void* pipeline_context) { &context->condition_not_empty_scanner, &context->condition_not_full_scanner, &context->cancelled)) { chunk_destroy(current_chunk); - pipeline_cancel(context); - parallel_scanner_destroy(scanner); - prepared_scanner_destroy(&prepared); - protocol_session_unbind(); - return thrd_error; + failed = true; + break; } } - if (parallel_scanner_failed(scanner)) { + if (dirs_mode) + directory_scanner_destroy(dscanner); + else parallel_scanner_destroy(scanner); + if (failed) { prepared_scanner_destroy(&prepared); mtx_lock(&context->mutex_scanner); context->scanner_done = true; @@ -943,7 +1051,6 @@ static int scan_directory_multithreaded(void* pipeline_context) { cnd_signal(&context->condition_not_empty_scanner); mtx_unlock(&context->mutex_scanner); - parallel_scanner_destroy(scanner); prepared_scanner_destroy(&prepared); protocol_session_unbind(); return thrd_success; diff --git a/src/client/scanner.c b/src/client/scanner.c index fe8bd43..2876ab3 100644 --- a/src/client/scanner.c +++ b/src/client/scanner.c @@ -306,6 +306,12 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo scanner->file_list = options->file_list; scanner->base_filters = options->base_filters; scanner->per_dir_filters = options->per_dir_filters; + scanner->dirs_mode = options->dirs; + scanner->relative_mode = options->relative && options->file_list != NULL; + scanner->dirs_root_emitted = false; + scanner->list_index = 0; + scanner->dirs_batch = NULL; + scanner->dirs_batch_size = 0; scanner->filter_nodes = NULL; if (scanner->base_filters || scanner->per_dir_filters) { scanner->filter_nodes = array_list_create(filter_node_destroy); @@ -372,6 +378,8 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_ false, NULL, NULL, + false, + false, false}; return directory_scanner_create_with_options(root_directory, &options); } @@ -387,6 +395,7 @@ void directory_scanner_destroy(DirectoryScanner* scanner) { free(scanner->current_rel); free(scanner->root_path); array_list_delete(scanner->filter_nodes); + array_list_delete(scanner->dirs_batch); queue_destroy(scanner->directories); free(scanner); } @@ -448,7 +457,174 @@ static int open_next_directory(DirectoryScanner* scanner) { return 1; } +/* ---- --dirs mode ---- + With -d the scanner transfers directory entries and never recurses into + contents. A plain `-d ` sends only the source-root directory mirror + (created empty at the destination). With -d + --files-from exactly the + listed items are sent: listed directories become empty directory entries and + listed regular files are transferred as files; nothing else is scanned, so + no descent into a listed directory can happen. */ + +/* Build the File for the transfer root directory itself (the `-d ` and + * "." cases). */ +static File* dirs_root_dir_file(DirectoryScanner* scanner) { + struct stat st; + if (stat(scanner->root_path, &st) != 0 || !S_ISDIR(st.st_mode)) { + log_perror("Could not stat source directory"); + scanner->failed = true; + return NULL; + } + File* file = file_create(scanner->root_path); + if (!file) { + scanner->failed = true; + return NULL; + } + file->is_dir = true; + if (scanner->use_metadata) { + file->metadata = file_metadata_create(&st); + if (!file->metadata) { + file_destroy(file); + scanner->failed = true; + return NULL; + } + } + return file; +} + +/* Map one normalized --files-from entry to a File (a directory entry or a + * regular file to transfer), or NULL to skip the entry. */ +static File* dirs_file_for_entry(DirectoryScanner* scanner, const char* entry) { + if (entry[0] == '\0') { + /* "." (whole tree): under -R the bare receive root is the destination and + there is nothing to create for the root itself; otherwise mirror the + source-root directory (empty). */ + if (scanner->relative_mode) + return NULL; + return dirs_root_dir_file(scanner); + } + char* abs_path = path_cat(scanner->root_path, entry); + if (!abs_path) { + scanner->failed = true; + return NULL; + } + struct stat link_stats; + if (lstat(abs_path, &link_stats) != 0) { + log_message(LOG_LEVEL_ERROR, "--dirs listed entry is not present under the source: %s", entry); + free(abs_path); + scanner->failed = true; + return NULL; + } + struct stat effective = link_stats; + if (S_ISLNK(link_stats.st_mode)) { + /* A symlink is transferred (following its referent) only when a link + resolution option is active, mirroring the regular scanner. */ + bool resolve = scanner->follow_symlinks || scanner->copy_links || scanner->safe_links || + scanner->copy_unsafe_links; + if (!resolve || stat(abs_path, &effective) != 0) { + free(abs_path); + return NULL; + } + } + bool is_dir = S_ISDIR(effective.st_mode); + bool is_file = S_ISREG(effective.st_mode); + if (!is_dir && !is_file) { + free(abs_path); + return NULL; + } + File* file = file_create(abs_path); + free(abs_path); + if (!file) { + scanner->failed = true; + return NULL; + } + file->is_dir = is_dir; + file->data->size = is_file ? (unsigned long long)effective.st_size : 0; + if (scanner->relative_mode) { + file->send_path = str_dup(entry); + if (!file->send_path) { + file_destroy(file); + scanner->failed = true; + return NULL; + } + } + if (scanner->use_metadata) { + file->metadata = file_metadata_create(&effective); + if (!file->metadata) { + file_destroy(file); + scanner->failed = true; + return NULL; + } + } + return file; +} + +/* The next File from the --dirs generator, or NULL when exhausted. */ +static File* dirs_next_file(DirectoryScanner* scanner) { + if (!scanner->file_list) { + if (scanner->dirs_root_emitted) + return NULL; + scanner->dirs_root_emitted = true; + return dirs_root_dir_file(scanner); + } + while (scanner->list_index < scanner->file_list->count) { + const char* entry = scanner->file_list->entries[scanner->list_index++]; + File* file = dirs_file_for_entry(scanner, entry); + if (scanner->failed) + return NULL; + if (file) + return file; + } + return NULL; +} + +static Chunk* dirs_flush_batch(DirectoryScanner* scanner) { + if (!scanner->dirs_batch || scanner->dirs_batch->size == 0) { + array_list_delete(scanner->dirs_batch); + scanner->dirs_batch = NULL; + scanner->dirs_batch_size = 0; + return NULL; + } + ArrayList* batch = scanner->dirs_batch; + scanner->dirs_batch = NULL; + scanner->dirs_batch_size = 0; + Chunk* chunk = chunk_data_to_chunk(batch); + if (!chunk) + scanner->failed = true; + return chunk; +} + +static Chunk* directory_scanner_next_dirs(DirectoryScanner* scanner) { + while (scanner->dirs_batch == NULL || scanner->dirs_batch_size <= scanner->chunk_size) { + if (!scanner->dirs_batch) { + scanner->dirs_batch = array_list_create(file_destroy); + if (!scanner->dirs_batch) { + scanner->failed = true; + return NULL; + } + scanner->dirs_batch_size = 0; + } + File* file = dirs_next_file(scanner); + if (scanner->failed) { + dirs_flush_batch(scanner); + return NULL; + } + if (!file) { + return dirs_flush_batch(scanner); + } + if (!array_list_add(scanner->dirs_batch, file)) { + file_destroy(file); + scanner->failed = true; + dirs_flush_batch(scanner); + return NULL; + } + scanner->dirs_batch_size += file->data ? file->data->size : 0; + } + return dirs_flush_batch(scanner); +} + Chunk* directory_scanner_next(DirectoryScanner* scanner) { + if (scanner && scanner->dirs_mode) + return directory_scanner_next_dirs(scanner); ArrayList* chunk_data = array_list_create(file_destroy); if (!chunk_data) { scanner->failed = true; @@ -477,16 +653,27 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) { if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) continue; - ScannerOptions options = {scanner->use_metadata, scanner->chunk_size, - scanner->exclude_patterns, scanner->exclude_count, - scanner->include_patterns, scanner->include_count, - scanner->max_size, scanner->min_size, - scanner->max_depth, 0, - scanner->follow_symlinks, scanner->copy_links, - scanner->safe_links, scanner->copy_unsafe_links, - scanner->checksum, scanner->one_file_system, - scanner->file_list, scanner->base_filters, - scanner->per_dir_filters}; + ScannerOptions options = {scanner->use_metadata, + scanner->chunk_size, + scanner->exclude_patterns, + scanner->exclude_count, + scanner->include_patterns, + scanner->include_count, + scanner->max_size, + scanner->min_size, + scanner->max_depth, + 0, + scanner->follow_symlinks, + scanner->copy_links, + scanner->safe_links, + scanner->copy_unsafe_links, + scanner->checksum, + scanner->one_file_system, + scanner->file_list, + scanner->base_filters, + scanner->per_dir_filters, + false, + false}; ScannerEntry inspected; int inspection = scanner_inspect_entry(&options, scanner->current_path, scanner->current_path, entry->d_name, &inspected); @@ -511,13 +698,23 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) { bool passes_selection = entry_passes_selection(scanner->file_list, scanner->base_filters, scanner->current_node, rel, entry->d_name, is_dir, scanner->per_dir_filters); + /* With -R + --files-from the wire/destination path is the entry's bare + relative path; keep `rel` alive to attach it to a transferred file. */ + char* rel_copy = scanner->relative_mode ? str_dup(rel) : NULL; free(rel); + if (rel_copy == NULL && scanner->relative_mode) { + free(cur_path); + scanner->failed = true; + break; + } if (!passes_selection) { + free(rel_copy); free(cur_path); continue; } if (is_dir) { + free(rel_copy); if (!scanner_same_filesystem(scanner->one_file_system, scanner->root_dev, stats.st_dev)) { free(cur_path); continue; @@ -533,38 +730,45 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) { free(cur_path); } else { if (scanner->max_depth > 0 && scanner->current_depth + 1 > scanner->max_depth) { + free(rel_copy); free(cur_path); continue; } File* file = file_create(cur_path); + free(cur_path); if (file == NULL) { - free(cur_path); + free(rel_copy); scanner->failed = true; continue; } file->data->size = stats.st_size; + if (scanner->relative_mode) { + file->send_path = rel_copy; + rel_copy = NULL; + } if (scanner->use_metadata) file->metadata = file_metadata_create(&stats); if (scanner->use_metadata && !file->metadata) { + free(rel_copy); file_destroy(file); - free(cur_path); scanner->failed = true; break; } if (!array_list_add(chunk_data, file)) { + free(rel_copy); file_destroy(file); scanner->failed = true; break; } chunk_data_size += file->data->size; if (chunk_data_size > scanner->chunk_size) { - free(cur_path); + free(rel_copy); Chunk* result = chunk_data_to_chunk(chunk_data); if (!result) scanner->failed = true; return result; } - free(cur_path); + free(rel_copy); } } @@ -792,12 +996,15 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo } bool passes = entry_passes_selection(options->file_list, options->base_filters, root_node, rel, entry->d_name, is_dir, options->per_dir_filters); - free(rel); + /* -R + --files-from: root-level files keep their bare relative send path. */ + bool use_rel = options->relative && options->file_list != NULL; if (!passes) { + free(rel); free(cur_path); return; } if (is_dir) { + free(rel); if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) { free(cur_path); return; @@ -811,18 +1018,25 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo File* file = file_create(cur_path); free(cur_path); if (!file) { + free(rel); ps->failed = true; return; } file->data->size = st.st_size; + if (use_rel) { + file->send_path = rel; + rel = NULL; + } if (options->use_metadata) file->metadata = file_metadata_create(&st); if (options->use_metadata && !file->metadata) { + free(rel); file_destroy(file); ps->failed = true; return; } if (!array_list_add(root_files, file)) { + free(rel); file_destroy(file); ps->failed = true; } diff --git a/src/client/scanner.h b/src/client/scanner.h index 566daf3..05d67a3 100644 --- a/src/client/scanner.h +++ b/src/client/scanner.h @@ -35,6 +35,8 @@ typedef struct { const FileListSet* file_list; /* --files-from allow-set, or NULL */ const FilterRuleList* base_filters; /* command-line + -C rules, or NULL */ bool per_dir_filters; /* -F: read .rsync-filter per directory */ + bool dirs; /* -d/--dirs: transfer dir entries, no recursion */ + bool relative; /* -R/--relative (dest rel paths, with --files-from) */ } ScannerOptions; /* Internal per-scanner filter state. FilterNode chains represent the ordered @@ -73,6 +75,14 @@ typedef struct { const FileListSet* file_list; const FilterRuleList* base_filters; bool per_dir_filters; + /* --dirs / -R state for the directory-entry generator (dirs_mode replaces + the recursive scan). */ + bool dirs_mode; + bool relative_mode; /* file_list && relative: send bare relative wire paths */ + bool dirs_root_emitted; + int list_index; + ArrayList* dirs_batch; /* owned when non-NULL */ + unsigned long long dirs_batch_size; } DirectoryScanner; typedef struct { diff --git a/src/client/usage.c b/src/client/usage.c index a77ad8a..e9579ba 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -26,8 +26,17 @@ void print_usage(void) { printf(" --delete Delete files on receiver not in source\n"); printf(" --ignore-existing Skip files that already exist on receiver\n"); printf(" --delay-updates Put updated files into place only at the end of transfer\n"); - printf( - " --dirs, --old-dirs, --old-d Transfer directories without recursing (not implemented)\n"); + printf(" --dirs, -d, --old-dirs, --old-d Transfer the named directory entries without\n"); + printf(" recursing into their contents (-d mirrors the source\n"); + printf(" directory empty; with --files-from listed dirs are created\n"); + printf(" empty and listed files are transferred)\n"); + printf(" -R, --relative With --files-from, preserve each listed entry's relative path\n"); + printf(" below the destination root instead of mirroring the full\n"); + printf(" source path (no effect without --files-from)\n"); + printf(" --no-implied-dirs With -R --files-from, refuse to place a listed file whose\n"); + printf(" parent directory is not itself listed\n"); + printf(" --mkpath Create the destination root directory on the server when it\n"); + printf(" does not exist yet\n"); printf(" --del Alias for --delete-during (not implemented)\n"); printf(" --exclude Exclude files matching pattern\n"); printf(" --include Only include files matching pattern\n"); diff --git a/src/server/receiver.c b/src/server/receiver.c index 5cf25b7..b203be6 100644 --- a/src/server/receiver.c +++ b/src/server/receiver.c @@ -133,7 +133,8 @@ int receiver_process(Config* config, int file_descriptor, const ReceiverSink* si if (!receive_status(file_descriptor, &status)) return -1; while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK || - status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH) { + status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH || + status == STATUS_MKDIR) { if (status == STATUS_KEEPALIVE) { if (!send_status(file_descriptor, STATUS_KEEPALIVE)) return -1; @@ -156,6 +157,10 @@ int receiver_process(Config* config, int file_descriptor, const ReceiverSink* si if (!receiver_process_batch(config, file_descriptor)) return -1; goto next; + } else if (status == STATUS_MKDIR) { + File* dir = file_receive_directory(file_descriptor); + if (!dir || !sink->store_file(dir, sink->context)) + goto receive_error; } else { File* file = file_receive(config, file_descriptor); if (!file) { @@ -208,7 +213,7 @@ static bool receiver_save_file(File* file, void* context_pointer) { } else { result = file_save_to_disk_full(context->config->receive_root_directory, file, context->config); } - if (result != FILE_SAVE_ERROR && context->config->remove_source_files && + if (result != FILE_SAVE_ERROR && context->config->remove_source_files && !file->is_dir && !receiver_outcomes_append(&context->outcomes, (unsigned char)result)) { file_destroy(file); return false; diff --git a/src/server/server.c b/src/server/server.c index fa0b001..bdfe5ef 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -63,6 +63,21 @@ static bool path_is_within(const char* root, const char* path) { return strncmp(root, path, n) == 0 && (path[n] == '\0' || path[n] == '/'); } +/* --mkpath contract: when the client's destination root directory does not + exist yet on the server side, --mkpath tells the server to create it (and + any missing leading components) below the authorized root at connection + start. Without --mkpath the destination root must already exist: a missing + root is rejected up front instead of being silently invented by a later + write. Both paths are confined to the authorized root by the secure file + helpers. */ +static bool ensure_receive_root(const Config* config) { + if (!config || !config->receive_root_directory) + return false; + if (config->mkpath) + return file_ensure_directory_secure(config->receive_root_directory); + return file_directory_exists_secure(config->receive_root_directory); +} + static bool __attribute__((unused)) configure_authorization(const char* root) { char resolved[PATH_MAX]; if (!root) { @@ -165,6 +180,17 @@ void handler(int file_descriptor) { return; } config->use_delete = config->use_delete && allow_delete; + /* --mkpath: create the destination root (and its missing leading components) + before anything else; without it the root must pre-exist. A failure here + aborts the connection cleanly before any file data is exchanged. */ + if (!ensure_receive_root(config)) { + log_message(LOG_LEVEL_ERROR, "destination root is not available: %s", + config->receive_root_directory); + config_delete(config); + close(file_descriptor); + protocol_session_unbind(); + return; + } /* A --delay-updates transfer stages under a private 0700 directory inside the receive root. Create it up front (wiping leftovers of any previously interrupted delayed transfer) so a fully-skipped run also starts clean. */ diff --git a/src/shared/chunk.c b/src/shared/chunk.c index 7c258ba..4a40487 100644 --- a/src/shared/chunk.c +++ b/src/shared/chunk.c @@ -65,7 +65,7 @@ void chunk_destroy(void* item) { static unsigned long long per_file_serialize_size(File* file, bool use_metadata) { unsigned long long size = sizeof(size_t); - size_t path_len = strlen(file->path); + size_t path_len = strlen(file_wire_path(file)); unsigned long long metadata_size = use_metadata ? sizeof(int) + (file->metadata ? FILE_METADATA_WIRE_SIZE : 0) : 0; if ((unsigned long long)path_len > ULLONG_MAX - size) @@ -74,6 +74,10 @@ static unsigned long long per_file_serialize_size(File* file, bool use_metadata) if (metadata_size > ULLONG_MAX - size) return 0; size += metadata_size; + /* Entry type marker: 0 = regular file, 1 = explicit directory entry. */ + if (sizeof(int) > ULLONG_MAX - size) + return 0; + size += sizeof(int); if (sizeof(size_t) > ULLONG_MAX - size) return 0; size += sizeof(size_t); @@ -89,7 +93,8 @@ Data* chunk_serialize(Chunk* chunk, bool use_metadata) { for (int i = 0; i < chunk->element_count; i++) { if (!chunk->items[i] || !chunk->items[i]->path || !chunk->items[i]->data || (chunk->items[i]->data->size > 0 && !chunk->items[i]->data->data) || - chunk->items[i]->path[0] == '\0' || has_path_traversal(chunk->items[i]->path)) + chunk->items[i]->path[0] == '\0' || has_path_traversal(chunk->items[i]->path) || + (file_wire_path(chunk->items[i]))[0] == '\0') return NULL; unsigned long long file_size = per_file_serialize_size(chunk->items[i], use_metadata); if (file_size == 0 || file_size > ULLONG_MAX - data_size || data_size + file_size > SIZE_MAX) @@ -104,19 +109,25 @@ Data* chunk_serialize(Chunk* chunk, bool use_metadata) { char* data_pointer = data->data; for (int i = 0; i < chunk->element_count; i++) { File* file = chunk->items[i]; - size_t path_len = strlen(file->path); + const char* wire_path = file_wire_path(file); + size_t path_len = strlen(wire_path); memcpy(data_pointer, &path_len, sizeof(size_t)); data_pointer += sizeof(size_t); - memcpy(data_pointer, file->path, path_len); + memcpy(data_pointer, wire_path, path_len); data_pointer += path_len; + int entry_type = file->is_dir ? 1 : 0; + memcpy(data_pointer, &entry_type, sizeof(int)); + data_pointer += sizeof(int); + if (use_metadata) metadata_to_buf(&data_pointer, file->metadata); size_t file_data_size = file->data->size; memcpy(data_pointer, &file_data_size, sizeof(size_t)); data_pointer += sizeof(size_t); - memcpy(data_pointer, file->data->data, file_data_size); + if (file_data_size > 0) + memcpy(data_pointer, file->data->data, file_data_size); data_pointer += file_data_size; } return data; @@ -187,6 +198,24 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) { return NULL; } + if (remaining_size < sizeof(int)) { + log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for entry type"); + file_destroy(file); + array_list_delete(files); + return NULL; + } + int entry_type; + memcpy(&entry_type, data_pointer, sizeof(int)); + if (entry_type != 0 && entry_type != 1) { + log_message(LOG_LEVEL_ERROR, "Invalid chunk format: bad entry type"); + file_destroy(file); + array_list_delete(files); + return NULL; + } + file->is_dir = entry_type == 1; + data_pointer += sizeof(int); + remaining_size -= sizeof(int); + if (use_metadata) { if (remaining_size < sizeof(int)) { log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for metadata"); diff --git a/src/shared/config.c b/src/shared/config.c index ffdcca7..ed5fdfe 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -102,6 +102,9 @@ static void config_set_defaults(Config* config) { config->prune_empty_dirs = false; config->one_file_system = false; config->relative = false; + config->no_implied_dirs = false; + config->dirs = false; + config->mkpath = false; config->rsh_command = NULL; config->rsync_path = NULL; config->old_args = false; @@ -159,7 +162,8 @@ static bool validate_received_config(const Config* config) { valid_wire_bool(config->use_fsync) && valid_wire_bool(config->append_verify) && valid_wire_bool(config->delete_excluded) && valid_wire_bool(config->delete_after) && valid_wire_bool(config->relative) && valid_wire_bool(config->prune_empty_dirs) && - valid_wire_bool(config->delay_updates) && !(config->delay_updates && config->inplace) && + valid_wire_bool(config->delay_updates) && valid_wire_bool(config->mkpath) && + !(config->delay_updates && config->inplace) && !(config->delay_updates && delay_updates_staging_name_conflict(config->backup_dir)) && valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) && valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) && @@ -307,7 +311,7 @@ static bool send_selection_options(int fd, const Config* c) { send_int(fd, c->use_fsync) && send_int(fd, c->append_verify) && send_int(fd, c->delete_excluded) && send_int(fd, c->delete_after) && send_n_data(fd, &c->max_delete, sizeof(c->max_delete)) && send_int(fd, c->relative) && - send_int(fd, c->prune_empty_dirs); + send_int(fd, c->prune_empty_dirs) && send_int(fd, c->mkpath); } static bool send_skip_compress_options(int fd, const Config* c) { @@ -412,6 +416,8 @@ static bool receive_selection_options(int fd, Config* c) { return false; if (!receive_wire_bool(fd, &c->prune_empty_dirs)) return false; + if (!receive_wire_bool(fd, &c->mkpath)) + return false; return true; } diff --git a/src/shared/config.h b/src/shared/config.h index 904333b..1b86c9c 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -115,7 +115,19 @@ typedef struct Config { bool per_dir_filter; /* -F: apply per-directory .rsync-filter files */ bool prune_empty_dirs; bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */ + /* -R/--relative: crosses the wire; with --files-from listed entries keep + * their bare relative destination path (no source-root mirror prefix). */ bool relative; + /* --no-implied-dirs: client-only. With -R + --files-from, refuse to place a + * listed file whose ancestor directory is not itself explicitly listed. */ + bool no_implied_dirs; + /* -d/--dirs: client-only. Transfer the directory entries named by the + * source argument / --files-from list without recursing into contents. */ + bool dirs; + /* --mkpath: crosses the wire. Tells the server to create the destination + * root directory (and missing leading components below its authorized root) + * at connection start instead of requiring it to already exist. */ + bool mkpath; // Issue #130: Remote shell/connection options char* rsh_command; @@ -162,7 +174,7 @@ typedef struct Config { DelayUpdatesContext* delay_context; } Config; -#define PROTOCOL_VERSION "2.6.0" +#define PROTOCOL_VERSION "2.7.0" #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) Config* config_create(void); diff --git a/src/shared/file.c b/src/shared/file.c index 8d5e54a..8cc13b3 100644 --- a/src/shared/file.c +++ b/src/shared/file.c @@ -73,6 +73,7 @@ File* file_create(const char* path) { memcpy(file->path, path, path_len); file->path[path_len] = '\0'; + file->send_path = NULL; file->data = data_create_reserve(0); if (file->data == NULL) { free(file->path); @@ -81,6 +82,7 @@ File* file_create(const char* path) { } file->metadata = NULL; file->skip = false; + file->is_dir = false; return file; } @@ -94,6 +96,8 @@ void file_destroy(void* item) { file->metadata = NULL; free(file->path); file->path = NULL; + free(file->send_path); + file->send_path = NULL; free(file); } @@ -323,6 +327,27 @@ bool file_ensure_directory_secure(const char* path) { return ok; } +/* True when `path` resolves to an existing directory below the authorized root + * (never creating anything). Used by the server to decide whether a client's + * destination root already exists. */ +bool file_directory_exists_secure(const char* path) { + if (!path) + return false; + char* leaf = NULL; + int parent_fd = file_open_secure_parent(path, &leaf, false); + if (parent_fd < 0) + return false; + int dir_fd = openat(parent_fd, leaf, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (dir_fd < 0 && errno == ENOENT) + dir_fd = -1; + bool ok = dir_fd >= 0; + if (dir_fd >= 0) + close(dir_fd); + close(parent_fd); + free(leaf); + return ok; +} + bool file_rename_secure(const char* old_path, const char* new_path) { char *old_leaf = NULL, *new_leaf = NULL; int old_parent = file_open_secure_parent(old_path, &old_leaf, false); diff --git a/src/shared/file.h b/src/shared/file.h index c5e1721..38c9f01 100644 --- a/src/shared/file.h +++ b/src/shared/file.h @@ -30,6 +30,7 @@ bool file_stat_secure(const char* path, struct stat* st); bool file_destination_is_newer_secure(const char* path, const FileMetadata* metadata); int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs); bool file_ensure_directory_secure(const char* path); +bool file_directory_exists_secure(const char* path); bool file_rename_secure(const char* old_path, const char* new_path); /* Open a private 0700 directory (creating it on demand) that must live below the authorized root. Used for the --temp-dir scratch directory and the diff --git a/src/shared/file_receive.c b/src/shared/file_receive.c index 6f2f7c8..2c4a9c0 100644 --- a/src/shared/file_receive.c +++ b/src/shared/file_receive.c @@ -119,6 +119,24 @@ FileSaveResult file_save_to_disk_full(const char* root_directory, const File* fi return FILE_SAVE_ERROR; } + /* Explicit directory entries (--dirs) carry an empty payload; the entry is + created as a directory under the receive root, applying the same secure + mkdir-parent semantics as regular writes. Directories are created + immediately (they are never staged by --delay-updates, matching rsync, + where directory creation is not delayed). */ + if (file->is_dir) { + if (file->path[0] == '\0' || has_path_traversal(file->path)) { + log_message(LOG_LEVEL_ERROR, "Invalid directory path received"); + return FILE_SAVE_ERROR; + } + char* dir_path = path_cat(root_directory, file->path); + if (!dir_path) + return FILE_SAVE_ERROR; + bool ok = file_ensure_directory_secure(dir_path); + free(dir_path); + return ok ? FILE_SAVE_WRITTEN : FILE_SAVE_ERROR; + } + /* These options arrive from the client. They are names below the server root, never independent filesystem roots. --temp-dir is confined exactly like --backup-dir/--partial-dir: an absolute or `..`-escaping scratch @@ -749,6 +767,31 @@ File* file_receive(const Config* config, int file_descriptor) { return file; } +/* Receive an explicit directory entry (--dirs): a STATUS_MKDIR frame carries + only the destination path; the entry carries no payload. The same path + validation as a regular file applies (non-empty, relative-or-mirrored, no + traversal), and the created File is routed through the regular store_file + sink so single-threaded and -m receivers handle directories identically. */ +File* file_receive_directory(int file_descriptor) { + char* path = receive_str(file_descriptor); + if (path == NULL) + return NULL; + if (path[0] == '\0' || has_path_traversal(path)) { + char* escaped_path = output_escape(path, log_get_8_bit_output()); + log_message(LOG_LEVEL_ERROR, "Invalid received directory path: %s", + escaped_path ? escaped_path : ""); + free(escaped_path); + free(path); + return NULL; + } + File* file = file_create(path); + free(path); + if (file == NULL) + return NULL; + file->is_dir = true; + return file; +} + int receive_manifest(int fd, const Config* config, int* next_status) { if (!config) { send_status(fd, STATUS_ERROR); diff --git a/src/shared/file_receive.h b/src/shared/file_receive.h index 179bdfd..2b7e689 100644 --- a/src/shared/file_receive.h +++ b/src/shared/file_receive.h @@ -8,6 +8,7 @@ /* Server-side file receive/save path. */ File* file_receive(const Config* config, int file_descriptor); +File* file_receive_directory(int file_descriptor); File* receive_incremental_check(int fd, const Config* config, bool* skipped); int receive_manifest(int fd, const Config* config, int* next_status); diff --git a/src/shared/file_send.c b/src/shared/file_send.c index d0546e8..8f6cebe 100644 --- a/src/shared/file_send.c +++ b/src/shared/file_send.c @@ -41,7 +41,7 @@ bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_ } data_to_send = compressed_data; } - if (send_path && !send_str(file_descriptor, file->path)) { + if (send_path && !send_str(file_descriptor, file_wire_path(file))) { data_destroy(compressed_data); return false; } @@ -73,7 +73,7 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta send_path, skip_suffixes, skip_count, compression_threads); - if (send_path && !send_str(file_descriptor, file->path)) + if (send_path && !send_str(file_descriptor, file_wire_path(file))) return false; if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; diff --git a/src/shared/file_types.h b/src/shared/file_types.h index a791fe9..c831813 100644 --- a/src/shared/file_types.h +++ b/src/shared/file_types.h @@ -17,9 +17,24 @@ typedef struct { typedef struct { char* path; + /* Sender-side override for the path transmitted on the wire (and used for + * the delete manifest / change output). NULL means "use `path`". With + * -R + --files-from this holds the entry's bare relative destination path, + * while `path` stays the absolute local source path the client reads from. + * Never populated on the receiver. */ + char* send_path; Data* data; FileMetadata* metadata; bool skip; + /* True when this entry is an explicit directory entry (--dirs mode): the + * receiver creates the directory instead of writing a regular file. */ + bool is_dir; } File; +/* The path that should be sent on the wire and used for the receiver-side + * destination layout (see send_path). */ +static inline const char* file_wire_path(const File* file) { + return file && file->send_path ? file->send_path : (file ? file->path : NULL); +} + #endif diff --git a/src/shared/multiprocessing.c b/src/shared/multiprocessing.c index 06c16f0..b573feb 100644 --- a/src/shared/multiprocessing.c +++ b/src/shared/multiprocessing.c @@ -295,8 +295,9 @@ int write_thread(void* pipeline_context) { } } /* Record the per-file outcome so a --remove-source-files sender learns - which sources were actually written versus skipped on the receiver. */ - if (context->config->remove_source_files && + which sources were actually written versus skipped on the receiver. + Explicit directory entries have no source and are never acknowledged. */ + if (context->config->remove_source_files && !file->is_dir && !receiver_outcomes_append(&context->outcomes, (unsigned char)result)) { file_destroy(file); pipeline_context_receiver_note_bytes_released(context, file_bytes); diff --git a/src/shared/protocol.h b/src/shared/protocol.h index 2b4f66f..d29d2fa 100644 --- a/src/shared/protocol.h +++ b/src/shared/protocol.h @@ -67,7 +67,10 @@ enum NET_STATUS { STATUS_DELTA_DATA, STATUS_KEEPALIVE, STATUS_ABORT, - STATUS_CHECK_BATCH + STATUS_CHECK_BATCH, + /* An explicit directory entry (--dirs): the sender transmits only the path; + * the receiver creates the directory below the receive root. */ + STATUS_MKDIR }; void io_set_fds(int read_fd, int write_fd); diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index 5565ffc..0026c37 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -1635,6 +1635,254 @@ class TestFilesFrom: transfers its whole subtree. The manifest (and thus --delete) derives from what was actually sent.""" + +def _make_relative_source(name): + """A small tree used by the -R/--dirs/--no-implied-dirs tests.""" + source = os.path.join(TEST_DATA_DIR, name) + clean_dir(source) + entries = { + "top.txt": b"top\n", + "a/b.txt": b"nested\n", + "sub/x.txt": b"x\n", + "sub/y.txt": b"y\n", + "dir1/keep.txt": b"dir content\n", + } + for rel, content in entries.items(): + full = os.path.join(source, rel) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "wb") as fh: + fh.write(content) + return source + + +def _write_rel_list(rel_text): + path = os.path.join(TEST_DATA_DIR, "rel_list.txt") + with open(path, "wb") as fh: + fh.write(rel_text) + return path + + +class TestRelativeFilesFrom: + """-R/--relative with --files-from keeps each listed entry's bare relative + destination path below the destination root instead of mirroring the full + source path. Without -R the layout is unchanged (full source mirror).""" + + @pytest.mark.parametrize("mt", [False, True]) + def test_relative_files_from_keeps_relative_layout(self, shared_server, mt): + source = _make_relative_source("rel_src") + dest = os.path.join(TEST_DATA_DIR, "rel_dst") + clean_dir(dest) + lst = _write_rel_list(b"top.txt\nsub/x.txt\n") + flags = ["--files-from", lst, "-R"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"-R files-from sync failed: {result.stderr[:200]}" + assert _read_file(os.path.join(dest, "sub", "x.txt")) == b"x\n", \ + "listed file must land at /sub/x.txt" + assert _read_file(os.path.join(dest, "top.txt")) == b"top\n", \ + "top-level listed file must land at /top.txt" + assert not os.path.exists(os.path.join(dest, "sub", "y.txt")) + # The source-root mirror must not be reproduced under -R. + assert not os.path.exists(get_dest_received_dir(dest, source)), \ + "-R must not mirror the full source path" + + @pytest.mark.parametrize("mt", [False, True]) + def test_relative_without_files_from_has_no_effect(self, shared_server, mt): + """-R alone (no --files-from) must leave the normal full-source mirror + layout untouched.""" + source = _make_relative_source("rel_only_src") + dest = os.path.join(TEST_DATA_DIR, "rel_only_dst") + clean_dir(dest) + flags = ["-R"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"-R alone sync failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + mismatches, missing = verify_transfer(source, received) + assert not missing and not mismatches + + @pytest.mark.parametrize("mt", [False, True]) + def test_without_relative_layout_unchanged(self, shared_server, mt): + source = _make_relative_source("rel_noR_src") + dest = os.path.join(TEST_DATA_DIR, "rel_noR_dst") + clean_dir(dest) + lst = _write_rel_list(b"sub/x.txt\n") + flags = ["--files-from", lst] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"files-from sync failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, "sub", "x.txt")) == b"x\n", \ + "without -R the full source mirror layout is preserved" + assert not os.path.exists(os.path.join(dest, "sub")), \ + "bare relative layout must not appear without -R" + + def test_relative_delete_manifest_stays_consistent(self): + """--delete derives from the sent (-R) relative paths, so a later + subset run removes unlisted relative entries but keeps listed ones.""" + source = _make_relative_source("rel_del_src") + dest = os.path.join(TEST_DATA_DIR, "rel_del_dst") + clean_dir(dest) + with ServerManager() as server: + server.start(extra_args=["--allow-delete"]) + lst = _write_rel_list(b"sub/x.txt\nsub/y.txt\n") + result, _ = run_client(source, dest, flags=["--files-from", lst, "-R"], + port=server.port) + assert result.returncode == 0, f"seed -R sync failed: {result.stderr[:200]}" + assert os.path.isfile(os.path.join(dest, "sub", "y.txt")) + + subset = _write_rel_list(b"sub/x.txt\n") + result, _ = run_client(source, dest, + flags=["--files-from", subset, "-R", "--delete"], + port=server.port) + assert result.returncode == 0, f"-R delete sync failed: {result.stderr[:200]}" + assert os.path.isfile(os.path.join(dest, "sub", "x.txt")), "listed file was deleted" + assert not os.path.exists(os.path.join(dest, "sub", "y.txt")), \ + "unlisted relative file was not deleted" + + +class TestNoImpliedDirs: + """--no-implied-dirs (only meaningful with -R + --files-from) refuses to + place a listed file whose parent directory is not itself listed.""" + + def _make(self): + return _make_relative_source("noimplied_src") + + @pytest.mark.parametrize("mt", [False, True]) + def test_implied_dir_only_fails_entry(self, shared_server, mt): + source = self._make() + dest = os.path.join(TEST_DATA_DIR, "noimplied_dst") + clean_dir(dest) + lst = _write_rel_list(b"a/b.txt\n") # "a" itself is not listed + flags = ["--files-from", lst, "-R", "--no-implied-dirs"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode != 0, "implied parent directory was not rejected" + assert "--no-implied-dirs" in (result.stderr or result.stdout) + assert not os.path.exists(os.path.join(dest, "a", "b.txt")) + + @pytest.mark.parametrize("mt", [False, True]) + def test_listed_dir_allows_file(self, shared_server, mt): + source = self._make() + dest = os.path.join(TEST_DATA_DIR, "noimplied_ok_dst") + clean_dir(dest) + lst = _write_rel_list(b"a\na/b.txt\n") + flags = ["--files-from", lst, "-R", "--no-implied-dirs"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"listed dir + file sync failed: {result.stderr[:200]}" + assert _read_file(os.path.join(dest, "a", "b.txt")) == b"nested\n" + + @pytest.mark.parametrize("mt", [False, True]) + def test_no_implied_dirs_without_relative_changes_nothing(self, shared_server, mt): + source = self._make() + dest = os.path.join(TEST_DATA_DIR, "noimplied_noR_dst") + clean_dir(dest) + lst = _write_rel_list(b"a/b.txt\n") + flags = ["--files-from", lst, "--no-implied-dirs"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, "--no-implied-dirs without -R changed behavior" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, "a", "b.txt")) == b"nested\n" + + +class TestDirs: + """-d/--dirs (and the --old-dirs/--old-d aliases) transfer directory entries + without recursing into their contents.""" + + def _make(self): + return _make_relative_source("dirs_src") + + def _assert_only_empty_mirror(self, dest, source): + mirror = get_dest_received_dir(dest, source) + assert os.path.isdir(mirror), "source-root mirror directory was not created" + files = [] + for root, _dirs, names in os.walk(mirror): + files.extend(os.path.relpath(os.path.join(root, n), mirror) for n in names) + assert files == [], f"--dirs descended into contents: {files}" + + @pytest.mark.parametrize("flag", ["--dirs", "-d", "--old-dirs", "--old-d"]) + @pytest.mark.parametrize("mt", [False, True]) + def test_dirs_transfers_empty_dir_only(self, shared_server, flag, mt): + source = self._make() + dest = os.path.join(TEST_DATA_DIR, "dirs_dst") + clean_dir(dest) + flags = [flag] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"{flag} sync failed: {result.stderr[:200]}" + self._assert_only_empty_mirror(dest, source) + + @pytest.mark.parametrize("mt", [False, True]) + def test_dirs_with_files_from(self, shared_server, mt): + source = self._make() + dest = os.path.join(TEST_DATA_DIR, "dirs_ff_dst") + clean_dir(dest) + # A listed directory is created empty; a listed file is transferred. + lst = _write_rel_list(b"dir1\nsub/x.txt\n") + flags = ["--files-from", lst, "--dirs", "-R"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"dirs files-from sync failed: {result.stderr[:200]}" + assert os.path.isdir(os.path.join(dest, "dir1")), "listed dir was not created" + assert not os.path.exists(os.path.join(dest, "dir1", "keep.txt")), \ + "--dirs must not descend into a listed directory" + assert _read_file(os.path.join(dest, "sub", "x.txt")) == b"x\n", \ + "listed file content was not transferred" + assert not os.path.exists(os.path.join(dest, "sub", "y.txt")), \ + "unlisted file appeared" + + @pytest.mark.parametrize("mt", [False, True]) + def test_dirs_with_files_from_mirror_layout(self, shared_server, mt): + """Without -R the dirs+files-from entries still mirror the source path.""" + source = self._make() + dest = os.path.join(TEST_DATA_DIR, "dirs_ff_noR_dst") + clean_dir(dest) + lst = _write_rel_list(b"dir1\n") + flags = ["--files-from", lst, "--dirs"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"dirs files-from no-R sync failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + assert os.path.isdir(os.path.join(received, "dir1")), "mirrored dir entry not created" + assert not os.path.exists(os.path.join(received, "dir1", "keep.txt")), \ + "--dirs must not descend into a listed directory" + assert not os.path.exists(os.path.join(received, "sub")), \ + "unlisted subtree appeared" + + +class TestMkpath: + """--mkpath tells the server to create the destination root directory (and + missing leading components) when it does not exist yet; without it a missing + destination root fails the transfer.""" + + def _transfer(self, dest, mt, mkpath): + source = _make_relative_source("mkpath_src") + flags = ["--mkpath"] if mkpath else [] + if mt: + flags += ["-m"] + result, _ = run_client(source, dest, flags=flags, port=self.server.port) + return result + + @pytest.mark.parametrize("mt", [False, True]) + def test_missing_root_fails_without_mkpath(self, mt): + source = _make_relative_source("mkpath_fail_src") + dest = os.path.join(TEST_DATA_DIR, "mkpath_missing_dst") + shutil.rmtree(dest, ignore_errors=True) + with ServerManager() as server: + server.start() + flags = ["-m"] if mt else [] + result, _ = run_client(source, dest, flags=flags, port=server.port) + assert result.returncode != 0, "missing destination root did not fail without --mkpath" + assert not os.path.exists(dest), "missing root was created without --mkpath" + + @pytest.mark.parametrize("mt", [False, True]) + def test_mkpath_creates_missing_root(self, mt): + source = _make_relative_source("mkpath_ok_src") + dest = os.path.join(TEST_DATA_DIR, "deep", "mkpath_dst") + shutil.rmtree(os.path.join(TEST_DATA_DIR, "deep"), ignore_errors=True) + with ServerManager() as server: + server.start() + flags = ["--mkpath"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=server.port) + assert result.returncode == 0, f"--mkpath sync failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, "sub", "x.txt")) == b"x\n", \ + "file not transferred into the --mkpath-created root" + + class TestFilters: """--filter/-C/-F rule layer: excludes prune, ordering is first-match-wins, the default with no matching rule is include, and legacy --exclude remains diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index dbb68eb..425d8aa 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -547,34 +547,54 @@ static void test_parse_args_unknown_option() { config_delete(cfg); } -/* Directory aliases must report the unsupported directory-only behavior clearly. */ -static void test_parse_args_rejects_dirs_aliases() { - static const char* const options[] = {"--dirs", "--old-dirs", "--old-d"}; - +/* -d/--dirs and the rsync --old-dirs/--old-d aliases all enable directory-only + * transfers (--dirs maps every spelling onto the same config field). */ +static void test_parse_args_dirs_aliases() { + static const char* const options[] = {"--dirs", "-d", "--old-dirs", "--old-d"}; for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) { Config* cfg = config_create(); char* argv[] = {"fastsync", (char*)options[i], "/src", "/dst"}; int positional_args[2]; int positional_count = 0; - FILE* log_file = tmpfile(); - char log_buffer[256] = {0}; - - EXPECT_NOT_NULL(log_file); - log_set_file(log_file); - - EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), -1); - fflush(log_file); - rewind(log_file); - EXPECT_TRUE(fread(log_buffer, 1, sizeof(log_buffer) - 1, log_file) > 0); - EXPECT_TRUE(strstr(log_buffer, options[i]) != NULL); - EXPECT_TRUE(strstr(log_buffer, "directory-only transfer is not implemented") != NULL); - EXPECT_TRUE(strstr(log_buffer, "requires --dirs") == NULL); - log_set_file(NULL); - fclose(log_file); + EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->dirs); config_delete(cfg); } } +/* -R/--relative, --no-implied-dirs and --mkpath are plain boolean flags. */ +static void test_parse_args_relative_no_implied_mkpath() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "-R", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->relative); + config_delete(cfg); + + cfg = config_create(); + positional_count = 0; + char* long_argv[] = {"fastsync", "--relative", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 4, long_argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->relative); + config_delete(cfg); + + cfg = config_create(); + positional_count = 0; + char* noimplied_argv[] = {"fastsync", "--no-implied-dirs", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 4, noimplied_argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->no_implied_dirs); + EXPECT_FALSE(cfg->relative); + config_delete(cfg); + + cfg = config_create(); + positional_count = 0; + char* mkpath_argv[] = {"fastsync", "--mkpath", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 4, mkpath_argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->mkpath); + config_delete(cfg); +} + /* --del is recognized as the rsync alias, but its timing mode is not implemented. */ static void test_parse_args_delete_during_alias_unimplemented() { static const char* const options[] = {"--del", "--delete-during"}; @@ -609,8 +629,6 @@ static void test_parse_args_rejects_unimplemented_options() { "--delete-after", "--max-delete", "--prune-empty-dirs", - "-R", - "--relative", "-e", "--rsh", "--rsync-path", @@ -1522,7 +1540,8 @@ void test_client_cli() { test_parse_args_max_alloc_sizes(); test_parse_args_rejects_invalid_max_alloc(); test_parse_args_unknown_option(); - test_parse_args_rejects_dirs_aliases(); + test_parse_args_dirs_aliases(); + test_parse_args_relative_no_implied_mkpath(); test_parse_args_delete_during_alias_unimplemented(); test_parse_args_rejects_unimplemented_options(); test_parse_args_quiet(); diff --git a/tests/test_config.c b/tests/test_config.c index 147bdd9..a89b498 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -137,6 +137,8 @@ static void test_config_send_receive() { send_cfg->existing = true; send_cfg->ignore_existing = true; send_cfg->delay_updates = true; + send_cfg->relative = true; + send_cfg->mkpath = true; send_cfg->skip_compress_set = true; send_cfg->skip_compress_count = 1; send_cfg->skip_compress_suffixes = calloc(1, sizeof(char*)); @@ -194,6 +196,10 @@ static void test_config_send_receive() { ok = false; if (!recv_cfg->delay_updates) ok = false; + if (!recv_cfg->relative) + ok = false; + if (!recv_cfg->mkpath) + ok = false; if (!recv_cfg->skip_compress_set || recv_cfg->skip_compress_count != 1 || strcmp(recv_cfg->skip_compress_suffixes[0], ".zip") != 0) ok = false; diff --git a/tests/test_file.c b/tests/test_file.c index 1bc5d8b..4677eaa 100644 --- a/tests/test_file.c +++ b/tests/test_file.c @@ -886,6 +886,42 @@ static void test_inplace_overwrite_truncates_shorter_payload() { rmdir(root); } +/* Explicit directory entries (--dirs) create the directory under the receive + root through the same save funnel, creating parents as needed, and reject + traversal the same way a file path does. */ +static void test_dir_entry_save_to_disk() { + const char* root = "test_dir_entry_root"; + EXPECT_EQ_INT(mkdir(root, 0755), 0); + + Config* config = config_create(); + EXPECT_NOT_NULL(config); + + File* dir = file_create("alpha/beta/gamma"); + EXPECT_NOT_NULL(dir); + dir->is_dir = true; + EXPECT_EQ_INT(file_save_to_disk_full(root, dir, config), FILE_SAVE_WRITTEN); + EXPECT_EQ_INT(file_save_to_disk_full(root, dir, config), FILE_SAVE_WRITTEN); + file_destroy(dir); + + struct stat st; + EXPECT_EQ_INT(stat("test_dir_entry_root/alpha/beta/gamma", &st), 0); + EXPECT_TRUE(S_ISDIR(st.st_mode)); + + /* The directory-entry save path never follows or escapes. */ + File* evil = file_create("../dir_entry_escape"); + EXPECT_NOT_NULL(evil); + evil->is_dir = true; + EXPECT_EQ_INT(file_save_to_disk_full(root, evil, config), FILE_SAVE_ERROR); + file_destroy(evil); + EXPECT_EQ_INT(lstat("../dir_entry_escape", &st), -1); + + config_delete(config); + rmdir("test_dir_entry_root/alpha/beta/gamma"); + rmdir("test_dir_entry_root/alpha/beta"); + rmdir("test_dir_entry_root/alpha"); + rmdir(root); +} + void test_file() { test_file_create(); test_file_destroy_null(); @@ -906,6 +942,7 @@ void test_file() { test_file_content_to_buffer(); test_file_save_to_disk_path_traversal(); test_file_save_to_disk_deep_traversal(); + test_dir_entry_save_to_disk(); if (!is_running_under_valgrind()) { // Fork tests are skipped under valgrind because the parent process runs // orders of magnitude slower than the child (parent is instrumented, child diff --git a/tests/test_scanner.c b/tests/test_scanner.c index e31f5e5..87fb497 100644 --- a/tests/test_scanner.c +++ b/tests/test_scanner.c @@ -397,8 +397,9 @@ static void test_parallel_scanner_root_chunks_without_workers() { create_test_file(file1, "a"); create_test_file(file2, "b"); - ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0, 0, 0, - false, false, false, false, false, false, NULL, NULL, false}; + ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, + 0, 0, 0, false, false, false, false, + false, false, NULL, NULL, false, false, false}; ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options, NULL); EXPECT_NOT_NULL(scanner); @@ -1078,6 +1079,190 @@ static void test_per_dir_filter_override(bool parallel) { rmdir(root); } +typedef struct { + char rel[512]; + char send[512]; + bool is_dir; +} ScanInfo; + +/* Collect every scanner entry below `root` into `out` (at most `max`), mapping + * paths to their root-relative form and capturing send_path and is_dir. */ +static int collect_scan_info(const char* root, const ScannerOptions* options, ScanInfo out[], + int max) { + DirectoryScanner* scanner = directory_scanner_create_with_options(root, options); + if (!scanner) + return -1; + size_t root_len = strlen(root); + while (root_len > 0 && root[root_len - 1] == '/') + root_len--; + int count = 0; + Chunk* chunk; + while ((chunk = directory_scanner_next(scanner)) != NULL) { + for (int i = 0; i < chunk->element_count && count < max; i++) { + const File* f = chunk->items[i]; + const char* rel = f->path + root_len; + if (*rel == '/') + rel++; + snprintf(out[count].rel, sizeof(out[count].rel), "%s", rel); + snprintf(out[count].send, sizeof(out[count].send), "%s", f->send_path ? f->send_path : ""); + out[count].is_dir = f->is_dir; + count++; + } + chunk_destroy(chunk); + } + bool failed = directory_scanner_failed(scanner); + directory_scanner_destroy(scanner); + return failed ? -1 : count; +} + +static bool scan_info_present(const ScanInfo* infos, int count, const char* rel, bool is_dir, + const char* send) { + for (int i = 0; i < count; i++) { + if (strcmp(infos[i].rel, rel) == 0 && infos[i].is_dir == is_dir && + strcmp(infos[i].send, send ? send : "") == 0) + return true; + } + return false; +} + +/* Parallel variant of collect_scan_info; drains `scanner` fully and destroys + * it. */ +static int collect_scan_info_parallel(ParallelScanner* scanner, const char* root, ScanInfo out[], + int max) { + if (!scanner) + return -1; + size_t root_len = strlen(root); + while (root_len > 0 && root[root_len - 1] == '/') + root_len--; + int count = 0; + Chunk* chunk; + while ((chunk = parallel_scanner_next(scanner)) != NULL) { + for (int i = 0; i < chunk->element_count && count < max; i++) { + const File* f = chunk->items[i]; + const char* rel = f->path + root_len; + if (*rel == '/') + rel++; + snprintf(out[count].rel, sizeof(out[count].rel), "%s", rel); + snprintf(out[count].send, sizeof(out[count].send), "%s", f->send_path ? f->send_path : ""); + out[count].is_dir = f->is_dir; + count++; + } + chunk_destroy(chunk); + } + bool failed = parallel_scanner_failed(scanner); + parallel_scanner_destroy(scanner); + return failed ? -1 : count; +} + +/* -d without --files-from emits exactly the source-root directory (empty) and + * never descends. */ +static void test_dirs_no_descent() { + const char* root = "test_scan_dirs_root"; + EXPECT_EQ_INT(mkdir(root, 0755), 0); + EXPECT_EQ_INT(mkdir("test_scan_dirs_root/sub", 0755), 0); + create_test_file("test_scan_dirs_root/a.txt", "a"); + create_test_file("test_scan_dirs_root/sub/b.txt", "b"); + + ScannerOptions options = {0}; + options.dirs = true; + ScanInfo infos[8]; + int count = collect_scan_info(root, &options, infos, 8); + EXPECT_EQ_INT(count, 1); + EXPECT_TRUE(scan_info_present(infos, count, "", true, NULL)); + EXPECT_FALSE(scan_info_present(infos, count, "a.txt", false, "")); + EXPECT_FALSE(scan_info_present(infos, count, "sub/b.txt", false, "")); + + unlink("test_scan_dirs_root/a.txt"); + unlink("test_scan_dirs_root/sub/b.txt"); + rmdir("test_scan_dirs_root/sub"); + rmdir(root); +} + +/* -d with --files-from transfers exactly the listed directory (empty) and the + * listed file; nothing is descended into. */ +static void test_dirs_files_from() { + const char* root = "test_scan_dirs_ff"; + const char* list_path = "test_scan_dirs_ff.list"; + EXPECT_EQ_INT(mkdir(root, 0755), 0); + EXPECT_EQ_INT(mkdir("test_scan_dirs_ff/sub", 0755), 0); + create_test_file("test_scan_dirs_ff/sub/keep.txt", "keep"); + create_test_file("test_scan_dirs_ff/sub/skip.bin", "skip"); + create_test_file("test_scan_dirs_ff/top.txt", "top"); + + char err[160]; + create_test_file(list_path, "sub\nsub/keep.txt\n"); + FileListSet* set = file_list_load(list_path, false, err, sizeof(err)); + EXPECT_NOT_NULL(set); + + for (int relative = 0; relative <= 1; relative++) { + ScannerOptions options = {0}; + options.dirs = true; + options.file_list = set; + options.relative = relative != 0; + ScanInfo infos[8]; + int count = collect_scan_info(root, &options, infos, 8); + EXPECT_EQ_INT(count, 2); + if (relative) { + EXPECT_TRUE(scan_info_present(infos, count, "sub", true, "sub")); + EXPECT_TRUE(scan_info_present(infos, count, "sub/keep.txt", false, "sub/keep.txt")); + } else { + EXPECT_TRUE(scan_info_present(infos, count, "sub", true, NULL)); + EXPECT_TRUE(scan_info_present(infos, count, "sub/keep.txt", false, NULL)); + } + EXPECT_FALSE(scan_info_present(infos, count, "sub/skip.bin", false, "")); + EXPECT_FALSE(scan_info_present(infos, count, "top.txt", false, "")); + } + file_list_destroy(set); + remove(list_path); + unlink("test_scan_dirs_ff/sub/keep.txt"); + unlink("test_scan_dirs_ff/sub/skip.bin"); + unlink("test_scan_dirs_ff/top.txt"); + rmdir("test_scan_dirs_ff/sub"); + rmdir(root); +} + +/* -R with --files-from (no -d): every file keeps its bare relative path as the + * send_path while the local scan path stays absolute-under-root. */ +static void test_files_from_relative_send_path() { + const char* root = "test_scan_rel_ff"; + const char* list_path = "test_scan_rel_ff.list"; + EXPECT_EQ_INT(mkdir(root, 0755), 0); + EXPECT_EQ_INT(mkdir("test_scan_rel_ff/sub", 0755), 0); + create_test_file("test_scan_rel_ff/root.txt", "root"); + create_test_file("test_scan_rel_ff/sub/keep.txt", "keep"); + + char err[160]; + create_test_file(list_path, "root.txt\nsub/keep.txt\n"); + FileListSet* set = file_list_load(list_path, false, err, sizeof(err)); + EXPECT_NOT_NULL(set); + + for (int parallel = 0; parallel <= 1; parallel++) { + ScannerOptions options = {0}; + options.file_list = set; + options.relative = true; + if (parallel) + options.num_threads = 2; + ScanInfo infos[8]; + int count; + if (parallel) { + ParallelScanner* scanner = parallel_scanner_create_with_options(root, &options, NULL); + EXPECT_NOT_NULL(scanner); + count = collect_scan_info_parallel(scanner, root, infos, 8); + } else { + count = collect_scan_info(root, &options, infos, 8); + } + EXPECT_EQ_INT(count, 2); + EXPECT_TRUE(scan_info_present(infos, count, "root.txt", false, "root.txt")); + EXPECT_TRUE(scan_info_present(infos, count, "sub/keep.txt", false, "sub/keep.txt")); + } + file_list_destroy(set); + remove(list_path); + unlink("test_scan_rel_ff/root.txt"); + unlink("test_scan_rel_ff/sub/keep.txt"); + rmdir("test_scan_rel_ff/sub"); + rmdir(root); +} + void test_scanner() { test_scanner_single_file(); test_scanner_multiple_files(); @@ -1110,4 +1295,7 @@ void test_scanner() { test_scanner_path_relative(); test_per_dir_filter_override(false); test_per_dir_filter_override(true); + test_dirs_no_descent(); + test_dirs_files_from(); + test_files_from_relative_send_path(); }