Add rsync-compatible --existing #240

Closed
TapTap wants to merge 2 commits from feat/existing into dev
11 changed files with 133 additions and 20 deletions
+2 -1
View File
@@ -102,6 +102,7 @@ partial, alternate, and planned behavior.
| `--max-size <n>` | Skip files larger than n bytes | | `--max-size <n>` | Skip files larger than n bytes |
| `--min-size <n>` | Skip files smaller than n bytes | | `--min-size <n>` | Skip files smaller than n bytes |
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `-s`. | | `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `-s`. |
| `--existing` | Skip files not already present at the destination; update existing files normally. |
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second | | `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) | | `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
| `--timeout <sec>` | I/O timeout in seconds (default: 30) | | `--timeout <sec>` | I/O timeout in seconds (default: 30) |
@@ -465,7 +466,7 @@ defaults to the current directory. |
## Protocol and Security ## Protocol and Security
FastSync protocol version `2.2.0` is shared by the client and server. The FastSync protocol version `2.3.0` is shared by the client and server. The
current protocol is sender-driven and includes configuration negotiation, current protocol is sender-driven and includes configuration negotiation,
incremental checks, checksums, manifests, keep-alives, abort handling, and incremental checks, checksums, manifests, keep-alives, abort handling, and
FastSync-native delta messages. Client and server versions must currently FastSync-native delta messages. Client and server versions must currently
+6 -6
View File
@@ -6,11 +6,11 @@ This document maps rsync's full feature set to FastSync's current implementation
| Status | Count | Description | | Status | Count | Description |
|--------|-------|-------------| |--------|-------|-------------|
| ✅ Implemented | 34 | Feature works end-to-end | | ✅ Implemented | 35 | Feature works end-to-end |
| 🔀 Alt Arg | 3 | Functionality exists but under different flag/semantics | | 🔀 Alt Arg | 3 | Functionality exists but under different flag/semantics |
| ⚠️ Partial | 1 | Flag parsed/stored but behavior incomplete | | ⚠️ Partial | 3 | Flag parsed/stored but behavior incomplete |
| ❌ Not Implemented | 98 | Flag not recognized or no behavior | | ❌ Not Implemented | 97 | Flag not recognized or no behavior |
| **Total** | **136** | | | **Total** | **138** | |
--- ---
@@ -60,7 +60,7 @@ This document maps rsync's full feature set to FastSync's current implementation
| `-I`, `--ignore-times` | Don't skip files matching size+time | ❌ Not Implemented | | | `-I`, `--ignore-times` | Don't skip files matching size+time | ❌ Not Implemented | |
| `--size-only` | Skip based on size only | ❌ Not Implemented | | | `--size-only` | Skip based on size only | ❌ Not Implemented | |
| `-@`, `--modify-window=NUM` | Mod-time comparison accuracy | ❌ Not Implemented | | | `-@`, `--modify-window=NUM` | Mod-time comparison accuracy | ❌ Not Implemented | |
| `--existing` | Skip creating new files on receiver | ❌ Not Implemented | | | `--existing` | Skip creating new files on receiver | ✅ Implemented | Existing destination files continue through normal update handling |
| `--ignore-existing` | Skip updating existing files | ❌ Not Implemented | | | `--ignore-existing` | Skip updating existing files | ❌ Not Implemented | |
| `--remove-source-files` | Sender removes synced files | ❌ Not Implemented | | | `--remove-source-files` | Sender removes synced files | ❌ Not Implemented | |
@@ -247,7 +247,7 @@ Ranked by user demand, implementation complexity, and interoperability impact:
| 1 | `--whole-file` / `-W` | Low | High — users expect opt-out of delta | | 1 | `--whole-file` / `-W` | Low | High — users expect opt-out of delta |
| 2 | `--ignore-times` / `-I` | Low | Medium — useful for forcing re-transfer | | 2 | `--ignore-times` / `-I` | Low | Medium — useful for forcing re-transfer |
| 3 | `--size-only` | Low | Medium — common migration scenario | | 3 | `--size-only` | Low | Medium — common migration scenario |
| 4 | `--existing` / `--ignore-existing` | Low | Medium — common sync patterns | | 4 | `--ignore-existing` | Low | Medium — common sync patterns |
| 5 | `--remove-source-files` | Low | High — common for moves/backup | | 5 | `--remove-source-files` | Low | High — common for moves/backup |
| 6 | `--delete-during` | Medium | High — performance improvement | | 6 | `--delete-during` | Medium | High — performance improvement |
| 7 | `--delay-updates` | Medium | High — atomic updates | | 7 | `--delay-updates` | Medium | High — atomic updates |
+1
View File
@@ -153,6 +153,7 @@ static const OptionEntry OPTION_TABLE[] = {
{"--sparse", "-S", OPT_FLAG, offsetof(Config, preserve_sparse)}, {"--sparse", "-S", OPT_FLAG, offsetof(Config, preserve_sparse)},
{"--inplace", NULL, OPT_FLAG, offsetof(Config, inplace)}, {"--inplace", NULL, OPT_FLAG, offsetof(Config, inplace)},
{"--checksum", NULL, OPT_FLAG, offsetof(Config, checksum)}, {"--checksum", NULL, OPT_FLAG, offsetof(Config, checksum)},
{"--existing", NULL, OPT_FLAG, offsetof(Config, existing)},
{"--source-dir", NULL, OPT_STRING, offsetof(Config, send_directory)}, {"--source-dir", NULL, OPT_STRING, offsetof(Config, send_directory)},
{"--dest-dir", NULL, OPT_STRING, offsetof(Config, receive_root_directory)}, {"--dest-dir", NULL, OPT_STRING, offsetof(Config, receive_root_directory)},
+1
View File
@@ -28,6 +28,7 @@ void print_usage(void) {
printf(" --max-size <n> Skip files larger than n bytes\n"); printf(" --max-size <n> Skip files larger than n bytes\n");
printf(" --min-size <n> Skip files smaller than n bytes\n"); printf(" --min-size <n> Skip files smaller than n bytes\n");
printf(" --incremental Skip files unchanged since last transfer\n"); printf(" --incremental Skip files unchanged since last transfer\n");
printf(" --existing Skip files not already present at destination\n");
printf(" --delta Delta transfer for changed files (requires --incremental)\n"); printf(" --delta Delta transfer for changed files (requires --incremental)\n");
printf(" --delta-block <n> Delta block size in bytes (default: %d)\n", printf(" --delta-block <n> Delta block size in bytes (default: %d)\n",
DELTA_BLOCK_SIZE_DEFAULT); DELTA_BLOCK_SIZE_DEFAULT);
+14 -11
View File
@@ -68,6 +68,7 @@ static void config_set_defaults(Config* config) {
config->debug_level = 0; config->debug_level = 0;
config->list_only = false; config->list_only = false;
config->human_readable = false; config->human_readable = false;
config->existing = false;
config->update = false; config->update = false;
config->inplace = false; config->inplace = false;
config->append = false; config->append = false;
@@ -123,12 +124,13 @@ static bool validate_received_config(const Config* config) {
valid_wire_bool(config->copy_unsafe_links) && valid_wire_bool(config->copy_unsafe_links) &&
valid_wire_bool(config->preserve_hard_links) && valid_wire_bool(config->preserve_acls) && valid_wire_bool(config->preserve_hard_links) && valid_wire_bool(config->preserve_acls) &&
valid_wire_bool(config->preserve_xattrs) && valid_wire_bool(config->preserve_devices) && valid_wire_bool(config->preserve_xattrs) && valid_wire_bool(config->preserve_devices) &&
valid_wire_bool(config->preserve_sparse) && valid_wire_bool(config->update) && valid_wire_bool(config->preserve_sparse) && valid_wire_bool(config->existing) &&
valid_wire_bool(config->inplace) && valid_wire_bool(config->append) && valid_wire_bool(config->update) && valid_wire_bool(config->inplace) &&
valid_wire_bool(config->append_verify) && valid_wire_bool(config->delete_excluded) && valid_wire_bool(config->append) && valid_wire_bool(config->append_verify) &&
valid_wire_bool(config->delete_after) && valid_wire_bool(config->relative) && valid_wire_bool(config->delete_excluded) && valid_wire_bool(config->delete_after) &&
valid_wire_bool(config->prune_empty_dirs) && valid_wire_bool(config->partial) && valid_wire_bool(config->relative) && valid_wire_bool(config->prune_empty_dirs) &&
valid_wire_bool(config->delete_before) && valid_wire_bool(config->checksum) && valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
valid_wire_bool(config->checksum) &&
(!config->use_compression || (!config->use_compression ||
(config->compression_level >= 1 && config->compression_level <= 22)) && (config->compression_level >= 1 && config->compression_level <= 22)) &&
config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE && config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE &&
@@ -243,10 +245,11 @@ static bool send_file_options(int fd, const Config* c) {
} }
static bool send_selection_options(int fd, const Config* c) { static bool send_selection_options(int fd, const Config* c) {
return send_int(fd, c->update) && send_int(fd, c->inplace) && send_int(fd, c->append) && return send_int(fd, c->existing) && send_int(fd, c->update) && send_int(fd, c->inplace) &&
send_int(fd, c->append_verify) && send_int(fd, c->delete_excluded) && send_int(fd, c->append) && send_int(fd, c->append_verify) &&
send_int(fd, c->delete_after) && send_n_data(fd, &c->max_delete, sizeof(c->max_delete)) && send_int(fd, c->delete_excluded) && send_int(fd, c->delete_after) &&
send_int(fd, c->relative) && send_int(fd, c->prune_empty_dirs); send_n_data(fd, &c->max_delete, sizeof(c->max_delete)) && send_int(fd, c->relative) &&
send_int(fd, c->prune_empty_dirs);
} }
static bool send_resume_options(int fd, const Config* c) { static bool send_resume_options(int fd, const Config* c) {
@@ -304,7 +307,7 @@ static bool receive_file_options(int fd, Config* c) {
} }
static bool receive_selection_options(int fd, Config* c) { static bool receive_selection_options(int fd, Config* c) {
bool* flags[] = {&c->update, &c->inplace, &c->append, bool* flags[] = {&c->existing, &c->update, &c->inplace, &c->append,
&c->append_verify, &c->delete_excluded, &c->delete_after}; &c->append_verify, &c->delete_excluded, &c->delete_after};
for (size_t i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) { for (size_t i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
if (!receive_wire_bool(fd, flags[i])) if (!receive_wire_bool(fd, flags[i]))
+2 -1
View File
@@ -76,6 +76,7 @@ typedef struct Config {
bool human_readable; bool human_readable;
// Issue #127: Transfer modes // Issue #127: Transfer modes
bool existing;
bool update; bool update;
bool inplace; bool inplace;
bool append; bool append;
@@ -128,7 +129,7 @@ typedef struct Config {
char* compress_choice; char* compress_choice;
} Config; } Config;
#define PROTOCOL_VERSION "2.2.0" #define PROTOCOL_VERSION "2.3.0"
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
Config* config_create(void); Config* config_create(void);
+19 -1
View File
@@ -29,6 +29,7 @@ bool file_save_to_disk(const char* root_directory, const File* file, const Confi
const char* backup_dir = (config && config->backup_dir) ? config->backup_dir : NULL; const char* backup_dir = (config && config->backup_dir) ? config->backup_dir : NULL;
const char* partial_dir = (config && config->partial_dir) ? config->partial_dir : NULL; const char* partial_dir = (config && config->partial_dir) ? config->partial_dir : NULL;
char *confined_backup = NULL, *confined_partial = NULL, *disk_path = NULL; char *confined_backup = NULL, *confined_partial = NULL, *disk_path = NULL;
char* destination_path = NULL;
char *backup_path = NULL, *parent_copy = NULL; char *backup_path = NULL, *parent_copy = NULL;
if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data) || if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data) ||
@@ -54,13 +55,27 @@ bool file_save_to_disk(const char* root_directory, const File* file, const Confi
const char* actual_root = const char* actual_root =
(partial_dir && config && config->partial) ? confined_partial : root_directory; (partial_dir && config && config->partial) ? confined_partial : root_directory;
destination_path = path_cat(root_directory, file->path);
disk_path = path_cat(actual_root, file->path); disk_path = path_cat(actual_root, file->path);
if (disk_path == NULL) { if (destination_path == NULL || disk_path == NULL) {
free(confined_backup); free(confined_backup);
free(confined_partial); free(confined_partial);
free(destination_path);
free(disk_path);
return false; return false;
} }
/* --existing checks the final destination, not a temporary partial path. */
if (config && config->existing && !file_path_exists_secure(destination_path)) {
free(confined_backup);
free(confined_partial);
free(destination_path);
free(disk_path);
return true;
}
free(destination_path);
destination_path = NULL;
/* --update is receiver-side policy: never replace a newer destination. */ /* --update is receiver-side policy: never replace a newer destination. */
if (config && config->update) { if (config && config->update) {
struct stat destination_stat; struct stat destination_stat;
@@ -68,6 +83,7 @@ bool file_save_to_disk(const char* root_directory, const File* file, const Confi
destination_stat.st_mtime > file->metadata->mtime_sec) { destination_stat.st_mtime > file->metadata->mtime_sec) {
free(confined_backup); free(confined_backup);
free(confined_partial); free(confined_partial);
free(destination_path);
free(disk_path); free(disk_path);
return true; return true;
} }
@@ -109,6 +125,7 @@ bool file_save_to_disk(const char* root_directory, const File* file, const Confi
free(backup_path); free(backup_path);
free(confined_backup); free(confined_backup);
free(confined_partial); free(confined_partial);
free(destination_path);
free(disk_path); free(disk_path);
return ok; return ok;
@@ -117,6 +134,7 @@ fail:
free(backup_path); free(backup_path);
free(confined_backup); free(confined_backup);
free(confined_partial); free(confined_partial);
free(destination_path);
free(disk_path); free(disk_path);
return false; return false;
} }
+28
View File
@@ -214,6 +214,34 @@ class TestIncremental:
assert f.read() == b"hello world\n" assert f.read() == b"hello world\n"
class TestExisting:
def test_existing_updates_existing_and_skips_new(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0, f"Initial sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
source_file = os.path.join(SOURCE_DIR, "small.txt")
new_source_file = os.path.join(SOURCE_DIR, "new-existing-test.txt")
with open(source_file, "wb") as f:
f.write(b"updated existing content\n")
with open(new_source_file, "wb") as f:
f.write(b"this file must not be created\n")
try:
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["-M", "--existing"], port=shared_server.port)
assert result.returncode == 0, f"--existing sync failed: {(result.stderr or result.stdout)[:200]}"
with open(os.path.join(received, "small.txt"), "rb") as f:
assert f.read() == b"updated existing content\n"
assert not os.path.exists(os.path.join(received, "new-existing-test.txt"))
finally:
os.unlink(new_source_file)
with open(source_file, "wb") as f:
f.write(b"hello world\n")
class TestDelete: class TestDelete:
def test_delete_removes_extra_files(self, shared_server): def test_delete_removes_extra_files(self, shared_server):
clean_dir(DEST_DIR) clean_dir(DEST_DIR)
+12
View File
@@ -339,6 +339,17 @@ static void test_parse_args_archive() {
config_delete(cfg); config_delete(cfg);
} }
static void test_parse_args_existing() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--existing", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->existing);
config_delete(cfg);
}
void test_client_cli() { void test_client_cli() {
test_validate_config_required_paths(); test_validate_config_required_paths();
test_validate_config_incompatible_options(); test_validate_config_incompatible_options();
@@ -360,4 +371,5 @@ void test_client_cli() {
test_parse_args_unknown_option(); test_parse_args_unknown_option();
test_parse_args_rejects_unimplemented_options(); test_parse_args_rejects_unimplemented_options();
test_parse_args_archive(); test_parse_args_archive();
test_parse_args_existing();
} }
+3
View File
@@ -126,6 +126,7 @@ static void test_config_send_receive() {
send_cfg->use_metadata = true; send_cfg->use_metadata = true;
send_cfg->compression_level = 5; send_cfg->compression_level = 5;
send_cfg->chunk_size = 1024; send_cfg->chunk_size = 1024;
send_cfg->existing = true;
/* Use socketpair for bidirectional communication */ /* Use socketpair for bidirectional communication */
int p[2]; int p[2];
@@ -160,6 +161,8 @@ static void test_config_send_receive() {
ok = false; ok = false;
if (recv_cfg->chunk_size != 1024) if (recv_cfg->chunk_size != 1024)
ok = false; ok = false;
if (!recv_cfg->existing)
ok = false;
} }
config_delete(recv_cfg); config_delete(recv_cfg);
close(p[0]); close(p[0]);
+45
View File
@@ -88,6 +88,50 @@ static void test_file_save_to_disk() {
rmdir("test_save_tmp"); rmdir("test_save_tmp");
} }
static void test_file_save_to_disk_existing() {
const char* root = "test_existing_tmp";
const char* existing_path = "test_existing_tmp/existing.txt";
const char* missing_path = "test_existing_tmp/missing.txt";
EXPECT_TRUE(file_write_to_disk(existing_path, "old", 3, false, false));
Config* cfg = config_create();
EXPECT_NOT_NULL(cfg);
cfg->existing = true;
File* existing = file_create("existing.txt");
EXPECT_NOT_NULL(existing);
existing->data->data = malloc(3);
EXPECT_NOT_NULL(existing->data->data);
memcpy(existing->data->data, "new", 3);
existing->data->size = 3;
EXPECT_TRUE(file_save_to_disk(root, existing, cfg));
file_destroy(existing);
File* missing = file_create("missing.txt");
EXPECT_NOT_NULL(missing);
missing->data->data = malloc(7);
EXPECT_NOT_NULL(missing->data->data);
memcpy(missing->data->data, "skipped", 7);
missing->data->size = 7;
EXPECT_TRUE(file_save_to_disk(root, missing, cfg));
file_destroy(missing);
FILE* fp = fopen(existing_path, "rb");
char content[4] = {0};
EXPECT_NOT_NULL(fp);
// cppcheck-suppress knownConditionTrueFalse
if (fp) {
EXPECT_EQ_INT((int)fread(content, 1, 3, fp), 3);
fclose(fp);
}
EXPECT_EQ_STR(content, "new");
EXPECT_EQ_INT(access(missing_path, F_OK), -1);
config_delete(cfg);
unlink(existing_path);
rmdir(root);
}
static void test_file_write_to_disk_basic() { static void test_file_write_to_disk_basic() {
const char* content = "Basic file_write_to_disk test"; const char* content = "Basic file_write_to_disk test";
EXPECT_TRUE(file_write_to_disk("test_file_write_to_disk_basic.txt", content, strlen(content), EXPECT_TRUE(file_write_to_disk("test_file_write_to_disk_basic.txt", content, strlen(content),
@@ -458,6 +502,7 @@ void test_file() {
test_file_load_data(); test_file_load_data();
test_file_load_data_missing_file(); test_file_load_data_missing_file();
test_file_save_to_disk(); test_file_save_to_disk();
test_file_save_to_disk_existing();
test_file_write_to_disk_basic(); test_file_write_to_disk_basic();
test_file_write_to_disk_creates_dirs(); test_file_write_to_disk_creates_dirs();
test_file_write_to_disk_does_not_follow_symlink(); test_file_write_to_disk_does_not_follow_symlink();