TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Merge remote-tracking branch 'origin/feat/modify-window' into dev

Browse Source 
# Conflicts:
#	RSYNC_COMPAT.md
#	src/client/client_cli.c
#	src/client/usage.c
#	src/server/receiver.c
#	src/server/server.c
#	src/shared/config.c
#	src/shared/config.h
#	src/shared/file_receive.c
#	tests/integration/test_features.py
#	tests/test_client_cli.c
#	tests/test_config.c
This commit is contained in:
TapTap
2026-09-04 17:40:16 +02:00
parent e42df1bde3 e37d5b9438
commit 6d10116d50
16 changed files with 262 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
RSYNC_COMPAT.md
+1 -1
View File
@@ -60,7 +60,7 @@ This document maps rsync's full feature set to FastSync's current implementation
| `--min-size=SIZE` | Skip files smaller than SIZE | ✅ Implemented | `min_size` in scanner |
| `-I`, `--ignore-times` | Don't skip files matching size+time | ❌ Not Implemented | |
| `--size-only` | Skip based on size only | ✅ Implemented | With `--incremental`, ignores mtime |
| `-@`, `--modify-window=NUM` | Mod-time comparison accuracy | ❌ Not Implemented | |
| `-@`, `--modify-window=NUM` | Mod-time comparison accuracy | ✅ Implemented | Whole-second tolerance with nanosecond-aware comparisons |
| `--existing` | Skip creating new files on receiver | ❌ Not Implemented | |
| `--ignore-existing` | Skip updating existing files | ❌ Not Implemented | |
| `--remove-source-files` | Sender removes regular files after confirmed transfer | ✅ Implemented | |
src/client/client_cli.c
+13
View File
@@ -208,6 +208,7 @@ static const OptionEntry OPTION_TABLE[] = {
{"--incremental", NULL, OPT_FLAG, offsetof(Config, use_incremental)},
{"--size-only", NULL, OPT_FLAG, offsetof(Config, size_only)},
{"--ignore-times", "-I", OPT_FLAG, offsetof(Config, ignore_times)},
{"--modify-window", "-@", OPT_NONNEG_INT, offsetof(Config, modify_window)},
{"--delta", NULL, OPT_FLAG, offsetof(Config, use_delta)},
{"--whole-file", "-W", OPT_FLAG, offsetof(Config, whole_file)},
{"--save-to-disk", NULL, OPT_FLAG, offsetof(Config, save_to_disk)},
@@ -292,6 +293,18 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
bool verbose = false;
protocol_set_8_bit_output(config->eight_bit_output);
for (int i = 1; i < argc; i++) {
const char* modify_window_prefix = "--modify-window=";
if (strncmp(argv[i], modify_window_prefix, strlen(modify_window_prefix)) == 0) {
if (set_nonneg_int_option(&config->modify_window, argv[i] + strlen(modify_window_prefix),
"--modify-window") != 0)
return -1;
continue;
}
if (strncmp(argv[i], "-@", 2) == 0 && argv[i][2] != '\0') {
if (set_nonneg_int_option(&config->modify_window, argv[i] + 2, "-@") != 0)
return -1;
continue;
}
const OptionEntry* entry = find_table_option(argv[i]);
if (entry) {
if (entry->kind != OPT_FLAG) {
src/client/client_send.c
+3
View File
@@ -286,10 +286,13 @@ static int incremental_check(Client* client, File* file, const Config* config,
return -1;
unsigned long long fsize = file->data->size;
long long mtime = file->metadata ? file->metadata->mtime_sec : 0;
long long mtime_nsec = file->metadata ? file->metadata->mtime_nsec : 0;
if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize)))
return -1;
if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime)))
return -1;
if (!send_n_data(client->file_descriptor, &mtime_nsec, sizeof(mtime_nsec)))
return -1;
if (config->checksum) {
uint64_t checksum;
if (!file_checksum(file, &checksum) ||
src/client/usage.c
+1
View File
@@ -32,6 +32,7 @@ void print_usage(void) {
printf(" --incremental Skip files unchanged since last transfer\n");
printf(" --size-only Skip incremental files matching in size, ignoring mtime\n");
printf(" -I, --ignore-times Transfer files even when size and mtime match\n");
printf(" -@, --modify-window <sec> Modification time tolerance\n");
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
printf(" -W, --whole-file Transfer changed files without delta processing\n");
printf(" --delta-block <n> Delta block size in bytes (default: %d)\n",
src/server/receiver.c
+16 -3
View File
@@ -2,6 +2,7 @@
#include "chunk.h"
#include "log.h"
#include "metadata.h"
#include "protocol.h"
#include "utils.h"
#include <stdlib.h>
@@ -37,9 +38,13 @@ static bool receiver_process_batch(Config* config, int file_descriptor) {
return false;
unsigned long long check_size;
long long check_mtime;
long long check_mtime_nsec;
if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) ||
!receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) {
!receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime)) ||
!receive_n_data(file_descriptor, &check_mtime_nsec, sizeof(check_mtime_nsec)) ||
check_mtime_nsec < 0 || check_mtime_nsec >= 1000000000LL) {
free(check_path);
send_status(file_descriptor, STATUS_ERROR);
return false;
}
if (!utils_valid_batch_path(check_path)) {
@@ -60,8 +65,16 @@ static bool receiver_process_batch(Config* config, int file_descriptor) {
}
struct stat st;
bool has_old = file_stat_secure(full_path, &st);
bool match = !config->ignore_times && has_old && (unsigned long long)st.st_size == check_size &&
(config->size_only || (long long)st.st_mtime == check_mtime);
long long old_mtime_nsec = 0;
if (has_old) {
#ifdef __linux__
old_mtime_nsec = st.st_mtim.tv_nsec;
#endif
}
bool match =
!config->ignore_times && has_old && (unsigned long long)st.st_size == check_size &&
metadata_mtime_matches(st.st_mtime, old_mtime_nsec, (time_t)check_mtime,
(long)check_mtime_nsec, config->modify_window);
bool sent = send_status(file_descriptor, match ? STATUS_OK : STATUS_NEXT);
free(full_path);
free(check_path);
src/server/server.c
+16 -1
View File
@@ -2,6 +2,7 @@
#include "chunk.h"
#include "file.h"
#include "log.h"
#include "metadata.h"
#include "multiprocessing.h"
#include "queue.h"
#include "receiver.h"
@@ -161,6 +162,13 @@ int receive_files(Config* config, int fd) {
free(check_path);
return -1;
}
long long check_mtime_nsec;
if (!receive_n_data(fd, &check_mtime_nsec, sizeof(check_mtime_nsec)) ||
check_mtime_nsec < 0 || check_mtime_nsec >= 1000000000LL) {
free(check_path);
send_status(fd, STATUS_ERROR);
return -1;
}
if (!utils_valid_batch_path(check_path)) {
free(check_path);
send_status(fd, STATUS_ERROR);
@@ -174,9 +182,16 @@ int receive_files(Config* config, int fd) {
return -1;
}
bool has_old = full_path && file_stat_secure(full_path, &st);
long long old_mtime_nsec = 0;
if (has_old) {
#ifdef __linux__
old_mtime_nsec = st.st_mtim.tv_nsec;
#endif
}
bool match = !config->ignore_times && has_old &&
(unsigned long long)st.st_size == check_size &&
(config->size_only || (long long)st.st_mtime == check_mtime);
metadata_mtime_matches(st.st_mtime, old_mtime_nsec, (time_t)check_mtime,
(long)check_mtime_nsec, config->modify_window);
bool sent = send_status(fd, match ? STATUS_OK : STATUS_NEXT);
free(full_path);
free(check_path);
src/shared/config.c
+7 -2
View File
@@ -39,6 +39,7 @@ static void config_set_defaults(Config* config) {
config->size_only = false;
config->use_delta = false;
config->whole_file = false;
config->modify_window = 0;
config->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT;
config->delta_max_file_size = DELTA_MAX_FILE_SIZE;
config->use_tls = false;
@@ -141,7 +142,8 @@ static bool validate_received_config(const Config* config) {
config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE &&
config->delta_block_size >= DELTA_BLOCK_SIZE_MIN &&
config->delta_block_size <= DELTA_BLOCK_SIZE_MAX &&
config->delta_max_file_size <= DELTA_MAX_FILE_SIZE && config->max_delete >= 0;
config->delta_max_file_size <= DELTA_MAX_FILE_SIZE && config->modify_window >= 0 &&
config->max_delete >= 0;
}
Config* config_create(void) {
@@ -264,7 +266,8 @@ static bool send_resume_options(int fd, const Config* c) {
return send_str(fd, c->temp_dir ? c->temp_dir : "") && send_int(fd, c->partial) &&
send_str(fd, c->partial_dir ? c->partial_dir : "") &&
send_str(fd, c->suffix ? c->suffix : "") && send_int(fd, c->delete_before) &&
send_int(fd, c->checksum) && send_str(fd, c->compress_choice ? c->compress_choice : "");
send_int(fd, c->checksum) && send_int(fd, c->modify_window) &&
send_str(fd, c->compress_choice ? c->compress_choice : "");
}
static bool receive_core_fields(int fd, Config* c) {
@@ -347,6 +350,8 @@ static bool receive_resume_options(int fd, Config* c) {
return false;
if (!receive_wire_bool(fd, &c->checksum))
return false;
if (!receive_n_data(fd, &c->modify_window, sizeof(c->modify_window)))
return false;
c->compress_choice = receive_str(fd);
return c->compress_choice != NULL;
}
src/shared/config.h
+1
View File
@@ -39,6 +39,7 @@ typedef struct Config {
bool size_only;
bool use_delta;
bool whole_file;
int modify_window;
uint32_t delta_block_size;
unsigned long long delta_max_file_size;
bool use_tls;
src/shared/file_receive.c
+13 -1
View File
@@ -314,12 +314,19 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
unsigned long long check_size;
long long check_mtime;
long long check_mtime_nsec;
uint64_t check_checksum = 0;
if (!receive_n_data(fd, &check_size, sizeof(check_size)) ||
!receive_n_data(fd, &check_mtime, sizeof(check_mtime))) {
free(check_path);
return NULL;
}
if (!receive_n_data(fd, &check_mtime_nsec, sizeof(check_mtime_nsec)) || check_mtime_nsec < 0 ||
check_mtime_nsec >= 1000000000LL) {
free(check_path);
send_status(fd, STATUS_ERROR);
return NULL;
}
if (config->checksum && !receive_n_data(fd, &check_checksum, sizeof(check_checksum))) {
free(check_path);
return NULL;
@@ -388,7 +395,12 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
free(old_data);
old_data = NULL;
} else if (match && !config->size_only) {
match = (long long)st.st_mtime == check_mtime;
long long old_mtime_nsec = 0;
#ifdef __linux__
old_mtime_nsec = st.st_mtim.tv_nsec;
#endif
match = metadata_mtime_matches(st.st_mtime, old_mtime_nsec, (time_t)check_mtime,
(long)check_mtime_nsec, config->modify_window);
}
if (match) {
src/shared/metadata.c
+21
View File
@@ -25,6 +25,27 @@ typedef char static_assert_mode_t_fits[(sizeof(mode_t) <= sizeof(int32_t)) ? 1 :
typedef char static_assert_uid_t_fits[(sizeof(uid_t) <= sizeof(int32_t)) ? 1 : -1];
typedef char static_assert_gid_t_fits[(sizeof(gid_t) <= sizeof(int32_t)) ? 1 : -1];
bool metadata_mtime_matches(time_t left_sec, long left_nsec, time_t right_sec, long right_nsec,
int modify_window) {
int64_t left = (int64_t)left_sec;
int64_t right = (int64_t)right_sec;
int64_t seconds;
int64_t nanoseconds;
if (left > right || (left == right && left_nsec >= right_nsec)) {
seconds = left - right;
nanoseconds = (int64_t)left_nsec - (int64_t)right_nsec;
} else {
seconds = right - left;
nanoseconds = (int64_t)right_nsec - (int64_t)left_nsec;
}
if (nanoseconds < 0) {
seconds--;
nanoseconds += 1000000000LL;
}
return seconds < modify_window || (seconds == modify_window && nanoseconds == 0);
}
void metadata_to_buf(char** buf, const FileMetadata* m) {
int32_t present = (m != NULL) ? 1 : 0;
memcpy(*buf, &present, sizeof(present));
src/shared/metadata.h
+5
View File
@@ -5,6 +5,7 @@
#include <stdbool.h>
#include <stdint.h>
#include <sys/stat.h>
#include <time.h>
/*
* Wire format (introduced in protocol version 2.0.0):
@@ -32,4 +33,8 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok);
void file_restore_metadata(const char* path, const FileMetadata* metadata);
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata);
/* Compare timestamps using rsync's whole-second modification window. */
bool metadata_mtime_matches(time_t left_sec, long left_nsec, time_t right_sec, long right_nsec,
int modify_window);
#endif
tests/integration/test_features.py
+81
View File
@@ -371,6 +371,87 @@ class TestIncremental:
with open(received_file, "rb") as f:
assert f.read() == b"hello world\n"
def test_modify_window_allows_subsecond_mtime_difference(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
source_file = os.path.join(SOURCE_DIR, "small.txt")
received_file = os.path.join(received, "small.txt")
source_stat = os.stat(source_file)
with open(received_file, "wb") as f:
f.write(b"modified!!!\n")
os.utime(received_file, ns=(source_stat.st_atime_ns,
source_stat.st_mtime_ns - 1500000000))
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["-M", "--incremental", "--modify-window=2"],
port=shared_server.port)
assert result.returncode == 0, f"Modify-window sync failed: {result.stderr[:200]}"
with open(received_file, "rb") as f:
assert f.read() == b"modified!!!\n"
def test_whole_file_disables_delta_and_keeps_compression(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0
source_file = os.path.join(SOURCE_DIR, "medium.txt")
with open(source_file, "wb") as f:
f.write(b"whole-file replacement\n" * 5000)
result, _ = run_client(
SOURCE_DIR,
DEST_DIR,
flags=["-M", "--incremental", "--delta", "-W", "-c"],
port=shared_server.port,
)
assert result.returncode == 0, f"Whole-file sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
source_file = os.path.join(SOURCE_DIR, "small.txt")
received_file = os.path.join(received, "small.txt")
source_stat = os.stat(source_file)
with open(received_file, "wb") as f:
f.write(b"stale data!\n")
os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime))
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["-M", "--incremental", "--ignore-times"],
port=shared_server.port)
assert result.returncode == 0, f"Ignore-times sync failed: {result.stderr[:200]}"
with open(received_file, "rb") as f:
assert f.read() == b"hello world\n"
def test_modify_window_allows_subsecond_mtime_difference(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
assert result.returncode == 0
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
source_file = os.path.join(SOURCE_DIR, "small.txt")
received_file = os.path.join(received, "small.txt")
source_stat = os.stat(source_file)
with open(received_file, "wb") as f:
f.write(b"modified!!!\n")
os.utime(received_file, ns=(source_stat.st_atime_ns,
source_stat.st_mtime_ns - 1500000000))
result, _ = run_client(SOURCE_DIR, DEST_DIR,
flags=["-M", "--incremental", "--modify-window=2"],
port=shared_server.port)
assert result.returncode == 0, f"Modify-window sync failed: {result.stderr[:200]}"
with open(received_file, "rb") as f:
assert f.read() == b"modified!!!\n"
def test_whole_file_disables_delta_and_keeps_compression(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
tests/test_client_cli.c
+39
View File
@@ -327,6 +327,43 @@ static void test_parse_args_debug_flags_validation() {
}
}
static void test_parse_args_modify_window() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--modify-window=3", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->modify_window, 3);
config_delete(cfg);
cfg = config_create();
char* short_argv[] = {"fastsync", "-@", "7", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, short_argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->modify_window, 7);
config_delete(cfg);
cfg = config_create();
char* attached_argv[] = {"fastsync", "-@11", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, attached_argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->modify_window, 11);
config_delete(cfg);
}
static void test_parse_args_rejects_invalid_modify_window() {
const char* values[] = {"-1", "not-a-number", ""};
for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--modify-window", (char*)values[i], "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
}
/* Test parse_args unknown option returns error */
static void test_parse_args_unknown_option() {
Config* cfg = config_create();
@@ -591,6 +628,8 @@ void test_client_cli() {
test_parse_args_debug_flags();
test_parse_args_debug_help();
test_parse_args_debug_flags_validation();
test_parse_args_modify_window();
test_parse_args_rejects_invalid_modify_window();
test_parse_args_unknown_option();
test_parse_args_rejects_unimplemented_options();
test_parse_args_quiet();
tests/test_config.c
+3
View File
@@ -131,6 +131,7 @@ static void test_config_send_receive() {
send_cfg->compression_level = 5;
send_cfg->chunk_size = 1024;
send_cfg->eight_bit_output = true;
send_cfg->modify_window = 4;
/* Use socketpair for bidirectional communication */
int p[2];
@@ -175,6 +176,8 @@ static void test_config_send_receive() {
ok = false;
if (!recv_cfg->whole_file)
ok = false;
if (recv_cfg->modify_window != 4)
ok = false;
}
config_delete(recv_cfg);
close(p[0]);
tests/test_metadata.c
+13
View File
@@ -122,6 +122,18 @@ static void test_metadata_rejects_invalid_values() {
close(p[1]);
}
static void test_metadata_mtime_window() {
EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 101, 600000000, 2));
EXPECT_FALSE(metadata_mtime_matches(100, 100000000, 102, 600000000, 2));
EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 102, 100000000, 2));
EXPECT_TRUE(metadata_mtime_matches(100, 900000000, 102, 100000000, 2));
EXPECT_FALSE(metadata_mtime_matches(100, 100000000, 102, 900000000, 2));
EXPECT_TRUE(metadata_mtime_matches(100, 900000000, 102, 900000000, 2));
EXPECT_FALSE(metadata_mtime_matches(100, 900000000, 101, 100000001, 0));
EXPECT_FALSE(metadata_mtime_matches(100, 100000000, 100, 100000001, 0));
EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 100, 100000000, 0));
}
static void test_file_restore_metadata() {
const char* path = "temp_meta_restore_test.txt";
const char* content = "test content";
@@ -151,5 +163,6 @@ void test_metadata() {
test_metadata_send_receive_roundtrip();
test_metadata_send_null();
test_metadata_rejects_invalid_values();
test_metadata_mtime_window();
test_file_restore_metadata();
}
tests/test_server.c
+29
View File
@@ -182,11 +182,40 @@ static void test_receive_manifest_rejects_traversal() {
config_delete(cfg);
}
static void test_receive_incremental_check_rejects_invalid_nanoseconds() {
Config* cfg = config_create();
EXPECT_NOT_NULL(cfg);
cfg->receive_root_directory = str_dup("/tmp/dst");
int p[2];
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
io_set_fds(p[0], p[1]);
EXPECT_TRUE(send_str(p[1], "file.txt"));
unsigned long long size = 0;
long long mtime = 100;
long long mtime_nsec = 1000000000LL;
EXPECT_TRUE(send_n_data(p[1], &size, sizeof(size)));
EXPECT_TRUE(send_n_data(p[1], &mtime, sizeof(mtime)));
EXPECT_TRUE(send_n_data(p[1], &mtime_nsec, sizeof(mtime_nsec)));
bool skipped = false;
EXPECT_NULL(receive_incremental_check(p[0], cfg, &skipped));
Status status;
EXPECT_TRUE(receive_status(p[1], &status));
EXPECT_EQ_INT(status, STATUS_ERROR);
EXPECT_FALSE(skipped);
close(p[0]);
close(p[1]);
config_delete(cfg);
}
void test_server() {
if (!is_running_under_valgrind()) {
test_receive_files_finished();
test_receive_files_single_file();
test_receive_files_abort();
test_receive_manifest_rejects_traversal();
test_receive_incremental_check_rejects_invalid_nanoseconds();
}
}