feat: add skip-compress option
CI / lint (pull_request) Successful in 12s
CI / sanitizers (address) (pull_request) Successful in 36s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 15s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s

This commit is contained in:
2026-09-03 17:56:27 +02:00
parent 190fc5d300
commit 2a08a7ba5c
14 changed files with 211 additions and 27 deletions
+1 -1
View File
@@ -174,7 +174,7 @@ This document maps rsync's full feature set to FastSync's current implementation
| `--compress-choice=STR` | Choose compression algorithm | ❌ Not Implemented | Removed because it had no effect; FastSync always uses zstd |
| `--compress-level=NUM` | Set compression level | ✅ Implemented | 1-22, default 5 |
| `--compress-threads=NUM` | Set compression threads | ❌ Not Implemented | |
| `--skip-compress=LIST` | Skip compress for suffixes | ❌ Not Implemented | Internal skip for hardcoded types; not user-configurable |
| `--skip-compress=LIST` | Skip compress for suffixes | ✅ Implemented | Comma-separated, case-insensitive suffix list; empty list skips none |
## 13. Connectivity
+29
View File
@@ -119,6 +119,29 @@ static int config_add_pattern(char*** patterns, int* count, const char* value,
return 0;
}
static int parse_skip_compress(Config* config, const char* value) {
char* list = str_dup(value);
if (!list)
return -1;
config->skip_compress_set = true;
for (char* token = strtok(list, ","); token; token = strtok(NULL, ",")) {
while (*token == ' ' || *token == '\t')
token++;
size_t len = strlen(token);
while (len > 0 && (token[len - 1] == ' ' || token[len - 1] == '\t'))
token[--len] = '\0';
if (len == 0)
continue;
if (config_add_pattern(&config->skip_compress_suffixes, &config->skip_compress_count, token,
"--skip-compress") != 0) {
free(list);
return -1;
}
}
free(list);
return 0;
}
typedef enum {
OPT_FLAG,
OPT_STRING,
@@ -361,6 +384,12 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
log_message(LOG_LEVEL_ERROR, "--compress-level must be between 1 and 22");
return -1;
}
} else if (strncmp(argv[i], "--skip-compress=", 16) == 0) {
if (parse_skip_compress(config, argv[i] + 16) != 0)
return -1;
} else if (opt_is(argv[i], "--skip-compress", NULL) && i + 1 < argc) {
if (parse_skip_compress(config, argv[++i]) != 0)
return -1;
} else if (argv[i][0] == '-') {
fprintf(stderr, "Unknown option: %s\n", argv[i]);
print_usage();
+23 -12
View File
@@ -234,7 +234,9 @@ static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* c
return send_status(client->file_descriptor, STATUS_NEXT) ? 1 : -1;
Data* to_send = delta_data;
if (config->use_compression) {
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
if (config->use_compression && !compression_should_skip_with_suffixes(
file->path, config->skip_compress_suffixes, skip_count)) {
to_send = data_compress(delta_data, config->compression_level);
data_destroy(delta_data);
if (!to_send)
@@ -251,20 +253,23 @@ static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* c
return ok ? 0 : -1;
}
typedef bool (*file_send_fn)(File*, int, bool, int, bool);
// Send a single file directly (non-incremental path).
static bool send_file_direct(File* file, int fd, bool use_metadata, int compression_level) {
static bool send_file_direct(File* file, int fd, bool use_metadata, int compression_level,
const Config* config) {
if (!send_status(fd, STATUS_NEXT))
return false;
return file_send_single_calls(file, fd, use_metadata, compression_level, true);
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
return file_send_single_calls_with_skip(file, fd, use_metadata, compression_level, true,
config->skip_compress_suffixes, skip_count);
}
// Send a single file directly via sendfile (non-incremental path).
static bool send_file_direct_sendfile(File* file, int fd, bool use_metadata) {
static bool send_file_direct_sendfile(File* file, int fd, bool use_metadata, const Config* config) {
if (!send_status(fd, STATUS_NEXT))
return false;
return file_send_sendfile(file, fd, use_metadata, 0, true);
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
return file_send_sendfile_with_skip(file, fd, use_metadata, 0, true,
config->skip_compress_suffixes, skip_count);
}
// Process one file in a chunk: either via incremental check or direct send.
@@ -275,10 +280,12 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
if (!use_incremental) {
if (use_sendfile) {
return send_file_direct_sendfile(file, client->file_descriptor, config->use_metadata) ? 0
return send_file_direct_sendfile(file, client->file_descriptor, config->use_metadata, config)
? 0
: -1;
}
return send_file_direct(file, client->file_descriptor, config->use_metadata, compression_level)
return send_file_direct(file, client->file_descriptor, config->use_metadata, compression_level,
config)
? 0
: -1;
}
@@ -304,13 +311,14 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
return -1;
}
// Fall through: send full file via sendfile (pass 0 for compression_level)
if (!file_send_sendfile(file, client->file_descriptor, config->use_metadata, 0, false))
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
if (!file_send_sendfile_with_skip(file, client->file_descriptor, config->use_metadata, 0, false,
config->skip_compress_suffixes, skip_count))
return -1;
return 0;
}
// Incremental path with single_calls (supports compression and delta)
file_send_fn send_fn = (file_send_fn)file_send_single_calls;
DeltaSignature* sig = NULL;
int rc = incremental_check(client, file, config, &sig);
if (rc < 0) {
@@ -338,7 +346,10 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
return -1;
}
}
if (!send_fn(file, client->file_descriptor, config->use_metadata, compression_level, false))
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
if (!file_send_single_calls_with_skip(file, client->file_descriptor, config->use_metadata,
compression_level, false, config->skip_compress_suffixes,
skip_count))
return -1;
return 0;
}
+1
View File
@@ -69,6 +69,7 @@ void print_usage(void) {
printf(" -S, --sparse Handle sparse files efficiently\n");
printf(" --inplace Update files in-place (no temp+rename)\n");
printf(" --compress-level <n> Compression level (default: 5)\n");
printf(" --skip-compress=LIST Skip compression for comma-separated suffixes\n");
printf(" --help Show this help\n");
printf(" -V, --version Show version\n");
}
+13 -3
View File
@@ -11,17 +11,27 @@
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
#define MAX_DECOMPRESSED_SIZE (100ULL * 1024 * 1024) /* 100 MB hard ceiling */
static const char* SKIP_COMPRESSION_EXTENSIONS[] = {".jpg", ".jpeg", ".png", ".gif", ".mp4", ".mkv",
static char* SKIP_COMPRESSION_EXTENSIONS[] = {".jpg", ".jpeg", ".png", ".gif", ".mp4", ".mkv",
".zip", ".gz", ".xz", ".zst", NULL};
bool compression_should_skip(const char* path) {
return compression_should_skip_with_suffixes(path, NULL, -1);
}
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count) {
if (!path)
return false;
const char* dot = strrchr(path, '.');
if (!dot)
return false;
for (int i = 0; SKIP_COMPRESSION_EXTENSIONS[i]; i++) {
if (strcasecmp(dot, SKIP_COMPRESSION_EXTENSIONS[i]) == 0)
if (count < 0) {
suffixes = SKIP_COMPRESSION_EXTENSIONS;
count = 0;
while (SKIP_COMPRESSION_EXTENSIONS[count])
count++;
}
for (int i = 0; i < count; i++) {
if (strcasecmp(dot, suffixes[i]) == 0)
return true;
}
return false;
+1
View File
@@ -8,5 +8,6 @@ Data* data_compress(Data* data_to_compress, int compression_level);
Data* data_decompress(Data* compressed_data);
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
bool compression_should_skip(const char* path);
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
#endif
+35 -3
View File
@@ -98,6 +98,9 @@ static void config_set_defaults(Config* config) {
config->server_mode = false;
config->checksum = false;
config->compress_choice = NULL;
config->skip_compress_suffixes = NULL;
config->skip_compress_count = 0;
config->skip_compress_set = false;
}
static bool valid_wire_bool(int value) {
@@ -134,7 +137,8 @@ static bool validate_received_config(const Config* config) {
config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE &&
config->delta_block_size >= DELTA_BLOCK_SIZE_MIN &&
config->delta_block_size <= DELTA_BLOCK_SIZE_MAX &&
config->delta_max_file_size <= DELTA_MAX_FILE_SIZE && config->max_delete >= 0;
config->delta_max_file_size <= DELTA_MAX_FILE_SIZE && config->max_delete >= 0 &&
config->skip_compress_count >= 0 && config->skip_compress_count <= 10000;
}
Config* config_create(void) {
@@ -208,6 +212,9 @@ void config_delete(Config* config) {
free(config->bind_address);
free(config->daemon_config);
free(config->compress_choice);
for (int i = 0; i < config->skip_compress_count; i++)
free(config->skip_compress_suffixes[i]);
free(config->skip_compress_suffixes);
if (config->filters) {
array_list_delete(config->filters);
}
@@ -249,11 +256,22 @@ static bool send_selection_options(int fd, const Config* c) {
send_int(fd, c->relative) && send_int(fd, c->prune_empty_dirs);
}
static bool send_skip_compress_options(int fd, const Config* c) {
if (!send_int(fd, c->skip_compress_set) || !send_int(fd, c->skip_compress_count))
return false;
for (int i = 0; i < c->skip_compress_count; i++) {
if (!send_str(fd, c->skip_compress_suffixes[i]))
return false;
}
return true;
}
static bool send_resume_options(int fd, const Config* c) {
return send_str(fd, c->temp_dir ? c->temp_dir : "") && send_int(fd, c->partial) &&
send_str(fd, c->partial_dir ? c->partial_dir : "") &&
send_str(fd, c->suffix ? c->suffix : "") && send_int(fd, c->delete_before) &&
send_int(fd, c->checksum) && send_str(fd, c->compress_choice ? c->compress_choice : "");
send_int(fd, c->checksum) && send_str(fd, c->compress_choice ? c->compress_choice : "") &&
send_skip_compress_options(fd, c);
}
static bool receive_core_fields(int fd, Config* c) {
@@ -330,7 +348,21 @@ static bool receive_resume_options(int fd, Config* c) {
if (!receive_wire_bool(fd, &c->checksum))
return false;
c->compress_choice = receive_str(fd);
return c->compress_choice != NULL;
if (!c->compress_choice || !receive_wire_bool(fd, &c->skip_compress_set) ||
!receive_int(fd, &c->skip_compress_count) || c->skip_compress_count < 0 ||
c->skip_compress_count > 10000)
return false;
if (c->skip_compress_count > 0) {
c->skip_compress_suffixes = calloc((size_t)c->skip_compress_count, sizeof(char*));
if (!c->skip_compress_suffixes)
return false;
for (int i = 0; i < c->skip_compress_count; i++) {
c->skip_compress_suffixes[i] = receive_str(fd);
if (!c->skip_compress_suffixes[i])
return false;
}
}
return true;
}
bool config_send(int file_descriptor, const Config* config) {
+4
View File
@@ -126,6 +126,10 @@ typedef struct Config {
// PR #184: Compression algorithm negotiation
char* compress_choice;
char** skip_compress_suffixes;
int skip_compress_count;
bool skip_compress_set;
} Config;
#define PROTOCOL_VERSION "2.2.0"
+12 -3
View File
@@ -169,7 +169,10 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
}
Data* raw_delta = delta_data;
if (config->use_compression) {
if (config->use_compression &&
!compression_should_skip_with_suffixes(
check_path, config->skip_compress_suffixes,
config->skip_compress_set ? config->skip_compress_count : -1)) {
raw_delta = data_decompress_limited(delta_data, MAX_RECEIVE_FILE_SIZE);
data_destroy(delta_data);
if (!raw_delta) {
@@ -272,7 +275,10 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
return NULL;
}
if (config->use_compression) {
if (config->use_compression &&
!compression_should_skip_with_suffixes(
file->path, config->skip_compress_suffixes,
config->skip_compress_set ? config->skip_compress_count : -1)) {
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_FILE_SIZE);
data_destroy(file_data);
if (uncompressed == NULL) {
@@ -500,7 +506,10 @@ File* file_receive(const Config* config, int file_descriptor) {
file_destroy(file);
return NULL;
}
if (config->use_compression && !compression_should_skip(file->path)) {
if (config->use_compression &&
!compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
config->skip_compress_set ? config->skip_compress_count
: -1)) {
Data* file_data_uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_FILE_SIZE);
data_destroy(file_data);
if (file_data_uncompressed == NULL) {
+18 -3
View File
@@ -19,11 +19,19 @@
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path) {
return file_send_single_calls_with_skip(file, file_descriptor, use_metadata, compression_level,
send_path, NULL, -1);
}
bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path,
char* const* skip_suffixes, int skip_count) {
if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data))
return false;
const Data* data_to_send = file->data;
Data* compressed_data = NULL;
if (compression_level > 0 && !compression_should_skip(file->path)) {
if (compression_level > 0 &&
!compression_should_skip_with_suffixes(file->path, skip_suffixes, skip_count)) {
compressed_data = data_compress(file->data, compression_level);
if (compressed_data == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
@@ -49,11 +57,18 @@ bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
bool send_path) {
return file_send_sendfile_with_skip(file, file_descriptor, use_metadata, compression_level,
send_path, NULL, -1);
}
bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path, char* const* skip_suffixes,
int skip_count) {
if (!file || !file->path || !file->data)
return false;
if (compression_level > 0)
return file_send_single_calls(file, file_descriptor, use_metadata, compression_level,
send_path);
return file_send_single_calls_with_skip(file, file_descriptor, use_metadata, compression_level,
send_path, skip_suffixes, skip_count);
if (send_path && !send_str(file_descriptor, file->path))
return false;
+6
View File
@@ -8,7 +8,13 @@
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path);
bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path,
char* const* skip_suffixes, int skip_count);
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
bool send_path);
bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path, char* const* skip_suffixes,
int skip_count);
#endif
+29
View File
@@ -52,6 +52,35 @@ class TestArchiveMode:
assert not mismatches, f"Mismatch: {mismatches}"
class TestSkipCompress:
def test_skip_compress_case_insensitive(self, shared_server):
clean_dir(DEST_DIR)
with open(os.path.join(SOURCE_DIR, "skip-case.TXT"), "wb") as f:
f.write((b"skip compression case test\n" * 100))
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-c", "--skip-compress=.txt"],
port=shared_server.port,
)
assert result.returncode == 0, f"Skip-compress sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
with open(os.path.join(received, "skip-case.TXT"), "rb") as f:
assert f.read() == b"skip compression case test\n" * 100
def test_skip_compress_empty_list(self, shared_server):
clean_dir(DEST_DIR)
result, _ = run_client(
SOURCE_DIR, DEST_DIR,
flags=["-c", "--skip-compress="],
port=shared_server.port,
)
assert result.returncode == 0, f"Empty skip-compress sync failed: {(result.stderr or result.stdout)[:200]}"
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
assert not mismatches, f"Mismatch: {mismatches}"
class TestExclude:
def test_exclude_single(self, shared_server):
clean_dir(DEST_DIR)
+28
View File
@@ -247,6 +247,32 @@ static void test_parse_args_valid_compression_level() {
config_delete(cfg);
}
static void test_parse_args_skip_compress() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--skip-compress=.ZIP, .GZ", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->skip_compress_set);
EXPECT_EQ_INT(cfg->skip_compress_count, 2);
EXPECT_EQ_STR(cfg->skip_compress_suffixes[0], ".ZIP");
EXPECT_EQ_STR(cfg->skip_compress_suffixes[1], ".GZ");
config_delete(cfg);
}
static void test_parse_args_empty_skip_compress() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--skip-compress=", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->skip_compress_set);
EXPECT_EQ_INT(cfg->skip_compress_count, 0);
config_delete(cfg);
}
/* Test parse_args unknown option returns error */
static void test_parse_args_unknown_option() {
Config* cfg = config_create();
@@ -357,6 +383,8 @@ void test_client_cli() {
test_parse_args_invalid_server_port();
test_parse_args_invalid_compression_level();
test_parse_args_valid_compression_level();
test_parse_args_skip_compress();
test_parse_args_empty_skip_compress();
test_parse_args_unknown_option();
test_parse_args_rejects_unimplemented_options();
test_parse_args_archive();
+9
View File
@@ -55,6 +55,14 @@ static void test_data_compress_decompress_large() {
data_destroy(decompressed);
}
static void test_skip_compress_suffix_matching() {
char* suffixes[] = {".ZIP", ".GZ"};
EXPECT_TRUE(compression_should_skip_with_suffixes("archive.zip", suffixes, 2));
EXPECT_TRUE(compression_should_skip_with_suffixes("backup.TAR.GZ", suffixes, 2));
EXPECT_FALSE(compression_should_skip_with_suffixes("notes.txt", suffixes, 2));
EXPECT_FALSE(compression_should_skip_with_suffixes("archive.zip", suffixes, 0));
}
static void test_chunk_compress_decompress_roundtrip() {
char* path1 = "temp_comp_test_1.txt";
char* content1 = "chunk compression test file 1";
@@ -115,5 +123,6 @@ static void test_chunk_compress_decompress_roundtrip() {
void test_compression() {
test_data_compress_decompress_roundtrip();
test_data_compress_decompress_large();
test_skip_compress_suffix_matching();
test_chunk_compress_decompress_roundtrip();
}