Merge remote-tracking branch 'origin/feat/skip-compress' into dev
This commit is contained in:
@@ -345,6 +345,7 @@ features without changing the meaning of ordinary compatibility options.
|
|||||||
| `--compress-level <n>` | Set the zstd compression level. |
|
| `--compress-level <n>` | Set the zstd compression level. |
|
||||||
| `--zc <alg>` | Alias for `--compress-choice`. FastSync supports `zstd` and `none`. |
|
| `--zc <alg>` | Alias for `--compress-choice`. FastSync supports `zstd` and `none`. |
|
||||||
| `--zl <n>` | Alias for `--compress-level`. |
|
| `--zl <n>` | Alias for `--compress-level`. |
|
||||||
|
| `--skip-compress <list>` | Skip compression for comma-separated suffixes; incompatible with `-s`. |
|
||||||
| `--chunk-size <bytes>` | Set the transfer chunk size. |
|
| `--chunk-size <bytes>` | Set the transfer chunk size. |
|
||||||
| `-s` | Enable FastSync chunk serialization. |
|
| `-s` | Enable FastSync chunk serialization. |
|
||||||
| `-f`, `--sendfile` | Use TCP `sendfile()` zero-copy transfer. Incompatible with compression and chunk serialization. |
|
| `-f`, `--sendfile` | Use TCP `sendfile()` zero-copy transfer. Incompatible with compression and chunk serialization. |
|
||||||
|
|||||||
+8
-6
@@ -175,7 +175,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
| `--compress-choice=STR`, `--zc=STR` | Choose compression algorithm | ✅ Implemented | FastSync supports `zstd` and `none` |
|
| `--compress-choice=STR`, `--zc=STR` | Choose compression algorithm | ✅ Implemented | FastSync supports `zstd` and `none` |
|
||||||
| `--compress-level=NUM`, `--zl=NUM` | Set compression level | ✅ Implemented | 1-22, default 5 |
|
| `--compress-level=NUM`, `--zl=NUM` | Set compression level | ✅ Implemented | 1-22, default 5 |
|
||||||
| `--compress-threads=NUM` | Set compression threads | ❌ Not Implemented | |
|
| `--compress-threads=NUM` | Set compression threads | ❌ Not Implemented | |
|
||||||
| `--skip-compress=LIST` | Skip compress for suffixes | ❌ Not Implemented | Internal skip for hardcoded types; not user-configurable |
|
| `--skip-compress=LIST` | Skip compress for suffixes | ✅ Implemented | Comma-separated, case-insensitive suffix list; empty list skips none; incompatible with FastSync chunk serialization (`-s`) |
|
||||||
|
|
||||||
## 13. Connectivity
|
## 13. Connectivity
|
||||||
|
|
||||||
@@ -249,11 +249,13 @@ Ranked by user demand, implementation complexity, and interoperability impact:
|
|||||||
| 2 | `--ignore-times` / `-I` | Low | Medium — useful for forcing re-transfer |
|
| 2 | `--ignore-times` / `-I` | Low | Medium — useful for forcing re-transfer |
|
||||||
| 3 | `--size-only` | Low | Medium — common migration scenario |
|
| 3 | `--size-only` | Low | Medium — common migration scenario |
|
||||||
| 4 | `--ignore-existing` | Low | Medium — common sync patterns |
|
| 4 | `--ignore-existing` | Low | Medium — common sync patterns |
|
||||||
| 5 | `--delete-during` | Medium | High — performance improvement |
|
| 5 | `--existing` | Low | Medium — common sync patterns |
|
||||||
| 6 | `--delay-updates` | Medium | High — atomic updates |
|
| 6 | `--remove-source-files` | Low | High — common for moves/backup |
|
||||||
| 7 | `--chmod` | Low | Medium — permission flexibility |
|
| 7 | `--delete-during` | Medium | High — performance improvement |
|
||||||
| 8 | `--executability` / `-E` | Low | Low — simple flag |
|
| 8 | `--delay-updates` | Medium | High — atomic updates |
|
||||||
| 9 | `--skip-compress` | Low | Medium — performance tuning |
|
| 9 | `--chmod` | Low | Medium — permission flexibility |
|
||||||
|
| 10 | `--executability` / `-E` | Low | Low — simple flag |
|
||||||
|
| 11 | `--skip-compress` | Low | Medium — performance tuning |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -197,6 +197,29 @@ static int config_add_pattern(char*** patterns, int* count, const char* value,
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int parse_skip_compress(Config* config, const char* value) {
|
||||||
|
char* list = str_dup(value);
|
||||||
|
if (!list)
|
||||||
|
return -1;
|
||||||
|
config->skip_compress_set = true;
|
||||||
|
for (char* token = strtok(list, ","); token; token = strtok(NULL, ",")) {
|
||||||
|
while (*token == ' ' || *token == '\t')
|
||||||
|
token++;
|
||||||
|
size_t len = strlen(token);
|
||||||
|
while (len > 0 && (token[len - 1] == ' ' || token[len - 1] == '\t'))
|
||||||
|
token[--len] = '\0';
|
||||||
|
if (len == 0)
|
||||||
|
continue;
|
||||||
|
if (config_add_pattern(&config->skip_compress_suffixes, &config->skip_compress_count, token,
|
||||||
|
"--skip-compress") != 0) {
|
||||||
|
free(list);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
free(list);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
typedef enum {
|
typedef enum {
|
||||||
OPT_FLAG,
|
OPT_FLAG,
|
||||||
OPT_NOOP,
|
OPT_NOOP,
|
||||||
@@ -557,6 +580,12 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
} else if (opt_is(argv[i], "-T", NULL) && i + 1 < argc) {
|
} else if (opt_is(argv[i], "-T", NULL) && i + 1 < argc) {
|
||||||
if (set_positive_int_option(&config->timeout, argv[++i], "-T") != 0)
|
if (set_positive_int_option(&config->timeout, argv[++i], "-T") != 0)
|
||||||
return -1;
|
return -1;
|
||||||
|
} else if (strncmp(argv[i], "--skip-compress=", 16) == 0) {
|
||||||
|
if (parse_skip_compress(config, argv[i] + 16) != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (opt_is(argv[i], "--skip-compress", NULL) && i + 1 < argc) {
|
||||||
|
if (parse_skip_compress(config, argv[++i]) != 0)
|
||||||
|
return -1;
|
||||||
} else if (argv[i][0] == '-') {
|
} else if (argv[i][0] == '-') {
|
||||||
char* escaped = output_escape(argv[i], false);
|
char* escaped = output_escape(argv[i], false);
|
||||||
fprintf(stderr, "Unknown option: %s\n", escaped ? escaped : "<allocation failed>");
|
fprintf(stderr, "Unknown option: %s\n", escaped ? escaped : "<allocation failed>");
|
||||||
|
|||||||
+24
-13
@@ -351,7 +351,9 @@ static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* c
|
|||||||
return send_status(client->file_descriptor, STATUS_NEXT) ? 1 : -1;
|
return send_status(client->file_descriptor, STATUS_NEXT) ? 1 : -1;
|
||||||
|
|
||||||
Data* to_send = delta_data;
|
Data* to_send = delta_data;
|
||||||
if (config->use_compression) {
|
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
|
||||||
|
if (config->use_compression && !compression_should_skip_with_suffixes(
|
||||||
|
file->path, config->skip_compress_suffixes, skip_count)) {
|
||||||
to_send = data_compress(delta_data, config->compression_level);
|
to_send = data_compress(delta_data, config->compression_level);
|
||||||
data_destroy(delta_data);
|
data_destroy(delta_data);
|
||||||
if (!to_send)
|
if (!to_send)
|
||||||
@@ -368,20 +370,23 @@ static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* c
|
|||||||
return ok ? 0 : -1;
|
return ok ? 0 : -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
typedef bool (*file_send_fn)(File*, int, bool, int, bool);
|
|
||||||
|
|
||||||
// Send a single file directly (non-incremental path).
|
// Send a single file directly (non-incremental path).
|
||||||
static bool send_file_direct(File* file, int fd, bool use_metadata, int compression_level) {
|
static bool send_file_direct(File* file, int fd, bool use_metadata, int compression_level,
|
||||||
|
const Config* config) {
|
||||||
if (!send_status(fd, STATUS_NEXT))
|
if (!send_status(fd, STATUS_NEXT))
|
||||||
return false;
|
return false;
|
||||||
return file_send_single_calls(file, fd, use_metadata, compression_level, true);
|
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
|
||||||
|
return file_send_single_calls_with_skip(file, fd, use_metadata, compression_level, true,
|
||||||
|
config->skip_compress_suffixes, skip_count);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send a single file directly via sendfile (non-incremental path).
|
// Send a single file directly via sendfile (non-incremental path).
|
||||||
static bool send_file_direct_sendfile(File* file, int fd, bool use_metadata) {
|
static bool send_file_direct_sendfile(File* file, int fd, bool use_metadata, const Config* config) {
|
||||||
if (!send_status(fd, STATUS_NEXT))
|
if (!send_status(fd, STATUS_NEXT))
|
||||||
return false;
|
return false;
|
||||||
return file_send_sendfile(file, fd, use_metadata, 0, true);
|
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
|
||||||
|
return file_send_sendfile_with_skip(file, fd, use_metadata, 0, true,
|
||||||
|
config->skip_compress_suffixes, skip_count);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process one file in a chunk: either via incremental check or direct send.
|
// Process one file in a chunk: either via incremental check or direct send.
|
||||||
@@ -392,10 +397,12 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
|
|
||||||
if (!use_incremental) {
|
if (!use_incremental) {
|
||||||
if (use_sendfile) {
|
if (use_sendfile) {
|
||||||
return send_file_direct_sendfile(file, client->file_descriptor, config->use_metadata) ? 0
|
return send_file_direct_sendfile(file, client->file_descriptor, config->use_metadata, config)
|
||||||
: -1;
|
? 0
|
||||||
|
: -1;
|
||||||
}
|
}
|
||||||
return send_file_direct(file, client->file_descriptor, config->use_metadata, compression_level)
|
return send_file_direct(file, client->file_descriptor, config->use_metadata, compression_level,
|
||||||
|
config)
|
||||||
? 0
|
? 0
|
||||||
: -1;
|
: -1;
|
||||||
}
|
}
|
||||||
@@ -421,13 +428,14 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
// Fall through: send full file via sendfile (pass 0 for compression_level)
|
// Fall through: send full file via sendfile (pass 0 for compression_level)
|
||||||
if (!file_send_sendfile(file, client->file_descriptor, config->use_metadata, 0, false))
|
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
|
||||||
|
if (!file_send_sendfile_with_skip(file, client->file_descriptor, config->use_metadata, 0, false,
|
||||||
|
config->skip_compress_suffixes, skip_count))
|
||||||
return -1;
|
return -1;
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Incremental path with single_calls (supports compression and delta)
|
// Incremental path with single_calls (supports compression and delta)
|
||||||
file_send_fn send_fn = (file_send_fn)file_send_single_calls;
|
|
||||||
DeltaSignature* sig = NULL;
|
DeltaSignature* sig = NULL;
|
||||||
int rc = incremental_check(client, file, config, &sig);
|
int rc = incremental_check(client, file, config, &sig);
|
||||||
if (rc < 0) {
|
if (rc < 0) {
|
||||||
@@ -455,7 +463,10 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!send_fn(file, client->file_descriptor, config->use_metadata, compression_level, false))
|
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
|
||||||
|
if (!file_send_single_calls_with_skip(file, client->file_descriptor, config->use_metadata,
|
||||||
|
compression_level, false, config->skip_compress_suffixes,
|
||||||
|
skip_count))
|
||||||
return -1;
|
return -1;
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,6 +23,11 @@ bool validate_config(const Config* config) {
|
|||||||
log_message(LOG_LEVEL_ERROR, "--incremental is not supported with -s (chunk serialization)");
|
log_message(LOG_LEVEL_ERROR, "--incremental is not supported with -s (chunk serialization)");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
if (config->skip_compress_set && config->use_chunk_serialization) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"--skip-compress cannot be combined with -s (chunk serialization)");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (config->use_delta && !config->whole_file && !config->use_incremental) {
|
if (config->use_delta && !config->whole_file && !config->use_incremental) {
|
||||||
log_message(LOG_LEVEL_ERROR, "--delta requires --incremental");
|
log_message(LOG_LEVEL_ERROR, "--delta requires --incremental");
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@@ -89,6 +89,7 @@ void print_usage(void) {
|
|||||||
printf(" --fsync Fsync every written file before publication\n");
|
printf(" --fsync Fsync every written file before publication\n");
|
||||||
printf(" --compress-level <n> Compression level (default: 5)\n");
|
printf(" --compress-level <n> Compression level (default: 5)\n");
|
||||||
printf(" --zl <n> Alias for --compress-level\n");
|
printf(" --zl <n> Alias for --compress-level\n");
|
||||||
|
printf(" --skip-compress=LIST Skip compression for comma-separated suffixes\n");
|
||||||
printf(" --help Show this help\n");
|
printf(" --help Show this help\n");
|
||||||
printf(" -V, --version Show version\n");
|
printf(" -V, --version Show version\n");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,17 +11,27 @@
|
|||||||
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
|
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
|
||||||
#define MAX_DECOMPRESSED_SIZE (100ULL * 1024 * 1024) /* 100 MB hard ceiling */
|
#define MAX_DECOMPRESSED_SIZE (100ULL * 1024 * 1024) /* 100 MB hard ceiling */
|
||||||
|
|
||||||
static const char* SKIP_COMPRESSION_EXTENSIONS[] = {".jpg", ".jpeg", ".png", ".gif", ".mp4", ".mkv",
|
static char* SKIP_COMPRESSION_EXTENSIONS[] = {".jpg", ".jpeg", ".png", ".gif", ".mp4", ".mkv",
|
||||||
".zip", ".gz", ".xz", ".zst", NULL};
|
".zip", ".gz", ".xz", ".zst", NULL};
|
||||||
|
|
||||||
bool compression_should_skip(const char* path) {
|
bool compression_should_skip(const char* path) {
|
||||||
|
return compression_should_skip_with_suffixes(path, NULL, -1);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count) {
|
||||||
if (!path)
|
if (!path)
|
||||||
return false;
|
return false;
|
||||||
const char* dot = strrchr(path, '.');
|
const char* dot = strrchr(path, '.');
|
||||||
if (!dot)
|
if (!dot)
|
||||||
return false;
|
return false;
|
||||||
for (int i = 0; SKIP_COMPRESSION_EXTENSIONS[i]; i++) {
|
if (count < 0) {
|
||||||
if (strcasecmp(dot, SKIP_COMPRESSION_EXTENSIONS[i]) == 0)
|
suffixes = SKIP_COMPRESSION_EXTENSIONS;
|
||||||
|
count = 0;
|
||||||
|
while (SKIP_COMPRESSION_EXTENSIONS[count])
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
if (strcasecmp(dot, suffixes[i]) == 0)
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@@ -8,5 +8,6 @@ Data* data_compress(Data* data_to_compress, int compression_level);
|
|||||||
Data* data_decompress(Data* compressed_data);
|
Data* data_decompress(Data* compressed_data);
|
||||||
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
|
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
|
||||||
bool compression_should_skip(const char* path);
|
bool compression_should_skip(const char* path);
|
||||||
|
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+39
-3
@@ -110,6 +110,9 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->checksum = false;
|
config->checksum = false;
|
||||||
config->compress_choice = NULL;
|
config->compress_choice = NULL;
|
||||||
config->chmod_spec = NULL;
|
config->chmod_spec = NULL;
|
||||||
|
config->skip_compress_suffixes = NULL;
|
||||||
|
config->skip_compress_count = 0;
|
||||||
|
config->skip_compress_set = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool valid_wire_bool(int value) {
|
static bool valid_wire_bool(int value) {
|
||||||
@@ -145,13 +148,15 @@ static bool validate_received_config(const Config* config) {
|
|||||||
valid_wire_bool(config->relative) && valid_wire_bool(config->prune_empty_dirs) &&
|
valid_wire_bool(config->relative) && valid_wire_bool(config->prune_empty_dirs) &&
|
||||||
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
|
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
|
||||||
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
|
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
|
||||||
|
!(config->skip_compress_set && config->use_chunk_serialization) &&
|
||||||
(!config->use_compression ||
|
(!config->use_compression ||
|
||||||
(config->compression_level >= 1 && config->compression_level <= 22)) &&
|
(config->compression_level >= 1 && config->compression_level <= 22)) &&
|
||||||
config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE &&
|
config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE &&
|
||||||
config->delta_block_size >= DELTA_BLOCK_SIZE_MIN &&
|
config->delta_block_size >= DELTA_BLOCK_SIZE_MIN &&
|
||||||
config->delta_block_size <= DELTA_BLOCK_SIZE_MAX &&
|
config->delta_block_size <= DELTA_BLOCK_SIZE_MAX &&
|
||||||
config->delta_max_file_size <= DELTA_MAX_FILE_SIZE && config->modify_window >= 0 &&
|
config->delta_max_file_size <= DELTA_MAX_FILE_SIZE && config->modify_window >= 0 &&
|
||||||
config->max_delete >= 0 &&
|
config->max_delete >= 0 && config->skip_compress_count >= 0 &&
|
||||||
|
config->skip_compress_count <= 10000 &&
|
||||||
(!config->chmod_spec || !*config->chmod_spec ||
|
(!config->chmod_spec || !*config->chmod_spec ||
|
||||||
chmod_apply(0, config->chmod_spec, &(mode_t){0}));
|
chmod_apply(0, config->chmod_spec, &(mode_t){0}));
|
||||||
}
|
}
|
||||||
@@ -228,6 +233,11 @@ void config_delete(Config* config) {
|
|||||||
free(config->daemon_config);
|
free(config->daemon_config);
|
||||||
free(config->compress_choice);
|
free(config->compress_choice);
|
||||||
free(config->chmod_spec);
|
free(config->chmod_spec);
|
||||||
|
if (config->skip_compress_suffixes) {
|
||||||
|
for (int i = 0; i < config->skip_compress_count; i++)
|
||||||
|
free(config->skip_compress_suffixes[i]);
|
||||||
|
free(config->skip_compress_suffixes);
|
||||||
|
}
|
||||||
if (config->filters) {
|
if (config->filters) {
|
||||||
array_list_delete(config->filters);
|
array_list_delete(config->filters);
|
||||||
}
|
}
|
||||||
@@ -275,13 +285,24 @@ static bool send_selection_options(int fd, const Config* c) {
|
|||||||
send_int(fd, c->prune_empty_dirs);
|
send_int(fd, c->prune_empty_dirs);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool send_skip_compress_options(int fd, const Config* c) {
|
||||||
|
if (!send_int(fd, c->skip_compress_set) || !send_int(fd, c->skip_compress_count))
|
||||||
|
return false;
|
||||||
|
for (int i = 0; i < c->skip_compress_count; i++) {
|
||||||
|
if (!send_str(fd, c->skip_compress_suffixes[i]))
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
static bool send_resume_options(int fd, const Config* c) {
|
static bool send_resume_options(int fd, const Config* c) {
|
||||||
return send_str(fd, c->temp_dir ? c->temp_dir : "") && send_int(fd, c->partial) &&
|
return send_str(fd, c->temp_dir ? c->temp_dir : "") && send_int(fd, c->partial) &&
|
||||||
send_str(fd, c->partial_dir ? c->partial_dir : "") &&
|
send_str(fd, c->partial_dir ? c->partial_dir : "") &&
|
||||||
send_str(fd, c->suffix ? c->suffix : "") && send_int(fd, c->delete_before) &&
|
send_str(fd, c->suffix ? c->suffix : "") && send_int(fd, c->delete_before) &&
|
||||||
send_int(fd, c->checksum) && send_int(fd, c->modify_window) &&
|
send_int(fd, c->checksum) && send_int(fd, c->modify_window) &&
|
||||||
send_str(fd, c->compress_choice ? c->compress_choice : "") &&
|
send_str(fd, c->compress_choice ? c->compress_choice : "") &&
|
||||||
send_str(fd, c->chmod_spec ? c->chmod_spec : "");
|
send_str(fd, c->chmod_spec ? c->chmod_spec : "") &&
|
||||||
|
send_skip_compress_options(fd, c);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool receive_core_fields(int fd, Config* c) {
|
static bool receive_core_fields(int fd, Config* c) {
|
||||||
@@ -372,7 +393,22 @@ static bool receive_resume_options(int fd, Config* c) {
|
|||||||
if (!c->compress_choice)
|
if (!c->compress_choice)
|
||||||
return false;
|
return false;
|
||||||
c->chmod_spec = receive_str(fd);
|
c->chmod_spec = receive_str(fd);
|
||||||
return c->chmod_spec != NULL;
|
if (!c->chmod_spec ||
|
||||||
|
!receive_wire_bool(fd, &c->skip_compress_set) ||
|
||||||
|
!receive_int(fd, &c->skip_compress_count) || c->skip_compress_count < 0 ||
|
||||||
|
c->skip_compress_count > 10000)
|
||||||
|
return false;
|
||||||
|
if (c->skip_compress_count > 0) {
|
||||||
|
c->skip_compress_suffixes = calloc((size_t)c->skip_compress_count, sizeof(char*));
|
||||||
|
if (!c->skip_compress_suffixes)
|
||||||
|
return false;
|
||||||
|
for (int i = 0; i < c->skip_compress_count; i++) {
|
||||||
|
c->skip_compress_suffixes[i] = receive_str(fd);
|
||||||
|
if (!c->skip_compress_suffixes[i])
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool config_send(int file_descriptor, const Config* config) {
|
bool config_send(int file_descriptor, const Config* config) {
|
||||||
|
|||||||
@@ -137,6 +137,10 @@ typedef struct Config {
|
|||||||
// PR #184: Compression algorithm negotiation
|
// PR #184: Compression algorithm negotiation
|
||||||
char* compress_choice;
|
char* compress_choice;
|
||||||
char* chmod_spec;
|
char* chmod_spec;
|
||||||
|
|
||||||
|
char** skip_compress_suffixes;
|
||||||
|
int skip_compress_count;
|
||||||
|
bool skip_compress_set;
|
||||||
} Config;
|
} Config;
|
||||||
|
|
||||||
#define PROTOCOL_VERSION "2.3.0"
|
#define PROTOCOL_VERSION "2.3.0"
|
||||||
|
|||||||
@@ -217,7 +217,10 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
|||||||
}
|
}
|
||||||
|
|
||||||
Data* raw_delta = delta_data;
|
Data* raw_delta = delta_data;
|
||||||
if (config->use_compression) {
|
if (config->use_compression &&
|
||||||
|
!compression_should_skip_with_suffixes(
|
||||||
|
check_path, config->skip_compress_suffixes,
|
||||||
|
config->skip_compress_set ? config->skip_compress_count : -1)) {
|
||||||
raw_delta = data_decompress_limited(delta_data, MAX_RECEIVE_FILE_SIZE);
|
raw_delta = data_decompress_limited(delta_data, MAX_RECEIVE_FILE_SIZE);
|
||||||
data_destroy(delta_data);
|
data_destroy(delta_data);
|
||||||
if (!raw_delta) {
|
if (!raw_delta) {
|
||||||
@@ -320,7 +323,10 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config->use_compression) {
|
if (config->use_compression &&
|
||||||
|
!compression_should_skip_with_suffixes(
|
||||||
|
file->path, config->skip_compress_suffixes,
|
||||||
|
config->skip_compress_set ? config->skip_compress_count : -1)) {
|
||||||
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_FILE_SIZE);
|
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_FILE_SIZE);
|
||||||
data_destroy(file_data);
|
data_destroy(file_data);
|
||||||
if (uncompressed == NULL) {
|
if (uncompressed == NULL) {
|
||||||
@@ -519,7 +525,10 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config->use_compression) {
|
if (config->use_compression &&
|
||||||
|
!compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
|
||||||
|
config->skip_compress_set ? config->skip_compress_count
|
||||||
|
: -1)) {
|
||||||
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_FILE_SIZE);
|
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_FILE_SIZE);
|
||||||
data_destroy(file_data);
|
data_destroy(file_data);
|
||||||
if (uncompressed == NULL) {
|
if (uncompressed == NULL) {
|
||||||
@@ -569,7 +578,10 @@ File* file_receive(const Config* config, int file_descriptor) {
|
|||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
if (config->use_compression && !compression_should_skip(file->path)) {
|
if (config->use_compression &&
|
||||||
|
!compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
|
||||||
|
config->skip_compress_set ? config->skip_compress_count
|
||||||
|
: -1)) {
|
||||||
Data* file_data_uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_FILE_SIZE);
|
Data* file_data_uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_FILE_SIZE);
|
||||||
data_destroy(file_data);
|
data_destroy(file_data);
|
||||||
if (file_data_uncompressed == NULL) {
|
if (file_data_uncompressed == NULL) {
|
||||||
|
|||||||
+18
-3
@@ -19,11 +19,19 @@
|
|||||||
|
|
||||||
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
|
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
|
||||||
int compression_level, bool send_path) {
|
int compression_level, bool send_path) {
|
||||||
|
return file_send_single_calls_with_skip(file, file_descriptor, use_metadata, compression_level,
|
||||||
|
send_path, NULL, -1);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||||
|
int compression_level, bool send_path,
|
||||||
|
char* const* skip_suffixes, int skip_count) {
|
||||||
if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data))
|
if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data))
|
||||||
return false;
|
return false;
|
||||||
const Data* data_to_send = file->data;
|
const Data* data_to_send = file->data;
|
||||||
Data* compressed_data = NULL;
|
Data* compressed_data = NULL;
|
||||||
if (compression_level > 0 && !compression_should_skip(file->path)) {
|
if (compression_level > 0 &&
|
||||||
|
!compression_should_skip_with_suffixes(file->path, skip_suffixes, skip_count)) {
|
||||||
compressed_data = data_compress(file->data, compression_level);
|
compressed_data = data_compress(file->data, compression_level);
|
||||||
if (compressed_data == NULL) {
|
if (compressed_data == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
|
log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
|
||||||
@@ -49,11 +57,18 @@ bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
|
|||||||
|
|
||||||
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
|
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
|
||||||
bool send_path) {
|
bool send_path) {
|
||||||
|
return file_send_sendfile_with_skip(file, file_descriptor, use_metadata, compression_level,
|
||||||
|
send_path, NULL, -1);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||||
|
int compression_level, bool send_path, char* const* skip_suffixes,
|
||||||
|
int skip_count) {
|
||||||
if (!file || !file->path || !file->data)
|
if (!file || !file->path || !file->data)
|
||||||
return false;
|
return false;
|
||||||
if (compression_level > 0)
|
if (compression_level > 0)
|
||||||
return file_send_single_calls(file, file_descriptor, use_metadata, compression_level,
|
return file_send_single_calls_with_skip(file, file_descriptor, use_metadata, compression_level,
|
||||||
send_path);
|
send_path, skip_suffixes, skip_count);
|
||||||
|
|
||||||
if (send_path && !send_str(file_descriptor, file->path))
|
if (send_path && !send_str(file_descriptor, file->path))
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@@ -8,7 +8,13 @@
|
|||||||
|
|
||||||
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
|
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
|
||||||
int compression_level, bool send_path);
|
int compression_level, bool send_path);
|
||||||
|
bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||||
|
int compression_level, bool send_path,
|
||||||
|
char* const* skip_suffixes, int skip_count);
|
||||||
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
|
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
|
||||||
bool send_path);
|
bool send_path);
|
||||||
|
bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||||
|
int compression_level, bool send_path, char* const* skip_suffixes,
|
||||||
|
int skip_count);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -228,6 +228,64 @@ class TestCompressionChoice:
|
|||||||
assert not mismatches, f"Mismatch: {mismatches}"
|
assert not mismatches, f"Mismatch: {mismatches}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestSkipCompress:
|
||||||
|
def test_skip_compress_case_insensitive(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
with open(os.path.join(SOURCE_DIR, "skip-case.TXT"), "wb") as f:
|
||||||
|
f.write((b"skip compression case test\n" * 100))
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-c", "--skip-compress=.txt"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Skip-compress sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
with open(os.path.join(received, "skip-case.TXT"), "rb") as f:
|
||||||
|
assert f.read() == b"skip compression case test\n" * 100
|
||||||
|
|
||||||
|
def test_skip_compress_empty_list(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-c", "--skip-compress="],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Empty skip-compress sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
||||||
|
assert not missing, f"Missing: {missing}"
|
||||||
|
assert not mismatches, f"Mismatch: {mismatches}"
|
||||||
|
|
||||||
|
def test_skip_compress_incremental_full_fallback(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
path = os.path.join(SOURCE_DIR, "incremental-skip.TXT")
|
||||||
|
with open(path, "wb") as f:
|
||||||
|
f.write(b"original skipped content\n")
|
||||||
|
flags = ["-c", "-M", "--skip-compress=.txt"]
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Initial sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
with open(path, "wb") as f:
|
||||||
|
f.write(b"updated skipped content\n")
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=flags + ["--incremental"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Incremental sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
with open(os.path.join(received, "incremental-skip.TXT"), "rb") as f:
|
||||||
|
assert f.read() == b"updated skipped content\n"
|
||||||
|
|
||||||
|
def test_skip_compress_rejects_chunk_serialization(self, shared_server):
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-c", "-s", "--skip-compress=.txt"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert "cannot be combined" in (result.stderr or result.stdout)
|
||||||
|
|
||||||
|
|
||||||
class TestExclude:
|
class TestExclude:
|
||||||
def test_exclude_single(self, shared_server):
|
def test_exclude_single(self, shared_server):
|
||||||
clean_dir(DEST_DIR)
|
clean_dir(DEST_DIR)
|
||||||
|
|||||||
@@ -39,6 +39,9 @@ static void test_validate_config_incompatible_options() {
|
|||||||
cfg->use_incremental = true;
|
cfg->use_incremental = true;
|
||||||
cfg->use_chunk_serialization = true;
|
cfg->use_chunk_serialization = true;
|
||||||
EXPECT_FALSE(validate_config(cfg));
|
EXPECT_FALSE(validate_config(cfg));
|
||||||
|
cfg->use_incremental = false;
|
||||||
|
cfg->skip_compress_set = true;
|
||||||
|
EXPECT_FALSE(validate_config(cfg));
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -426,6 +429,32 @@ static void test_parse_args_rejects_invalid_modify_window() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_parse_args_skip_compress() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--skip-compress=.ZIP, .GZ", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->skip_compress_set);
|
||||||
|
EXPECT_EQ_INT(cfg->skip_compress_count, 2);
|
||||||
|
EXPECT_EQ_STR(cfg->skip_compress_suffixes[0], ".ZIP");
|
||||||
|
EXPECT_EQ_STR(cfg->skip_compress_suffixes[1], ".GZ");
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_parse_args_empty_skip_compress() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
char* argv[] = {"fastsync", "--skip-compress=", "/src", "/dst"};
|
||||||
|
int positional_args[2];
|
||||||
|
int positional_count = 0;
|
||||||
|
|
||||||
|
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||||
|
EXPECT_TRUE(cfg->skip_compress_set);
|
||||||
|
EXPECT_EQ_INT(cfg->skip_compress_count, 0);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
/* Test parse_args unknown option returns error */
|
/* Test parse_args unknown option returns error */
|
||||||
static void test_parse_args_unknown_option() {
|
static void test_parse_args_unknown_option() {
|
||||||
Config* cfg = config_create();
|
Config* cfg = config_create();
|
||||||
@@ -807,6 +836,8 @@ void test_client_cli() {
|
|||||||
test_parse_args_debug_flags_validation();
|
test_parse_args_debug_flags_validation();
|
||||||
test_parse_args_modify_window();
|
test_parse_args_modify_window();
|
||||||
test_parse_args_rejects_invalid_modify_window();
|
test_parse_args_rejects_invalid_modify_window();
|
||||||
|
test_parse_args_skip_compress();
|
||||||
|
test_parse_args_empty_skip_compress();
|
||||||
test_parse_args_unknown_option();
|
test_parse_args_unknown_option();
|
||||||
test_parse_args_rejects_unimplemented_options();
|
test_parse_args_rejects_unimplemented_options();
|
||||||
test_parse_args_quiet();
|
test_parse_args_quiet();
|
||||||
|
|||||||
@@ -55,6 +55,14 @@ static void test_data_compress_decompress_large() {
|
|||||||
data_destroy(decompressed);
|
data_destroy(decompressed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_skip_compress_suffix_matching() {
|
||||||
|
char* suffixes[] = {".ZIP", ".GZ"};
|
||||||
|
EXPECT_TRUE(compression_should_skip_with_suffixes("archive.zip", suffixes, 2));
|
||||||
|
EXPECT_TRUE(compression_should_skip_with_suffixes("backup.TAR.GZ", suffixes, 2));
|
||||||
|
EXPECT_FALSE(compression_should_skip_with_suffixes("notes.txt", suffixes, 2));
|
||||||
|
EXPECT_FALSE(compression_should_skip_with_suffixes("archive.zip", suffixes, 0));
|
||||||
|
}
|
||||||
|
|
||||||
static void test_chunk_compress_decompress_roundtrip() {
|
static void test_chunk_compress_decompress_roundtrip() {
|
||||||
char* path1 = "temp_comp_test_1.txt";
|
char* path1 = "temp_comp_test_1.txt";
|
||||||
char* content1 = "chunk compression test file 1";
|
char* content1 = "chunk compression test file 1";
|
||||||
@@ -115,5 +123,6 @@ static void test_chunk_compress_decompress_roundtrip() {
|
|||||||
void test_compression() {
|
void test_compression() {
|
||||||
test_data_compress_decompress_roundtrip();
|
test_data_compress_decompress_roundtrip();
|
||||||
test_data_compress_decompress_large();
|
test_data_compress_decompress_large();
|
||||||
|
test_skip_compress_suffix_matching();
|
||||||
test_chunk_compress_decompress_roundtrip();
|
test_chunk_compress_decompress_roundtrip();
|
||||||
}
|
}
|
||||||
|
|||||||
+9
-2
@@ -121,7 +121,7 @@ static void test_config_send_receive() {
|
|||||||
send_cfg->receive_root_directory = str_dup("/send/dst");
|
send_cfg->receive_root_directory = str_dup("/send/dst");
|
||||||
send_cfg->save_to_disk = true;
|
send_cfg->save_to_disk = true;
|
||||||
send_cfg->use_multithreading = true;
|
send_cfg->use_multithreading = true;
|
||||||
send_cfg->use_chunk_serialization = true;
|
send_cfg->use_chunk_serialization = false;
|
||||||
send_cfg->use_compression = true;
|
send_cfg->use_compression = true;
|
||||||
send_cfg->use_metadata = true;
|
send_cfg->use_metadata = true;
|
||||||
send_cfg->use_executability = true;
|
send_cfg->use_executability = true;
|
||||||
@@ -135,6 +135,10 @@ static void test_config_send_receive() {
|
|||||||
send_cfg->modify_window = 4;
|
send_cfg->modify_window = 4;
|
||||||
send_cfg->existing = true;
|
send_cfg->existing = true;
|
||||||
send_cfg->ignore_existing = true;
|
send_cfg->ignore_existing = true;
|
||||||
|
send_cfg->skip_compress_set = true;
|
||||||
|
send_cfg->skip_compress_count = 1;
|
||||||
|
send_cfg->skip_compress_suffixes = calloc(1, sizeof(char*));
|
||||||
|
send_cfg->skip_compress_suffixes[0] = str_dup(".zip");
|
||||||
|
|
||||||
/* Use socketpair for bidirectional communication */
|
/* Use socketpair for bidirectional communication */
|
||||||
int p[2];
|
int p[2];
|
||||||
@@ -163,7 +167,7 @@ static void test_config_send_receive() {
|
|||||||
ok = false;
|
ok = false;
|
||||||
if (!recv_cfg->use_multithreading)
|
if (!recv_cfg->use_multithreading)
|
||||||
ok = false;
|
ok = false;
|
||||||
if (!recv_cfg->use_chunk_serialization)
|
if (recv_cfg->use_chunk_serialization)
|
||||||
ok = false;
|
ok = false;
|
||||||
if (recv_cfg->compression_level != 5)
|
if (recv_cfg->compression_level != 5)
|
||||||
ok = false;
|
ok = false;
|
||||||
@@ -187,6 +191,9 @@ static void test_config_send_receive() {
|
|||||||
ok = false;
|
ok = false;
|
||||||
if (!recv_cfg->ignore_existing)
|
if (!recv_cfg->ignore_existing)
|
||||||
ok = false;
|
ok = false;
|
||||||
|
if (!recv_cfg->skip_compress_set || recv_cfg->skip_compress_count != 1 ||
|
||||||
|
strcmp(recv_cfg->skip_compress_suffixes[0], ".zip") != 0)
|
||||||
|
ok = false;
|
||||||
}
|
}
|
||||||
config_delete(recv_cfg);
|
config_delete(recv_cfg);
|
||||||
close(p[0]);
|
close(p[0]);
|
||||||
|
|||||||
Reference in New Issue
Block a user