1 Commits

Author SHA1 Message Date
TapTap c220dc0739 docs: clarify rsync compatibility roadmap
CI / lint (pull_request) Successful in 30s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 17s
CI / coverage (pull_request) Successful in 32s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 34s
2026-08-15 12:49:07 +02:00
13 changed files with 83 additions and 210 deletions
+17 -22
View File
@@ -45,7 +45,7 @@ replacement for every rsync feature or protocol mode.
- Recursive directory scanning. - Recursive directory scanning.
- Rsync-style source and destination arguments. - Rsync-style source and destination arguments.
- SSH transport using `user@host:destination` paths below the remote authorized root. - SSH transport using `user@host:/path` destinations.
- TCP client/server transfers. - TCP client/server transfers.
- Dry runs, excludes, includes, size filters, backups, statistics, and - Dry runs, excludes, includes, size filters, backups, statistics, and
bandwidth limiting. bandwidth limiting.
@@ -79,10 +79,8 @@ partial, alternate, and planned behavior.
### Build ### Build
Requirements: C11 compiler, CMake 3.22 or newer, xxHash, zstd, OpenSSL, Requirements: C11 compiler, CMake 3.22 or newer, zstd, OpenSSL, pthreads,
pthreads, and an SSH client for SSH transport. The first CMake configure fetches and an SSH client for SSH transport.
xxHash from GitHub, so network access is required unless the dependency is
already cached.
```bash ```bash
cmake -B build -S . cmake -B build -S .
@@ -100,12 +98,10 @@ cmake --build build -j$(nproc)
### SSH transfer ### SSH transfer
The remote host must have `fastsync-server` available in `PATH`, or use The remote host must have `fastsync-server` available in `PATH`, or use
`--fastsync-server-path`. SSH starts `fastsync-server --stdio` in its remote `--fastsync-server-path`.
working directory, so use a destination below that directory unless the
remote server is otherwise configured with a matching authorized root.
```bash ```bash
./build/client /path/to/source user@host:destination ./build/client /path/to/source user@host:/path/to/destination
``` ```
### TCP transfer ### TCP transfer
@@ -113,7 +109,7 @@ remote server is otherwise configured with a matching authorized root.
Start the FastSync server: Start the FastSync server:
```bash ```bash
./build/server --destination-root /path/to -p 8080 ./build/server -p 8080
``` ```
Then run the client: Then run the client:
@@ -127,7 +123,7 @@ Then run the client:
### TLS transfer ### TLS transfer
```bash ```bash
./build/server --destination-root /path/to --tls --cert server.pem --key server-key.pem -p 8443 ./build/server --tls --cert server.pem --key server-key.pem -p 8443
./build/client --tls --cert client.pem --key client-key.pem --ca ca.pem \ ./build/client --tls --cert client.pem --key client-key.pem --ca ca.pem \
--server-host example.com --server-port 8443 \ --server-host example.com --server-port 8443 \
--source-dir /path/to/source --dest-dir /path/to/destination \ --source-dir /path/to/source --dest-dir /path/to/destination \
@@ -144,30 +140,30 @@ FastSync-native are optional performance or transport extensions.
./build/client /source/ /destination/ ./build/client /source/ /destination/
# Archive-style synchronization (current FastSync archive behavior) # Archive-style synchronization (current FastSync archive behavior)
./build/client -a /source/ user@host:destination/ ./build/client -a /source/ user@host:/destination/
# Preview a transfer without changing the destination # Preview a transfer without changing the destination
./build/client -n /source/ /destination/ ./build/client -n /source/ /destination/
# Exclude temporary and object files # Exclude temporary and object files
./build/client --exclude '*.tmp' --exclude '*.o' \ ./build/client --exclude '*.tmp' --exclude '*.o' \
/source/ user@host:destination/ /source/ user@host:/destination/
# Remove destination entries not present in the source # Remove destination entries not present in the source
./build/client --delete /source/ user@host:destination/ ./build/client --delete /source/ user@host:/destination/
# Skip unchanged files using size and modification time # Skip unchanged files using size and modification time
./build/client --incremental /source/ user@host:destination/ ./build/client --incremental /source/ user@host:/destination/
# Verify content when size and time are not sufficient # Verify content when size and time are not sufficient
./build/client --incremental --checksum /source/ user@host:destination/ ./build/client --incremental --checksum /source/ user@host:/destination/
# Preserve supported mode and timestamp metadata # Preserve supported mode and timestamp metadata
./build/client -M /source/ user@host:destination/ ./build/client -M /source/ user@host:/destination/
# Keep backups of overwritten destination files # Keep backups of overwritten destination files
./build/client --backup --backup-dir backups \ ./build/client --backup --backup-dir /backups \
/source/ user@host:destination/ /source/ user@host:/destination/
``` ```
## FastSync Extensions ## FastSync Extensions
@@ -221,8 +217,8 @@ before FastSync can claim full rsync CLI compatibility.
| `--backup` | Back up overwritten files. | | `--backup` | Back up overwritten files. |
| `--backup-dir <dir>` | Store backups under a separate directory. | | `--backup-dir <dir>` | Store backups under a separate directory. |
| `--suffix <suffix>` | Set the backup filename suffix. | | `--suffix <suffix>` | Set the backup filename suffix. |
| `--partial` | Select partial-transfer handling. With `--partial-dir`, completed files are written there; resumable transfers are not implemented. | | `--partial` | Keep received data under the configured partial location. Full resume semantics are planned. |
| `--partial-dir <dir>` | Set a relative partial-transfer directory below the server destination root; use with `--partial`. | | `--partial-dir <dir>` | Set the partial transfer directory. |
| `--inplace` | Write directly to the destination instead of using a temporary file. | | `--inplace` | Write directly to the destination instead of using a temporary file. |
### Metadata and links ### Metadata and links
@@ -273,7 +269,6 @@ before FastSync can claim full rsync CLI compatibility.
| `--cert <path>` | TLS certificate file. | | `--cert <path>` | TLS certificate file. |
| `--key <path>` | TLS private key file. | | `--key <path>` | TLS private key file. |
| `--ca <path>` | CA file for peer verification. | | `--ca <path>` | CA file for peer verification. |
| `--destination-root <path>` | Confine received files to this server-side root; defaults to the current directory. |
| `--allow-delete` | Permit client delete manifests. Deletion is refused by default. | | `--allow-delete` | Permit client delete manifests. Deletion is refused by default. |
| `-v`, `--verbose` | Enable debug logging. | | `-v`, `--verbose` | Enable debug logging. |
| `--help` | Print server usage. | | `--help` | Print server usage. |
+2 -2
View File
@@ -152,14 +152,14 @@ This document maps rsync's full feature set to FastSync's current implementation
| Flag | Rsync Description | FastSync Status | Notes | | Flag | Rsync Description | FastSync Status | Notes |
|------|-------------------|-----------------|-------| |------|-------------------|-----------------|-------|
| `-S`, `--sparse` | Sparse block handling | ⚠️ Partial | Flag is accepted, but full hole preservation is not implemented | | `-S`, `--sparse` | Sparse block handling | ✅ Implemented | `preserve_sparse` config field |
| `--preallocate` | Allocate dest files before writing | ❌ Not Implemented | | | `--preallocate` | Allocate dest files before writing | ❌ Not Implemented | |
## 11. Checksum & Comparison ## 11. Checksum & Comparison
| Flag | Rsync Description | FastSync Status | Notes | | Flag | Rsync Description | FastSync Status | Notes |
|------|-------------------|-----------------|-------| |------|-------------------|-----------------|-------|
| `--checksum` | Skip based on checksum | Implemented | With `--incremental`, compares xxHash64 content checksums; `-c` remains compression | | `--checksum` | Skip based on checksum | ❌ Not Implemented | Removed because it had no effect; `-c` means compression |
| `--checksum-choice=STR` | Choose checksum algorithm | ❌ Not Implemented | xxHash used internally | | `--checksum-choice=STR` | Choose checksum algorithm | ❌ Not Implemented | xxHash used internally |
| `--compare-dest=DIR` | Compare dest files relative to DIR | ❌ Not Implemented | Removed because it had no effect | | `--compare-dest=DIR` | Compare dest files relative to DIR | ❌ Not Implemented | Removed because it had no effect |
| `--copy-dest=DIR` | Include copies of unchanged files | ❌ Not Implemented | Removed because it had no effect | | `--copy-dest=DIR` | Include copies of unchanged files | ❌ Not Implemented | Removed because it had no effect |
+8 -26
View File
@@ -60,7 +60,6 @@ static Client* connect_transfer_client(const Config* config) {
connected = client_connect(client, config->server_host, config->server_port); connected = client_connect(client, config->server_host, config->server_port);
} }
if (!connected) { if (!connected) {
client_disconnect(client);
client_delete(client); client_delete(client);
return NULL; return NULL;
} }
@@ -209,11 +208,8 @@ static int incremental_check(Client* client, File* file, const Config* config,
static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* config) { static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* config) {
Delta* delta = delta_compute(file->data->data, file->data->size, sig, config->delta_block_size); Delta* delta = delta_compute(file->data->data, file->data->size, sig, config->delta_block_size);
if (!delta) { if (!delta)
if (!send_status(client->file_descriptor, STATUS_NEXT))
return -1;
return 1; return 1;
}
if (!delta_is_worthwhile(delta, file->data->size)) { if (!delta_is_worthwhile(delta, file->data->size)) {
delta_destroy(delta); delta_destroy(delta);
@@ -379,7 +375,6 @@ static int send_chunks_multithreaded(void* pipeline_context) {
if (context->config->transport == TRANSPORT_TCP) if (context->config->transport == TRANSPORT_TCP)
fprintf(stderr, "Error: could not connect to server%s\n", fprintf(stderr, "Error: could not connect to server%s\n",
context->config->use_tls ? " via TLS" : ""); context->config->use_tls ? " via TLS" : "");
pipeline_cancel(context);
mark_sender_done(context); mark_sender_done(context);
return thrd_error; return thrd_error;
} }
@@ -388,7 +383,6 @@ static int send_chunks_multithreaded(void* pipeline_context) {
protocol_session_set_ssl(&session, (SSL*)client->ssl); protocol_session_set_ssl(&session, (SSL*)client->ssl);
protocol_session_bind(&session); protocol_session_bind(&session);
if (!config_send(client->file_descriptor, context->config)) { if (!config_send(client->file_descriptor, context->config)) {
pipeline_cancel(context);
disconnect_transfer_client(client); disconnect_transfer_client(client);
mark_sender_done(context); mark_sender_done(context);
protocol_session_unbind(); protocol_session_unbind();
@@ -400,13 +394,6 @@ static int send_chunks_multithreaded(void* pipeline_context) {
context->queue_loader, &context->mutex_loader, &context->condition_not_empty_loader, context->queue_loader, &context->mutex_loader, &context->condition_not_empty_loader,
&context->condition_not_full_loader, &context->loader_done); &context->condition_not_full_loader, &context->loader_done);
if (current_chunk == NULL) { if (current_chunk == NULL) {
if (atomic_load(&context->cancelled)) {
pipeline_cancel(context);
disconnect_transfer_client(client);
mark_sender_done(context);
protocol_session_unbind();
return thrd_error;
}
if (context->config->use_delete) { if (context->config->use_delete) {
if (send_delete_manifest(client->file_descriptor, context->manifest) != 0) if (send_delete_manifest(client->file_descriptor, context->manifest) != 0)
goto send_fail; goto send_fail;
@@ -519,10 +506,9 @@ static int load_files_multithreaded(void* pipeline_context) {
if (f->data->size > STREAM_THRESHOLD) if (f->data->size > STREAM_THRESHOLD)
continue; continue;
if (!file_load_data(f)) { if (!file_load_data(f)) {
log_message(LOG_LEVEL_ERROR, "Failed to load file data"); log_message(LOG_LEVEL_ERROR, "Failed to load file data, skipping");
chunk_destroy(chunk); file_destroy(f);
pipeline_cancel(context); chunk->items[i] = NULL;
return thrd_error;
} }
} }
} }
@@ -531,7 +517,9 @@ static int load_files_multithreaded(void* pipeline_context) {
&context->condition_not_full_loader, &context->condition_not_full_loader,
&context->cancelled)) { &context->cancelled)) {
chunk_destroy(chunk); chunk_destroy(chunk);
pipeline_cancel(context); atomic_store(&context->cancelled, true);
cnd_broadcast(&context->condition_not_full_loader);
cnd_broadcast(&context->condition_not_empty_loader);
return thrd_error; return thrd_error;
} }
} }
@@ -627,8 +615,7 @@ int send_files(Config* config) {
continue; continue;
if (!file_load_data(f)) { if (!file_load_data(f)) {
log_message(LOG_LEVEL_ERROR, "Failed to load file data"); log_message(LOG_LEVEL_ERROR, "Failed to load file data");
chunk_destroy(current_chunk); continue;
goto send_fail;
} }
} }
} }
@@ -658,7 +645,6 @@ int send_files(Config* config) {
if (config->use_delete) { if (config->use_delete) {
if (send_delete_manifest(client->file_descriptor, manifest) != 0) { if (send_delete_manifest(client->file_descriptor, manifest) != 0) {
array_list_delete(manifest); array_list_delete(manifest);
manifest = NULL;
goto send_fail; goto send_fail;
} }
array_list_delete(manifest); array_list_delete(manifest);
@@ -722,10 +708,6 @@ int send_files_multithreaded(Config* config) {
} }
if (config->use_delete) if (config->use_delete)
context->manifest = create_transfer_manifest(config); context->manifest = create_transfer_manifest(config);
if (config->use_delete && !context->manifest) {
pipeline_context_sender_destroy(context);
return 1;
}
thrd_t scanner, loader, sender; thrd_t scanner, loader, sender;
bool scanner_created = false; bool scanner_created = false;
+13 -26
View File
@@ -363,8 +363,6 @@ static int parallel_worker_thread(void* arg) {
cnd_broadcast(&wa->ps->result_not_empty); cnd_broadcast(&wa->ps->result_not_empty);
cnd_broadcast(&wa->ps->result_not_full); cnd_broadcast(&wa->ps->result_not_full);
mtx_unlock(&wa->ps->result_mutex); mtx_unlock(&wa->ps->result_mutex);
for (int j = i; j < wa->dir_count; j++)
free(wa->dirs[j]);
break; break;
} }
Chunk* chunk; Chunk* chunk;
@@ -400,18 +398,6 @@ static int parallel_worker_thread(void* arg) {
return thrd_success; return thrd_success;
} }
static void parallel_scanner_creation_failed(ParallelScanner* ps) {
mtx_lock(&ps->result_mutex);
ps->failed = true;
atomic_store(&ps->cancelled, true);
ps->expected_threads = ps->created_threads;
if (ps->completed >= ps->expected_threads)
ps->done = true;
cnd_broadcast(&ps->result_not_empty);
cnd_broadcast(&ps->result_not_full);
mtx_unlock(&ps->result_mutex);
}
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory, ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
const ScannerOptions* options) { const ScannerOptions* options) {
if (!root_directory || !options) if (!root_directory || !options)
@@ -534,6 +520,7 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
void** items = array_list_to_array(batch); void** items = array_list_to_array(batch);
if (!items) { if (!items) {
ps->failed = true; ps->failed = true;
batch->item_destroyer = file_destroy;
array_list_delete(batch); array_list_delete(batch);
batch = NULL; batch = NULL;
break; break;
@@ -542,13 +529,11 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
free(items); free(items);
if (!c) { if (!c) {
ps->failed = true; ps->failed = true;
batch->item_destroyer = file_destroy;
array_list_delete(batch); array_list_delete(batch);
batch = NULL; batch = NULL;
break; break;
} }
int batch_start = i - batch->size + 1;
for (int j = batch_start; j <= i; j++)
root_files->items[j] = NULL;
batch->item_destroyer = NULL; batch->item_destroyer = NULL;
array_list_delete(batch); array_list_delete(batch);
batch = NULL; batch = NULL;
@@ -575,6 +560,7 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
array_list_delete(batch); array_list_delete(batch);
} }
ps->initial_chunk = first; ps->initial_chunk = first;
root_files->item_destroyer = NULL;
} }
array_list_delete(root_files); array_list_delete(root_files);
@@ -601,14 +587,14 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
break; break;
ParallelWorkerArg* wa = calloc(1, sizeof(ParallelWorkerArg)); ParallelWorkerArg* wa = calloc(1, sizeof(ParallelWorkerArg));
if (!wa) { if (!wa) {
parallel_scanner_creation_failed(ps); ps->failed = true;
break; break;
} }
wa->ps = ps; wa->ps = ps;
wa->dirs = calloc(count, sizeof(char*)); wa->dirs = calloc(count, sizeof(char*));
if (!wa->dirs) { if (!wa->dirs) {
free(wa); free(wa);
parallel_scanner_creation_failed(ps); ps->failed = true;
break; break;
} }
bool dup_ok = true; bool dup_ok = true;
@@ -622,7 +608,7 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
free(wa->dirs[j]); free(wa->dirs[j]);
free(wa->dirs); free(wa->dirs);
free(wa); free(wa);
parallel_scanner_creation_failed(ps); ps->failed = true;
break; break;
} }
wa->dir_count = count; wa->dir_count = count;
@@ -634,7 +620,13 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
free(wa->dirs[j]); free(wa->dirs[j]);
free(wa->dirs); free(wa->dirs);
free(wa); free(wa);
parallel_scanner_creation_failed(ps); ps->failed = true;
atomic_store(&ps->cancelled, true);
ps->expected_threads = ps->created_threads;
mtx_lock(&ps->result_mutex);
cnd_broadcast(&ps->result_not_empty);
cnd_broadcast(&ps->result_not_full);
mtx_unlock(&ps->result_mutex);
break; break;
} }
ps->num_threads++; ps->num_threads++;
@@ -667,11 +659,6 @@ Chunk* parallel_scanner_next(ParallelScanner* ps) {
} }
if (ps->num_threads == 0) { if (ps->num_threads == 0) {
mtx_lock(&ps->result_mutex); mtx_lock(&ps->result_mutex);
if (!queue_is_empty(ps->result_queue)) {
Chunk* chunk = queue_dequeue(ps->result_queue);
mtx_unlock(&ps->result_mutex);
return chunk;
}
ps->done = true; ps->done = true;
mtx_unlock(&ps->result_mutex); mtx_unlock(&ps->result_mutex);
return NULL; return NULL;
-1
View File
@@ -106,7 +106,6 @@ void handler(int file_descriptor) {
protocol_session_unbind(); protocol_session_unbind();
return; return;
} }
context->session.total_allocated_bytes = session.total_allocated_bytes;
thrd_t receiver, writer; thrd_t receiver, writer;
bool receiver_created = thrd_create(&receiver, receive_thread, context) == thrd_success; bool receiver_created = thrd_create(&receiver, receive_thread, context) == thrd_success;
bool writer_created = false; bool writer_created = false;
+3 -42
View File
@@ -1,5 +1,4 @@
#include <stddef.h> #include <stddef.h>
#include <stdint.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
@@ -15,7 +14,6 @@
/* Maximum individual file data size within a chunk (64 MB) */ /* Maximum individual file data size within a chunk (64 MB) */
#define MAX_FILE_DATA_SIZE (64ULL * 1024 * 1024) #define MAX_FILE_DATA_SIZE (64ULL * 1024 * 1024)
#define MAX_CHUNK_FILES (1024 * 1024)
Chunk* chunk_create(File** items, int element_count) { Chunk* chunk_create(File** items, int element_count) {
Chunk* chunk = (Chunk*)malloc(sizeof(Chunk)); Chunk* chunk = (Chunk*)malloc(sizeof(Chunk));
@@ -94,11 +92,6 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
size_t remaining_size = data->size; size_t remaining_size = data->size;
while (remaining_size > 0) { while (remaining_size > 0) {
if (files->size >= MAX_CHUNK_FILES) {
log_message(LOG_LEVEL_ERROR, "Chunk contains too many files");
array_list_delete(files);
return NULL;
}
if (remaining_size < sizeof(size_t)) { if (remaining_size < sizeof(size_t)) {
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for path length"); log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for path length");
array_list_delete(files); array_list_delete(files);
@@ -110,7 +103,7 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
data_pointer += sizeof(size_t); data_pointer += sizeof(size_t);
remaining_size -= sizeof(size_t); remaining_size -= sizeof(size_t);
if (path_len > SIZE_MAX - 1 || remaining_size < path_len) { if (remaining_size < path_len) {
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for path"); log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for path");
array_list_delete(files); array_list_delete(files);
return NULL; return NULL;
@@ -129,15 +122,10 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
File* file = file_create(path); File* file = file_create(path);
free(path); free(path);
if (file == NULL) {
array_list_delete(files);
return NULL;
}
if (use_metadata) { if (use_metadata) {
if (remaining_size < sizeof(int)) { if (remaining_size < sizeof(int)) {
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for metadata"); log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for metadata");
file_destroy(file);
array_list_delete(files); array_list_delete(files);
return NULL; return NULL;
} }
@@ -146,7 +134,6 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
memcpy(&present_flag, data_pointer, sizeof(int)); memcpy(&present_flag, data_pointer, sizeof(int));
if (present_flag && remaining_size < sizeof(int) + FILE_METADATA_WIRE_SIZE) { if (present_flag && remaining_size < sizeof(int) + FILE_METADATA_WIRE_SIZE) {
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for metadata body"); log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for metadata body");
file_destroy(file);
array_list_delete(files); array_list_delete(files);
return NULL; return NULL;
} }
@@ -154,16 +141,10 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
remaining_size -= sizeof(int); remaining_size -= sizeof(int);
if (file->metadata) if (file->metadata)
remaining_size -= FILE_METADATA_WIRE_SIZE; remaining_size -= FILE_METADATA_WIRE_SIZE;
else if (present_flag) {
file_destroy(file);
array_list_delete(files);
return NULL;
}
} }
if (remaining_size < sizeof(size_t)) { if (remaining_size < sizeof(size_t)) {
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for data size"); log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for data size");
file_destroy(file);
array_list_delete(files); array_list_delete(files);
return NULL; return NULL;
} }
@@ -175,7 +156,6 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
if (remaining_size < file_data_size) { if (remaining_size < file_data_size) {
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for file content"); log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for file content");
file_destroy(file);
array_list_delete(files); array_list_delete(files);
return NULL; return NULL;
} }
@@ -184,48 +164,29 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
if (file_data_size > MAX_FILE_DATA_SIZE) { if (file_data_size > MAX_FILE_DATA_SIZE) {
log_message(LOG_LEVEL_ERROR, "File data size %zu exceeds maximum %llu", file_data_size, log_message(LOG_LEVEL_ERROR, "File data size %zu exceeds maximum %llu", file_data_size,
(unsigned long long)MAX_FILE_DATA_SIZE); (unsigned long long)MAX_FILE_DATA_SIZE);
file_destroy(file);
array_list_delete(files); array_list_delete(files);
return NULL; return NULL;
} }
void* file_data = malloc(file_data_size > 0 ? file_data_size : 1); void* file_data = malloc(file_data_size);
if (file_data == NULL) { if (file_data == NULL) {
perror("Could not allocate memory for file data"); perror("Could not allocate memory for file data");
file_destroy(file);
array_list_delete(files); array_list_delete(files);
return NULL; return NULL;
} }
memcpy(file_data, data_pointer, file_data_size); memcpy(file_data, data_pointer, file_data_size);
data_destroy(file->data); data_destroy(file->data);
file->data = data_create(file_data, file_data_size); file->data = data_create(file_data, file_data_size);
if (file->data == NULL) {
file_destroy(file);
array_list_delete(files);
return NULL;
}
data_pointer += file_data_size; data_pointer += file_data_size;
remaining_size -= file_data_size; remaining_size -= file_data_size;
if (!array_list_add(files, file)) { array_list_add(files, file);
file_destroy(file);
array_list_delete(files);
return NULL;
}
} }
File** file_array = (File**)array_list_to_array(files); File** file_array = (File**)array_list_to_array(files);
if (file_array == NULL) {
array_list_delete(files);
return NULL;
}
Chunk* chunk = chunk_create(file_array, files->size); Chunk* chunk = chunk_create(file_array, files->size);
free(file_array); free(file_array);
if (chunk == NULL) {
array_list_delete(files);
return NULL;
}
files->item_destroyer = NULL; files->item_destroyer = NULL;
array_list_delete(files); array_list_delete(files);
+2 -14
View File
@@ -130,20 +130,9 @@ bool file_store_write_secure(const char* path, const void* data, unsigned long l
ok = file_restore_metadata_fd(fd, metadata); ok = file_restore_metadata_fd(fd, metadata);
} }
} else { } else {
int tmp_size = snprintf(NULL, 0, ".%s.tmp.%ld.%u", leaf, (long)getpid(), 99U); char tmp[NAME_MAX];
if (tmp_size < 0) {
close(dirfd);
free(leaf);
return false;
}
char* tmp = malloc((size_t)tmp_size + 1);
if (!tmp) {
close(dirfd);
free(leaf);
return false;
}
for (unsigned int i = 0; i < 100 && !ok; ++i) { for (unsigned int i = 0; i < 100 && !ok; ++i) {
snprintf(tmp, (size_t)tmp_size + 1, ".%s.tmp.%ld.%u", leaf, (long)getpid(), i); snprintf(tmp, sizeof(tmp), ".%s.tmp.%ld.%u", leaf, (long)getpid(), i);
fd = openat(dirfd, tmp, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600); fd = openat(dirfd, tmp, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600);
if (fd < 0) if (fd < 0)
continue; continue;
@@ -161,7 +150,6 @@ bool file_store_write_secure(const char* path, const void* data, unsigned long l
if (!ok) if (!ok)
unlinkat(dirfd, tmp, 0); unlinkat(dirfd, tmp, 0);
} }
free(tmp);
} }
if (fd >= 0) if (fd >= 0)
close(fd); close(fd);
-9
View File
@@ -183,15 +183,6 @@ int write_thread(void* pipeline_context) {
bool save_to_disk = context->config->save_to_disk; bool save_to_disk = context->config->save_to_disk;
char* root_directory = str_dup(context->config->receive_root_directory); char* root_directory = str_dup(context->config->receive_root_directory);
mtx_unlock(&context->mutex); mtx_unlock(&context->mutex);
if (save_to_disk && !root_directory) {
mtx_lock(&context->mutex);
atomic_store(&context->cancelled, true);
context->receiver_done = true;
cnd_broadcast(&context->condition_not_full);
cnd_broadcast(&context->condition_not_empty);
mtx_unlock(&context->mutex);
return thrd_error;
}
while (true) { while (true) {
File* file = File* file =
+37 -30
View File
@@ -19,7 +19,6 @@ static __thread int io_read_fd = -1;
static __thread int io_write_fd = -1; static __thread int io_write_fd = -1;
static __thread SSL* io_ssl; static __thread SSL* io_ssl;
static __thread ProtocolSession* bound_session; static __thread ProtocolSession* bound_session;
static __thread ProtocolSession legacy_io_session = {.read_fd = -1, .write_fd = -1};
static unsigned long long io_bwlimit = 0; static unsigned long long io_bwlimit = 0;
static long long bw_tokens = 0; static long long bw_tokens = 0;
@@ -27,6 +26,8 @@ static struct timespec bw_last_refill = {0, 0};
static mtx_t bw_mutex; static mtx_t bw_mutex;
static once_flag bw_mutex_once = ONCE_FLAG_INIT; static once_flag bw_mutex_once = ONCE_FLAG_INIT;
static __thread unsigned long long total_allocated_bytes = 0;
void io_set_fds(int read_fd, int write_fd) { void io_set_fds(int read_fd, int write_fd) {
bound_session = NULL; bound_session = NULL;
io_read_fd = read_fd; io_read_fd = read_fd;
@@ -34,11 +35,7 @@ void io_set_fds(int read_fd, int write_fd) {
/* A descriptor switch starts a new transport; never reuse a TLS object /* A descriptor switch starts a new transport; never reuse a TLS object
belonging to a previous connection or test pipe. */ belonging to a previous connection or test pipe. */
io_ssl = NULL; io_ssl = NULL;
legacy_io_session.read_fd = read_fd; total_allocated_bytes = 0;
legacy_io_session.write_fd = write_fd;
legacy_io_session.ssl = NULL;
legacy_io_session.total_allocated_bytes = 0;
protocol_session_set_bwlimit(&legacy_io_session, io_bwlimit);
} }
void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd) { void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd) {
@@ -129,30 +126,32 @@ SSL* io_get_ssl(void) {
return io_ssl; return io_ssl;
} }
static ProtocolSession* legacy_session(int read_fd, int write_fd) { static ProtocolSession* legacy_session(void) {
static __thread ProtocolSession session;
if (bound_session) if (bound_session)
return bound_session; return bound_session;
int target_read_fd = io_read_fd != -1 ? io_read_fd : read_fd; session.read_fd = io_read_fd;
int target_write_fd = io_write_fd != -1 ? io_write_fd : write_fd; session.write_fd = io_write_fd;
if (legacy_io_session.read_fd != target_read_fd || session.ssl = io_ssl;
legacy_io_session.write_fd != target_write_fd) { session.bwlimit = io_bwlimit;
legacy_io_session.read_fd = target_read_fd; session.bw_tokens = (unsigned long long)(bw_tokens < 0 ? 0 : bw_tokens);
legacy_io_session.write_fd = target_write_fd; session.bw_last_refill_sec = bw_last_refill.tv_sec;
legacy_io_session.total_allocated_bytes = 0; session.bw_last_refill_nsec = bw_last_refill.tv_nsec;
protocol_session_set_bwlimit(&legacy_io_session, io_bwlimit); return &session;
} else if (legacy_io_session.bwlimit != io_bwlimit) {
protocol_session_set_bwlimit(&legacy_io_session, io_bwlimit);
}
legacy_io_session.ssl = io_ssl;
return &legacy_io_session;
} }
bool send_n_data(int file_descriptor, const void* data, size_t data_size) { bool send_n_data(int file_descriptor, const void* data, size_t data_size) {
return protocol_send_n_data(legacy_session(-1, file_descriptor), data, data_size); ProtocolSession* session = legacy_session();
if (session->write_fd == -1)
session->write_fd = file_descriptor;
return protocol_send_n_data(session, data, data_size);
} }
bool receive_n_data(int file_descriptor, void* data, size_t data_size) { bool receive_n_data(int file_descriptor, void* data, size_t data_size) {
return protocol_receive_n_data(legacy_session(file_descriptor, -1), data, data_size); ProtocolSession* session = legacy_session();
if (session->read_fd == -1)
session->read_fd = file_descriptor;
return protocol_receive_n_data(session, data, data_size);
} }
static int deadline_remaining_ms(const struct timespec* deadline) { static int deadline_remaining_ms(const struct timespec* deadline) {
@@ -404,26 +403,34 @@ bool protocol_receive_status(ProtocolSession* session, Status* status) {
} }
bool send_str(int fd, const char* data) { bool send_str(int fd, const char* data) {
return protocol_send_str(legacy_session(-1, fd), data); (void)fd;
return protocol_send_str(legacy_session(), data);
} }
char* receive_str(int fd) { char* receive_str(int fd) {
return protocol_receive_str(legacy_session(fd, -1)); (void)fd;
return protocol_receive_str(legacy_session());
} }
bool send_data(int fd, const Data* data) { bool send_data(int fd, const Data* data) {
return protocol_send_data(legacy_session(-1, fd), data); (void)fd;
return protocol_send_data(legacy_session(), data);
} }
Data* receive_data(int fd) { Data* receive_data(int fd) {
return protocol_receive_data(legacy_session(fd, -1)); (void)fd;
return protocol_receive_data(legacy_session());
} }
bool send_int(int fd, int data) { bool send_int(int fd, int data) {
return protocol_send_int(legacy_session(-1, fd), data); (void)fd;
return protocol_send_int(legacy_session(), data);
} }
bool receive_int(int fd, int* data) { bool receive_int(int fd, int* data) {
return protocol_receive_int(legacy_session(fd, -1), data); (void)fd;
return protocol_receive_int(legacy_session(), data);
} }
bool send_status(int fd, Status status) { bool send_status(int fd, Status status) {
return protocol_send_status(legacy_session(-1, fd), status); (void)fd;
return protocol_send_status(legacy_session(), status);
} }
bool receive_status(int fd, Status* status) { bool receive_status(int fd, Status* status) {
return protocol_receive_status(legacy_session(fd, -1), status); (void)fd;
return protocol_receive_status(legacy_session(), status);
} }
-4
View File
@@ -151,10 +151,6 @@ int tcp_get_contimeout_sec(void) {
return g_contimeout_sec; return g_contimeout_sec;
} }
int tcp_get_timeout_sec(void) {
return g_timeout_sec;
}
static void tcp_apply_socket_timeout(int fd) { static void tcp_apply_socket_timeout(int fd) {
struct timeval tv; struct timeval tv;
tv.tv_sec = g_timeout_sec; tv.tv_sec = g_timeout_sec;
-1
View File
@@ -35,6 +35,5 @@ void client_disconnect(Client* client);
void client_delete(Client* client); void client_delete(Client* client);
void tcp_set_timeouts(int timeout_sec, int contimeout_sec); void tcp_set_timeouts(int timeout_sec, int contimeout_sec);
int tcp_get_contimeout_sec(void); int tcp_get_contimeout_sec(void);
int tcp_get_timeout_sec(void);
#endif #endif
+1 -4
View File
@@ -9,7 +9,6 @@
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <time.h>
#include <unistd.h> #include <unistd.h>
bool tls_global_init(void) { bool tls_global_init(void) {
@@ -93,7 +92,6 @@ static SSL* wrap_fd_with_ssl(int fd, SSL_CTX* ctx, bool is_server, const char* h
} }
// Retry SSL_accept/SSL_connect on WANT_READ/WANT_WRITE (non-blocking handshake) // Retry SSL_accept/SSL_connect on WANT_READ/WANT_WRITE (non-blocking handshake)
time_t deadline = time(NULL) + (is_server ? tcp_get_timeout_sec() : tcp_get_contimeout_sec());
int ret; int ret;
do { do {
if (is_server) if (is_server)
@@ -103,8 +101,7 @@ static SSL* wrap_fd_with_ssl(int fd, SSL_CTX* ctx, bool is_server, const char* h
if (ret <= 0) { if (ret <= 0) {
int ssl_err = SSL_get_error(ssl, ret); int ssl_err = SSL_get_error(ssl, ret);
if ((ssl_err == SSL_ERROR_WANT_READ || ssl_err == SSL_ERROR_WANT_WRITE) && if (ssl_err == SSL_ERROR_WANT_READ || ssl_err == SSL_ERROR_WANT_WRITE)
time(NULL) < deadline)
continue; continue;
log_message(LOG_LEVEL_ERROR, "SSL %s failed", is_server ? "accept" : "connect"); log_message(LOG_LEVEL_ERROR, "SSL %s failed", is_server ? "accept" : "connect");
log_ssl_errors(); log_ssl_errors();
-29
View File
@@ -385,34 +385,6 @@ static void test_scanner_no_patterns() {
rmdir(dir); rmdir(dir);
} }
static void test_parallel_scanner_root_chunks_without_workers() {
const char* dir = "test_parallel_scan_root";
const char* file1 = "test_parallel_scan_root/a.txt";
const char* file2 = "test_parallel_scan_root/b.txt";
EXPECT_EQ_INT(mkdir(dir, 0755), 0);
create_test_file(file1, "a");
create_test_file(file2, "b");
ParallelScanner* scanner = parallel_scanner_create(dir, false, 1, NULL, 0, NULL, 0, 0, 0, 0, 0,
false, false, false, false, false);
EXPECT_NOT_NULL(scanner);
int total_files = 0;
Chunk* chunk;
while ((chunk = parallel_scanner_next(scanner)) != NULL) {
total_files += chunk->element_count;
chunk_destroy(chunk);
}
EXPECT_EQ_INT(total_files, 2);
EXPECT_FALSE(parallel_scanner_failed(scanner));
parallel_scanner_destroy(scanner);
unlink(file1);
unlink(file2);
rmdir(dir);
}
void test_scanner() { void test_scanner() {
test_scanner_single_file(); test_scanner_single_file();
test_scanner_multiple_files(); test_scanner_multiple_files();
@@ -427,5 +399,4 @@ void test_scanner() {
test_scanner_size_range(); test_scanner_size_range();
test_scanner_mixed_patterns(); test_scanner_mixed_patterns();
test_scanner_no_patterns(); test_scanner_no_patterns();
test_parallel_scanner_root_chunks_without_workers();
} }