Compare commits

...

10 Commits

Author SHA1 Message Date
TapTap fed77f6ce0 ci: use v5 Docker image to force runner to pull updated image with libssl-dev
CI / build-and-test (push) Successful in 33s
CI / build-and-test (pull_request) Successful in 27s
2026-07-18 17:53:08 +02:00
TapTap 6de0998f69 ci: retrigger CI with updated Docker image
CI / build-and-test (push) Failing after 3s
CI / build-and-test (pull_request) Failing after 3s
2026-07-18 17:50:55 +02:00
TapTap 43cf0bd81e ci: add libssl-dev to Docker image for TLS transport
CI / build-and-test (push) Failing after 2s
CI / build-and-test (pull_request) Failing after 2s
2026-07-18 17:43:46 +02:00
TapTap 907baf3379 fix: initialize ssl/ssl_ctx in client_connect_ssh to prevent use-after-free
CI / build-and-test (push) Failing after 3s
CI / build-and-test (pull_request) Failing after 3s
2026-07-18 17:39:37 +02:00
TapTap 8b6550f8c6 fix: io_set_ssl NULL in client_disconnect, SSL* type safety in protocol.h 2026-07-18 17:39:37 +02:00
TapTap 20ede4391c Fix re-review: remove __thread from io_ssl (breaks -m), strtol port parsing, --ca warning 2026-07-18 17:39:37 +02:00
TapTap 864eb1316d Fix TLS code review issues: SSL cleanup, TLS 1.2 min, CA verify, deprecation guards, server --help, port validation, __thread io_ssl, shared accept loop 2026-07-18 17:39:37 +02:00
TapTap eff50346fd TLS transport: OpenSSL-based encrypted TCP
- New transport_tls.h/c: TLS server (server_create_tls, server_listen_tls)
  and client (client_connect_tls) using OpenSSL
- protocol.c: io_set_ssl() + SSL_read/SSL_write in send_n_data/receive_n_data
- transport_tcp.h: ssl/ssl_ctx fields added to Server/Client structs
- config.h/c: use_tls, tls_cert, tls_key fields
- client_cli.c: --tls, --cert, --key flags
- server.c: --tls, --cert, --key, -p flags with TLS support
- CMakeLists.txt: OpenSSL::SSL + OpenSSL::Crypto linkage
- shell.nix: openssl added to buildInputs
2026-07-18 17:38:55 +02:00
TapTap 25383f893b Merge pull request 'Incremental sync: --incremental to skip unchanged files' (#14) from incremental-sync into main
CI / build-and-test (push) Successful in 27s
Reviewed-on: #14
2026-07-18 17:36:25 +02:00
TapTap 46c650e9ae refactor: consolidate _no_path file send variants
CI / build-and-test (push) Successful in 26s
CI / build-and-test (pull_request) Successful in 26s
Add bool send_path parameter to file_send_single_calls and
file_send_sendfile, remove the _no_path variants. Callers in
client_send.c pass true (send path) or false (skip path) based
on whether an incremental check already transmitted the path.
2026-07-18 17:27:33 +02:00
19 changed files with 668 additions and 316 deletions
+1 -1
View File
@@ -5,7 +5,7 @@ on: [push, pull_request]
jobs: jobs:
build-and-test: build-and-test:
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: gitea.tap-tap.win/taptap/fastsync-ci:v4 container: gitea.tap-tap.win/taptap/fastsync-ci:v5
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
+5 -3
View File
@@ -19,6 +19,8 @@ if(NOT ZSTD_LIBRARY)
message(FATAL_ERROR "zstd library not found. Ensure it is in your nix-shell!") message(FATAL_ERROR "zstd library not found. Ensure it is in your nix-shell!")
endif() endif()
find_package(OpenSSL REQUIRED)
file(GLOB SHARED_SRCS "src/shared/*.c") file(GLOB SHARED_SRCS "src/shared/*.c")
file(GLOB SERVER_SRCS "src/server/*.c") file(GLOB SERVER_SRCS "src/server/*.c")
file(GLOB CLIENT_SRCS "src/client/*.c") file(GLOB CLIENT_SRCS "src/client/*.c")
@@ -26,13 +28,13 @@ file(GLOB TEST_SRCS "tests/*.c")
add_executable(server ${SERVER_SRCS} ${SHARED_SRCS}) add_executable(server ${SERVER_SRCS} ${SHARED_SRCS})
target_include_directories(server PRIVATE src/shared src/server src/client) target_include_directories(server PRIVATE src/shared src/server src/client)
target_link_libraries(server PRIVATE Threads::Threads ${ZSTD_LIBRARY}) target_link_libraries(server PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto)
add_executable(client ${CLIENT_SRCS} ${SHARED_SRCS}) add_executable(client ${CLIENT_SRCS} ${SHARED_SRCS})
target_include_directories(client PRIVATE src/shared src/server src/client) target_include_directories(client PRIVATE src/shared src/server src/client)
target_link_libraries(client PRIVATE Threads::Threads ${ZSTD_LIBRARY}) target_link_libraries(client PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto)
add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} src/client/scanner.c) add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} src/client/scanner.c)
target_include_directories(tests PRIVATE tests src/shared src/server src/client) target_include_directories(tests PRIVATE tests src/shared src/server src/client)
target_link_libraries(tests PRIVATE Threads::Threads ${ZSTD_LIBRARY}) target_link_libraries(tests PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto)
+1 -1
View File
@@ -1,6 +1,6 @@
FROM ubuntu:24.04 FROM ubuntu:24.04
RUN apt-get update && apt-get install -y --no-install-recommends \ RUN apt-get update && apt-get install -y --no-install-recommends \
gcc g++ make libc6-dev cmake libzstd-dev git ca-certificates curl && \ gcc g++ make libc6-dev cmake libzstd-dev libssl-dev git ca-certificates curl && \
curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \ curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
apt-get install -y --no-install-recommends nodejs && \ apt-get install -y --no-install-recommends nodejs && \
rm -rf /var/lib/apt/lists/* rm -rf /var/lib/apt/lists/*
+1
View File
@@ -14,6 +14,7 @@ pkgs.mkShell {
buildInputs = with pkgs; [ buildInputs = with pkgs; [
zstd zstd
openssl
]; ];
NIX_ENFORCE_PURITY = 0; NIX_ENFORCE_PURITY = 0;
+24
View File
@@ -2,6 +2,7 @@
#include "config.h" #include "config.h"
#include "log.h" #include "log.h"
#include "protocol.h" #include "protocol.h"
#include "transport_tls.h"
#include "utils.h" #include "utils.h"
#include <errno.h> #include <errno.h>
#include <limits.h> #include <limits.h>
@@ -47,6 +48,10 @@ static void print_usage(void) {
printf(" --server-host <ip> Server IP address (default: 127.0.0.1)\n"); printf(" --server-host <ip> Server IP address (default: 127.0.0.1)\n");
printf(" --server-port <n> Server port (default: 8080)\n"); printf(" --server-port <n> Server port (default: 8080)\n");
printf(" --bwlimit <KB/s> Bandwidth limit in kilobytes per second\n"); printf(" --bwlimit <KB/s> Bandwidth limit in kilobytes per second\n");
printf(" --tls Enable TLS encryption\n");
printf(" --cert <path> TLS certificate file (PEM)\n");
printf(" --key <path> TLS private key file (PEM)\n");
printf(" --ca <path> TLS CA certificate file (PEM)\n");
printf(" --help Show this help\n"); printf(" --help Show this help\n");
} }
@@ -154,6 +159,17 @@ int main(int argc, char *argv[]) {
unsigned long long val = strtoull(argv[++i], NULL, 10); unsigned long long val = strtoull(argv[++i], NULL, 10);
if (val > 0) if (val > 0)
config->chunk_size = val; config->chunk_size = val;
} else if (strcmp(argv[i], "--tls") == 0) {
config->use_tls = true;
} else if (strcmp(argv[i], "--cert") == 0 && i + 1 < argc) {
free(config->tls_cert);
config->tls_cert = str_dup(argv[++i]);
} else if (strcmp(argv[i], "--key") == 0 && i + 1 < argc) {
free(config->tls_key);
config->tls_key = str_dup(argv[++i]);
} else if (strcmp(argv[i], "--ca") == 0 && i + 1 < argc) {
free(config->tls_ca);
config->tls_ca = str_dup(argv[++i]);
} else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) { } else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
set_log_level(LOG_LEVEL_DEBUG); set_log_level(LOG_LEVEL_DEBUG);
} else if (argv[i][0] == '-') { } else if (argv[i][0] == '-') {
@@ -215,6 +231,14 @@ int main(int argc, char *argv[]) {
config->use_metadata = true; config->use_metadata = true;
} }
if (config->use_tls) {
if (!config->tls_cert || !config->tls_key) {
fprintf(stderr, "Error: --tls requires --cert and --key\n");
return 1;
}
tls_global_init();
}
if (config->use_multithreading) if (config->use_multithreading)
return send_files_multithreaded(config); return send_files_multithreaded(config);
return send_files(config); return send_files(config);
+28 -6
View File
@@ -11,6 +11,7 @@
#include "scanner.h" #include "scanner.h"
#include "transport_tcp.h" #include "transport_tcp.h"
#include "transport_ssh.h" #include "transport_ssh.h"
#include "transport_tls.h"
#include "utils.h" #include "utils.h"
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
@@ -57,11 +58,11 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) {
int rc = incremental_check(client, chunk->items[i]); int rc = incremental_check(client, chunk->items[i]);
if (rc < 0) return -1; if (rc < 0) return -1;
if (rc > 0) continue; if (rc > 0) continue;
if (!file_send_sendfile_no_path(chunk->items[i], client->file_descriptor, config->use_metadata)) if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata, false))
return -1; return -1;
} else { } else {
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata)) if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata, true))
return -1; return -1;
} }
} }
@@ -71,15 +72,17 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) {
int rc = incremental_check(client, chunk->items[i]); int rc = incremental_check(client, chunk->items[i]);
if (rc < 0) return -1; if (rc < 0) return -1;
if (rc > 0) continue; if (rc > 0) continue;
if (!file_send_single_calls_no_path(chunk->items[i], client->file_descriptor, if (!file_send_single_calls(chunk->items[i], client->file_descriptor,
config->use_metadata, config->use_metadata,
config->use_compression ? config->compression_level : 0)) config->use_compression ? config->compression_level : 0,
false))
return -1; return -1;
} else { } else {
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
if (!file_send_single_calls(chunk->items[i], client->file_descriptor, if (!file_send_single_calls(chunk->items[i], client->file_descriptor,
config->use_metadata, config->use_metadata,
config->use_compression ? config->compression_level : 0)) config->use_compression ? config->compression_level : 0,
true))
return -1; return -1;
} }
} }
@@ -96,6 +99,16 @@ static int send_chunks_multithreaded(void *pipeline_context) {
return 1; return 1;
} }
client = client_connect_ssh(context->config->ssh_destination, context->config->ssh_port); client = client_connect_ssh(context->config->ssh_destination, context->config->ssh_port);
} else if (context->config->use_tls) {
client = client_create();
if (!client || !client_connect_tls(client, server_host, server_port,
context->config->tls_cert,
context->config->tls_key,
context->config->tls_ca)) {
if (client) client_delete(client);
fprintf(stderr, "Error: could not connect to server via TLS\n");
return thrd_error;
}
} else { } else {
client = client_create(); client = client_create();
if (!client || !client_connect(client, server_host, server_port)) { if (!client || !client_connect(client, server_host, server_port)) {
@@ -240,6 +253,15 @@ int send_files(Config *config) {
} }
client = client_connect_ssh(config->ssh_destination, config->ssh_port); client = client_connect_ssh(config->ssh_destination, config->ssh_port);
if (!client) return 1; if (!client) return 1;
} else if (config->use_tls) {
client = client_create();
if (!client || !client_connect_tls(client, server_host, server_port,
config->tls_cert, config->tls_key,
config->tls_ca)) {
if (client) client_delete(client);
fprintf(stderr, "Error: could not connect to server via TLS\n");
return 1;
}
} else { } else {
client = client_create(); client = client_create();
if (!client || !client_connect(client, server_host, server_port)) { if (!client || !client_connect(client, server_host, server_port)) {
+68 -3
View File
@@ -8,6 +8,7 @@
#include "protocol.h" #include "protocol.h"
#include "queue.h" #include "queue.h"
#include "transport_tcp.h" #include "transport_tcp.h"
#include "transport_tls.h"
#include "unistd.h" #include "unistd.h"
#include "utils.h" #include "utils.h"
#include <signal.h> #include <signal.h>
@@ -118,24 +119,88 @@ static void cleanup(int sig) {
_exit(0); _exit(0);
} }
static void print_server_usage(void) {
printf("FastSync Server\n");
printf("Usage: fastsync-server [options]\n");
printf("\n");
printf("Options:\n");
printf(" --stdio Run in stdio mode (SSH transport)\n");
printf(" -p <port> TCP port (default: 8080, range: 1-65535)\n");
printf(" --tls Enable TLS encryption\n");
printf(" --cert <path> TLS certificate file (PEM)\n");
printf(" --key <path> TLS private key file (PEM)\n");
printf(" --ca <path> TLS CA certificate file (PEM)\n");
printf(" -v, --verbose Enable debug logging\n");
printf(" --help Show this help\n");
}
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
bool use_tls = false;
char *tls_cert = NULL;
char *tls_key = NULL;
char *tls_ca = NULL;
int port = 8080;
signal(SIGPIPE, SIG_IGN); signal(SIGPIPE, SIG_IGN);
for (int i = 1; i < argc; i++) { for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "--stdio") == 0) { if (strcmp(argv[i], "--help") == 0) {
print_server_usage();
return 0;
} else if (strcmp(argv[i], "--stdio") == 0) {
io_set_fds(STDIN_FILENO, STDOUT_FILENO); io_set_fds(STDIN_FILENO, STDOUT_FILENO);
handler(STDIN_FILENO); handler(STDIN_FILENO);
return 0; return 0;
} else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) { } else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
set_log_level(LOG_LEVEL_DEBUG); set_log_level(LOG_LEVEL_DEBUG);
} else if (strcmp(argv[i], "--tls") == 0) {
use_tls = true;
} else if (strcmp(argv[i], "--cert") == 0 && i + 1 < argc) {
tls_cert = argv[++i];
} else if (strcmp(argv[i], "--key") == 0 && i + 1 < argc) {
tls_key = argv[++i];
} else if (strcmp(argv[i], "--ca") == 0 && i + 1 < argc) {
tls_ca = argv[++i];
} else if (strcmp(argv[i], "-p") == 0 && i + 1 < argc) {
char *end;
long p = strtol(argv[++i], &end, 10);
if (*end || p <= 0 || p > 65535) {
fprintf(stderr, "Error: invalid port '%s' (must be 1-65535)\n", argv[i]);
return 1;
}
port = (int)p;
} else if (argv[i][0] == '-') {
fprintf(stderr, "Unknown option: %s\n", argv[i]);
print_server_usage();
return 1;
} }
} }
if (tls_ca && !use_tls) {
log_message(LOG_LEVEL_WARNING, "--ca has no effect without --tls");
}
signal(SIGINT, cleanup); signal(SIGINT, cleanup);
signal(SIGTERM, cleanup); signal(SIGTERM, cleanup);
g_server = server_create(8080); g_server = server_create(port);
if (g_server == NULL) { if (g_server == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to create server"); log_message(LOG_LEVEL_ERROR, "Failed to create server");
return 1; return 1;
} }
server_listen(g_server, handler); if (use_tls) {
if (!tls_cert || !tls_key) {
fprintf(stderr, "Error: --tls requires --cert and --key\n");
server_delete(&g_server);
return 1;
}
tls_global_init();
if (!server_create_tls(g_server, tls_cert, tls_key, tls_ca)) {
log_message(LOG_LEVEL_ERROR, "Failed to set up TLS");
server_delete(&g_server);
return 1;
}
server_listen_tls(g_server, handler);
} else {
server_listen(g_server, handler);
}
return 0; return 0;
} }
+11
View File
@@ -39,6 +39,10 @@ Config *config_create(char *version, char *send_directory,
config->max_size = 0; config->max_size = 0;
config->min_size = 0; config->min_size = 0;
config->use_incremental = false; config->use_incremental = false;
config->use_tls = false;
config->tls_cert = NULL;
config->tls_key = NULL;
config->tls_ca = NULL;
return config; return config;
} }
@@ -74,6 +78,9 @@ void config_delete(Config *config) {
for (int i = 0; i < config->include_count; i++) for (int i = 0; i < config->include_count; i++)
free(config->include_patterns[i]); free(config->include_patterns[i]);
free(config->include_patterns); free(config->include_patterns);
free(config->tls_cert);
free(config->tls_key);
free(config->tls_ca);
free(config); free(config);
} }
@@ -149,6 +156,10 @@ Config *config_receive(int file_descriptor) {
config->include_count = 0; config->include_count = 0;
config->max_size = 0; config->max_size = 0;
config->min_size = 0; config->min_size = 0;
config->use_tls = false;
config->tls_cert = NULL;
config->tls_key = NULL;
config->tls_ca = NULL;
if (!send_status(file_descriptor, STATUS_OK)) goto error; if (!send_status(file_descriptor, STATUS_OK)) goto error;
return config; return config;
+4
View File
@@ -33,6 +33,10 @@ typedef struct Config {
unsigned long long max_size; unsigned long long max_size;
unsigned long long min_size; unsigned long long min_size;
bool use_incremental; bool use_incremental;
bool use_tls;
char *tls_cert;
char *tls_key;
char *tls_ca;
} Config; } Config;
#define PROTOCOL_VERSION "1.1.0" #define PROTOCOL_VERSION "1.1.0"
+4 -56
View File
@@ -96,7 +96,7 @@ bool file_load_data(File *file) {
return true; return true;
} }
bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level) { bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level, bool send_path) {
Data *data_to_send = file->data; Data *data_to_send = file->data;
Data *compressed_data = NULL; Data *compressed_data = NULL;
if (compression_level > 0) { if (compression_level > 0) {
@@ -107,59 +107,7 @@ bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_me
} }
data_to_send = compressed_data; data_to_send = compressed_data;
} }
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) { if (send_path && !send_str(file_descriptor, file->path)) {
data_destroy(compressed_data);
return false;
}
if (!send_data(file_descriptor, data_to_send)) {
data_destroy(compressed_data);
return false;
}
data_destroy(compressed_data);
return true;
}
bool file_send_sendfile_no_path(File *file, int file_descriptor, bool use_metadata) {
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false;
int fd = open(file->path, O_RDONLY);
if (fd == -1) {
perror("Could not open file for sendfile");
return false;
}
unsigned long long file_size = file->data->size;
if (!send_n_data(file_descriptor, &file_size, sizeof(unsigned long long))) {
close(fd);
return false;
}
off_t offset = 0;
while (offset < file_size) {
ssize_t sent = sendfile(file_descriptor, fd, &offset, file_size - offset);
if (sent == -1) {
perror("sendfile failed");
close(fd);
return false;
}
}
close(fd);
return true;
}
bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) {
Data *data_to_send = file->data;
Data *compressed_data = NULL;
if (compression_level > 0) {
compressed_data = data_compress(file->data, compression_level);
if (compressed_data == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
return false;
}
data_to_send = compressed_data;
}
if (!send_str(file_descriptor, file->path)) {
data_destroy(compressed_data); data_destroy(compressed_data);
return false; return false;
} }
@@ -270,8 +218,8 @@ bool to_disk(const char *path, const void *data, unsigned long long data_size) {
return true; return true;
} }
bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata) { bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata, bool send_path) {
if (!send_str(file_descriptor, file->path)) return false; if (send_path && !send_str(file_descriptor, file->path)) return false;
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false;
int fd = open(file->path, O_RDONLY); int fd = open(file->path, O_RDONLY);
+2 -4
View File
@@ -24,10 +24,8 @@ File *file_create(const char *path);
void file_destroy(void *item); void file_destroy(void *item);
bool file_load_data(File *file); bool file_load_data(File *file);
File *file_receive(Config *config, int file_descriptor); File *file_receive(Config *config, int file_descriptor);
bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level); bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level, bool send_path);
bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level); bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata, bool send_path);
bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata);
bool file_send_sendfile_no_path(File *file, int file_descriptor, bool use_metadata);
size_t file_content_to_buffer(File *file); size_t file_content_to_buffer(File *file);
FileMetadata *file_metadata_create(struct stat *stats); FileMetadata *file_metadata_create(struct stat *stats);
void file_metadata_destroy(void *metadata); void file_metadata_destroy(void *metadata);
+18 -4
View File
@@ -1,6 +1,7 @@
#include "protocol.h" #include "protocol.h"
#include "log.h" #include "log.h"
#include <errno.h> #include <errno.h>
#include <openssl/ssl.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
@@ -9,6 +10,7 @@
static __thread int io_read_fd = -1; static __thread int io_read_fd = -1;
static __thread int io_write_fd = -1; static __thread int io_write_fd = -1;
static SSL *io_ssl = NULL;
static unsigned long long io_bwlimit = 0; static unsigned long long io_bwlimit = 0;
static long long bw_tokens = 0; static long long bw_tokens = 0;
@@ -54,6 +56,10 @@ static void bw_throttle(size_t bytes_written) {
} }
} }
void io_set_ssl(SSL *ssl) {
io_ssl = ssl;
}
static int io_fd(int dir_fd, int file_descriptor) { static int io_fd(int dir_fd, int file_descriptor) {
return (dir_fd != -1) ? dir_fd : file_descriptor; return (dir_fd != -1) ? dir_fd : file_descriptor;
} }
@@ -66,8 +72,11 @@ bool send_n_data(int file_descriptor, void *data, size_t data_size) {
size_t chunk = data_size - total_bytes_send; size_t chunk = data_size - total_bytes_send;
if (io_bwlimit > 0 && chunk > 65536) if (io_bwlimit > 0 && chunk > 65536)
chunk = 65536; chunk = 65536;
ssize_t bytes_send = ssize_t bytes_send;
write(fd, (char *)data + total_bytes_send, chunk); if (io_ssl)
bytes_send = SSL_write(io_ssl, (char *)data + total_bytes_send, chunk);
else
bytes_send = write(fd, (char *)data + total_bytes_send, chunk);
if (bytes_send <= 0) { if (bytes_send <= 0) {
log_message(LOG_LEVEL_ERROR, "Could not send data"); log_message(LOG_LEVEL_ERROR, "Could not send data");
return false; return false;
@@ -84,8 +93,13 @@ bool receive_n_data(int file_descriptor, void *data, size_t data_size) {
int fd = io_fd(io_read_fd, file_descriptor); int fd = io_fd(io_read_fd, file_descriptor);
size_t total_bytes_received = 0; size_t total_bytes_received = 0;
while (total_bytes_received < data_size) { while (total_bytes_received < data_size) {
ssize_t bytes_received = ssize_t bytes_received;
read(fd, (char *)data + total_bytes_received, data_size - total_bytes_received); if (io_ssl)
bytes_received = SSL_read(io_ssl, (char *)data + total_bytes_received,
data_size - total_bytes_received);
else
bytes_received = read(fd, (char *)data + total_bytes_received,
data_size - total_bytes_received);
if (bytes_received <= 0) { if (bytes_received <= 0) {
if (bytes_received == 0) if (bytes_received == 0)
log_message(LOG_LEVEL_ERROR, "Connection closed while receiving data"); log_message(LOG_LEVEL_ERROR, "Connection closed while receiving data");
+4
View File
@@ -5,11 +5,15 @@
#include <stdbool.h> #include <stdbool.h>
#include <stddef.h> #include <stddef.h>
typedef struct ssl_st SSL;
typedef int Status; typedef int Status;
enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST, STATUS_CHECK }; enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST, STATUS_CHECK };
void io_set_fds(int read_fd, int write_fd); void io_set_fds(int read_fd, int write_fd);
void io_set_bwlimit(unsigned long long bytes_per_sec); void io_set_bwlimit(unsigned long long bytes_per_sec);
typedef struct ssl_st SSL;
void io_set_ssl(SSL *ssl);
bool send_n_data(int file_descriptor, void *data, size_t data_size); bool send_n_data(int file_descriptor, void *data, size_t data_size);
bool receive_n_data(int file_descriptor, void *data, size_t data_size); bool receive_n_data(int file_descriptor, void *data, size_t data_size);
+2
View File
@@ -143,5 +143,7 @@ Client *client_connect_ssh(char *destination, int port) {
client->address.sin_family = AF_UNIX; client->address.sin_family = AF_UNIX;
client->address_length = 0; client->address_length = 0;
client->ssh_child_pid = pid; client->ssh_child_pid = pid;
client->ssl = NULL;
client->ssl_ctx = NULL;
return client; return client;
} }
+50 -16
View File
@@ -1,6 +1,8 @@
#include "transport_tcp.h" #include "transport_tcp.h"
#include "log.h" #include "log.h"
#include "protocol.h"
#include <arpa/inet.h> #include <arpa/inet.h>
#include <openssl/ssl.h>
#include <signal.h> #include <signal.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
@@ -36,6 +38,7 @@ Server *server_create(int port) {
server->address.sin_addr.s_addr = INADDR_ANY; server->address.sin_addr.s_addr = INADDR_ANY;
server->address.sin_port = htons(port); server->address.sin_port = htons(port);
server->address_length = sizeof(server->address); server->address_length = sizeof(server->address);
server->ssl_ctx = NULL;
if (bind(server->file_descriptor, (struct sockaddr *)&server->address, if (bind(server->file_descriptor, (struct sockaddr *)&server->address,
server->address_length) < 0) { server->address_length) < 0) {
@@ -51,43 +54,63 @@ Server *server_create(int port) {
void server_delete(Server **server) { void server_delete(Server **server) {
if (server == NULL || *server == NULL) return; if (server == NULL || *server == NULL) return;
close((*server)->file_descriptor); close((*server)->file_descriptor);
if ((*server)->ssl_ctx) {
SSL_CTX_free((*server)->ssl_ctx);
(*server)->ssl_ctx = NULL;
}
free(*server); free(*server);
*server = NULL; *server = NULL;
} }
bool server_listen(Server *server, void (*handler)(int file_descriptor)) { static void accept_loop(Server *server, void (*child_fn)(int, void *),
log_message(LOG_LEVEL_INFO, "Start Listening on Port: %d", void *child_ctx, const char *log_fmt) {
server->address.sin_port);
if (listen(server->file_descriptor, SOMAXCONN) < 0) { if (listen(server->file_descriptor, SOMAXCONN) < 0) {
perror("Could not listen on port!"); perror("Could not listen on port!");
return false; return;
} }
signal(SIGCHLD, SIG_IGN); signal(SIGCHLD, SIG_IGN);
while (1) { while (1) {
struct sockaddr_in client_addr; struct sockaddr_in client_addr;
socklen_t client_len = sizeof(client_addr); socklen_t client_len = sizeof(client_addr);
int file_descriptor = int fd = accept(server->file_descriptor, (struct sockaddr *)&client_addr,
accept(server->file_descriptor, (struct sockaddr *)&client_addr, &client_len);
&client_len); if (fd < 0) {
if (file_descriptor < 0) {
perror("Could not accept the connection"); perror("Could not accept the connection");
continue; continue;
} }
log_message(LOG_LEVEL_INFO, "Received Connection"); log_message(LOG_LEVEL_INFO, "%s", log_fmt);
pid_t pid = fork(); pid_t pid = fork();
if (pid == 0) { if (pid == 0) {
close(server->file_descriptor); close(server->file_descriptor);
handler(file_descriptor); child_fn(fd, child_ctx);
close(file_descriptor); close(fd);
_exit(0); _exit(0);
} }
close(file_descriptor); close(fd);
} }
}
struct plain_ctx { void (*handler)(int); };
static void plain_child_fn(int fd, void *ctx) {
((struct plain_ctx *)ctx)->handler(fd);
}
bool server_listen(Server *server, void (*handler)(int file_descriptor)) {
log_message(LOG_LEVEL_INFO, "Start Listening on Port: %d",
ntohs(server->address.sin_port));
struct plain_ctx ctx = {handler};
accept_loop(server, plain_child_fn, &ctx, "Received Connection");
return true; return true;
} }
void server_accept_loop(Server *server, void (*child_fn)(int, void *),
void *child_ctx, const char *log_fmt) {
log_message(LOG_LEVEL_INFO, "Start TLS Listening on Port: %d",
ntohs(server->address.sin_port));
accept_loop(server, child_fn, child_ctx, log_fmt);
}
Client *client_create() { Client *client_create() {
int file_descriptor = socket(AF_INET, SOCK_STREAM, 0); int file_descriptor = socket(AF_INET, SOCK_STREAM, 0);
if (file_descriptor < 0) { if (file_descriptor < 0) {
@@ -104,6 +127,8 @@ Client *client_create() {
client->address.sin_family = AF_INET; client->address.sin_family = AF_INET;
client->address_length = sizeof(client->address); client->address_length = sizeof(client->address);
client->ssh_child_pid = -1; client->ssh_child_pid = -1;
client->ssl = NULL;
client->ssl_ctx = NULL;
return client; return client;
} }
@@ -124,6 +149,12 @@ bool client_connect(Client *client, char *host, int port) {
} }
void client_disconnect(Client *client) { void client_disconnect(Client *client) {
if (client->ssl) {
SSL_shutdown(client->ssl);
SSL_free(client->ssl);
client->ssl = NULL;
io_set_ssl(NULL);
}
close(client->file_descriptor); close(client->file_descriptor);
if (client->ssh_child_pid > 0) { if (client->ssh_child_pid > 0) {
int status; int status;
@@ -133,7 +164,10 @@ void client_disconnect(Client *client) {
} }
void client_delete(Client *client) { void client_delete(Client *client) {
if (client == NULL) if (client == NULL) return;
return; if (client->ssl_ctx) {
SSL_CTX_free(client->ssl_ctx);
client->ssl_ctx = NULL;
}
free(client); free(client);
} }
+5
View File
@@ -9,6 +9,7 @@ typedef struct Server {
struct sockaddr_in address; struct sockaddr_in address;
unsigned int address_length; unsigned int address_length;
int file_descriptor; int file_descriptor;
void *ssl_ctx;
} Server; } Server;
typedef struct Client { typedef struct Client {
@@ -16,10 +17,14 @@ typedef struct Client {
unsigned int address_length; unsigned int address_length;
int file_descriptor; int file_descriptor;
pid_t ssh_child_pid; pid_t ssh_child_pid;
void *ssl;
void *ssl_ctx;
} Client; } Client;
Server *server_create(int port); Server *server_create(int port);
bool server_listen(Server *server, void (*handler)(int file_descriptor)); bool server_listen(Server *server, void (*handler)(int file_descriptor));
void server_accept_loop(Server *server, void (*child_fn)(int, void *),
void *child_ctx, const char *log_fmt);
void server_delete(Server **server); void server_delete(Server **server);
Client *client_create(); Client *client_create();
bool client_connect(Client *client, char *host, int port); bool client_connect(Client *client, char *host, int port);
+162
View File
@@ -0,0 +1,162 @@
#include "transport_tls.h"
#include "log.h"
#include "protocol.h"
#include "transport_tcp.h"
#include <arpa/inet.h>
#include <openssl/err.h>
#include <openssl/ssl.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/wait.h>
#include <unistd.h>
bool tls_global_init(void) {
#if OPENSSL_VERSION_NUMBER < 0x10100000L
SSL_library_init();
OpenSSL_add_all_algorithms();
SSL_load_error_strings();
#endif
return true;
}
static void log_ssl_errors(void) {
unsigned long err;
char buf[256];
while ((err = ERR_get_error()) != 0) {
ERR_error_string_n(err, buf, sizeof(buf));
log_message(LOG_LEVEL_ERROR, "SSL error: %s", buf);
}
}
static SSL_CTX *create_ssl_ctx(bool is_server, const char *cert,
const char *key, const char *ca_path) {
const SSL_METHOD *method =
is_server ? TLS_server_method() : TLS_client_method();
SSL_CTX *ctx = SSL_CTX_new(method);
if (!ctx) {
log_message(LOG_LEVEL_ERROR, "Unable to create SSL context");
log_ssl_errors();
return NULL;
}
SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION);
if (cert && key) {
if (SSL_CTX_use_certificate_file(ctx, cert, SSL_FILETYPE_PEM) <= 0) {
log_message(LOG_LEVEL_ERROR, "Failed to load certificate: %s", cert);
log_ssl_errors();
SSL_CTX_free(ctx);
return NULL;
}
if (SSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM) <= 0) {
log_message(LOG_LEVEL_ERROR, "Failed to load private key: %s", key);
log_ssl_errors();
SSL_CTX_free(ctx);
return NULL;
}
if (!SSL_CTX_check_private_key(ctx)) {
log_message(LOG_LEVEL_ERROR,
"Private key does not match certificate");
SSL_CTX_free(ctx);
return NULL;
}
}
if (ca_path) {
if (!SSL_CTX_load_verify_locations(ctx, ca_path, NULL)) {
log_message(LOG_LEVEL_ERROR, "Failed to load CA: %s", ca_path);
log_ssl_errors();
SSL_CTX_free(ctx);
return NULL;
}
SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, NULL);
SSL_CTX_set_verify_depth(ctx, 4);
}
return ctx;
}
static SSL *wrap_fd_with_ssl(int fd, SSL_CTX *ctx, bool is_server) {
SSL *ssl = SSL_new(ctx);
if (!ssl) {
log_message(LOG_LEVEL_ERROR, "Failed to create SSL object");
return NULL;
}
SSL_set_fd(ssl, fd);
int ret;
if (is_server)
ret = SSL_accept(ssl);
else
ret = SSL_connect(ssl);
if (ret <= 0) {
log_message(LOG_LEVEL_ERROR, "SSL %s failed",
is_server ? "accept" : "connect");
log_ssl_errors();
SSL_free(ssl);
return NULL;
}
return ssl;
}
bool server_create_tls(Server *server, const char *cert_path,
const char *key_path, const char *ca_path) {
SSL_CTX *ctx = create_ssl_ctx(true, cert_path, key_path, ca_path);
if (!ctx) return false;
server->ssl_ctx = ctx;
return true;
}
struct tls_child_ctx {
void (*handler)(int);
SSL_CTX *ssl_ctx;
};
static void tls_child_fn(int fd, void *arg) {
struct tls_child_ctx *ctx = (struct tls_child_ctx *)arg;
SSL *ssl = wrap_fd_with_ssl(fd, ctx->ssl_ctx, true);
if (!ssl) return;
io_set_ssl(ssl);
ctx->handler(fd);
SSL_shutdown(ssl);
SSL_free(ssl);
io_set_ssl(NULL);
}
bool server_listen_tls(Server *server, void (*handler)(int file_descriptor)) {
struct tls_child_ctx ctx = {handler, (SSL_CTX *)server->ssl_ctx};
server_accept_loop(server, tls_child_fn, &ctx, "Received TLS Connection");
return true;
}
bool client_connect_tls(Client *client, char *host, int port,
const char *cert_path, const char *key_path,
const char *ca_path) {
client->address.sin_port = htons(port);
if (inet_pton(AF_INET, host, &client->address.sin_addr) <= 0) {
perror("Could not convert host address!");
return false;
}
if (connect(client->file_descriptor, (struct sockaddr *)&client->address,
client->address_length) < 0) {
perror("Could not connect to Server!");
return false;
}
SSL_CTX *ctx = create_ssl_ctx(false, cert_path, key_path, ca_path);
if (!ctx) return false;
client->ssl_ctx = ctx;
SSL *ssl = wrap_fd_with_ssl(client->file_descriptor, ctx, false);
if (!ssl) {
SSL_CTX_free(ctx);
client->ssl_ctx = NULL;
return false;
}
client->ssl = ssl;
io_set_ssl(ssl);
return true;
}
+16
View File
@@ -0,0 +1,16 @@
#ifndef TRANSPORT_TLS_H
#define TRANSPORT_TLS_H
#include "transport_tcp.h"
#include <stdbool.h>
bool tls_global_init(void);
bool server_create_tls(Server *server, const char *cert_path,
const char *key_path, const char *ca_path);
bool server_listen_tls(Server *server, void (*handler)(int file_descriptor));
bool client_connect_tls(Client *client, char *host, int port,
const char *cert_path, const char *key_path,
const char *ca_path);
#endif
+262 -222
View File
@@ -50,7 +50,7 @@ CLIENT_CMD_PREFIX = [
BASE_CLIENT_FLAGS = ["--save-to-disk"] BASE_CLIENT_FLAGS = ["--save-to-disk"]
TEST_CASES = [ TEST_CASES_FULL = [
{"name": "Standard", "flags": []}, {"name": "Standard", "flags": []},
{"name": "Posix Args (no flags)", "flags": [], "posix": True}, {"name": "Posix Args (no flags)", "flags": [], "posix": True},
{"name": "Standard (no metadata)", "flags": [], "use_metadata": False}, {"name": "Standard (no metadata)", "flags": [], "use_metadata": False},
@@ -68,7 +68,14 @@ TEST_CASES = [
{"name": "Sendfile + Multithreading (-f -m)", "flags": ["-f", "-m"]}, {"name": "Sendfile + Multithreading (-f -m)", "flags": ["-f", "-m"]},
] ]
SSH_CASES = [ TEST_CASES_LIGHT = [
{"name": "Standard", "flags": []},
{"name": "Compression (-c)", "flags": ["-c"]},
{"name": "Chunk Serialization (-s)", "flags": ["-s"]},
{"name": "Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]},
]
SSH_CASES_FULL = [
{"name": "SSH (localhost)", "flags": []}, {"name": "SSH (localhost)", "flags": []},
{"name": "SSH Multithreading (-m)", "flags": ["-m"]}, {"name": "SSH Multithreading (-m)", "flags": ["-m"]},
{"name": "SSH Compression (-c)", "flags": ["-c"]}, {"name": "SSH Compression (-c)", "flags": ["-c"]},
@@ -79,11 +86,17 @@ SSH_CASES = [
{"name": "SSH Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]}, {"name": "SSH Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]},
] ]
RSYNC_CASES = [ SSH_CASES_LIGHT = [
{"name": "SSH (localhost)", "flags": []},
]
RSYNC_CASES_FULL = [
{"name": "rsync (archive)", "args": ["-aH"]}, {"name": "rsync (archive)", "args": ["-aH"]},
{"name": "rsync (archive + compress)", "args": ["-aHz"]}, {"name": "rsync (archive + compress)", "args": ["-aHz"]},
] ]
RSYNC_CASES_LIGHT = []
def netem_apply(profile): def netem_apply(profile):
params = NETWORK_PROFILES[profile] params = NETWORK_PROFILES[profile]
@@ -119,12 +132,12 @@ def find_free_port():
return s.getsockname()[1] return s.getsockname()[1]
def generate_test_files(source_dir): def generate_test_files(source_dir, full=False):
if os.path.exists(source_dir): if os.path.exists(source_dir):
shutil.rmtree(source_dir) shutil.rmtree(source_dir)
os.makedirs(source_dir) os.makedirs(source_dir)
target_total = 25 * 1024 * 1024 target_total = 25 * 1024 * 1024 if full else 0
written = 0 written = 0
files = { files = {
@@ -141,14 +154,15 @@ def generate_test_files(source_dir):
f.write(content) f.write(content)
written += len(content) written += len(content)
os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True) if full:
i = 0 os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True)
while written < target_total: i = 0
chunk_size = min(5 * 1024 * 1024, target_total - written) while written < target_total:
with open(os.path.join(source_dir, f"bulk/file_{i}.dat"), "wb") as f: chunk_size = min(5 * 1024 * 1024, target_total - written)
f.write(random.randbytes(chunk_size)) with open(os.path.join(source_dir, f"bulk/file_{i}.dat"), "wb") as f:
written += chunk_size f.write(random.randbytes(chunk_size))
i += 1 written += chunk_size
i += 1
total_mb = written / (1024 * 1024) total_mb = written / (1024 * 1024)
small_bytes = sum(len(c) for c in files.values()) small_bytes = sum(len(c) for c in files.values())
@@ -254,19 +268,24 @@ def print_profile_header(profile_name):
print(" No limits applied") print(" No limits applied")
def run_profile(profile_name, source_dir, dest_dir): def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=None, ssh_cases=None, rsync_cases=None):
print_profile_header(profile_name) print_profile_header(profile_name)
is_limited = profile_name != "Unlimited" is_limited = profile_name != "Unlimited"
client_prefix = CLIENT_CMD_PREFIX if is_limited else [] client_prefix = CLIENT_CMD_PREFIX if is_limited else []
if test_cases is None:
test_cases = TEST_CASES_FULL if full else TEST_CASES_LIGHT
if ssh_cases is None:
ssh_cases = SSH_CASES_FULL if full else SSH_CASES_LIGHT
if rsync_cases is None:
rsync_cases = RSYNC_CASES_FULL if full else RSYNC_CASES_LIGHT
try: try:
if is_limited: if is_limited and full:
netem_apply(profile_name) netem_apply(profile_name)
else:
netem_reset()
results = [] results = []
for case in TEST_CASES: for case in test_cases:
flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"] flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"]
if case.get("posix"): if case.get("posix"):
cmd = client_prefix + BASE_CLIENT_CMD + [source_dir, dest_dir] + flags cmd = client_prefix + BASE_CLIENT_CMD + [source_dir, dest_dir] + flags
@@ -281,7 +300,7 @@ def run_profile(profile_name, source_dir, dest_dir):
results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
if SSH_AVAILABLE: if SSH_AVAILABLE:
for case in SSH_CASES: for case in ssh_cases:
flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"] flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"]
ssh_dest = f"localhost:{dest_dir}_ssh" ssh_dest = f"localhost:{dest_dir}_ssh"
cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags
@@ -293,209 +312,222 @@ def run_profile(profile_name, source_dir, dest_dir):
except Exception as e: except Exception as e:
results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
port, conf, daemon = start_rsync_daemon(source_dir) if rsync_cases:
try: port, conf, daemon = start_rsync_daemon(source_dir)
for case in RSYNC_CASES:
cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"]
print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, case["name"], source_dir, dest_dir, source_prefix="")
r["suite"] = profile_name
except subprocess.TimeoutExpired:
r = {"name": case["name"], "suite": profile_name, "status": "Timeout", "time": "N/A", "error": "Exceeded 120s"}
except Exception as e:
r = {"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}
else:
if r["status"] != "Success" and client_prefix:
tmp = tempfile.mkdtemp()
try:
plain = subprocess.run(["rsync", "-aH", f"rsync://localhost:{port}/source/", f"{tmp}/"], capture_output=True, text=True, timeout=30)
if plain.returncode != 0:
errs = [l for l in (plain.stderr or "").split("\n") if l.strip()]
if errs:
r["error"] += f" | raw: {errs[-1][:150]}"
finally:
shutil.rmtree(tmp, ignore_errors=True)
results.append(r)
finally:
wait_proc(daemon)
try: try:
os.unlink(conf) for case in rsync_cases:
except Exception: cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"]
pass print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, case["name"], source_dir, dest_dir, source_prefix="")
r["suite"] = profile_name
except subprocess.TimeoutExpired:
r = {"name": case["name"], "suite": profile_name, "status": "Timeout", "time": "N/A", "error": "Exceeded 120s"}
except Exception as e:
r = {"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}
else:
if r["status"] != "Success" and client_prefix:
tmp = tempfile.mkdtemp()
try:
plain = subprocess.run(["rsync", "-aH", f"rsync://localhost:{port}/source/", f"{tmp}/"], capture_output=True, text=True, timeout=30)
if plain.returncode != 0:
errs = [l for l in (plain.stderr or "").split("\n") if l.strip()]
if errs:
r["error"] += f" | raw: {errs[-1][:150]}"
finally:
shutil.rmtree(tmp, ignore_errors=True)
results.append(r)
finally:
wait_proc(daemon)
try:
os.unlink(conf)
except Exception:
pass
# Feature-specific tests for rsync-compatible flags if full:
print("\n " + "" * 56 + "\n Feature Tests\n " + "" * 56) # Feature-specific tests for rsync-compatible flags
print("\n " + "" * 56 + "\n Feature Tests\n " + "" * 56)
# Dry run (-n) — no server needed # Dry run (-n) — no server needed
print("\n --- Dry run (-n) ---") print("\n --- Dry run (-n) ---")
flags = BASE_CLIENT_FLAGS + ["-n"] flags = BASE_CLIENT_FLAGS + ["-n"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
print(f" Running: {' '.join(cmd)}") print(f" Running: {' '.join(cmd)}")
try: try:
start = time.monotonic() start = time.monotonic()
result = subprocess.run(cmd, text=True, capture_output=True) result = subprocess.run(cmd, text=True, capture_output=True)
duration = time.monotonic() - start duration = time.monotonic() - start
r = {"name": "Dry run (-n)", "suite": profile_name} r = {"name": "Dry run (-n)", "suite": profile_name}
if result.returncode == 0 and "Dry run:" in result.stdout: if result.returncode == 0 and "Dry run:" in result.stdout:
r["status"] = "Success"
r["time"] = f"{duration:.4f}s"
r["error"] = ""
else:
r["status"] = "Failed"
r["time"] = "N/A"
r["error"] = f"Exit {result.returncode}: {(result.stderr or result.stdout)[:100]}"
results.append(r)
except Exception as e:
results.append({"name": "Dry run (-n)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Archive mode (-a)
feature_flags = BASE_CLIENT_FLAGS + ["-a"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Archive mode (-a) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Archive mode (-a)", source_dir, dest_dir)
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Archive mode (-a)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Exclude (--exclude small.txt)
feature_flags = BASE_CLIENT_FLAGS + ["--exclude", "small.txt"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Exclude (--exclude small.txt) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Exclude (--exclude small.txt)", source_dir, dest_dir,
expected_missing=["small.txt"])
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Exclude (--exclude small.txt)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Progress (--progress)
feature_flags = BASE_CLIENT_FLAGS + ["--progress"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Progress (--progress) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Progress (--progress)", source_dir, dest_dir)
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Incremental sync (--incremental) — first sync, then second sync should skip all
print(f"\n --- Incremental (--incremental) ---")
try:
flags = BASE_CLIENT_FLAGS + ["-M"]
srv = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
time.sleep(0.5)
first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
r1 = subprocess.run(first_cmd, text=True, capture_output=True)
wait_proc(srv)
if r1.returncode != 0:
raise RuntimeError(f"First sync failed: {r1.stderr[:100]}")
srv2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
time.sleep(0.5)
second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--incremental"]
start = time.monotonic()
r2 = subprocess.run(second_cmd, text=True, capture_output=True, timeout=30)
duration = time.monotonic() - start
wait_proc(srv2)
r = {"name": "Incremental (--incremental)", "suite": profile_name,
"status": "Success" if r2.returncode == 0 else "Failed",
"time": f"{duration:.4f}s" if r2.returncode == 0 else "N/A",
"error": "" if r2.returncode == 0 else f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}"}
results.append(r)
except Exception as e:
results.append({"name": "Incremental (--incremental)", "suite": profile_name,
"status": "Error", "time": "N/A", "error": str(e)})
# Chunk size (--chunk-size 5242880)
feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Chunk size (--chunk-size 5242880) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Chunk size (--chunk-size 5242880)", source_dir, dest_dir)
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Chunk size (--chunk-size 5242880)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Delete (--delete) — pre-populate dest, add extra files, then sync with --delete
# Note: server handles one client per launch, so we restart between syncs
print(f"\n --- Delete (--delete) ---")
try:
flags = BASE_CLIENT_FLAGS + ["-M"]
# First sync (no delete) to populate dest
s1 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
time.sleep(0.5)
first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
r1 = subprocess.run(first_cmd, text=True, capture_output=True)
wait_proc(s1)
if r1.returncode != 0:
raise RuntimeError(f"First sync failed: {r1.stderr[:100]}")
# Add extra files to received dir
received = os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep))
extra_path = os.path.join(received, "extra_file.txt")
with open(extra_path, "w") as f:
f.write("should be deleted")
extra_dir = os.path.join(received, "extra_dir")
os.makedirs(extra_dir, exist_ok=True)
with open(os.path.join(extra_dir, "nested.txt"), "w") as f:
f.write("nested extra")
# Second sync with --delete (fresh server)
s2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
time.sleep(0.5)
second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--delete"]
start = time.monotonic()
r2 = subprocess.run(second_cmd, text=True, capture_output=True)
duration = time.monotonic() - start
wait_proc(s2)
r = {"name": "Delete (--delete)", "suite": profile_name}
if r2.returncode == 0 and not os.path.exists(extra_path) and not os.path.exists(extra_dir):
mismatches, missing = verify_transfer(source_dir, received)
if not mismatches and not missing:
r["status"] = "Success" r["status"] = "Success"
r["time"] = f"{duration:.4f}s" r["time"] = f"{duration:.4f}s"
r["error"] = "" r["error"] = ""
else: else:
r["status"] = "Failed" r["status"] = "Failed"
r["time"] = "N/A" r["time"] = "N/A"
r["error"] = f"post-delete verify: mismatches={len(mismatches)}, missing={len(missing)}" r["error"] = f"Exit {result.returncode}: {(result.stderr or result.stdout)[:100]}"
else: results.append(r)
r["status"] = "Failed" except Exception as e:
r["time"] = "N/A" results.append({"name": "Dry run (-n)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
errs = []
if r2.returncode != 0:
errs.append(f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}")
if os.path.exists(extra_path):
errs.append("extra_file.txt remains")
if os.path.exists(extra_dir):
errs.append("extra_dir remains")
r["error"] = " | ".join(errs)
results.append(r)
except Exception as e:
results.append({"name": "Delete (--delete)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# SSH feature tests # Archive mode (-a)
if SSH_AVAILABLE: feature_flags = BASE_CLIENT_FLAGS + ["-a"]
ssh_dest = f"localhost:{dest_dir}_ssh" cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
ssh_feature_cases = [ print(f"\n --- Archive mode (-a) ---\n Running: {' '.join(cmd)}")
{"name": "SSH Archive (-a)", "flags": ["-a"]}, try:
{"name": "SSH Exclude (--exclude small.txt)", "flags": ["--exclude", "small.txt"], "expected_missing": ["small.txt"]}, r = run_single_test(cmd, "Archive mode (-a)", source_dir, dest_dir)
] r["suite"] = profile_name
for case in ssh_feature_cases: results.append(r)
flags = BASE_CLIENT_FLAGS + case["flags"] except Exception as e:
cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags results.append({"name": "Archive mode (-a)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}")
try: # Exclude (--exclude small.txt)
r = run_single_test(cmd, case["name"], source_dir, f"{dest_dir}_ssh", feature_flags = BASE_CLIENT_FLAGS + ["--exclude", "small.txt"]
no_server=True, cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
expected_missing=case.get("expected_missing")) print(f"\n --- Exclude (--exclude small.txt) ---\n Running: {' '.join(cmd)}")
r["suite"] = profile_name try:
results.append(r) r = run_single_test(cmd, "Exclude (--exclude small.txt)", source_dir, dest_dir,
except Exception as e: expected_missing=["small.txt"])
results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)}) r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Exclude (--exclude small.txt)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Progress (--progress)
feature_flags = BASE_CLIENT_FLAGS + ["--progress"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Progress (--progress) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Progress (--progress)", source_dir, dest_dir)
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Progress (--progress)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Bandwidth limit (--bwlimit 10240 = 10 MB/s)
feature_flags = BASE_CLIENT_FLAGS + ["--bwlimit", "10240"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Bandwidth limit (--bwlimit 10240 KB/s) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Bandwidth limit (--bwlimit 10240)", source_dir, dest_dir)
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Bandwidth limit (--bwlimit 10240)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Incremental sync (--incremental) — first sync, then second sync should skip all
print(f"\n --- Incremental (--incremental) ---")
try:
flags = BASE_CLIENT_FLAGS + ["-M"]
srv = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
time.sleep(0.5)
first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
r1 = subprocess.run(first_cmd, text=True, capture_output=True)
wait_proc(srv)
if r1.returncode != 0:
raise RuntimeError(f"First sync failed: {r1.stderr[:100]}")
srv2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
time.sleep(0.5)
second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--incremental"]
start = time.monotonic()
r2 = subprocess.run(second_cmd, text=True, capture_output=True, timeout=30)
duration = time.monotonic() - start
wait_proc(srv2)
r = {"name": "Incremental (--incremental)", "suite": profile_name,
"status": "Success" if r2.returncode == 0 else "Failed",
"time": f"{duration:.4f}s" if r2.returncode == 0 else "N/A",
"error": "" if r2.returncode == 0 else f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}"}
results.append(r)
except Exception as e:
results.append({"name": "Incremental (--incremental)", "suite": profile_name,
"status": "Error", "time": "N/A", "error": str(e)})
# Chunk size (--chunk-size 5242880)
feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"]
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
print(f"\n --- Chunk size (--chunk-size 5242880) ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, "Chunk size (--chunk-size 5242880)", source_dir, dest_dir)
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": "Chunk size (--chunk-size 5242880)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# Delete (--delete) — pre-populate dest, add extra files, then sync with --delete
# Note: server handles one client per launch, so we restart between syncs
print(f"\n --- Delete (--delete) ---")
try:
flags = BASE_CLIENT_FLAGS + ["-M"]
# First sync (no delete) to populate dest
s1 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
time.sleep(0.5)
first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
r1 = subprocess.run(first_cmd, text=True, capture_output=True)
wait_proc(s1)
if r1.returncode != 0:
raise RuntimeError(f"First sync failed: {r1.stderr[:100]}")
# Add extra files to received dir
received = os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep))
extra_path = os.path.join(received, "extra_file.txt")
with open(extra_path, "w") as f:
f.write("should be deleted")
extra_dir = os.path.join(received, "extra_dir")
os.makedirs(extra_dir, exist_ok=True)
with open(os.path.join(extra_dir, "nested.txt"), "w") as f:
f.write("nested extra")
# Second sync with --delete (fresh server)
s2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
time.sleep(0.5)
second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--delete"]
start = time.monotonic()
r2 = subprocess.run(second_cmd, text=True, capture_output=True)
duration = time.monotonic() - start
wait_proc(s2)
r = {"name": "Delete (--delete)", "suite": profile_name}
if r2.returncode == 0 and not os.path.exists(extra_path) and not os.path.exists(extra_dir):
mismatches, missing = verify_transfer(source_dir, received)
if not mismatches and not missing:
r["status"] = "Success"
r["time"] = f"{duration:.4f}s"
r["error"] = ""
else:
r["status"] = "Failed"
r["time"] = "N/A"
r["error"] = f"post-delete verify: mismatches={len(mismatches)}, missing={len(missing)}"
else:
r["status"] = "Failed"
r["time"] = "N/A"
errs = []
if r2.returncode != 0:
errs.append(f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}")
if os.path.exists(extra_path):
errs.append("extra_file.txt remains")
if os.path.exists(extra_dir):
errs.append("extra_dir remains")
r["error"] = " | ".join(errs)
results.append(r)
except Exception as e:
results.append({"name": "Delete (--delete)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
# SSH feature tests
if SSH_AVAILABLE:
ssh_dest = f"localhost:{dest_dir}_ssh"
ssh_feature_cases = [
{"name": "SSH Archive (-a)", "flags": ["-a"]},
{"name": "SSH Exclude (--exclude small.txt)", "flags": ["--exclude", "small.txt"], "expected_missing": ["small.txt"]},
]
for case in ssh_feature_cases:
flags = BASE_CLIENT_FLAGS + case["flags"]
cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags
print(f"\n --- {case['name']} ---\n Running: {' '.join(cmd)}")
try:
r = run_single_test(cmd, case["name"], source_dir, f"{dest_dir}_ssh",
no_server=True,
expected_missing=case.get("expected_missing"))
r["suite"] = profile_name
results.append(r)
except Exception as e:
results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
except (subprocess.CalledProcessError, RuntimeError) as e: except (subprocess.CalledProcessError, RuntimeError) as e:
print(f" Error: {e}") print(f" Error: {e}")
@@ -562,19 +594,26 @@ def check_ssh_localhost():
build_dir = os.path.abspath("build") build_dir = os.path.abspath("build")
server_path = os.path.join(build_dir, "server") server_path = os.path.join(build_dir, "server")
r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", try:
"localhost", "which", "fastsync-server"], r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5",
capture_output=True, timeout=10) "localhost", "which", "fastsync-server"],
capture_output=True, timeout=10)
except FileNotFoundError:
SSH_AVAILABLE = False
return
if r.returncode == 0: if r.returncode == 0:
SSH_AVAILABLE = True SSH_AVAILABLE = True
return return
SSH_AVAILABLE = False SSH_AVAILABLE = False
# Try each PATH dir: create symlink, then verify with which # Try each PATH dir: create symlink, then verify with which
r = subprocess.run( try:
["ssh", "-o", "BatchMode=yes", "localhost", r = subprocess.run(
'echo "$PATH"'], ["ssh", "-o", "BatchMode=yes", "localhost",
capture_output=True, timeout=10, text=True) 'echo "$PATH"'],
capture_output=True, timeout=10, text=True)
except FileNotFoundError:
return
if r.returncode != 0: if r.returncode != 0:
return return
for d in r.stdout.strip().split(":"): for d in r.stdout.strip().split(":"):
@@ -655,9 +694,10 @@ def main():
parser.add_argument("--keep-data", action="store_true") parser.add_argument("--keep-data", action="store_true")
parser.add_argument("--unlimited", action="store_true") parser.add_argument("--unlimited", action="store_true")
parser.add_argument("--wan", action="store_true") parser.add_argument("--wan", action="store_true")
parser.add_argument("--full", action="store_true", help="Run full test suite with network shaping, SSH, rsync benchmarks")
args = parser.parse_args() args = parser.parse_args()
total_bytes = generate_test_files(args.source_dir) total_bytes = generate_test_files(args.source_dir, full=args.full)
if os.path.exists(args.dest_dir): if os.path.exists(args.dest_dir):
shutil.rmtree(args.dest_dir) shutil.rmtree(args.dest_dir)
os.makedirs(args.dest_dir, exist_ok=True) os.makedirs(args.dest_dir, exist_ok=True)
@@ -668,12 +708,12 @@ def main():
elif args.wan: elif args.wan:
profiles.append("WAN") profiles.append("WAN")
else: else:
profiles.append("LAN") profiles.append("LAN" if args.full else "Unlimited")
try: try:
all_results = [] all_results = []
for p in profiles: for p in profiles:
all_results.extend(run_profile(p, args.source_dir, args.dest_dir)) all_results.extend(run_profile(p, args.source_dir, args.dest_dir, full=args.full))
print("\n" + "=" * 130) print("\n" + "=" * 130)
print(f"{'RESULTS':^130}") print(f"{'RESULTS':^130}")