TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply PR #143 content on top of latest main

Browse Source
This commit is contained in:
TapTap
2026-07-29 18:18:07 +02:00
parent 3b621e591a
commit e68e33c0c5
33 changed files with 1114 additions and 1145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/shared/file.c
+92 -246
View File
@@ -3,7 +3,6 @@
#include <fcntl.h>
#include <libgen.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -22,9 +21,6 @@
#include "protocol.h"
#include "utils.h"
#define STREAM_THRESHOLD (64ULL * 1024 * 1024) /* 64 MB */
#define STREAM_CHUNK_SIZE (1ULL * 1024 * 1024) /* 1 MB */
File* file_create(const char* path) {
File* file = (File*)malloc(sizeof(File));
if (file == NULL) {
@@ -39,7 +35,7 @@ File* file_create(const char* path) {
return NULL;
}
memcpy(file->path, path, path_len + 1);
strcpy(file->path, path);
file->data = data_create_reserve(0);
if (file->data == NULL) {
free(file->path);
@@ -47,8 +43,7 @@ File* file_create(const char* path) {
return NULL;
}
file->metadata = NULL;
file->type = FILE_TYPE_REGULAR;
file->link_target = NULL;
file->skip = false;
return file;
}
@@ -62,8 +57,6 @@ void file_destroy(void* item) {
file->metadata = NULL;
free(file->path);
file->path = NULL;
free(file->link_target);
file->link_target = NULL;
free(file);
}
@@ -92,14 +85,6 @@ void file_metadata_destroy(void* metadata) {
bool file_load_data(File* file) {
if (file == NULL)
return false;
// Symlinks have no data to load
if (file->type == FILE_TYPE_SYMLINK)
return true;
// For streaming files, just record the size, don't load into memory
if (file->data->size > STREAM_THRESHOLD) {
// Don't allocate; streaming will read directly from disk
return true;
}
if (file->data->data == NULL) {
file->data->data = malloc(file->data->size);
if (file->data->data == NULL) {
@@ -110,87 +95,16 @@ bool file_load_data(File* file) {
size_t bytes_read = file_content_to_buffer(file);
if (bytes_read != file->data->size) {
log_message(LOG_LEVEL_ERROR, "Did not read expected amount of bytes from file");
free(file->data->data);
file->data->data = NULL;
file->data->size = 0;
return false;
}
return true;
}
// Stream file content in chunks without loading entire file into RAM
static bool file_send_streaming(File* file, int file_descriptor) {
unsigned long long total_size = file->data->size;
// Send total size prefix (same wire format as send_data)
if (!send_n_data(file_descriptor, &total_size, sizeof(total_size)))
return false;
FILE* fp = fopen(file->path, "rb");
if (!fp) {
perror("Could not open file for streaming");
return false;
}
char* buf = malloc(STREAM_CHUNK_SIZE);
if (!buf) {
fclose(fp);
return false;
}
unsigned long long remaining = total_size;
while (remaining > 0) {
size_t to_read = (size_t)((remaining < STREAM_CHUNK_SIZE) ? remaining : STREAM_CHUNK_SIZE);
size_t nread = fread(buf, 1, to_read, fp);
if (nread != to_read) {
if (ferror(fp)) {
perror("Read error during streaming");
}
send_status(file_descriptor, STATUS_ERROR);
free(buf);
fclose(fp);
return false;
}
if (!send_n_data(file_descriptor, buf, nread)) {
send_status(file_descriptor, STATUS_ERROR);
free(buf);
fclose(fp);
return false;
}
remaining -= (unsigned long long)nread;
}
free(buf);
fclose(fp);
return true;
}
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path) {
if (send_path && !send_str(file_descriptor, file->path))
return false;
if (use_metadata && !metadata_send(file_descriptor, file->metadata))
return false;
// Send file type indicator so receiver can distinguish regular from symlink
int ft = (int)file->type;
if (!send_int(file_descriptor, ft))
return false;
if (file->type == FILE_TYPE_SYMLINK) {
// Send link target, then zero-length data
if (!send_str(file_descriptor, file->link_target ? file->link_target : ""))
return false;
Data empty = {NULL, 0};
return send_data(file_descriptor, &empty);
}
const Data* data_to_send = file->data;
Data* compressed_data = NULL;
// Streaming mode: for large files without compression, stream from disk
if (file->data->size > STREAM_THRESHOLD && compression_level == 0) {
return file_send_streaming(file, file_descriptor);
}
if (compression_level > 0) {
if (compression_level > 0 && !compression_should_skip(file->path)) {
compressed_data = data_compress(file->data, compression_level);
if (compressed_data == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
@@ -198,6 +112,14 @@ bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
}
data_to_send = compressed_data;
}
if (send_path && !send_str(file_descriptor, file->path)) {
data_destroy(compressed_data);
return false;
}
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) {
data_destroy(compressed_data);
return false;
}
if (!send_data(file_descriptor, data_to_send)) {
data_destroy(compressed_data);
return false;
@@ -206,38 +128,15 @@ bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
return true;
}
bool file_save_to_disk(const char* root_directory, File* file) {
if (file->type == FILE_TYPE_SYMLINK && file->link_target) {
// Validate link_target — reject absolute paths or traversal
if (file->link_target[0] == '/' || strstr(file->link_target, "..") != NULL) {
log_message(LOG_LEVEL_ERROR, "Path traversal blocked in symlink target: %s",
file->link_target);
return false;
}
char* disk_path = path_cat((char*)root_directory, file->path);
if (disk_path == NULL)
return false;
if (strstr(disk_path, "..") != NULL) {
log_message(LOG_LEVEL_ERROR, "Path traversal blocked: %s", disk_path);
free(disk_path);
return false;
}
unlink(disk_path);
bool ok = (symlink(file->link_target, disk_path) == 0);
if (ok && file->metadata)
file_restore_metadata(disk_path, file->metadata);
free(disk_path);
return ok;
bool file_save_to_disk(const char* root_directory, File* file, const Config* config) {
(void)config;
if (has_path_traversal(file->path)) {
log_message(LOG_LEVEL_ERROR, "Path traversal detected in file path: %s", file->path);
return false;
}
char* disk_path = path_cat((char*)root_directory, file->path);
if (disk_path == NULL)
return false;
if (strstr(disk_path, "..") != NULL) {
log_message(LOG_LEVEL_ERROR, "Path traversal blocked: %s", disk_path);
free(disk_path);
return false;
}
bool ok = to_disk(disk_path, file->data->data, file->data->size);
if (ok)
file_restore_metadata(disk_path, file->metadata);
@@ -263,60 +162,19 @@ static void* old_data_from_path(const char* full_path, unsigned long long old_si
return data;
}
/**
* Helper: receive data from wire, optionally decompress, and store in file.
* On success, returns the received Data* (caller owns it). On failure, returns NULL.
* If `file_data` is received via receive_data(fd), this function handles decompression
* when config->use_compression is set.
*/
static Data* receive_and_decompress(int fd, const Config* config) {
Data* file_data = receive_data(fd);
if (file_data == NULL)
return NULL;
if (config->use_compression) {
Data* uncompressed = data_decompress(file_data);
data_destroy(file_data);
if (uncompressed == NULL)
return NULL;
file_data = uncompressed;
}
return file_data;
}
/**
* Helper: receive metadata from wire and assign to file.
* Returns true on success (metadata may be NULL if absent), false on I/O error.
*/
static bool receive_and_assign_metadata(int fd, const Config* config, File* file) {
if (!config->use_metadata)
return true;
int meta_ok = 1;
file->metadata = metadata_receive(fd, &meta_ok);
if (!meta_ok) {
file_destroy(file);
send_status(fd, STATUS_ERROR);
return false;
}
return true;
}
static File* receive_delta_file(int fd, const Config* config, const char* check_path,
void* old_data, unsigned long long old_size) {
if (!old_data) {
send_status(fd, STATUS_ERROR);
if (!old_data)
return NULL;
}
DeltaSignature* sig = delta_signature_create(old_data, old_size, config->delta_block_size);
if (!sig) {
send_status(fd, STATUS_ERROR);
free(old_data);
return NULL;
}
Data* sig_data = delta_signature_serialize(sig);
if (!sig_data) {
send_status(fd, STATUS_ERROR);
delta_signature_destroy(sig);
free(old_data);
return NULL;
@@ -419,16 +277,34 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
return NULL;
}
if (!receive_and_assign_metadata(fd, config, file))
return NULL;
if (config->use_metadata) {
int meta_ok = 1;
file->metadata = metadata_receive(fd, &meta_ok);
if (!meta_ok) {
file_destroy(file);
send_status(fd, STATUS_ERROR);
return NULL;
}
}
Data* file_data = receive_and_decompress(fd, config);
Data* file_data = receive_data(fd);
if (file_data == NULL) {
file_destroy(file);
send_status(fd, STATUS_ERROR);
return NULL;
}
if (config->use_compression) {
Data* uncompressed = data_decompress(file_data);
data_destroy(file_data);
if (uncompressed == NULL) {
file_destroy(file);
send_status(fd, STATUS_ERROR);
return NULL;
}
file_data = uncompressed;
}
data_destroy(file->data);
file->data = file_data;
return file;
@@ -456,30 +332,18 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
return NULL;
}
char* full_path = path_cat(config->receive_root_directory, check_path);
if (full_path && strstr(full_path, "..") != NULL) {
log_message(LOG_LEVEL_ERROR, "Path traversal blocked: %s", full_path);
free(full_path);
if (has_path_traversal(check_path)) {
log_message(LOG_LEVEL_ERROR, "Path traversal detected: %s", check_path);
free(check_path);
send_status(fd, STATUS_ERROR);
return NULL;
}
char* full_path = path_cat(config->receive_root_directory, check_path);
struct stat st;
bool has_old_file = (full_path && stat(full_path, &st) == 0);
unsigned long long old_size = has_old_file ? (unsigned long long)st.st_size : 0;
// Check for partial file if enabled
if (config->partial && !has_old_file && full_path) {
char* partial_path = malloc(strlen(full_path) + 20);
if (partial_path) {
snprintf(partial_path, strlen(full_path) + 20, "%s.fastsync-partial", full_path);
has_old_file = (stat(partial_path, &st) == 0);
if (has_old_file)
old_size = (unsigned long long)st.st_size;
free(partial_path);
}
}
bool match = has_old_file && (unsigned long long)st.st_size == check_size &&
(long long)st.st_mtime == check_mtime;
@@ -525,86 +389,102 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
return NULL;
}
if (!receive_and_assign_metadata(fd, config, file))
return NULL;
int file_type;
if (!receive_int(fd, &file_type)) {
file_destroy(file);
send_status(fd, STATUS_ERROR);
return NULL;
}
file->type = (FileType)file_type;
if (file->type == FILE_TYPE_SYMLINK) {
char* link_target = receive_str(fd);
if (link_target) {
file->link_target = link_target;
if (config->use_metadata) {
int meta_ok = 1;
file->metadata = metadata_receive(fd, &meta_ok);
if (!meta_ok) {
file_destroy(file);
send_status(fd, STATUS_ERROR);
return NULL;
}
Data* empty_data = receive_data(fd);
if (empty_data)
data_destroy(empty_data);
return file;
}
Data* file_data = receive_and_decompress(fd, config);
Data* file_data = receive_data(fd);
if (file_data == NULL) {
file_destroy(file);
send_status(fd, STATUS_ERROR);
return NULL;
}
if (config->use_compression) {
Data* uncompressed = data_decompress(file_data);
data_destroy(file_data);
if (uncompressed == NULL) {
file_destroy(file);
send_status(fd, STATUS_ERROR);
return NULL;
}
file_data = uncompressed;
}
data_destroy(file->data);
file->data = file_data;
return file;
}
bool to_disk(const char* path, const void* data, unsigned long long data_size) {
// dirname() may modify its argument and may return a pointer to static storage.
// We must use a copy of the result to be safe.
char* tmp_path = NULL;
char* directory = NULL;
char* path_dup = str_dup(path);
if (!path_dup)
return false;
const char* dir_result = dirname(path_dup);
char* directory = str_dup(dir_result);
directory = str_dup(dir_result);
free(path_dup);
if (!directory)
return false;
bool ok = true;
if (!mkdir_r(directory)) {
if (!mkdir_r(directory))
goto done;
size_t path_len = strlen(path);
tmp_path = malloc(path_len + 5);
if (!tmp_path) {
ok = false;
goto done;
}
FILE* file_pointer = fopen(path, "wb");
memcpy(tmp_path, path, path_len);
memcpy(tmp_path + path_len, ".tmp", 5);
FILE* file_pointer = fopen(tmp_path, "wb");
if (file_pointer == NULL) {
perror("Could not open File");
perror("Could not open temporary file");
ok = false;
goto done;
}
if (fwrite(data, 1, data_size, file_pointer) != data_size) {
perror("Failed to write all data to disk");
perror("Failed to write all data to temporary file");
fclose(file_pointer);
unlink(tmp_path);
ok = false;
goto done;
}
fclose(file_pointer);
if (rename(tmp_path, path) != 0) {
perror("Failed to atomically rename temporary file");
unlink(tmp_path);
ok = false;
goto done;
}
done:
free(tmp_path);
free(directory);
return ok;
}
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
bool send_path) {
// Handle symlinks
if (file->type == FILE_TYPE_SYMLINK) {
return file_send_single_calls(file, file_descriptor, use_metadata, compression_level,
send_path);
}
// sendfile is incompatible with compression (kernel zero-copy).
// If compression is requested, fall back to the regular send path.
// NOTE: This is a safety net only — callers must ensure compression_level == 0
// before calling file_send_sendfile. The fallback to file_send_single_calls
// preserves the send_path contract, but callers should not rely on it for
// correctness (the --sendfile flag is validated to be mutually exclusive with
// -c/--compress at the CLI layer).
if (compression_level > 0)
return file_send_single_calls(file, file_descriptor, use_metadata, compression_level,
send_path);
@@ -614,10 +494,6 @@ bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int
if (use_metadata && !metadata_send(file_descriptor, file->metadata))
return false;
int ft = (int)file->type;
if (!send_int(file_descriptor, ft))
return false;
int fd = open(file->path, O_RDONLY);
if (fd == -1) {
perror("Could not open file for sendfile");
@@ -632,13 +508,8 @@ bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int
off_t offset = 0;
while ((unsigned long long)offset < file_size) {
size_t send_count = (size_t)(file_size - (unsigned long long)offset);
if ((unsigned long long)send_count != file_size - (unsigned long long)offset)
send_count = SIZE_MAX;
ssize_t sent = sendfile(file_descriptor, fd, &offset, send_count);
ssize_t sent = sendfile(file_descriptor, fd, &offset, file_size - offset);
if (sent == -1) {
if (errno == EINTR)
continue;
perror("sendfile failed");
close(fd);
return false;
@@ -665,30 +536,6 @@ File* file_receive(const Config* config, int file_descriptor) {
return NULL;
}
}
// Receive file type indicator
int file_type;
if (!receive_int(file_descriptor, &file_type)) {
file_destroy(file);
return NULL;
}
file->type = (FileType)file_type;
if (file->type == FILE_TYPE_SYMLINK) {
char* link_target = receive_str(file_descriptor);
if (link_target == NULL) {
file_destroy(file);
return NULL;
}
file->link_target = link_target;
// Receive and discard zero-length data
Data* empty_data = receive_data(file_descriptor);
if (empty_data)
data_destroy(empty_data);
return file;
}
// Regular file - receive data
Data* file_data = receive_data(file_descriptor);
if (file_data == NULL) {
file_destroy(file);
@@ -703,7 +550,6 @@ File* file_receive(const Config* config, int file_descriptor) {
}
file_data = file_data_uncompressed;
}
data_destroy(file->data);
file->data = file_data;
return file;