Incremental sync: --incremental to skip unchanged files #14

Merged
TapTap merged 6 commits from incremental-sync into main 2026-07-18 17:36:26 +02:00
5 changed files with 78 additions and 19 deletions
Showing only changes of commit f1af88a25c - Show all commits
+10
View File
@@ -205,6 +205,16 @@ int main(int argc, char *argv[]) {
return 1; return 1;
} }
if (config->use_incremental && config->use_chunk_serialization) {
fprintf(stderr, "Error: --incremental is not supported with -s (chunk serialization)\n");
return 1;
}
if (config->use_incremental && !config->use_metadata) {
log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --incremental");
config->use_metadata = true;
}
if (config->use_multithreading) if (config->use_multithreading)
return send_files_multithreaded(config); return send_files_multithreaded(config);
return send_files(config); return send_files(config);
+8 -4
View File
@@ -41,13 +41,16 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) {
if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1; if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1;
Status s; Status s;
if (!receive_status(client->file_descriptor, &s)) return -1; if (!receive_status(client->file_descriptor, &s)) return -1;
if (s == STATUS_ERROR) { log_message(LOG_LEVEL_ERROR, "Server reported error for file"); return -1; }
if (s == STATUS_OK) continue; if (s == STATUS_OK) continue;
if (s != STATUS_NEXT) return -1; if (s != STATUS_NEXT) { log_message(LOG_LEVEL_ERROR, "Unexpected server status"); return -1; }
if (!file_send_sendfile_no_path(chunk->items[i], client->file_descriptor, config->use_metadata))
return -1;
} else { } else {
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1; if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata))
return -1;
} }
if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata))
return -1;
} }
} else { } else {
for (int i = 0; i < chunk->element_count; i++) { for (int i = 0; i < chunk->element_count; i++) {
@@ -60,8 +63,9 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) {
if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1; if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1;
Status s; Status s;
if (!receive_status(client->file_descriptor, &s)) return -1; if (!receive_status(client->file_descriptor, &s)) return -1;
if (s == STATUS_ERROR) { log_message(LOG_LEVEL_ERROR, "Server reported error for file"); return -1; }
if (s == STATUS_OK) continue; if (s == STATUS_OK) continue;
if (s != STATUS_NEXT) return -1; if (s != STATUS_NEXT) { log_message(LOG_LEVEL_ERROR, "Unexpected server status"); return -1; }
if (!file_send_single_calls_no_path(chunk->items[i], client->file_descriptor, if (!file_send_single_calls_no_path(chunk->items[i], client->file_descriptor,
config->use_metadata, config->use_metadata,
config->use_compression ? config->compression_level : 0)) config->use_compression ? config->compression_level : 0))
+1 -1
View File
@@ -35,7 +35,7 @@ typedef struct Config {
bool use_incremental; bool use_incremental;
} Config; } Config;
#define PROTOCOL_VERSION "1.0.0" #define PROTOCOL_VERSION "1.1.0"
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
Config *config_create(char *version, char *send_directory, Config *config_create(char *version, char *send_directory,
+58 -14
View File
@@ -97,37 +97,81 @@ bool file_load_data(File *file) {
} }
bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level) { bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level) {
Data *data_to_send = file->data;
Data *compressed_data = NULL;
if (compression_level > 0) { if (compression_level > 0) {
Data *compressed_data = data_compress(file->data, compression_level); compressed_data = data_compress(file->data, compression_level);
if (compressed_data == NULL) { if (compressed_data == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to compress file data"); log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
return false; return false;
} }
data_destroy(file->data); data_to_send = compressed_data;
file->data = compressed_data;
} }
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) {
data_destroy(compressed_data);
return false;
}
if (!send_data(file_descriptor, data_to_send)) {
data_destroy(compressed_data);
return false;
}
data_destroy(compressed_data);
return true;
}
bool file_send_sendfile_no_path(File *file, int file_descriptor, bool use_metadata) {
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false;
if (!send_data(file_descriptor, file->data)) return false;
int fd = open(file->path, O_RDONLY);
if (fd == -1) {
perror("Could not open file for sendfile");
return false;
}
unsigned long long file_size = file->data->size;
if (!send_n_data(file_descriptor, &file_size, sizeof(unsigned long long))) {
close(fd);
return false;
}
off_t offset = 0;
while (offset < file_size) {
ssize_t sent = sendfile(file_descriptor, fd, &offset, file_size - offset);
if (sent == -1) {
perror("sendfile failed");
close(fd);
return false;
}
}
close(fd);
return true; return true;
} }
bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) { bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) {
Data *data_to_send = file->data;
Data *compressed_data = NULL;
if (compression_level > 0) { if (compression_level > 0) {
Data *compressed_data = data_compress(file->data, compression_level); compressed_data = data_compress(file->data, compression_level);
if (compressed_data == NULL) { if (compressed_data == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to compress file data"); log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
return false; return false;
} }
data_destroy(file->data); data_to_send = compressed_data;
if (compressed_data == NULL) {
log_message(LOG_LEVEL_ERROR, "Compression failed in file_send_single_calls");
exit(EXIT_FAILURE);
}
file->data = compressed_data;
} }
if (!send_str(file_descriptor, file->path)) return false; if (!send_str(file_descriptor, file->path)) {
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false; data_destroy(compressed_data);
if (!send_data(file_descriptor, file->data)) return false; return false;
}
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) {
data_destroy(compressed_data);
return false;
}
if (!send_data(file_descriptor, data_to_send)) {
data_destroy(compressed_data);
return false;
}
data_destroy(compressed_data);
return true; return true;
} }
+1
View File
@@ -27,6 +27,7 @@ File *file_receive(Config *config, int file_descriptor);
bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level); bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level);
bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level); bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level);
bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata); bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata);
bool file_send_sendfile_no_path(File *file, int file_descriptor, bool use_metadata);
size_t file_content_to_buffer(File *file); size_t file_content_to_buffer(File *file);
FileMetadata *file_metadata_create(struct stat *stats); FileMetadata *file_metadata_create(struct stat *stats);
void file_metadata_destroy(void *metadata); void file_metadata_destroy(void *metadata);