Incremental sync: --incremental to skip unchanged files #14
@@ -34,6 +34,7 @@ static void print_usage(void) {
|
||||
printf(" --include <pattern> Only include files matching pattern\n");
|
||||
printf(" --max-size <n> Skip files larger than n bytes\n");
|
||||
printf(" --min-size <n> Skip files smaller than n bytes\n");
|
||||
printf(" --incremental Skip files unchanged since last transfer\n");
|
||||
printf(" -m Enable multithreading\n");
|
||||
printf(" -s Enable chunk serialization\n");
|
||||
printf(" -f Enable sendfile (TCP only, not with -c or -s)\n");
|
||||
@@ -93,6 +94,8 @@ int main(int argc, char *argv[]) {
|
||||
config->max_size = strtoull(argv[++i], NULL, 10);
|
||||
} else if (strcmp(argv[i], "--min-size") == 0 && i + 1 < argc) {
|
||||
config->min_size = strtoull(argv[++i], NULL, 10);
|
||||
} else if (strcmp(argv[i], "--incremental") == 0) {
|
||||
config->use_incremental = true;
|
||||
} else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) {
|
||||
config->use_compression = true;
|
||||
log_message(LOG_LEVEL_INFO, "Enabled Compression");
|
||||
@@ -202,6 +205,16 @@ int main(int argc, char *argv[]) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (config->use_incremental && config->use_chunk_serialization) {
|
||||
fprintf(stderr, "Error: --incremental is not supported with -s (chunk serialization)\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (config->use_incremental && !config->use_metadata) {
|
||||
log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --incremental");
|
||||
config->use_metadata = true;
|
||||
}
|
||||
|
||||
if (config->use_multithreading)
|
||||
return send_files_multithreaded(config);
|
||||
return send_files(config);
|
||||
|
||||
@@ -18,6 +18,27 @@
|
||||
#include <threads.h>
|
||||
#include <time.h>
|
||||
|
||||
static int incremental_check(Client *client, File *file) {
|
||||
if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1;
|
||||
if (!send_str(client->file_descriptor, file->path)) return -1;
|
||||
unsigned long long fsize = file->data->size;
|
||||
long long mtime = file->metadata ? file->metadata->mtime_sec : 0;
|
||||
if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1;
|
||||
if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1;
|
||||
Status s;
|
||||
if (!receive_status(client->file_descriptor, &s)) return -1;
|
||||
if (s == STATUS_ERROR) {
|
||||
log_message(LOG_LEVEL_ERROR, "Server reported error for file");
|
||||
return -1;
|
||||
}
|
||||
if (s == STATUS_OK) return 1;
|
||||
if (s != STATUS_NEXT) {
|
||||
log_message(LOG_LEVEL_ERROR, "Unexpected server status");
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int send_chunk(Client *client, Chunk *chunk, Config *config) {
|
||||
if (config->use_chunk_serialization) {
|
||||
if (!send_status(client->file_descriptor, STATUS_CHUNK)) return -1;
|
||||
@@ -32,19 +53,39 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) {
|
||||
data_destroy(data);
|
||||
} else if (config->use_sendfile && !config->use_compression) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
|
||||
if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata))
|
||||
if (config->use_incremental) {
|
||||
int rc = incremental_check(client, chunk->items[i]);
|
||||
if (rc < 0) return -1;
|
||||
if (rc > 0) continue;
|
||||
if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata, false))
|
||||
return -1;
|
||||
} else {
|
||||
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
|
||||
if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata, true))
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
if (config->use_incremental) {
|
||||
int rc = incremental_check(client, chunk->items[i]);
|
||||
if (rc < 0) return -1;
|
||||
if (rc > 0) continue;
|
||||
if (!file_send_single_calls(chunk->items[i], client->file_descriptor,
|
||||
config->use_metadata,
|
||||
config->use_compression ? config->compression_level : 0,
|
||||
false))
|
||||
return -1;
|
||||
} else {
|
||||
if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
|
||||
if (!file_send_single_calls(chunk->items[i], client->file_descriptor,
|
||||
config->use_metadata,
|
||||
config->use_compression ? config->compression_level : 0))
|
||||
config->use_compression ? config->compression_level : 0,
|
||||
true))
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
+30
-64
@@ -1,11 +1,9 @@
|
||||
#include "array_list.h"
|
||||
#include "chunk.h"
|
||||
#include "compression.h"
|
||||
#include "config.h"
|
||||
#include "data.h"
|
||||
#include "file.h"
|
||||
#include "log.h"
|
||||
#include "metadata.h"
|
||||
#include "multiprocessing.h"
|
||||
#include "protocol.h"
|
||||
#include "queue.h"
|
||||
@@ -17,89 +15,57 @@
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
int receive_files(Config *config, int file_descriptor) {
|
||||
int receive_files(Config *config, int fd) {
|
||||
Status status;
|
||||
if (!receive_status(file_descriptor, &status)) return -1;
|
||||
while (status == STATUS_NEXT || status == STATUS_CHUNK) {
|
||||
if (status == STATUS_CHUNK) {
|
||||
Data *chunk_data = receive_data(file_descriptor);
|
||||
if (chunk_data == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data");
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
Data *data_to_process = chunk_data;
|
||||
if (config->use_compression) {
|
||||
data_to_process = data_decompress(chunk_data);
|
||||
data_destroy(chunk_data);
|
||||
if (data_to_process == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk");
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata);
|
||||
data_destroy(data_to_process);
|
||||
if (chunk == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping");
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
if (!receive_status(fd, &status)) return -1;
|
||||
|
||||
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) {
|
||||
if (status == STATUS_CHECK) {
|
||||
bool skipped;
|
||||
File *file = receive_incremental_check(fd, config, &skipped);
|
||||
if (skipped) goto next;
|
||||
if (file == NULL && !skipped) return -1;
|
||||
if (config->save_to_disk)
|
||||
file_save_to_disk(config->receive_root_directory, file);
|
||||
file_destroy(file);
|
||||
} else if (status == STATUS_CHUNK) {
|
||||
Chunk *chunk = receive_chunk_data(fd, config);
|
||||
if (chunk == NULL) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
if (config->save_to_disk) {
|
||||
char *disk_path = path_cat(config->receive_root_directory, chunk->items[i]->path);
|
||||
if (disk_path) {
|
||||
to_disk(disk_path, chunk->items[i]->data->data, chunk->items[i]->data->size);
|
||||
file_restore_metadata(disk_path, chunk->items[i]->metadata);
|
||||
free(disk_path);
|
||||
}
|
||||
}
|
||||
if (config->save_to_disk)
|
||||
file_save_to_disk(config->receive_root_directory, chunk->items[i]);
|
||||
}
|
||||
chunk_destroy(chunk);
|
||||
} else {
|
||||
File *file = file_receive(config, file_descriptor);
|
||||
File *file = file_receive(config, fd);
|
||||
if (file == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to receive file");
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
if (config->save_to_disk) {
|
||||
char *disk_path = path_cat(config->receive_root_directory, file->path);
|
||||
if (disk_path) {
|
||||
to_disk(disk_path, file->data->data, file->data->size);
|
||||
file_restore_metadata(disk_path, file->metadata);
|
||||
free(disk_path);
|
||||
}
|
||||
}
|
||||
if (config->save_to_disk)
|
||||
file_save_to_disk(config->receive_root_directory, file);
|
||||
file_destroy(file);
|
||||
}
|
||||
if (!receive_status(file_descriptor, &status)) {
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
next:
|
||||
if (!receive_status(fd, &status)) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (status == STATUS_MANIFEST) {
|
||||
int count;
|
||||
if (!receive_int(file_descriptor, &count)) return -1;
|
||||
ArrayList *manifest = array_list_create(free);
|
||||
if (manifest) {
|
||||
for (int i = 0; i < count; i++) {
|
||||
char *s = receive_str(file_descriptor);
|
||||
if (s) array_list_add(manifest, s);
|
||||
}
|
||||
fprintf(stderr, "Deleting files not in manifest...\n");
|
||||
delete_extras(config->receive_root_directory, manifest);
|
||||
array_list_delete(manifest);
|
||||
}
|
||||
if (!receive_status(file_descriptor, &status)) return -1;
|
||||
if (receive_manifest(fd, config, &status) != 0) return -1;
|
||||
}
|
||||
if (status != STATUS_FINISHED) {
|
||||
log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status");
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
send_status(file_descriptor, STATUS_OK);
|
||||
send_status(fd, STATUS_OK);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include "file.h"
|
||||
#include "log.h"
|
||||
#include "metadata.h"
|
||||
#include "protocol.h"
|
||||
|
||||
Chunk *chunk_create(File **items, int element_count) {
|
||||
Chunk *chunk = (Chunk *)malloc(sizeof(Chunk));
|
||||
@@ -178,5 +179,27 @@ Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata) {
|
||||
return compressed;
|
||||
}
|
||||
|
||||
Chunk *receive_chunk_data(int fd, Config *config) {
|
||||
Data *chunk_data = receive_data(fd);
|
||||
if (chunk_data == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data");
|
||||
return NULL;
|
||||
}
|
||||
Data *data_to_process = chunk_data;
|
||||
if (config->use_compression) {
|
||||
data_to_process = data_decompress(chunk_data);
|
||||
data_destroy(chunk_data);
|
||||
if (data_to_process == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk");
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata);
|
||||
data_destroy(data_to_process);
|
||||
if (chunk == NULL)
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping");
|
||||
return chunk;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#ifndef CHUNK_H
|
||||
#define CHUNK_H
|
||||
|
||||
#include "config.h"
|
||||
#include "data.h"
|
||||
#include "file.h"
|
||||
#include <stdbool.h>
|
||||
@@ -18,5 +19,6 @@ void chunk_destroy(void *chunk);
|
||||
Data *chunk_serialize(Chunk *chunk, bool use_metadata);
|
||||
Chunk *chunk_deserialize(Data *data, bool use_metadata);
|
||||
Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata);
|
||||
Chunk *receive_chunk_data(int fd, Config *config);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -38,6 +38,7 @@ Config *config_create(char *version, char *send_directory,
|
||||
config->include_count = 0;
|
||||
config->max_size = 0;
|
||||
config->min_size = 0;
|
||||
config->use_incremental = false;
|
||||
return config;
|
||||
}
|
||||
|
||||
@@ -89,6 +90,7 @@ bool config_send(int file_descriptor, Config *config) {
|
||||
if (!send_int(file_descriptor, (int)config->chunk_size)) return false;
|
||||
if (!send_int(file_descriptor, config->use_sendfile)) return false;
|
||||
if (!send_int(file_descriptor, config->use_delete)) return false;
|
||||
if (!send_int(file_descriptor, config->use_incremental)) return false;
|
||||
Status status;
|
||||
if (!receive_status(file_descriptor, &status)) return false;
|
||||
if (status != STATUS_OK) {
|
||||
@@ -134,6 +136,8 @@ Config *config_receive(int file_descriptor) {
|
||||
config->use_sendfile = tmp;
|
||||
if (!receive_int(file_descriptor, &tmp)) goto error;
|
||||
config->use_delete = tmp;
|
||||
if (!receive_int(file_descriptor, &tmp)) goto error;
|
||||
config->use_incremental = tmp;
|
||||
config->show_progress = false;
|
||||
config->dry_run = false;
|
||||
config->ssh_port = 22;
|
||||
|
||||
+2
-1
@@ -32,9 +32,10 @@ typedef struct Config {
|
||||
int include_count;
|
||||
unsigned long long max_size;
|
||||
unsigned long long min_size;
|
||||
bool use_incremental;
|
||||
} Config;
|
||||
|
||||
#define PROTOCOL_VERSION "1.0.0"
|
||||
#define PROTOCOL_VERSION "1.1.0"
|
||||
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
||||
|
||||
Config *config_create(char *version, char *send_directory,
|
||||
|
||||
+109
-13
@@ -96,26 +96,103 @@ bool file_load_data(File *file) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) {
|
||||
bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level, bool send_path) {
|
||||
Data *data_to_send = file->data;
|
||||
Data *compressed_data = NULL;
|
||||
if (compression_level > 0) {
|
||||
Data *compressed_data = data_compress(file->data, compression_level);
|
||||
compressed_data = data_compress(file->data, compression_level);
|
||||
if (compressed_data == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
|
||||
return false;
|
||||
}
|
||||
data_destroy(file->data);
|
||||
if (compressed_data == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Compression failed in file_send_single_calls");
|
||||
exit(EXIT_FAILURE);
|
||||
data_to_send = compressed_data;
|
||||
}
|
||||
file->data = compressed_data;
|
||||
if (send_path && !send_str(file_descriptor, file->path)) {
|
||||
data_destroy(compressed_data);
|
||||
return false;
|
||||
}
|
||||
if (!send_str(file_descriptor, file->path)) return false;
|
||||
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false;
|
||||
if (!send_data(file_descriptor, file->data)) return false;
|
||||
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) {
|
||||
data_destroy(compressed_data);
|
||||
return false;
|
||||
}
|
||||
if (!send_data(file_descriptor, data_to_send)) {
|
||||
data_destroy(compressed_data);
|
||||
return false;
|
||||
}
|
||||
data_destroy(compressed_data);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool file_save_to_disk(const char *root_directory, File *file) {
|
||||
char *disk_path = path_cat((char *)root_directory, file->path);
|
||||
if (disk_path == NULL) return false;
|
||||
bool ok = to_disk(disk_path, file->data->data, file->data->size);
|
||||
if (ok) file_restore_metadata(disk_path, file->metadata);
|
||||
free(disk_path);
|
||||
return ok;
|
||||
}
|
||||
|
||||
File *receive_incremental_check(int fd, Config *config, bool *skipped) { *skipped = false;
|
||||
char *check_path = receive_str(fd);
|
||||
if (check_path == NULL) { send_status(fd, STATUS_ERROR); return NULL; }
|
||||
|
||||
unsigned long long check_size;
|
||||
long long check_mtime;
|
||||
if (!receive_n_data(fd, &check_size, sizeof(check_size)) ||
|
||||
!receive_n_data(fd, &check_mtime, sizeof(check_mtime))) {
|
||||
free(check_path);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
char *full_path = path_cat(config->receive_root_directory, check_path);
|
||||
struct stat st;
|
||||
bool match = false;
|
||||
if (full_path && stat(full_path, &st) == 0 &&
|
||||
(unsigned long long)st.st_size == check_size &&
|
||||
(long long)st.st_mtime == check_mtime) {
|
||||
match = true;
|
||||
}
|
||||
free(full_path);
|
||||
|
||||
if (match) {
|
||||
if (!send_status(fd, STATUS_OK)) { free(check_path); return NULL; }
|
||||
free(check_path);
|
||||
*skipped = true;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (!send_status(fd, STATUS_NEXT)) { free(check_path); return NULL; }
|
||||
|
||||
File *file = file_create(check_path);
|
||||
free(check_path);
|
||||
if (file == NULL) { send_status(fd, STATUS_ERROR); return NULL; }
|
||||
|
||||
if (config->use_metadata) {
|
||||
int meta_ok = 1;
|
||||
file->metadata = metadata_receive(fd, &meta_ok);
|
||||
if (!meta_ok) { file_destroy(file); send_status(fd, STATUS_ERROR); return NULL; }
|
||||
}
|
||||
|
||||
Data *file_data = receive_data(fd);
|
||||
if (file_data == NULL) {
|
||||
file_destroy(file);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (config->use_compression) {
|
||||
Data *uncompressed = data_decompress(file_data);
|
||||
data_destroy(file_data);
|
||||
if (uncompressed == NULL) { file_destroy(file); send_status(fd, STATUS_ERROR); return NULL; }
|
||||
file_data = uncompressed;
|
||||
}
|
||||
|
||||
data_destroy(file->data);
|
||||
file->data = file_data;
|
||||
return file;
|
||||
}
|
||||
|
||||
bool to_disk(const char *path, const void *data, unsigned long long data_size) {
|
||||
char *directory = str_dup(path);
|
||||
char *dir_to_free = directory;
|
||||
@@ -141,8 +218,8 @@ bool to_disk(const char *path, const void *data, unsigned long long data_size) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata) {
|
||||
if (!send_str(file_descriptor, file->path)) return false;
|
||||
bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata, bool send_path) {
|
||||
if (send_path && !send_str(file_descriptor, file->path)) return false;
|
||||
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false;
|
||||
|
||||
int fd = open(file->path, O_RDONLY);
|
||||
@@ -178,7 +255,9 @@ File *file_receive(Config *config, int file_descriptor) {
|
||||
free(path);
|
||||
if (file == NULL) return NULL;
|
||||
if (config->use_metadata) {
|
||||
file->metadata = metadata_receive(file_descriptor);
|
||||
int meta_ok = 1;
|
||||
file->metadata = metadata_receive(file_descriptor, &meta_ok);
|
||||
if (!meta_ok) { file_destroy(file); return NULL; }
|
||||
}
|
||||
Data *file_data = receive_data(file_descriptor);
|
||||
if (file_data == NULL) {
|
||||
@@ -216,4 +295,21 @@ size_t file_content_to_buffer(File *file) {
|
||||
return bytes_read;
|
||||
}
|
||||
|
||||
int receive_manifest(int fd, Config *config, int *next_status) {
|
||||
int count;
|
||||
if (!receive_int(fd, &count)) return -1;
|
||||
ArrayList *manifest = array_list_create(free);
|
||||
if (manifest) {
|
||||
for (int i = 0; i < count; i++) {
|
||||
char *s = receive_str(fd);
|
||||
if (s) array_list_add(manifest, s);
|
||||
}
|
||||
fprintf(stderr, "Deleting files not in manifest...\n");
|
||||
delete_extras(config->receive_root_directory, manifest);
|
||||
array_list_delete(manifest);
|
||||
}
|
||||
if (!receive_status(fd, next_status)) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
||||
+5
-2
@@ -24,11 +24,14 @@ File *file_create(const char *path);
|
||||
void file_destroy(void *item);
|
||||
bool file_load_data(File *file);
|
||||
File *file_receive(Config *config, int file_descriptor);
|
||||
bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level);
|
||||
bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata);
|
||||
bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level, bool send_path);
|
||||
bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata, bool send_path);
|
||||
size_t file_content_to_buffer(File *file);
|
||||
FileMetadata *file_metadata_create(struct stat *stats);
|
||||
void file_metadata_destroy(void *metadata);
|
||||
bool to_disk(const char *path, const void *data, unsigned long long data_size);
|
||||
bool file_save_to_disk(const char *root_directory, File *file);
|
||||
File *receive_incremental_check(int fd, Config *config, bool *skipped);
|
||||
int receive_manifest(int fd, Config *config, int *next_status);
|
||||
|
||||
#endif
|
||||
|
||||
+10
-4
@@ -50,22 +50,28 @@ bool metadata_send(int file_descriptor, FileMetadata *m) {
|
||||
send_n_data(file_descriptor, &m->mtime_nsec, sizeof(long));
|
||||
}
|
||||
|
||||
FileMetadata *metadata_receive(int file_descriptor) {
|
||||
FileMetadata *metadata_receive(int file_descriptor, int *ok) {
|
||||
int present;
|
||||
if (!receive_n_data(file_descriptor, &present, sizeof(int)))
|
||||
if (!receive_n_data(file_descriptor, &present, sizeof(int))) {
|
||||
if (ok) *ok = 0;
|
||||
return NULL;
|
||||
if (!present)
|
||||
}
|
||||
if (!present) {
|
||||
if (ok) *ok = 1;
|
||||
return NULL;
|
||||
}
|
||||
FileMetadata *m = malloc(sizeof(FileMetadata));
|
||||
if (m == NULL) return NULL;
|
||||
if (m == NULL) { if (ok) *ok = 0; return NULL; }
|
||||
if (!receive_n_data(file_descriptor, &m->mode, sizeof(mode_t)) ||
|
||||
!receive_n_data(file_descriptor, &m->uid, sizeof(uid_t)) ||
|
||||
!receive_n_data(file_descriptor, &m->gid, sizeof(gid_t)) ||
|
||||
!receive_n_data(file_descriptor, &m->mtime_sec, sizeof(time_t)) ||
|
||||
!receive_n_data(file_descriptor, &m->mtime_nsec, sizeof(long))) {
|
||||
free(m);
|
||||
if (ok) *ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
if (ok) *ok = 1;
|
||||
return m;
|
||||
}
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
void metadata_to_buf(char **buf, FileMetadata *m);
|
||||
FileMetadata *metadata_from_buf(char **buf);
|
||||
bool metadata_send(int file_descriptor, FileMetadata *m);
|
||||
FileMetadata *metadata_receive(int file_descriptor);
|
||||
FileMetadata *metadata_receive(int file_descriptor, int *ok);
|
||||
void file_restore_metadata(const char *path, FileMetadata *metadata);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
#include "multiprocessing.h"
|
||||
#include "array_list.h"
|
||||
#include "chunk.h"
|
||||
#include "compression.h"
|
||||
#include "config.h"
|
||||
#include "data.h"
|
||||
#include "file.h"
|
||||
#include "log.h"
|
||||
#include "metadata.h"
|
||||
#include "protocol.h"
|
||||
#include "queue.h"
|
||||
#include "utils.h"
|
||||
@@ -85,26 +83,8 @@ void pipeline_context_receiver_destroy(PipelineContextReceiver *context) {
|
||||
|
||||
static void receive_chunk_enqueue(int file_descriptor,
|
||||
PipelineContextReceiver *context) {
|
||||
Data *chunk_data = receive_data(file_descriptor);
|
||||
if (chunk_data == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data");
|
||||
return;
|
||||
}
|
||||
Data *data_to_process = chunk_data;
|
||||
if (context->config->use_compression) {
|
||||
data_to_process = data_decompress(chunk_data);
|
||||
data_destroy(chunk_data);
|
||||
if (data_to_process == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk");
|
||||
return;
|
||||
}
|
||||
}
|
||||
Chunk *chunk = chunk_deserialize(data_to_process, context->config->use_metadata);
|
||||
data_destroy(data_to_process);
|
||||
if (chunk == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping");
|
||||
return;
|
||||
}
|
||||
Chunk *chunk = receive_chunk_data(file_descriptor, context->config);
|
||||
if (chunk == NULL) return;
|
||||
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
File *file = chunk->items[i];
|
||||
@@ -126,8 +106,17 @@ int receive_thread(void *pipeline_context) {
|
||||
|
||||
Status status;
|
||||
if (!receive_status(file_descriptor, &status)) return thrd_error;
|
||||
while (status == STATUS_NEXT || status == STATUS_CHUNK) {
|
||||
if (status == STATUS_CHUNK) {
|
||||
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) {
|
||||
if (status == STATUS_CHECK) {
|
||||
bool skipped;
|
||||
File *file = receive_incremental_check(file_descriptor, config, &skipped);
|
||||
if (!skipped) {
|
||||
if (file == NULL) return thrd_error;
|
||||
queue_enqueue_multithreaded(context->queue, file, &context->mutex,
|
||||
&context->condition_not_empty,
|
||||
&context->condition_not_full);
|
||||
}
|
||||
} else if (status == STATUS_CHUNK) {
|
||||
receive_chunk_enqueue(file_descriptor, context);
|
||||
} else {
|
||||
File *file = file_receive(config, file_descriptor);
|
||||
@@ -142,22 +131,7 @@ int receive_thread(void *pipeline_context) {
|
||||
if (!receive_status(file_descriptor, &status)) return thrd_error;
|
||||
}
|
||||
if (status == STATUS_MANIFEST) {
|
||||
int count;
|
||||
if (!receive_int(file_descriptor, &count)) return thrd_error;
|
||||
ArrayList *manifest = array_list_create(free);
|
||||
if (manifest) {
|
||||
for (int i = 0; i < count; i++) {
|
||||
char *s = receive_str(file_descriptor);
|
||||
if (s) {
|
||||
array_list_add(manifest, s);
|
||||
}
|
||||
}
|
||||
delete_extras(context->config->receive_root_directory, manifest);
|
||||
for (int i = 0; i < manifest->size; i++)
|
||||
free(manifest->items[i]);
|
||||
array_list_delete(manifest);
|
||||
}
|
||||
if (!receive_status(file_descriptor, &status)) return thrd_error;
|
||||
if (receive_manifest(file_descriptor, config, &status) != 0) return thrd_error;
|
||||
}
|
||||
mtx_lock(&context->mutex);
|
||||
context->receiver_done = true;
|
||||
@@ -182,14 +156,8 @@ int write_thread(void *pipeline_context) {
|
||||
free(root_directory);
|
||||
return thrd_success;
|
||||
}
|
||||
if (save_to_disk) {
|
||||
char *disk_path = path_cat(root_directory, file->path);
|
||||
if (disk_path) {
|
||||
to_disk(disk_path, file->data->data, file->data->size);
|
||||
file_restore_metadata(disk_path, file->metadata);
|
||||
free(disk_path);
|
||||
}
|
||||
}
|
||||
if (save_to_disk)
|
||||
file_save_to_disk(root_directory, file);
|
||||
file_destroy(file);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,6 +111,8 @@ static const char *status_to_string(Status status) {
|
||||
return "NEXT";
|
||||
case STATUS_CHUNK:
|
||||
return "CHUNK";
|
||||
case STATUS_CHECK:
|
||||
return "CHECK";
|
||||
default:
|
||||
return "UNKNOWN";
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
#include <stddef.h>
|
||||
|
||||
typedef int Status;
|
||||
enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST };
|
||||
enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST, STATUS_CHECK };
|
||||
|
||||
void io_set_fds(int read_fd, int write_fd);
|
||||
void io_set_bwlimit(unsigned long long bytes_per_sec);
|
||||
|
||||
@@ -415,6 +415,33 @@ def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=No
|
||||
except Exception as e:
|
||||
results.append({"name": "Bandwidth limit (--bwlimit 10240)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
|
||||
|
||||
# Incremental sync (--incremental) — first sync, then second sync should skip all
|
||||
print(f"\n --- Incremental (--incremental) ---")
|
||||
try:
|
||||
flags = BASE_CLIENT_FLAGS + ["-M"]
|
||||
srv = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
|
||||
time.sleep(0.5)
|
||||
first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
|
||||
r1 = subprocess.run(first_cmd, text=True, capture_output=True)
|
||||
wait_proc(srv)
|
||||
if r1.returncode != 0:
|
||||
raise RuntimeError(f"First sync failed: {r1.stderr[:100]}")
|
||||
srv2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
|
||||
time.sleep(0.5)
|
||||
second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--incremental"]
|
||||
start = time.monotonic()
|
||||
r2 = subprocess.run(second_cmd, text=True, capture_output=True, timeout=30)
|
||||
duration = time.monotonic() - start
|
||||
wait_proc(srv2)
|
||||
r = {"name": "Incremental (--incremental)", "suite": profile_name,
|
||||
"status": "Success" if r2.returncode == 0 else "Failed",
|
||||
"time": f"{duration:.4f}s" if r2.returncode == 0 else "N/A",
|
||||
"error": "" if r2.returncode == 0 else f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}"}
|
||||
results.append(r)
|
||||
except Exception as e:
|
||||
results.append({"name": "Incremental (--incremental)", "suite": profile_name,
|
||||
"status": "Error", "time": "N/A", "error": str(e)})
|
||||
|
||||
# Chunk size (--chunk-size 5242880)
|
||||
feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"]
|
||||
cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
|
||||
|
||||
Reference in New Issue
Block a user