feat(p6-batch): residual-batch write/read driver + codec
Implements the client-only residual-batch feature end-to-end:
- src/shared/batch.{c,h}: self-contained single-file batch codec using the
existing chunk_serialize/chunk_deserialize codec (byte-identical by
construction). Magic+format-version header (metadata mode is persisted into
the header so a batch is self-describing across machines), length-prefixed
chunk records, bounded reads that reject malformed/truncated/oversized
records cleanly.
- src/client/client_send.c: write_batch_from_source (deterministic separate
scan pass, loads every chunk's file images, emits header+records) and
apply_batch_to_dest (local apply to a destination root via
file_save_to_disk_full). No wire change, no server involved.
- src/client/client_validation.c: --write-batch XOR --only-write-batch;
--read-batch exclusive with both; --read-batch needs only a DEST,
--only-write-batch only a SOURCE.
- src/client/client_cli.c: main() drives the three batch modes without
connecting/transferring for read/only-write; --write-batch runs the live
transfer (single-threaded so the config survives) then emits the batch.
- tests/test_batch.{c,h} (unit: byte-identical roundtrip with and without
metadata; bad-magic/truncated/oversized rejection) + tests/integration/
test_batch.py (only-write no-server, read-batch no-source roundtrip,
--write-batch with a live transfer, conflict rejections).
- clang-format: realign PART-1 config.h comment block.
No PROTOCOL_VERSION bump, no config-frame field, no server flag.
This commit is contained in:
+49
-5
@@ -1611,10 +1611,33 @@ int main(int argc, char* argv[]) {
|
||||
}
|
||||
config->save_to_disk = true;
|
||||
} else if (positional_count == 1) {
|
||||
log_message(LOG_LEVEL_ERROR, "missing destination argument");
|
||||
print_usage();
|
||||
exit_code = 1;
|
||||
goto cleanup;
|
||||
if (config->read_batch) {
|
||||
/* --read-batch=<file> <dest>: the single positional is the destination
|
||||
(there is no source). */
|
||||
free(config->receive_root_directory);
|
||||
config->receive_root_directory = str_dup(argv[positional_args[0]]);
|
||||
if (!config->receive_root_directory) {
|
||||
log_message(LOG_LEVEL_ERROR, "memory allocation failed");
|
||||
exit_code = 1;
|
||||
goto cleanup;
|
||||
}
|
||||
config->save_to_disk = true;
|
||||
} else if (config->only_write_batch) {
|
||||
/* --only-write-batch=<file> <source>: the single positional is the
|
||||
source (there is no destination). */
|
||||
free(config->send_directory);
|
||||
config->send_directory = str_dup(argv[positional_args[0]]);
|
||||
if (!config->send_directory) {
|
||||
log_message(LOG_LEVEL_ERROR, "memory allocation failed");
|
||||
exit_code = 1;
|
||||
goto cleanup;
|
||||
}
|
||||
} else {
|
||||
log_message(LOG_LEVEL_ERROR, "missing destination argument");
|
||||
print_usage();
|
||||
exit_code = 1;
|
||||
goto cleanup;
|
||||
}
|
||||
} else {
|
||||
if (!config->send_directory && env_source) {
|
||||
config->send_directory = str_dup(env_source);
|
||||
@@ -1680,8 +1703,29 @@ int main(int argc, char* argv[]) {
|
||||
|
||||
tcp_set_timeouts(config->timeout, config->contimeout);
|
||||
|
||||
/* Phase 6 residual-batch driver modes. --read-batch / --only-write-batch are
|
||||
purely local (apply a batch file, or emit one from a scan): neither connects
|
||||
to nor transfers to a server. --write-batch runs the normal live transfer
|
||||
AND then emits the batch FILE from a separate deterministic scan pass. It
|
||||
drives the single-threaded transfer so the config outlives the run for that
|
||||
second pass (the -m path takes ownership of the config). */
|
||||
if (config->read_batch) {
|
||||
exit_code = apply_batch_to_dest(config, config->read_batch, config->receive_root_directory);
|
||||
goto cleanup;
|
||||
}
|
||||
if (config->only_write_batch) {
|
||||
exit_code = write_batch_from_source(config, config->only_write_batch);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Execute transfer */
|
||||
if (config->use_multithreading) {
|
||||
if (config->write_batch) {
|
||||
exit_code = send_files(config);
|
||||
if (exit_code == 0 && write_batch_from_source(config, config->write_batch) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "live transfer succeeded but batch emission failed");
|
||||
exit_code = 1;
|
||||
}
|
||||
} else if (config->use_multithreading) {
|
||||
exit_code = send_files_multithreaded(&config);
|
||||
} else {
|
||||
exit_code = send_files(config);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#include "client_send.h"
|
||||
#include "array_list.h"
|
||||
#include "batch.h"
|
||||
#include "change_list.h"
|
||||
#include "charset.h"
|
||||
#include "chunk.h"
|
||||
@@ -1718,6 +1719,78 @@ static int progress_thread_fn(void* arg) {
|
||||
return thrd_success;
|
||||
}
|
||||
|
||||
/* Phase 6 residual-batch (client-only). --write-batch=FILE / --only-write-batch
|
||||
* emit a self-contained single-file batch of a whole source tree from a
|
||||
* deterministic separate scan pass. Each chunk's file images are fully loaded
|
||||
* into memory (so chunk_serialize sees complete content, matching the -s wire
|
||||
* codec byte-for-byte) and written to FILE as a length-prefixed record. The
|
||||
* batch never crosses the wire and needs no server. Returns 0 on success. */
|
||||
int write_batch_from_source(const Config* config, const char* batch_path) {
|
||||
if (!config || !batch_path || !config->send_directory)
|
||||
return 1;
|
||||
PreparedScanner prepared;
|
||||
memset(&prepared, 0, sizeof(prepared));
|
||||
if (!prepare_scanner(config, 0, &prepared))
|
||||
return 1;
|
||||
DirectoryScanner* scanner =
|
||||
directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
||||
if (!scanner) {
|
||||
prepared_scanner_destroy(&prepared);
|
||||
return 1;
|
||||
}
|
||||
int fd = open(batch_path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
||||
if (fd < 0) {
|
||||
log_perror("could not create batch file");
|
||||
directory_scanner_destroy(scanner);
|
||||
prepared_scanner_destroy(&prepared);
|
||||
return 1;
|
||||
}
|
||||
bool ok = batch_write_header(fd, config);
|
||||
Chunk* chunk;
|
||||
while (ok && (chunk = directory_scanner_next(scanner)) != NULL) {
|
||||
for (int i = 0; i < chunk->element_count && ok; i++) {
|
||||
File* f = chunk->items[i];
|
||||
if (f == NULL || f->data == NULL)
|
||||
continue;
|
||||
if (f->data->size > 0 && f->data->data == NULL && !file_load_data(f)) {
|
||||
log_message(LOG_LEVEL_ERROR, "batch: failed to load data for %s",
|
||||
f->path ? f->path : "<no path>");
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (ok)
|
||||
ok = batch_write_chunk(fd, chunk);
|
||||
chunk_destroy(chunk);
|
||||
}
|
||||
if (ok && directory_scanner_failed(scanner))
|
||||
ok = false;
|
||||
if (directory_scanner_had_io_error(scanner))
|
||||
log_message(LOG_LEVEL_WARNING, "batch: source scan hit an unreadable directory");
|
||||
close(fd);
|
||||
directory_scanner_destroy(scanner);
|
||||
prepared_scanner_destroy(&prepared);
|
||||
if (!ok && batch_path[0] != '\0')
|
||||
unlink(batch_path); /* never leave a partial batch behind */
|
||||
return ok ? 0 : 1;
|
||||
}
|
||||
|
||||
/* Apply a batch FILE to DEST_ROOT (client-only, no server). Returns 0 on
|
||||
* success; a malformed/truncated/oversized record or an apply error fails the
|
||||
* whole apply. */
|
||||
int apply_batch_to_dest(const Config* config, const char* batch_path, const char* dest_root) {
|
||||
if (!batch_path || !dest_root)
|
||||
return 1;
|
||||
int fd = open(batch_path, O_RDONLY);
|
||||
if (fd < 0) {
|
||||
log_perror("could not open batch file");
|
||||
return 1;
|
||||
}
|
||||
int rc = batch_read_apply(fd, config, dest_root);
|
||||
close(fd);
|
||||
return rc;
|
||||
}
|
||||
|
||||
int send_files(Config* config) {
|
||||
if (config->list_only)
|
||||
return send_list_only(config);
|
||||
|
||||
@@ -9,5 +9,8 @@ int send_chunk(Client* client, Chunk* chunk, Config* config);
|
||||
int send_files(Config* config);
|
||||
/* Takes ownership only when *config is set to NULL on return. */
|
||||
int send_files_multithreaded(Config** config);
|
||||
/* Phase 6 residual-batch (client-only). See client_send.c. */
|
||||
int write_batch_from_source(const Config* config, const char* batch_path);
|
||||
int apply_batch_to_dest(const Config* config, const char* batch_path, const char* dest_root);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -8,7 +8,34 @@
|
||||
|
||||
/* Validate config after parsing. Returns true if valid. */
|
||||
bool validate_config(const Config* config) {
|
||||
if (!config->send_directory || !config->receive_root_directory) {
|
||||
/* Phase 6 residual-batch modes relax the normal source+destination pair: the
|
||||
batch driver is local and needs only what it consumes. --only-write-batch
|
||||
emits a batch from the source (no destination, no server);
|
||||
--read-batch applies a batch to the destination (no source, no server);
|
||||
--write-batch runs the live transfer AND emits a batch, so it keeps the
|
||||
full pair. */
|
||||
bool write_batch = config->write_batch != NULL;
|
||||
bool only_write_batch = config->only_write_batch != NULL;
|
||||
bool read_batch = config->read_batch != NULL;
|
||||
if ((write_batch && only_write_batch) || (write_batch && read_batch) ||
|
||||
(only_write_batch && read_batch)) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--write-batch, --only-write-batch, and --read-batch are mutually exclusive");
|
||||
return false;
|
||||
}
|
||||
if (read_batch) {
|
||||
if (!config->receive_root_directory) {
|
||||
log_message(LOG_LEVEL_ERROR, "--read-batch requires a destination directory");
|
||||
print_usage();
|
||||
return false;
|
||||
}
|
||||
} else if (only_write_batch) {
|
||||
if (!config->send_directory) {
|
||||
log_message(LOG_LEVEL_ERROR, "--only-write-batch requires a source directory");
|
||||
print_usage();
|
||||
return false;
|
||||
}
|
||||
} else if (!config->send_directory || !config->receive_root_directory) {
|
||||
log_message(LOG_LEVEL_ERROR, "source and destination directories are required");
|
||||
print_usage();
|
||||
return false;
|
||||
|
||||
Reference in New Issue
Block a user