feat(p6-batch): residual-batch write/read driver + codec
Implements the client-only residual-batch feature end-to-end:
- src/shared/batch.{c,h}: self-contained single-file batch codec using the
existing chunk_serialize/chunk_deserialize codec (byte-identical by
construction). Magic+format-version header (metadata mode is persisted into
the header so a batch is self-describing across machines), length-prefixed
chunk records, bounded reads that reject malformed/truncated/oversized
records cleanly.
- src/client/client_send.c: write_batch_from_source (deterministic separate
scan pass, loads every chunk's file images, emits header+records) and
apply_batch_to_dest (local apply to a destination root via
file_save_to_disk_full). No wire change, no server involved.
- src/client/client_validation.c: --write-batch XOR --only-write-batch;
--read-batch exclusive with both; --read-batch needs only a DEST,
--only-write-batch only a SOURCE.
- src/client/client_cli.c: main() drives the three batch modes without
connecting/transferring for read/only-write; --write-batch runs the live
transfer (single-threaded so the config survives) then emits the batch.
- tests/test_batch.{c,h} (unit: byte-identical roundtrip with and without
metadata; bad-magic/truncated/oversized rejection) + tests/integration/
test_batch.py (only-write no-server, read-batch no-source roundtrip,
--write-batch with a live transfer, conflict rejections).
- clang-format: realign PART-1 config.h comment block.
No PROTOCOL_VERSION bump, no config-frame field, no server flag.
This commit is contained in:
+45
-1
@@ -1611,10 +1611,33 @@ int main(int argc, char* argv[]) {
|
|||||||
}
|
}
|
||||||
config->save_to_disk = true;
|
config->save_to_disk = true;
|
||||||
} else if (positional_count == 1) {
|
} else if (positional_count == 1) {
|
||||||
|
if (config->read_batch) {
|
||||||
|
/* --read-batch=<file> <dest>: the single positional is the destination
|
||||||
|
(there is no source). */
|
||||||
|
free(config->receive_root_directory);
|
||||||
|
config->receive_root_directory = str_dup(argv[positional_args[0]]);
|
||||||
|
if (!config->receive_root_directory) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed");
|
||||||
|
exit_code = 1;
|
||||||
|
goto cleanup;
|
||||||
|
}
|
||||||
|
config->save_to_disk = true;
|
||||||
|
} else if (config->only_write_batch) {
|
||||||
|
/* --only-write-batch=<file> <source>: the single positional is the
|
||||||
|
source (there is no destination). */
|
||||||
|
free(config->send_directory);
|
||||||
|
config->send_directory = str_dup(argv[positional_args[0]]);
|
||||||
|
if (!config->send_directory) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed");
|
||||||
|
exit_code = 1;
|
||||||
|
goto cleanup;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
log_message(LOG_LEVEL_ERROR, "missing destination argument");
|
log_message(LOG_LEVEL_ERROR, "missing destination argument");
|
||||||
print_usage();
|
print_usage();
|
||||||
exit_code = 1;
|
exit_code = 1;
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
if (!config->send_directory && env_source) {
|
if (!config->send_directory && env_source) {
|
||||||
config->send_directory = str_dup(env_source);
|
config->send_directory = str_dup(env_source);
|
||||||
@@ -1680,8 +1703,29 @@ int main(int argc, char* argv[]) {
|
|||||||
|
|
||||||
tcp_set_timeouts(config->timeout, config->contimeout);
|
tcp_set_timeouts(config->timeout, config->contimeout);
|
||||||
|
|
||||||
|
/* Phase 6 residual-batch driver modes. --read-batch / --only-write-batch are
|
||||||
|
purely local (apply a batch file, or emit one from a scan): neither connects
|
||||||
|
to nor transfers to a server. --write-batch runs the normal live transfer
|
||||||
|
AND then emits the batch FILE from a separate deterministic scan pass. It
|
||||||
|
drives the single-threaded transfer so the config outlives the run for that
|
||||||
|
second pass (the -m path takes ownership of the config). */
|
||||||
|
if (config->read_batch) {
|
||||||
|
exit_code = apply_batch_to_dest(config, config->read_batch, config->receive_root_directory);
|
||||||
|
goto cleanup;
|
||||||
|
}
|
||||||
|
if (config->only_write_batch) {
|
||||||
|
exit_code = write_batch_from_source(config, config->only_write_batch);
|
||||||
|
goto cleanup;
|
||||||
|
}
|
||||||
|
|
||||||
/* Execute transfer */
|
/* Execute transfer */
|
||||||
if (config->use_multithreading) {
|
if (config->write_batch) {
|
||||||
|
exit_code = send_files(config);
|
||||||
|
if (exit_code == 0 && write_batch_from_source(config, config->write_batch) != 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "live transfer succeeded but batch emission failed");
|
||||||
|
exit_code = 1;
|
||||||
|
}
|
||||||
|
} else if (config->use_multithreading) {
|
||||||
exit_code = send_files_multithreaded(&config);
|
exit_code = send_files_multithreaded(&config);
|
||||||
} else {
|
} else {
|
||||||
exit_code = send_files(config);
|
exit_code = send_files(config);
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
#include "client_send.h"
|
#include "client_send.h"
|
||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
|
#include "batch.h"
|
||||||
#include "change_list.h"
|
#include "change_list.h"
|
||||||
#include "charset.h"
|
#include "charset.h"
|
||||||
#include "chunk.h"
|
#include "chunk.h"
|
||||||
@@ -1718,6 +1719,78 @@ static int progress_thread_fn(void* arg) {
|
|||||||
return thrd_success;
|
return thrd_success;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Phase 6 residual-batch (client-only). --write-batch=FILE / --only-write-batch
|
||||||
|
* emit a self-contained single-file batch of a whole source tree from a
|
||||||
|
* deterministic separate scan pass. Each chunk's file images are fully loaded
|
||||||
|
* into memory (so chunk_serialize sees complete content, matching the -s wire
|
||||||
|
* codec byte-for-byte) and written to FILE as a length-prefixed record. The
|
||||||
|
* batch never crosses the wire and needs no server. Returns 0 on success. */
|
||||||
|
int write_batch_from_source(const Config* config, const char* batch_path) {
|
||||||
|
if (!config || !batch_path || !config->send_directory)
|
||||||
|
return 1;
|
||||||
|
PreparedScanner prepared;
|
||||||
|
memset(&prepared, 0, sizeof(prepared));
|
||||||
|
if (!prepare_scanner(config, 0, &prepared))
|
||||||
|
return 1;
|
||||||
|
DirectoryScanner* scanner =
|
||||||
|
directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
||||||
|
if (!scanner) {
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
int fd = open(batch_path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
||||||
|
if (fd < 0) {
|
||||||
|
log_perror("could not create batch file");
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
bool ok = batch_write_header(fd, config);
|
||||||
|
Chunk* chunk;
|
||||||
|
while (ok && (chunk = directory_scanner_next(scanner)) != NULL) {
|
||||||
|
for (int i = 0; i < chunk->element_count && ok; i++) {
|
||||||
|
File* f = chunk->items[i];
|
||||||
|
if (f == NULL || f->data == NULL)
|
||||||
|
continue;
|
||||||
|
if (f->data->size > 0 && f->data->data == NULL && !file_load_data(f)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "batch: failed to load data for %s",
|
||||||
|
f->path ? f->path : "<no path>");
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (ok)
|
||||||
|
ok = batch_write_chunk(fd, chunk);
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
}
|
||||||
|
if (ok && directory_scanner_failed(scanner))
|
||||||
|
ok = false;
|
||||||
|
if (directory_scanner_had_io_error(scanner))
|
||||||
|
log_message(LOG_LEVEL_WARNING, "batch: source scan hit an unreadable directory");
|
||||||
|
close(fd);
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
|
if (!ok && batch_path[0] != '\0')
|
||||||
|
unlink(batch_path); /* never leave a partial batch behind */
|
||||||
|
return ok ? 0 : 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Apply a batch FILE to DEST_ROOT (client-only, no server). Returns 0 on
|
||||||
|
* success; a malformed/truncated/oversized record or an apply error fails the
|
||||||
|
* whole apply. */
|
||||||
|
int apply_batch_to_dest(const Config* config, const char* batch_path, const char* dest_root) {
|
||||||
|
if (!batch_path || !dest_root)
|
||||||
|
return 1;
|
||||||
|
int fd = open(batch_path, O_RDONLY);
|
||||||
|
if (fd < 0) {
|
||||||
|
log_perror("could not open batch file");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
int rc = batch_read_apply(fd, config, dest_root);
|
||||||
|
close(fd);
|
||||||
|
return rc;
|
||||||
|
}
|
||||||
|
|
||||||
int send_files(Config* config) {
|
int send_files(Config* config) {
|
||||||
if (config->list_only)
|
if (config->list_only)
|
||||||
return send_list_only(config);
|
return send_list_only(config);
|
||||||
|
|||||||
@@ -9,5 +9,8 @@ int send_chunk(Client* client, Chunk* chunk, Config* config);
|
|||||||
int send_files(Config* config);
|
int send_files(Config* config);
|
||||||
/* Takes ownership only when *config is set to NULL on return. */
|
/* Takes ownership only when *config is set to NULL on return. */
|
||||||
int send_files_multithreaded(Config** config);
|
int send_files_multithreaded(Config** config);
|
||||||
|
/* Phase 6 residual-batch (client-only). See client_send.c. */
|
||||||
|
int write_batch_from_source(const Config* config, const char* batch_path);
|
||||||
|
int apply_batch_to_dest(const Config* config, const char* batch_path, const char* dest_root);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -8,7 +8,34 @@
|
|||||||
|
|
||||||
/* Validate config after parsing. Returns true if valid. */
|
/* Validate config after parsing. Returns true if valid. */
|
||||||
bool validate_config(const Config* config) {
|
bool validate_config(const Config* config) {
|
||||||
if (!config->send_directory || !config->receive_root_directory) {
|
/* Phase 6 residual-batch modes relax the normal source+destination pair: the
|
||||||
|
batch driver is local and needs only what it consumes. --only-write-batch
|
||||||
|
emits a batch from the source (no destination, no server);
|
||||||
|
--read-batch applies a batch to the destination (no source, no server);
|
||||||
|
--write-batch runs the live transfer AND emits a batch, so it keeps the
|
||||||
|
full pair. */
|
||||||
|
bool write_batch = config->write_batch != NULL;
|
||||||
|
bool only_write_batch = config->only_write_batch != NULL;
|
||||||
|
bool read_batch = config->read_batch != NULL;
|
||||||
|
if ((write_batch && only_write_batch) || (write_batch && read_batch) ||
|
||||||
|
(only_write_batch && read_batch)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"--write-batch, --only-write-batch, and --read-batch are mutually exclusive");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (read_batch) {
|
||||||
|
if (!config->receive_root_directory) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--read-batch requires a destination directory");
|
||||||
|
print_usage();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} else if (only_write_batch) {
|
||||||
|
if (!config->send_directory) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--only-write-batch requires a source directory");
|
||||||
|
print_usage();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} else if (!config->send_directory || !config->receive_root_directory) {
|
||||||
log_message(LOG_LEVEL_ERROR, "source and destination directories are required");
|
log_message(LOG_LEVEL_ERROR, "source and destination directories are required");
|
||||||
print_usage();
|
print_usage();
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@@ -0,0 +1,160 @@
|
|||||||
|
#include "batch.h"
|
||||||
|
#include "data.h"
|
||||||
|
#include "file.h"
|
||||||
|
#include "file_receive.h"
|
||||||
|
#include "log.h"
|
||||||
|
#include <errno.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
|
/* Serialization metadata mode for the batch stream, captured from the config at
|
||||||
|
* batch_write_header time. The header persists it into the file so a batch is
|
||||||
|
* self-describing: batch_read_apply re-reads it from the file (not from the
|
||||||
|
* reading config), so a batch written with -M is applied identically by an
|
||||||
|
* invoking process regardless of its own -M setting. The batch driver is a
|
||||||
|
* single sequential scan pass within one thread, so this module-level flag is
|
||||||
|
* safe. */
|
||||||
|
static bool batch_metadata_mode = false;
|
||||||
|
|
||||||
|
static bool write_all_bytes(int fd, const void* data, size_t size) {
|
||||||
|
const unsigned char* p = (const unsigned char*)data;
|
||||||
|
size_t done = 0;
|
||||||
|
while (done < size) {
|
||||||
|
ssize_t n = write(fd, p + done, size - done);
|
||||||
|
if (n < 0 && errno == EINTR)
|
||||||
|
continue;
|
||||||
|
if (n <= 0)
|
||||||
|
return false;
|
||||||
|
done += (size_t)n;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool batch_write_header(int fd, const Config* config) {
|
||||||
|
if (fd < 0)
|
||||||
|
return false;
|
||||||
|
batch_metadata_mode = config != NULL && config->use_metadata;
|
||||||
|
if (!write_all_bytes(fd, BATCH_MAGIC, BATCH_MAGIC_LEN))
|
||||||
|
return false;
|
||||||
|
unsigned char version = BATCH_FORMAT_VERSION;
|
||||||
|
if (!write_all_bytes(fd, &version, 1))
|
||||||
|
return false;
|
||||||
|
unsigned char mode = batch_metadata_mode ? 1 : 0;
|
||||||
|
return write_all_bytes(fd, &mode, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool batch_write_chunk(int fd, Chunk* chunk) {
|
||||||
|
if (fd < 0 || chunk == NULL)
|
||||||
|
return false;
|
||||||
|
Data* serialized = chunk_serialize(chunk, batch_metadata_mode);
|
||||||
|
if (serialized == NULL)
|
||||||
|
return false;
|
||||||
|
bool ok = false;
|
||||||
|
unsigned long long length = (unsigned long long)serialized->size;
|
||||||
|
if (length > BATCH_MAX_RECORD) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "batch: record size %llu exceeds the %llu-byte cap", length,
|
||||||
|
(unsigned long long)BATCH_MAX_RECORD);
|
||||||
|
} else if (write_all_bytes(fd, &length, sizeof(length)) &&
|
||||||
|
(length == 0 || write_all_bytes(fd, serialized->data, (size_t)length))) {
|
||||||
|
ok = true;
|
||||||
|
}
|
||||||
|
data_destroy(serialized);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Read exactly `size` bytes. Returns true on success. On reaching EOF, sets
|
||||||
|
* *clean_eof only when no bytes had been read yet (a clean boundary) and returns
|
||||||
|
* that value, so a truncated record (EOF mid-read) yields false. */
|
||||||
|
static bool read_exact(int fd, void* data, size_t size, bool* clean_eof) {
|
||||||
|
unsigned char* p = (unsigned char*)data;
|
||||||
|
size_t done = 0;
|
||||||
|
while (done < size) {
|
||||||
|
ssize_t n = read(fd, p + done, size - done);
|
||||||
|
if (n < 0 && errno == EINTR)
|
||||||
|
continue;
|
||||||
|
if (n == 0) {
|
||||||
|
if (clean_eof)
|
||||||
|
*clean_eof = done == 0;
|
||||||
|
return done == 0;
|
||||||
|
}
|
||||||
|
if (n < 0)
|
||||||
|
return false;
|
||||||
|
done += (size_t)n;
|
||||||
|
}
|
||||||
|
if (clean_eof)
|
||||||
|
*clean_eof = false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
int batch_read_apply(int fd, const Config* config, const char* dest_root) {
|
||||||
|
if (fd < 0 || dest_root == NULL || dest_root[0] == '\0')
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
char magic[BATCH_MAGIC_LEN];
|
||||||
|
bool eof = false;
|
||||||
|
if (!read_exact(fd, magic, BATCH_MAGIC_LEN, &eof) || eof ||
|
||||||
|
memcmp(magic, BATCH_MAGIC, BATCH_MAGIC_LEN) != 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad magic)");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
unsigned char version;
|
||||||
|
if (!read_exact(fd, &version, 1, &eof) || eof || version != BATCH_FORMAT_VERSION) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad or missing format version)");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
unsigned char mode;
|
||||||
|
if (!read_exact(fd, &mode, 1, &eof) || eof || (mode != 0 && mode != 1)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad metadata flag)");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
bool use_metadata = mode == 1;
|
||||||
|
|
||||||
|
while (1) {
|
||||||
|
unsigned long long length;
|
||||||
|
if (!read_exact(fd, &length, sizeof(length), &eof)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "batch: truncated length prefix");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (eof)
|
||||||
|
break; /* clean end of stream */
|
||||||
|
if (length == 0 || length > BATCH_MAX_RECORD) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "batch: rejected record length %llu (valid range 1..%llu)",
|
||||||
|
length, (unsigned long long)BATCH_MAX_RECORD);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
char* record = (char*)malloc((size_t)length);
|
||||||
|
if (record == NULL) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "batch: could not allocate a %llu-byte record", length);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (!read_exact(fd, record, (size_t)length, &eof)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "batch: truncated chunk record");
|
||||||
|
free(record);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
Data* data = data_create(record, (size_t)length);
|
||||||
|
if (data == NULL)
|
||||||
|
return -1; /* data_create frees `record` on failure */
|
||||||
|
Chunk* chunk = chunk_deserialize(data, use_metadata);
|
||||||
|
data_destroy(data);
|
||||||
|
if (chunk == NULL) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "batch: rejected malformed chunk record");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
File* file = chunk->items[i];
|
||||||
|
chunk->items[i] = NULL;
|
||||||
|
if (file == NULL)
|
||||||
|
continue;
|
||||||
|
FileSaveResult result = file_save_to_disk_full(dest_root, file, config);
|
||||||
|
file_destroy(file);
|
||||||
|
if (result == FILE_SAVE_ERROR) {
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
#ifndef BATCH_H
|
||||||
|
#define BATCH_H
|
||||||
|
#include "chunk.h"
|
||||||
|
#include "config.h"
|
||||||
|
|
||||||
|
/* Phase 6 residual-batch codec. A residual batch is a self-contained
|
||||||
|
* single-file record of a whole source tree: a magic+format-version header
|
||||||
|
* followed by length-prefixed chunk blobs (each built with chunk_serialize),
|
||||||
|
* byte-identical by construction. The batch is a client-only driver feature:
|
||||||
|
* it never crosses the wire, so there is no PROTOCOL_VERSION bump and no server
|
||||||
|
* change. */
|
||||||
|
|
||||||
|
#define BATCH_MAGIC "FSTRESBATCH"
|
||||||
|
#define BATCH_MAGIC_LEN 11
|
||||||
|
#define BATCH_FORMAT_VERSION 1
|
||||||
|
/* A single deserialized record is bounded by the chunk codec's 64 MB cap
|
||||||
|
* (the same per-file data cap chunk_deserialize enforces). */
|
||||||
|
#define BATCH_MAX_RECORD (64ULL * 1024 * 1024)
|
||||||
|
|
||||||
|
bool batch_write_header(int fd, const Config* config);
|
||||||
|
bool batch_write_chunk(int fd, Chunk* chunk);
|
||||||
|
int batch_read_apply(int fd, const Config* config, const char* dest_root);
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
"""Residual-batch (client-only) driver tests.
|
||||||
|
|
||||||
|
--write-batch / --only-write-batch emit a self-contained batch file of a whole
|
||||||
|
source tree; --read-batch applies one locally. None of these cross the wire (no
|
||||||
|
PROTOCOL_VERSION bump, no config-frame field, no server flag): only --write-batch
|
||||||
|
also performs a live transfer and so needs a server.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from common import (
|
||||||
|
TEST_DATA_DIR,
|
||||||
|
run_client,
|
||||||
|
generate_test_files,
|
||||||
|
verify_transfer,
|
||||||
|
clean_dir,
|
||||||
|
get_dest_received_dir,
|
||||||
|
CLIENT_CMD,
|
||||||
|
)
|
||||||
|
|
||||||
|
SOURCE_DIR = os.path.join(TEST_DATA_DIR, "batch_source")
|
||||||
|
DEST1 = os.path.join(TEST_DATA_DIR, "batch_dest1")
|
||||||
|
DEST2 = os.path.join(TEST_DATA_DIR, "batch_dest2")
|
||||||
|
BATCH_FILE = os.path.join(TEST_DATA_DIR, "batch.bin")
|
||||||
|
|
||||||
|
BATCH_MAGIC = b"FSTRESBATCH"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module", autouse=True)
|
||||||
|
def setup_test_data():
|
||||||
|
generate_test_files(SOURCE_DIR, full=False)
|
||||||
|
clean_dir(DEST1)
|
||||||
|
clean_dir(DEST2)
|
||||||
|
yield
|
||||||
|
shutil.rmtree(SOURCE_DIR, ignore_errors=True)
|
||||||
|
shutil.rmtree(DEST1, ignore_errors=True)
|
||||||
|
shutil.rmtree(DEST2, ignore_errors=True)
|
||||||
|
for p in (BATCH_FILE,):
|
||||||
|
if os.path.exists(p):
|
||||||
|
os.unlink(p)
|
||||||
|
|
||||||
|
|
||||||
|
def _run(args):
|
||||||
|
return CLIENT_CMD + args
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_batch_no_server():
|
||||||
|
"""--only-write-batch emits a batch from the source with no destination and
|
||||||
|
no server connection."""
|
||||||
|
if os.path.exists(BATCH_FILE):
|
||||||
|
os.unlink(BATCH_FILE)
|
||||||
|
cmd = _run(["--only-write-batch", BATCH_FILE, SOURCE_DIR])
|
||||||
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
||||||
|
assert result.returncode == 0, (result.stdout, result.stderr)
|
||||||
|
with open(BATCH_FILE, "rb") as f:
|
||||||
|
assert f.read(len(BATCH_MAGIC)) == BATCH_MAGIC
|
||||||
|
# No destination was touched (nothing was created next to the batch).
|
||||||
|
assert not os.path.exists(os.path.join(DEST1, "small.txt"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_batch_roundtrip_no_source():
|
||||||
|
"""--read-batch applies an emitted batch to a fresh destination with no
|
||||||
|
source and no server; the tree is byte-identical to the source."""
|
||||||
|
received = get_dest_received_dir(DEST2, SOURCE_DIR)
|
||||||
|
clean_dir(DEST2)
|
||||||
|
cmd = _run(["--read-batch", BATCH_FILE, DEST2])
|
||||||
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
||||||
|
assert result.returncode == 0, (result.stdout, result.stderr)
|
||||||
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
||||||
|
assert not missing, f"Missing: {missing[:5]}"
|
||||||
|
assert not mismatches, f"Mismatch: {mismatches[:5]}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_write_batch_with_transfer(shared_server):
|
||||||
|
"""--write-batch runs a live transfer to a server AND emits the batch file."""
|
||||||
|
if os.path.exists(BATCH_FILE):
|
||||||
|
os.unlink(BATCH_FILE)
|
||||||
|
clean_dir(DEST1)
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST1,
|
||||||
|
flags=["--write-batch", BATCH_FILE], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stdout, result.stderr)
|
||||||
|
with open(BATCH_FILE, "rb") as f:
|
||||||
|
assert f.read(len(BATCH_MAGIC)) == BATCH_MAGIC
|
||||||
|
received = get_dest_received_dir(DEST1, SOURCE_DIR)
|
||||||
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
||||||
|
assert not missing, f"Missing: {missing[:5]}"
|
||||||
|
assert not mismatches, f"Mismatch: {mismatches[:5]}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_read_batch_requires_destination():
|
||||||
|
"""--read-batch with no positional destination fails cleanly."""
|
||||||
|
cmd = _run(["--read-batch", BATCH_FILE])
|
||||||
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
||||||
|
assert result.returncode != 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_only_write_batch_requires_source():
|
||||||
|
"""--only-write-batch with no source fails cleanly."""
|
||||||
|
cmd = _run(["--only-write-batch", BATCH_FILE])
|
||||||
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
||||||
|
assert result.returncode != 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_batch_modes_conflict():
|
||||||
|
"""The three batch flags are mutually exclusive."""
|
||||||
|
combos = [
|
||||||
|
["--write-batch", BATCH_FILE, "--only-write-batch", BATCH_FILE],
|
||||||
|
["--write-batch", BATCH_FILE, "--read-batch", BATCH_FILE],
|
||||||
|
["--only-write-batch", BATCH_FILE, "--read-batch", BATCH_FILE],
|
||||||
|
]
|
||||||
|
for flags in combos:
|
||||||
|
cmd = _run(["--source-dir", SOURCE_DIR, "--dest-dir", DEST1] + flags)
|
||||||
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
||||||
|
assert result.returncode != 0, \
|
||||||
|
f"expected conflict failure for {flags}: {result.stderr}"
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
#include "test_array_list.h"
|
#include "test_array_list.h"
|
||||||
|
#include "test_batch.h"
|
||||||
#include "test_chunk.h"
|
#include "test_chunk.h"
|
||||||
#include "test_change_list.h"
|
#include "test_change_list.h"
|
||||||
#include "test_checksum.h"
|
#include "test_checksum.h"
|
||||||
@@ -50,6 +51,7 @@ int main() {
|
|||||||
RUN_TEST(test_array_list);
|
RUN_TEST(test_array_list);
|
||||||
RUN_TEST(test_shared_utils);
|
RUN_TEST(test_shared_utils);
|
||||||
RUN_TEST(test_chunk);
|
RUN_TEST(test_chunk);
|
||||||
|
RUN_TEST(test_batch);
|
||||||
RUN_TEST(test_change_list);
|
RUN_TEST(test_change_list);
|
||||||
RUN_TEST(test_config);
|
RUN_TEST(test_config);
|
||||||
RUN_TEST(test_credentials);
|
RUN_TEST(test_credentials);
|
||||||
|
|||||||
@@ -0,0 +1,188 @@
|
|||||||
|
#include "batch.h"
|
||||||
|
#include "chunk.h"
|
||||||
|
#include "config.h"
|
||||||
|
#include "file.h"
|
||||||
|
#include "metadata.h"
|
||||||
|
#include "test_utils.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
|
static void batch_test_cleanup(void) {
|
||||||
|
unlink("batch_dest/batch_src.txt");
|
||||||
|
rmdir("batch_dest");
|
||||||
|
unlink("batch_src.txt");
|
||||||
|
unlink("batch.bin");
|
||||||
|
unlink("batch_bad.bin");
|
||||||
|
unlink("batch_trunc.bin");
|
||||||
|
unlink("batch_big.bin");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A batch round-trips a full file image byte-identically: write header+chunks,
|
||||||
|
* then apply the file to a fresh destination root and verify the content landed
|
||||||
|
* unchanged. */
|
||||||
|
static void test_batch_roundtrip() {
|
||||||
|
batch_test_cleanup();
|
||||||
|
EXPECT_EQ_INT(mkdir("batch_dest", 0755), 0);
|
||||||
|
|
||||||
|
const char* content = "residual batch full image payload\nwith \x01\x02\x03 bytes\n";
|
||||||
|
size_t content_len = strlen(content);
|
||||||
|
file_write_to_disk("batch_src.txt", content, content_len, false, false);
|
||||||
|
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat("batch_src.txt", &st), 0);
|
||||||
|
File* f = file_create("batch_src.txt");
|
||||||
|
EXPECT_NOT_NULL(f);
|
||||||
|
f->data->size = (unsigned long long)st.st_size;
|
||||||
|
EXPECT_TRUE(file_load_data(f));
|
||||||
|
File* files[1] = {f};
|
||||||
|
Chunk* chunk = chunk_create(files, 1);
|
||||||
|
EXPECT_NOT_NULL(chunk);
|
||||||
|
|
||||||
|
Config* config = config_create();
|
||||||
|
EXPECT_NOT_NULL(config);
|
||||||
|
|
||||||
|
int wfd = open("batch.bin", O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
||||||
|
EXPECT_TRUE(wfd >= 0);
|
||||||
|
EXPECT_TRUE(batch_write_header(wfd, config));
|
||||||
|
EXPECT_TRUE(batch_write_chunk(wfd, chunk));
|
||||||
|
EXPECT_EQ_INT(close(wfd), 0);
|
||||||
|
chunk_destroy(chunk); /* frees f */
|
||||||
|
|
||||||
|
int rfd = open("batch.bin", O_RDONLY);
|
||||||
|
EXPECT_TRUE(rfd >= 0);
|
||||||
|
EXPECT_EQ_INT(batch_read_apply(rfd, config, "batch_dest"), 0);
|
||||||
|
EXPECT_EQ_INT(close(rfd), 0);
|
||||||
|
|
||||||
|
char* dest_path = path_cat("batch_dest", "batch_src.txt");
|
||||||
|
EXPECT_NOT_NULL(dest_path);
|
||||||
|
FILE* df = fopen(dest_path, "rb");
|
||||||
|
EXPECT_NOT_NULL(df);
|
||||||
|
char buf[512];
|
||||||
|
size_t n = fread(buf, 1, sizeof(buf), df);
|
||||||
|
EXPECT_EQ_INT(fclose(df), 0);
|
||||||
|
EXPECT_EQ_INT((int)n, (int)content_len);
|
||||||
|
EXPECT_EQ_INT(n == content_len && memcmp(buf, content, content_len) == 0, 1);
|
||||||
|
free(dest_path);
|
||||||
|
|
||||||
|
config_delete(config);
|
||||||
|
batch_test_cleanup();
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_batch_roundtrip_metadata() {
|
||||||
|
batch_test_cleanup();
|
||||||
|
EXPECT_EQ_INT(mkdir("batch_dest", 0755), 0);
|
||||||
|
|
||||||
|
const char* content = "metadata-carrying batch image\n";
|
||||||
|
size_t content_len = strlen(content);
|
||||||
|
file_write_to_disk("batch_src.txt", content, content_len, false, false);
|
||||||
|
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat("batch_src.txt", &st), 0);
|
||||||
|
File* f = file_create("batch_src.txt");
|
||||||
|
EXPECT_NOT_NULL(f);
|
||||||
|
f->data->size = (unsigned long long)st.st_size;
|
||||||
|
EXPECT_TRUE(file_load_data(f));
|
||||||
|
f->metadata = file_metadata_create("batch_src.txt", &st, false, false);
|
||||||
|
EXPECT_NOT_NULL(f->metadata);
|
||||||
|
File* files[1] = {f};
|
||||||
|
Chunk* chunk = chunk_create(files, 1);
|
||||||
|
EXPECT_NOT_NULL(chunk);
|
||||||
|
|
||||||
|
Config* config = config_create();
|
||||||
|
EXPECT_NOT_NULL(config);
|
||||||
|
config->use_metadata = true;
|
||||||
|
|
||||||
|
int wfd = open("batch.bin", O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
||||||
|
EXPECT_TRUE(wfd >= 0);
|
||||||
|
EXPECT_TRUE(batch_write_header(wfd, config));
|
||||||
|
EXPECT_TRUE(batch_write_chunk(wfd, chunk));
|
||||||
|
EXPECT_EQ_INT(close(wfd), 0);
|
||||||
|
chunk_destroy(chunk); /* frees f */
|
||||||
|
|
||||||
|
int rfd = open("batch.bin", O_RDONLY);
|
||||||
|
EXPECT_TRUE(rfd >= 0);
|
||||||
|
EXPECT_EQ_INT(batch_read_apply(rfd, config, "batch_dest"), 0);
|
||||||
|
EXPECT_EQ_INT(close(rfd), 0);
|
||||||
|
|
||||||
|
char* dest_path = path_cat("batch_dest", "batch_src.txt");
|
||||||
|
EXPECT_NOT_NULL(dest_path);
|
||||||
|
FILE* df = fopen(dest_path, "rb");
|
||||||
|
EXPECT_NOT_NULL(df);
|
||||||
|
char buf[512];
|
||||||
|
size_t n = fread(buf, 1, sizeof(buf), df);
|
||||||
|
EXPECT_EQ_INT(fclose(df), 0);
|
||||||
|
EXPECT_EQ_INT((int)n, (int)content_len);
|
||||||
|
EXPECT_EQ_INT(memcmp(buf, content, content_len) == 0, 1);
|
||||||
|
free(dest_path);
|
||||||
|
|
||||||
|
config_delete(config);
|
||||||
|
batch_test_cleanup();
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A corrupt magic (and only 11 bytes of junk) is rejected, never applied. */
|
||||||
|
static void test_batch_reject_bad_magic() {
|
||||||
|
Config* config = config_create();
|
||||||
|
EXPECT_NOT_NULL(config);
|
||||||
|
int fd = open("batch_bad.bin", O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
||||||
|
EXPECT_TRUE(fd >= 0);
|
||||||
|
const char* garbage = "NOTABATCHFXV";
|
||||||
|
EXPECT_EQ_INT(write(fd, garbage, strlen(garbage)), (ssize_t)strlen(garbage));
|
||||||
|
EXPECT_EQ_INT(close(fd), 0);
|
||||||
|
fd = open("batch_bad.bin", O_RDONLY);
|
||||||
|
EXPECT_TRUE(fd >= 0);
|
||||||
|
EXPECT_EQ_INT(batch_read_apply(fd, config, "batch_dest"), -1);
|
||||||
|
EXPECT_EQ_INT(close(fd), 0);
|
||||||
|
config_delete(config);
|
||||||
|
unlink("batch_bad.bin");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A clean header with a length prefix promising 100 bytes but only 12 present
|
||||||
|
* is a truncated record and is rejected (never crashes, never applies). */
|
||||||
|
static void test_batch_reject_truncated() {
|
||||||
|
Config* config = config_create();
|
||||||
|
EXPECT_NOT_NULL(config);
|
||||||
|
int fd = open("batch_trunc.bin", O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
||||||
|
EXPECT_TRUE(fd >= 0);
|
||||||
|
EXPECT_TRUE(batch_write_header(fd, config));
|
||||||
|
unsigned long long length = 100;
|
||||||
|
EXPECT_EQ_INT(write(fd, &length, sizeof(length)), (ssize_t)sizeof(length));
|
||||||
|
const char* partial = "onlytwelvebytes";
|
||||||
|
EXPECT_EQ_INT(write(fd, partial, 15), (ssize_t)15);
|
||||||
|
EXPECT_EQ_INT(close(fd), 0);
|
||||||
|
fd = open("batch_trunc.bin", O_RDONLY);
|
||||||
|
EXPECT_TRUE(fd >= 0);
|
||||||
|
EXPECT_EQ_INT(batch_read_apply(fd, config, "batch_dest"), -1);
|
||||||
|
EXPECT_EQ_INT(close(fd), 0);
|
||||||
|
config_delete(config);
|
||||||
|
unlink("batch_trunc.bin");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A length prefix above the 64 MB cap is refused before any allocation. */
|
||||||
|
static void test_batch_reject_oversized() {
|
||||||
|
Config* config = config_create();
|
||||||
|
EXPECT_NOT_NULL(config);
|
||||||
|
int fd = open("batch_big.bin", O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
||||||
|
EXPECT_TRUE(fd >= 0);
|
||||||
|
EXPECT_TRUE(batch_write_header(fd, config));
|
||||||
|
unsigned long long length = BATCH_MAX_RECORD + 16U;
|
||||||
|
EXPECT_EQ_INT(write(fd, &length, sizeof(length)), (ssize_t)sizeof(length));
|
||||||
|
EXPECT_EQ_INT(close(fd), 0);
|
||||||
|
fd = open("batch_big.bin", O_RDONLY);
|
||||||
|
EXPECT_TRUE(fd >= 0);
|
||||||
|
EXPECT_EQ_INT(batch_read_apply(fd, config, "batch_dest"), -1);
|
||||||
|
EXPECT_EQ_INT(close(fd), 0);
|
||||||
|
config_delete(config);
|
||||||
|
unlink("batch_big.bin");
|
||||||
|
}
|
||||||
|
|
||||||
|
void test_batch() {
|
||||||
|
test_batch_roundtrip();
|
||||||
|
test_batch_roundtrip_metadata();
|
||||||
|
test_batch_reject_bad_magic();
|
||||||
|
test_batch_reject_truncated();
|
||||||
|
test_batch_reject_oversized();
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#ifndef TEST_BATCH_H
|
||||||
|
#define TEST_BATCH_H
|
||||||
|
|
||||||
|
void test_batch();
|
||||||
|
|
||||||
|
#endif
|
||||||
Reference in New Issue
Block a user