TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity
Files
bb6c788cf9e97a961a0de81537ed83eb76b5fe97
FastSync/src/shared/checksum.c
T
TapTap 67076bf218 feat(basis): rsync quick-check default + FastSync-only --verify-basis 
- default basis match is rsync's metadata quick-check (size + mtime; size-only
  drops mtime; -I disables), no mandatory content digest
- new long-only --verify-basis (wire bool, protocol stays 2.28.0) restores the
  strict whole-file content equality
- --copy-dest re-applies source attributes; basis-hit 256 MiB cap removed by
  streaming the copy/hash; basis miss keeps the normal payload bound
- compare/copy/link-dest rows -> caveat; tally 116/13/28
2026-09-19 15:55:51 +02:00

428 lines
12 KiB
C
Raw Blame History

#include "checksum.h"
#include "utils.h"
#include <fcntl.h>
#include <openssl/evp.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
/* delta.c owns the single XXH_IMPLEMENTATION that provides the xxHash symbols
* for the whole binary; this TU only needs the declarations. The streaming
* state structs and XXH3_update are exposed only with XXH_STATIC_LINKING_ONLY. */
#define XXH_STATIC_LINKING_ONLY
#include <xxhash.h>
/* ---------------------------------------------------------------------------
* Self-contained MD4 (RFC 1320). OpenSSL's MD4 lives in the legacy provider
* and is not guaranteed present, so FastSync carries its own implementation to
* keep --checksum-choice=md4 working on every build.
* ------------------------------------------------------------------------- */
typedef struct {
uint32_t state[4];
uint64_t bit_count;
uint8_t buffer[64];
size_t buffer_len;
} Md4Ctx;
static uint32_t md4_rotl(uint32_t x, int n) {
return (x << n) | (x >> (32 - n));
}
static void md4_transform(uint32_t state[4], const uint8_t block[64]) {
uint32_t x[16];
for (int i = 0; i < 16; i++)
x[i] = (uint32_t)block[i * 4] | ((uint32_t)block[i * 4 + 1] << 8) |
((uint32_t)block[i * 4 + 2] << 16) | ((uint32_t)block[i * 4 + 3] << 24);
uint32_t a = state[0], b = state[1], c = state[2], d = state[3];
#define F(x, y, z) (((x) & (y)) | (~(x) & (z)))
#define G(x, y, z) (((x) & (y)) | ((x) & (z)) | ((y) & (z)))
#define H(x, y, z) ((x) ^ (y) ^ (z))
#define ROUND1(a, b, c, d, k, s) a = md4_rotl(a + F(b, c, d) + x[k], s)
#define ROUND2(a, b, c, d, k, s) a = md4_rotl(a + G(b, c, d) + x[k] + 0x5a827999u, s)
#define ROUND3(a, b, c, d, k, s) a = md4_rotl(a + H(b, c, d) + x[k] + 0x6ed9eba1u, s)
ROUND1(a, b, c, d, 0, 3);
ROUND1(d, a, b, c, 1, 7);
ROUND1(c, d, a, b, 2, 11);
ROUND1(b, c, d, a, 3, 19);
ROUND1(a, b, c, d, 4, 3);
ROUND1(d, a, b, c, 5, 7);
ROUND1(c, d, a, b, 6, 11);
ROUND1(b, c, d, a, 7, 19);
ROUND1(a, b, c, d, 8, 3);
ROUND1(d, a, b, c, 9, 7);
ROUND1(c, d, a, b, 10, 11);
ROUND1(b, c, d, a, 11, 19);
ROUND1(a, b, c, d, 12, 3);
ROUND1(d, a, b, c, 13, 7);
ROUND1(c, d, a, b, 14, 11);
ROUND1(b, c, d, a, 15, 19);
ROUND2(a, b, c, d, 0, 3);
ROUND2(d, a, b, c, 4, 5);
ROUND2(c, d, a, b, 8, 9);
ROUND2(b, c, d, a, 12, 13);
ROUND2(a, b, c, d, 1, 3);
ROUND2(d, a, b, c, 5, 5);
ROUND2(c, d, a, b, 9, 9);
ROUND2(b, c, d, a, 13, 13);
ROUND2(a, b, c, d, 2, 3);
ROUND2(d, a, b, c, 6, 5);
ROUND2(c, d, a, b, 10, 9);
ROUND2(b, c, d, a, 14, 13);
ROUND2(a, b, c, d, 3, 3);
ROUND2(d, a, b, c, 7, 5);
ROUND2(c, d, a, b, 11, 9);
ROUND2(b, c, d, a, 15, 13);
ROUND3(a, b, c, d, 0, 3);
ROUND3(d, a, b, c, 8, 9);
ROUND3(c, d, a, b, 4, 11);
ROUND3(b, c, d, a, 12, 15);
ROUND3(a, b, c, d, 2, 3);
ROUND3(d, a, b, c, 10, 9);
ROUND3(c, d, a, b, 6, 11);
ROUND3(b, c, d, a, 14, 15);
ROUND3(a, b, c, d, 1, 3);
ROUND3(d, a, b, c, 9, 9);
ROUND3(c, d, a, b, 5, 11);
ROUND3(b, c, d, a, 13, 15);
ROUND3(a, b, c, d, 3, 3);
ROUND3(d, a, b, c, 11, 9);
ROUND3(c, d, a, b, 7, 11);
ROUND3(b, c, d, a, 15, 15);
#undef F
#undef G
#undef H
#undef ROUND1
#undef ROUND2
#undef ROUND3
state[0] += a;
state[1] += b;
state[2] += c;
state[3] += d;
}
static void md4_init(Md4Ctx* ctx) {
ctx->state[0] = 0x67452301u;
ctx->state[1] = 0xefcdab89u;
ctx->state[2] = 0x98badcfeu;
ctx->state[3] = 0x10325476u;
ctx->bit_count = 0;
ctx->buffer_len = 0;
}
static void md4_update(Md4Ctx* ctx, const uint8_t* data, size_t len) {
ctx->bit_count += (uint64_t)len * 8;
while (len > 0) {
size_t space = sizeof(ctx->buffer) - ctx->buffer_len;
size_t take = len < space ? len : space;
memcpy(ctx->buffer + ctx->buffer_len, data, take);
ctx->buffer_len += take;
data += take;
len -= take;
if (ctx->buffer_len == sizeof(ctx->buffer)) {
md4_transform(ctx->state, ctx->buffer);
ctx->buffer_len = 0;
}
}
}
static void md4_final(Md4Ctx* ctx, uint8_t out[16]) {
uint64_t bit_count = ctx->bit_count;
uint8_t pad = 0x80;
md4_update(ctx, &pad, 1);
uint8_t zero = 0;
while (ctx->buffer_len != 56)
md4_update(ctx, &zero, 1);
uint8_t length_le[8];
for (int i = 0; i < 8; i++)
length_le[i] = (uint8_t)((bit_count >> (8 * i)) & 0xff);
md4_update(ctx, length_le, sizeof(length_le));
for (int i = 0; i < 4; i++) {
out[i * 4] = (uint8_t)(ctx->state[i] & 0xff);
out[i * 4 + 1] = (uint8_t)((ctx->state[i] >> 8) & 0xff);
out[i * 4 + 2] = (uint8_t)((ctx->state[i] >> 16) & 0xff);
out[i * 4 + 3] = (uint8_t)((ctx->state[i] >> 24) & 0xff);
}
}
/* One-shot EVP digest (md5/sha1). Returns false when OpenSSL refuses. */
static bool evp_digest(const EVP_MD* md, const void* data, size_t size, uint8_t* out,
size_t out_capacity, size_t* out_len) {
static const uint8_t empty = 0;
const void* input = data ? data : &empty;
unsigned int digest_len = 0;
if (EVP_Digest(input, size, out, &digest_len, md, NULL) != 1)
return false;
if (digest_len > out_capacity)
return false;
*out_len = digest_len;
return true;
}
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
size_t out_capacity, size_t* out_len) {
if (!out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
return false;
if (data == NULL && size != 0)
return false;
switch (algo) {
case CHECKSUM_ALGO_XXH64: {
uint64_t digest = XXH64(data, size, seed);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
return true;
}
case CHECKSUM_ALGO_XXH3: {
uint64_t digest = XXH3_64bits_withSeed(data, size, seed);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
return true;
}
case CHECKSUM_ALGO_XXH128: {
XXH128_hash_t digest = XXH3_128bits_withSeed(data, size, seed);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
return true;
}
case CHECKSUM_ALGO_MD5:
/* md5 takes no seed; the caller's seed is deliberately ignored (documented
* in RSYNC_COMPAT.md). */
return evp_digest(EVP_md5(), data, size, out, out_capacity, out_len);
case CHECKSUM_ALGO_MD4: {
Md4Ctx ctx;
md4_init(&ctx);
md4_update(&ctx, (const uint8_t*)data, size);
md4_final(&ctx, out);
*out_len = 16;
return true;
}
case CHECKSUM_ALGO_SHA1:
/* sha1 takes no seed; the caller's seed is deliberately ignored. */
return evp_digest(EVP_sha1(), data, size, out, out_capacity, out_len);
case CHECKSUM_ALGO_NONE:
/* No checksum requested: an empty digest is the successful result. */
*out_len = 0;
return true;
}
return false;
}
bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, uint8_t* out,
size_t out_capacity, size_t* out_len) {
if (!path || !out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
return false;
int fd = open(path, O_RDONLY | O_CLOEXEC);
if (fd < 0)
return false;
bool ok = checksum_digest_fd(algo, seed, fd, out, out_capacity, out_len);
close(fd);
return ok;
}
bool checksum_digest_fd(ChecksumAlgo algo, uint64_t seed, int fd, uint8_t* out, size_t out_capacity,
size_t* out_len) {
if (fd < 0 || !out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
return false;
if (algo == CHECKSUM_ALGO_NONE) {
/* No checksum requested: nothing to read; an empty digest succeeds. */
*out_len = 0;
return true;
}
uint8_t buffer[64 * 1024];
bool ok = false;
lseek(fd, 0, SEEK_SET);
if (algo == CHECKSUM_ALGO_MD5 || algo == CHECKSUM_ALGO_SHA1) {
const EVP_MD* md = algo == CHECKSUM_ALGO_MD5 ? EVP_md5() : EVP_sha1();
EVP_MD_CTX* ctx = EVP_MD_CTX_new();
if (!ctx)
return false;
unsigned int digest_len = 0;
if (EVP_DigestInit_ex(ctx, md, NULL) == 1) {
ok = true;
ssize_t got;
while ((got = read(fd, buffer, sizeof(buffer))) > 0) {
if (EVP_DigestUpdate(ctx, buffer, (size_t)got) != 1) {
ok = false;
break;
}
}
if (got < 0)
ok = false;
if (ok && EVP_DigestFinal_ex(ctx, out, &digest_len) == 1 && digest_len <= out_capacity)
*out_len = digest_len;
else
ok = false;
}
EVP_MD_CTX_free(ctx);
return ok;
}
if (algo == CHECKSUM_ALGO_MD4) {
Md4Ctx ctx;
md4_init(&ctx);
ok = true;
ssize_t got;
while ((got = read(fd, buffer, sizeof(buffer))) > 0)
md4_update(&ctx, buffer, (size_t)got);
if (got < 0)
ok = false;
if (ok) {
md4_final(&ctx, out);
*out_len = 16;
}
return ok;
}
XXH64_state_t xxh64;
XXH3_state_t* xxh3 = NULL;
if (algo == CHECKSUM_ALGO_XXH64) {
XXH64_reset(&xxh64, seed);
} else if (algo == CHECKSUM_ALGO_XXH3 || algo == CHECKSUM_ALGO_XXH128) {
xxh3 = XXH3_createState();
if (!xxh3)
return false;
if (algo == CHECKSUM_ALGO_XXH3)
XXH3_64bits_reset_withSeed(xxh3, seed);
else
XXH3_128bits_reset_withSeed(xxh3, seed);
} else {
return false;
}
ok = true;
ssize_t got;
while ((got = read(fd, buffer, sizeof(buffer))) > 0) {
if (algo == CHECKSUM_ALGO_XXH64)
XXH64_update(&xxh64, buffer, (size_t)got);
else if (XXH3_64bits_update(xxh3, buffer, (size_t)got) == XXH_ERROR) {
ok = false;
break;
}
}
if (got < 0)
ok = false;
if (ok) {
if (algo == CHECKSUM_ALGO_XXH64) {
uint64_t digest = XXH64_digest(&xxh64);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
} else if (algo == CHECKSUM_ALGO_XXH3) {
uint64_t digest = XXH3_64bits_digest(xxh3);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
} else {
XXH128_hash_t digest = XXH3_128bits_digest(xxh3);
memcpy(out, &digest, sizeof(digest));
*out_len = sizeof(digest);
}
}
if (xxh3)
XXH3_freeState(xxh3);
return ok;
}
int checksum_algo_from_name(const char* name) {
if (!name)
return -1;
if (strcasecmp(name, "xxh64") == 0 || strcasecmp(name, "xxhash") == 0)
return (int)CHECKSUM_ALGO_XXH64;
if (strcasecmp(name, "xxh3") == 0)
return (int)CHECKSUM_ALGO_XXH3;
if (strcasecmp(name, "xxh128") == 0)
return (int)CHECKSUM_ALGO_XXH128;
if (strcasecmp(name, "md5") == 0)
return (int)CHECKSUM_ALGO_MD5;
if (strcasecmp(name, "md4") == 0)
return (int)CHECKSUM_ALGO_MD4;
if (strcasecmp(name, "sha1") == 0)
return (int)CHECKSUM_ALGO_SHA1;
if (strcasecmp(name, "none") == 0)
return (int)CHECKSUM_ALGO_NONE;
return -1;
}
const char* checksum_algo_name(ChecksumAlgo algo) {
switch (algo) {
case CHECKSUM_ALGO_XXH64:
return "xxh64";
case CHECKSUM_ALGO_XXH3:
return "xxh3";
case CHECKSUM_ALGO_XXH128:
return "xxh128";
case CHECKSUM_ALGO_MD5:
return "md5";
case CHECKSUM_ALGO_MD4:
return "md4";
case CHECKSUM_ALGO_SHA1:
return "sha1";
case CHECKSUM_ALGO_NONE:
return "none";
}
return "<unknown>";
}
bool checksum_algo_valid(int algo) {
return algo == (int)CHECKSUM_ALGO_XXH64 || algo == (int)CHECKSUM_ALGO_MD5 ||
algo == (int)CHECKSUM_ALGO_XXH3 || algo == (int)CHECKSUM_ALGO_XXH128 ||
algo == (int)CHECKSUM_ALGO_MD4 || algo == (int)CHECKSUM_ALGO_SHA1 ||
algo == (int)CHECKSUM_ALGO_NONE;
}
uint8_t checksum_digest_len(ChecksumAlgo algo) {
switch (algo) {
case CHECKSUM_ALGO_XXH64:
case CHECKSUM_ALGO_XXH3:
return 8;
case CHECKSUM_ALGO_XXH128:
case CHECKSUM_ALGO_MD5:
case CHECKSUM_ALGO_MD4:
return 16;
case CHECKSUM_ALGO_SHA1:
return 20;
case CHECKSUM_ALGO_NONE:
return 0;
}
return 0;
}
static ChecksumAlgo compiled_checksum_preference_first(void) {
/* rsync 3.4.1 default preference order; every entry is compiled in, so this
* resolves to xxh128. */
static const ChecksumAlgo preference[] = {
CHECKSUM_ALGO_XXH128, CHECKSUM_ALGO_XXH3, CHECKSUM_ALGO_XXH64, CHECKSUM_ALGO_MD5,
CHECKSUM_ALGO_MD4, CHECKSUM_ALGO_SHA1, CHECKSUM_ALGO_NONE,
};
for (size_t i = 0; i < sizeof(preference) / sizeof(preference[0]); i++) {
if (checksum_algo_valid((int)preference[i]))
return preference[i];
}
return CHECKSUM_ALGO_XXH64;
}
int checksum_choice_resolve(void) {
bool specified = false;
int env = env_choice_first("RSYNC_CHECKSUM_LIST", checksum_algo_from_name, &specified);
if (specified)
return env; /* -1 = the list named no supported checksum */
return (int)compiled_checksum_preference_first();
}
ChecksumAlgo checksum_negotiate_default(void) {
int resolved = checksum_choice_resolve();
return resolved >= 0 ? (ChecksumAlgo)resolved : compiled_checksum_preference_first();
}
Reference in New Issue View Git Blame Copy Permalink