fix: initialize ssl/ssl_ctx in client_connect_ssh to prevent use-after-free

- New transport_tls.h/c: TLS server (server_create_tls, server_listen_tls)
  and client (client_connect_tls) using OpenSSL
- protocol.c: io_set_ssl() + SSL_read/SSL_write in send_n_data/receive_n_data
- transport_tcp.h: ssl/ssl_ctx fields added to Server/Client structs
- config.h/c: use_tls, tls_cert, tls_key fields
- client_cli.c: --tls, --cert, --key flags
- server.c: --tls, --cert, --key, -p flags with TLS support
- CMakeLists.txt: OpenSSL::SSL + OpenSSL::Crypto linkage
- shell.nix: openssl added to buildInputs

.gitea/workflows/ci.yaml

@@ -0,0 +1,23 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    container: gitea.tap-tap.win/taptap/fastsync-ci:v4
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Configure
+        run: cmake -B build -S .
+
+      - name: Build
+        run: cmake --build build -j$(nproc)
+
+      - name: Unit Tests
+        run: ./build/tests
+
+      - name: Integration Tests
+        run: python3 test.py

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 4.1)
+cmake_minimum_required(VERSION 3.22)
 
 project(FastFileTransfer)
 

Dockerfile

@@ -0,0 +1,6 @@
+FROM ubuntu:24.04
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc g++ make libc6-dev cmake libzstd-dev git ca-certificates curl && \
+    curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
+    apt-get install -y --no-install-recommends nodejs && \
+    rm -rf /var/lib/apt/lists/*

shell.nix

@@ -8,6 +8,7 @@ pkgs.mkShell {
     cmake
     gnumake
     pkg-config
+    docker
     tea
   ];
 

src/client/client_cli.c

@@ -35,6 +35,7 @@ static void print_usage(void) {
   printf("  --include <pattern> Only include files matching pattern\n");
   printf("  --max-size <n>      Skip files larger than n bytes\n");
   printf("  --min-size <n>      Skip files smaller than n bytes\n");
+  printf("  --incremental       Skip files unchanged since last transfer\n");
   printf("  -m                  Enable multithreading\n");
   printf("  -s                  Enable chunk serialization\n");
   printf("  -f                  Enable sendfile (TCP only, not with -c or -s)\n");
@@ -98,6 +99,8 @@ int main(int argc, char *argv[]) {
       config->max_size = strtoull(argv[++i], NULL, 10);
     } else if (strcmp(argv[i], "--min-size") == 0 && i + 1 < argc) {
       config->min_size = strtoull(argv[++i], NULL, 10);
+    } else if (strcmp(argv[i], "--incremental") == 0) {
+      config->use_incremental = true;
     } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) {
       config->use_compression = true;
       log_message(LOG_LEVEL_INFO, "Enabled Compression");
@@ -218,6 +221,16 @@ int main(int argc, char *argv[]) {
     return 1;
   }
 
+  if (config->use_incremental && config->use_chunk_serialization) {
+    fprintf(stderr, "Error: --incremental is not supported with -s (chunk serialization)\n");
+    return 1;
+  }
+
+  if (config->use_incremental && !config->use_metadata) {
+    log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --incremental");
+    config->use_metadata = true;
+  }
+
   if (config->use_tls) {
     if (!config->tls_cert || !config->tls_key) {
       fprintf(stderr, "Error: --tls requires --cert and --key\n");

src/client/client_send.c

@@ -19,6 +19,27 @@
 #include <threads.h>
 #include <time.h>
 
+static int incremental_check(Client *client, File *file) {
+  if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1;
+  if (!send_str(client->file_descriptor, file->path)) return -1;
+  unsigned long long fsize = file->data->size;
+  long long mtime = file->metadata ? file->metadata->mtime_sec : 0;
+  if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1;
+  if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1;
+  Status s;
+  if (!receive_status(client->file_descriptor, &s)) return -1;
+  if (s == STATUS_ERROR) {
+    log_message(LOG_LEVEL_ERROR, "Server reported error for file");
+    return -1;
+  }
+  if (s == STATUS_OK) return 1;
+  if (s != STATUS_NEXT) {
+    log_message(LOG_LEVEL_ERROR, "Unexpected server status");
+    return -1;
+  }
+  return 0;
+}
+
 int send_chunk(Client *client, Chunk *chunk, Config *config) {
   if (config->use_chunk_serialization) {
     if (!send_status(client->file_descriptor, STATUS_CHUNK)) return -1;
@@ -33,19 +54,39 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) {
     data_destroy(data);
   } else if (config->use_sendfile && !config->use_compression) {
     for (int i = 0; i < chunk->element_count; i++) {
-      if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
-      if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata))
+      if (config->use_incremental) {
+        int rc = incremental_check(client, chunk->items[i]);
+        if (rc < 0) return -1;
+        if (rc > 0) continue;
+        if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata, false))
           return -1;
+      } else {
+        if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
+        if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata, true))
+          return -1;
+      }
     }
   } else {
     for (int i = 0; i < chunk->element_count; i++) {
+      if (config->use_incremental) {
+        int rc = incremental_check(client, chunk->items[i]);
+        if (rc < 0) return -1;
+        if (rc > 0) continue;
+        if (!file_send_single_calls(chunk->items[i], client->file_descriptor,
+                                   config->use_metadata,
+                                   config->use_compression ? config->compression_level : 0,
+                                   false))
+          return -1;
+      } else {
         if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
         if (!file_send_single_calls(chunk->items[i], client->file_descriptor,
                                config->use_metadata,
-                            config->use_compression ? config->compression_level : 0))
+                               config->use_compression ? config->compression_level : 0,
+                               true))
           return -1;
       }
     }
+  }
   return 0;
 }
 

src/server/server.c

@@ -1,11 +1,9 @@
 #include "array_list.h"
 #include "chunk.h"
-#include "compression.h"
 #include "config.h"
 #include "data.h"
 #include "file.h"
 #include "log.h"
-#include "metadata.h"
 #include "multiprocessing.h"
 #include "protocol.h"
 #include "queue.h"
@@ -18,89 +16,57 @@
 #include <stdlib.h>
 #include <string.h>
 
-int receive_files(Config *config, int file_descriptor) {
+int receive_files(Config *config, int fd) {
   Status status;
-  if (!receive_status(file_descriptor, &status)) return -1;
-  while (status == STATUS_NEXT || status == STATUS_CHUNK) {
-    if (status == STATUS_CHUNK) {
-      Data *chunk_data = receive_data(file_descriptor);
-      if (chunk_data == NULL) {
-        log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data");
-        send_status(file_descriptor, STATUS_ERROR);
-        return -1;
-      }
-      Data *data_to_process = chunk_data;
-      if (config->use_compression) {
-        data_to_process = data_decompress(chunk_data);
-        data_destroy(chunk_data);
-        if (data_to_process == NULL) {
-          log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk");
-          send_status(file_descriptor, STATUS_ERROR);
-          return -1;
-        }
-      }
-      Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata);
-      data_destroy(data_to_process);
-      if (chunk == NULL) {
-        log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping");
-        send_status(file_descriptor, STATUS_ERROR);
-        return -1;
-      }
+  if (!receive_status(fd, &status)) return -1;
 
+  while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) {
+    if (status == STATUS_CHECK) {
+      bool skipped;
+      File *file = receive_incremental_check(fd, config, &skipped);
+      if (skipped) goto next;
+      if (file == NULL && !skipped) return -1;
+      if (config->save_to_disk)
+        file_save_to_disk(config->receive_root_directory, file);
+      file_destroy(file);
+    } else if (status == STATUS_CHUNK) {
+      Chunk *chunk = receive_chunk_data(fd, config);
+      if (chunk == NULL) {
+        send_status(fd, STATUS_ERROR);
+        return -1;
+      }
       for (int i = 0; i < chunk->element_count; i++) {
-        if (config->save_to_disk) {
-          char *disk_path = path_cat(config->receive_root_directory, chunk->items[i]->path);
-          if (disk_path) {
-            to_disk(disk_path, chunk->items[i]->data->data, chunk->items[i]->data->size);
-            file_restore_metadata(disk_path, chunk->items[i]->metadata);
-            free(disk_path);
-          }
-        }
+        if (config->save_to_disk)
+          file_save_to_disk(config->receive_root_directory, chunk->items[i]);
       }
       chunk_destroy(chunk);
     } else {
-      File *file = file_receive(config, file_descriptor);
+      File *file = file_receive(config, fd);
       if (file == NULL) {
         log_message(LOG_LEVEL_ERROR, "Failed to receive file");
-        send_status(file_descriptor, STATUS_ERROR);
+        send_status(fd, STATUS_ERROR);
         return -1;
       }
-      if (config->save_to_disk) {
-        char *disk_path = path_cat(config->receive_root_directory, file->path);
-        if (disk_path) {
-          to_disk(disk_path, file->data->data, file->data->size);
-          file_restore_metadata(disk_path, file->metadata);
-          free(disk_path);
-        }
-      }
+      if (config->save_to_disk)
+        file_save_to_disk(config->receive_root_directory, file);
       file_destroy(file);
     }
-    if (!receive_status(file_descriptor, &status)) {
-      send_status(file_descriptor, STATUS_ERROR);
+  next:
+    if (!receive_status(fd, &status)) {
+      send_status(fd, STATUS_ERROR);
       return -1;
     }
   }
+
   if (status == STATUS_MANIFEST) {
-    int count;
-    if (!receive_int(file_descriptor, &count)) return -1;
-    ArrayList *manifest = array_list_create(free);
-    if (manifest) {
-      for (int i = 0; i < count; i++) {
-        char *s = receive_str(file_descriptor);
-        if (s) array_list_add(manifest, s);
-      }
-      fprintf(stderr, "Deleting files not in manifest...\n");
-      delete_extras(config->receive_root_directory, manifest);
-      array_list_delete(manifest);
-    }
-    if (!receive_status(file_descriptor, &status)) return -1;
+    if (receive_manifest(fd, config, &status) != 0) return -1;
   }
   if (status != STATUS_FINISHED) {
     log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status");
-    send_status(file_descriptor, STATUS_ERROR);
+    send_status(fd, STATUS_ERROR);
     return -1;
   }
-  send_status(file_descriptor, STATUS_OK);
+  send_status(fd, STATUS_OK);
   return 0;
 }
 

src/shared/chunk.c

@@ -10,6 +10,7 @@
 #include "file.h"
 #include "log.h"
 #include "metadata.h"
+#include "protocol.h"
 
 Chunk *chunk_create(File **items, int element_count) {
   Chunk *chunk = (Chunk *)malloc(sizeof(Chunk));
@@ -178,5 +179,27 @@ Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata) {
   return compressed;
 }
 
+Chunk *receive_chunk_data(int fd, Config *config) {
+  Data *chunk_data = receive_data(fd);
+  if (chunk_data == NULL) {
+    log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data");
+    return NULL;
+  }
+  Data *data_to_process = chunk_data;
+  if (config->use_compression) {
+    data_to_process = data_decompress(chunk_data);
+    data_destroy(chunk_data);
+    if (data_to_process == NULL) {
+      log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk");
+      return NULL;
+    }
+  }
+  Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata);
+  data_destroy(data_to_process);
+  if (chunk == NULL)
+    log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping");
+  return chunk;
+}
+
 
 

src/shared/chunk.h

@@ -1,6 +1,7 @@
 #ifndef CHUNK_H
 #define CHUNK_H
 
+#include "config.h"
 #include "data.h"
 #include "file.h"
 #include <stdbool.h>
@@ -18,5 +19,6 @@ void chunk_destroy(void *chunk);
 Data *chunk_serialize(Chunk *chunk, bool use_metadata);
 Chunk *chunk_deserialize(Data *data, bool use_metadata);
 Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata);
+Chunk *receive_chunk_data(int fd, Config *config);
 
 #endif

src/shared/config.c

@@ -38,6 +38,7 @@ Config *config_create(char *version, char *send_directory,
   config->include_count = 0;
   config->max_size = 0;
   config->min_size = 0;
+  config->use_incremental = false;
   config->use_tls = false;
   config->tls_cert = NULL;
   config->tls_key = NULL;
@@ -96,6 +97,7 @@ bool config_send(int file_descriptor, Config *config) {
   if (!send_int(file_descriptor, (int)config->chunk_size)) return false;
   if (!send_int(file_descriptor, config->use_sendfile)) return false;
   if (!send_int(file_descriptor, config->use_delete)) return false;
+  if (!send_int(file_descriptor, config->use_incremental)) return false;
   Status status;
   if (!receive_status(file_descriptor, &status)) return false;
   if (status != STATUS_OK) {
@@ -141,6 +143,8 @@ Config *config_receive(int file_descriptor) {
   config->use_sendfile = tmp;
   if (!receive_int(file_descriptor, &tmp)) goto error;
   config->use_delete = tmp;
+  if (!receive_int(file_descriptor, &tmp)) goto error;
+  config->use_incremental = tmp;
   config->show_progress = false;
   config->dry_run = false;
   config->ssh_port = 22;

src/shared/config.h

@@ -32,13 +32,14 @@ typedef struct Config {
   int include_count;
   unsigned long long max_size;
   unsigned long long min_size;
+  bool use_incremental;
   bool use_tls;
   char *tls_cert;
   char *tls_key;
   char *tls_ca;
 } Config;
 
-#define PROTOCOL_VERSION "1.0.0"
+#define PROTOCOL_VERSION "1.1.0"
 #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
 
 Config *config_create(char *version, char *send_directory,

src/shared/file.c

@@ -96,26 +96,103 @@ bool file_load_data(File *file) {
   return true;
 }
 
-bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) {
+bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level, bool send_path) {
+  Data *data_to_send = file->data;
+  Data *compressed_data = NULL;
   if (compression_level > 0) {
-    Data *compressed_data = data_compress(file->data, compression_level);
+    compressed_data = data_compress(file->data, compression_level);
     if (compressed_data == NULL) {
       log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
       return false;
     }
-    data_destroy(file->data);
-    if (compressed_data == NULL) {
-      log_message(LOG_LEVEL_ERROR, "Compression failed in file_send_single_calls");
-      exit(EXIT_FAILURE);
+    data_to_send = compressed_data;
   }
-    file->data = compressed_data;
+  if (send_path && !send_str(file_descriptor, file->path)) {
+    data_destroy(compressed_data);
+    return false;
   }
-  if (!send_str(file_descriptor, file->path)) return false;
-  if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false;
-  if (!send_data(file_descriptor, file->data)) return false;
+  if (use_metadata && !metadata_send(file_descriptor, file->metadata)) {
+    data_destroy(compressed_data);
+    return false;
+  }
+  if (!send_data(file_descriptor, data_to_send)) {
+    data_destroy(compressed_data);
+    return false;
+  }
+  data_destroy(compressed_data);
   return true;
 }
 
+bool file_save_to_disk(const char *root_directory, File *file) {
+  char *disk_path = path_cat((char *)root_directory, file->path);
+  if (disk_path == NULL) return false;
+  bool ok = to_disk(disk_path, file->data->data, file->data->size);
+  if (ok) file_restore_metadata(disk_path, file->metadata);
+  free(disk_path);
+  return ok;
+}
+
+File *receive_incremental_check(int fd, Config *config, bool *skipped) {  *skipped = false;
+  char *check_path = receive_str(fd);
+  if (check_path == NULL) { send_status(fd, STATUS_ERROR); return NULL; }
+
+  unsigned long long check_size;
+  long long check_mtime;
+  if (!receive_n_data(fd, &check_size, sizeof(check_size)) ||
+      !receive_n_data(fd, &check_mtime, sizeof(check_mtime))) {
+    free(check_path);
+    send_status(fd, STATUS_ERROR);
+    return NULL;
+  }
+
+  char *full_path = path_cat(config->receive_root_directory, check_path);
+  struct stat st;
+  bool match = false;
+  if (full_path && stat(full_path, &st) == 0 &&
+      (unsigned long long)st.st_size == check_size &&
+      (long long)st.st_mtime == check_mtime) {
+    match = true;
+  }
+  free(full_path);
+
+  if (match) {
+    if (!send_status(fd, STATUS_OK)) { free(check_path); return NULL; }
+    free(check_path);
+    *skipped = true;
+    return NULL;
+  }
+
+  if (!send_status(fd, STATUS_NEXT)) { free(check_path); return NULL; }
+
+  File *file = file_create(check_path);
+  free(check_path);
+  if (file == NULL) { send_status(fd, STATUS_ERROR); return NULL; }
+
+  if (config->use_metadata) {
+    int meta_ok = 1;
+    file->metadata = metadata_receive(fd, &meta_ok);
+    if (!meta_ok) { file_destroy(file); send_status(fd, STATUS_ERROR); return NULL; }
+  }
+
+  Data *file_data = receive_data(fd);
+  if (file_data == NULL) {
+    file_destroy(file);
+    send_status(fd, STATUS_ERROR);
+    return NULL;
+  }
+
+  if (config->use_compression) {
+    Data *uncompressed = data_decompress(file_data);
+    data_destroy(file_data);
+    if (uncompressed == NULL) { file_destroy(file); send_status(fd, STATUS_ERROR); return NULL; }
+    file_data = uncompressed;
+  }
+
+  data_destroy(file->data);
+  file->data = file_data;
+  return file;
+}
+
 bool to_disk(const char *path, const void *data, unsigned long long data_size) {
   char *directory = str_dup(path);
   char *dir_to_free = directory;
@@ -141,8 +218,8 @@ bool to_disk(const char *path, const void *data, unsigned long long data_size) {
   return true;
 }
 
-bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata) {
-  if (!send_str(file_descriptor, file->path)) return false;
+bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata, bool send_path) {
+  if (send_path && !send_str(file_descriptor, file->path)) return false;
   if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false;
 
   int fd = open(file->path, O_RDONLY);
@@ -178,7 +255,9 @@ File *file_receive(Config *config, int file_descriptor) {
   free(path);
   if (file == NULL) return NULL;
   if (config->use_metadata) {
-    file->metadata = metadata_receive(file_descriptor);
+    int meta_ok = 1;
+    file->metadata = metadata_receive(file_descriptor, &meta_ok);
+    if (!meta_ok) { file_destroy(file); return NULL; }
   }
   Data *file_data = receive_data(file_descriptor);
   if (file_data == NULL) {
@@ -216,4 +295,21 @@ size_t file_content_to_buffer(File *file) {
   return bytes_read;
 }
 
+int receive_manifest(int fd, Config *config, int *next_status) {
+  int count;
+  if (!receive_int(fd, &count)) return -1;
+  ArrayList *manifest = array_list_create(free);
+  if (manifest) {
+    for (int i = 0; i < count; i++) {
+      char *s = receive_str(fd);
+      if (s) array_list_add(manifest, s);
+    }
+    fprintf(stderr, "Deleting files not in manifest...\n");
+    delete_extras(config->receive_root_directory, manifest);
+    array_list_delete(manifest);
+  }
+  if (!receive_status(fd, next_status)) return -1;
+  return 0;
+}
+
 

src/shared/file.h

@@ -24,11 +24,14 @@ File *file_create(const char *path);
 void file_destroy(void *item);
 bool file_load_data(File *file);
 File *file_receive(Config *config, int file_descriptor);
-bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level);
-bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata);
+bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level, bool send_path);
+bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata, bool send_path);
 size_t file_content_to_buffer(File *file);
 FileMetadata *file_metadata_create(struct stat *stats);
 void file_metadata_destroy(void *metadata);
 bool to_disk(const char *path, const void *data, unsigned long long data_size);
+bool file_save_to_disk(const char *root_directory, File *file);
+File *receive_incremental_check(int fd, Config *config, bool *skipped);
+int receive_manifest(int fd, Config *config, int *next_status);
 
 #endif

src/shared/metadata.c

@@ -50,22 +50,28 @@ bool metadata_send(int file_descriptor, FileMetadata *m) {
          send_n_data(file_descriptor, &m->mtime_nsec, sizeof(long));
 }
 
-FileMetadata *metadata_receive(int file_descriptor) {
+FileMetadata *metadata_receive(int file_descriptor, int *ok) {
   int present;
-  if (!receive_n_data(file_descriptor, &present, sizeof(int)))
+  if (!receive_n_data(file_descriptor, &present, sizeof(int))) {
+    if (ok) *ok = 0;
     return NULL;
-  if (!present)
+  }
+  if (!present) {
+    if (ok) *ok = 1;
     return NULL;
+  }
   FileMetadata *m = malloc(sizeof(FileMetadata));
-  if (m == NULL) return NULL;
+  if (m == NULL) { if (ok) *ok = 0; return NULL; }
   if (!receive_n_data(file_descriptor, &m->mode, sizeof(mode_t)) ||
       !receive_n_data(file_descriptor, &m->uid, sizeof(uid_t)) ||
       !receive_n_data(file_descriptor, &m->gid, sizeof(gid_t)) ||
       !receive_n_data(file_descriptor, &m->mtime_sec, sizeof(time_t)) ||
       !receive_n_data(file_descriptor, &m->mtime_nsec, sizeof(long))) {
     free(m);
+    if (ok) *ok = 0;
     return NULL;
   }
+  if (ok) *ok = 1;
   return m;
 }
 

src/shared/metadata.h

@@ -10,7 +10,7 @@
 void metadata_to_buf(char **buf, FileMetadata *m);
 FileMetadata *metadata_from_buf(char **buf);
 bool metadata_send(int file_descriptor, FileMetadata *m);
-FileMetadata *metadata_receive(int file_descriptor);
+FileMetadata *metadata_receive(int file_descriptor, int *ok);
 void file_restore_metadata(const char *path, FileMetadata *metadata);
 
 #endif

src/shared/multiprocessing.c

@@ -1,12 +1,10 @@
 #include "multiprocessing.h"
 #include "array_list.h"
 #include "chunk.h"
-#include "compression.h"
 #include "config.h"
 #include "data.h"
 #include "file.h"
 #include "log.h"
-#include "metadata.h"
 #include "protocol.h"
 #include "queue.h"
 #include "utils.h"
@@ -85,26 +83,8 @@ void pipeline_context_receiver_destroy(PipelineContextReceiver *context) {
 
 static void receive_chunk_enqueue(int file_descriptor,
                                   PipelineContextReceiver *context) {
-  Data *chunk_data = receive_data(file_descriptor);
-  if (chunk_data == NULL) {
-    log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data");
-    return;
-  }
-  Data *data_to_process = chunk_data;
-  if (context->config->use_compression) {
-    data_to_process = data_decompress(chunk_data);
-    data_destroy(chunk_data);
-    if (data_to_process == NULL) {
-      log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk");
-      return;
-    }
-  }
-  Chunk *chunk = chunk_deserialize(data_to_process, context->config->use_metadata);
-  data_destroy(data_to_process);
-  if (chunk == NULL) {
-    log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping");
-    return;
-  }
+  Chunk *chunk = receive_chunk_data(file_descriptor, context->config);
+  if (chunk == NULL) return;
 
   for (int i = 0; i < chunk->element_count; i++) {
     File *file = chunk->items[i];
@@ -126,8 +106,17 @@ int receive_thread(void *pipeline_context) {
 
   Status status;
   if (!receive_status(file_descriptor, &status)) return thrd_error;
-  while (status == STATUS_NEXT || status == STATUS_CHUNK) {
-    if (status == STATUS_CHUNK) {
+  while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) {
+    if (status == STATUS_CHECK) {
+      bool skipped;
+      File *file = receive_incremental_check(file_descriptor, config, &skipped);
+      if (!skipped) {
+        if (file == NULL) return thrd_error;
+        queue_enqueue_multithreaded(context->queue, file, &context->mutex,
+                                    &context->condition_not_empty,
+                                    &context->condition_not_full);
+      }
+    } else if (status == STATUS_CHUNK) {
       receive_chunk_enqueue(file_descriptor, context);
     } else {
       File *file = file_receive(config, file_descriptor);
@@ -142,22 +131,7 @@ int receive_thread(void *pipeline_context) {
     if (!receive_status(file_descriptor, &status)) return thrd_error;
   }
   if (status == STATUS_MANIFEST) {
-    int count;
-    if (!receive_int(file_descriptor, &count)) return thrd_error;
-    ArrayList *manifest = array_list_create(free);
-    if (manifest) {
-      for (int i = 0; i < count; i++) {
-        char *s = receive_str(file_descriptor);
-        if (s) {
-          array_list_add(manifest, s);
-        }
-      }
-      delete_extras(context->config->receive_root_directory, manifest);
-      for (int i = 0; i < manifest->size; i++)
-        free(manifest->items[i]);
-      array_list_delete(manifest);
-    }
-    if (!receive_status(file_descriptor, &status)) return thrd_error;
+    if (receive_manifest(file_descriptor, config, &status) != 0) return thrd_error;
   }
   mtx_lock(&context->mutex);
   context->receiver_done = true;
@@ -182,14 +156,8 @@ int write_thread(void *pipeline_context) {
       free(root_directory);
       return thrd_success;
     }
-    if (save_to_disk) {
-      char *disk_path = path_cat(root_directory, file->path);
-      if (disk_path) {
-        to_disk(disk_path, file->data->data, file->data->size);
-        file_restore_metadata(disk_path, file->metadata);
-        free(disk_path);
-      }
-    }
+    if (save_to_disk)
+      file_save_to_disk(root_directory, file);
     file_destroy(file);
   }
 }

src/shared/protocol.c

@@ -125,6 +125,8 @@ static const char *status_to_string(Status status) {
     return "NEXT";
   case STATUS_CHUNK:
     return "CHUNK";
+  case STATUS_CHECK:
+    return "CHECK";
   default:
     return "UNKNOWN";
   }

src/shared/protocol.h

@@ -8,7 +8,7 @@
 typedef struct ssl_st SSL;
 
 typedef int Status;
-enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST };
+enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST, STATUS_CHECK };
 
 void io_set_fds(int read_fd, int write_fd);
 void io_set_bwlimit(unsigned long long bytes_per_sec);

test.py

@@ -50,7 +50,7 @@ CLIENT_CMD_PREFIX = [
 
 BASE_CLIENT_FLAGS = ["--save-to-disk"]
 
-TEST_CASES = [
+TEST_CASES_FULL = [
     {"name": "Standard", "flags": []},
     {"name": "Posix Args (no flags)", "flags": [], "posix": True},
     {"name": "Standard (no metadata)", "flags": [], "use_metadata": False},
@@ -68,7 +68,14 @@ TEST_CASES = [
     {"name": "Sendfile + Multithreading (-f -m)", "flags": ["-f", "-m"]},
 ]
 
-SSH_CASES = [
+TEST_CASES_LIGHT = [
+    {"name": "Standard", "flags": []},
+    {"name": "Compression (-c)", "flags": ["-c"]},
+    {"name": "Chunk Serialization (-s)", "flags": ["-s"]},
+    {"name": "Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]},
+]
+
+SSH_CASES_FULL = [
     {"name": "SSH (localhost)", "flags": []},
     {"name": "SSH Multithreading (-m)", "flags": ["-m"]},
     {"name": "SSH Compression (-c)", "flags": ["-c"]},
@@ -79,11 +86,17 @@ SSH_CASES = [
     {"name": "SSH Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]},
 ]
 
-RSYNC_CASES = [
+SSH_CASES_LIGHT = [
+    {"name": "SSH (localhost)", "flags": []},
+]
+
+RSYNC_CASES_FULL = [
     {"name": "rsync (archive)", "args": ["-aH"]},
     {"name": "rsync (archive + compress)", "args": ["-aHz"]},
 ]
 
+RSYNC_CASES_LIGHT = []
+
 
 def netem_apply(profile):
     params = NETWORK_PROFILES[profile]
@@ -119,12 +132,12 @@ def find_free_port():
         return s.getsockname()[1]
 
 
-def generate_test_files(source_dir):
+def generate_test_files(source_dir, full=False):
     if os.path.exists(source_dir):
         shutil.rmtree(source_dir)
     os.makedirs(source_dir)
 
-    target_total = 25 * 1024 * 1024
+    target_total = 25 * 1024 * 1024 if full else 0
     written = 0
 
     files = {
@@ -141,6 +154,7 @@ def generate_test_files(source_dir):
             f.write(content)
         written += len(content)
 
+    if full:
         os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True)
         i = 0
         while written < target_total:
@@ -254,19 +268,24 @@ def print_profile_header(profile_name):
         print("  No limits applied")
 
 
-def run_profile(profile_name, source_dir, dest_dir):
+def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=None, ssh_cases=None, rsync_cases=None):
     print_profile_header(profile_name)
     is_limited = profile_name != "Unlimited"
     client_prefix = CLIENT_CMD_PREFIX if is_limited else []
 
+    if test_cases is None:
+        test_cases = TEST_CASES_FULL if full else TEST_CASES_LIGHT
+    if ssh_cases is None:
+        ssh_cases = SSH_CASES_FULL if full else SSH_CASES_LIGHT
+    if rsync_cases is None:
+        rsync_cases = RSYNC_CASES_FULL if full else RSYNC_CASES_LIGHT
+
     try:
-        if is_limited:
+        if is_limited and full:
             netem_apply(profile_name)
-        else:
-            netem_reset()
 
         results = []
-        for case in TEST_CASES:
+        for case in test_cases:
             flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"]
             if case.get("posix"):
                 cmd = client_prefix + BASE_CLIENT_CMD + [source_dir, dest_dir] + flags
@@ -281,7 +300,7 @@ def run_profile(profile_name, source_dir, dest_dir):
                 results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
 
         if SSH_AVAILABLE:
-            for case in SSH_CASES:
+            for case in ssh_cases:
                 flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"]
                 ssh_dest = f"localhost:{dest_dir}_ssh"
                 cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags
@@ -293,9 +312,10 @@ def run_profile(profile_name, source_dir, dest_dir):
                 except Exception as e:
                     results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
     
+        if rsync_cases:
             port, conf, daemon = start_rsync_daemon(source_dir)
             try:
-            for case in RSYNC_CASES:
+                for case in rsync_cases:
                     cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"]
                     print(f"\n  --- {case['name']} ---\n    Running: {' '.join(cmd)}")
                     try:
@@ -324,6 +344,7 @@ def run_profile(profile_name, source_dir, dest_dir):
                 except Exception:
                     pass
 
+        if full:
             # Feature-specific tests for rsync-compatible flags
             print("\n  " + "─" * 56 + "\n  Feature Tests\n  " + "─" * 56)
 
@@ -394,6 +415,33 @@ def run_profile(profile_name, source_dir, dest_dir):
             except Exception as e:
                 results.append({"name": "Bandwidth limit (--bwlimit 10240)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
 
+            # Incremental sync (--incremental) — first sync, then second sync should skip all
+            print(f"\n  --- Incremental (--incremental) ---")
+            try:
+                flags = BASE_CLIENT_FLAGS + ["-M"]
+                srv = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
+                time.sleep(0.5)
+                first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
+                r1 = subprocess.run(first_cmd, text=True, capture_output=True)
+                wait_proc(srv)
+                if r1.returncode != 0:
+                    raise RuntimeError(f"First sync failed: {r1.stderr[:100]}")
+                srv2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
+                time.sleep(0.5)
+                second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--incremental"]
+                start = time.monotonic()
+                r2 = subprocess.run(second_cmd, text=True, capture_output=True, timeout=30)
+                duration = time.monotonic() - start
+                wait_proc(srv2)
+                r = {"name": "Incremental (--incremental)", "suite": profile_name,
+                     "status": "Success" if r2.returncode == 0 else "Failed",
+                     "time": f"{duration:.4f}s" if r2.returncode == 0 else "N/A",
+                     "error": "" if r2.returncode == 0 else f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}"}
+                results.append(r)
+            except Exception as e:
+                results.append({"name": "Incremental (--incremental)", "suite": profile_name,
+                                "status": "Error", "time": "N/A", "error": str(e)})
+
             # Chunk size (--chunk-size 5242880)
             feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"]
             cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
@@ -546,19 +594,26 @@ def check_ssh_localhost():
     build_dir = os.path.abspath("build")
     server_path = os.path.join(build_dir, "server")
 
+    try:
         r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5",
                             "localhost", "which", "fastsync-server"],
                            capture_output=True, timeout=10)
+    except FileNotFoundError:
+        SSH_AVAILABLE = False
+        return
     if r.returncode == 0:
         SSH_AVAILABLE = True
         return
 
     SSH_AVAILABLE = False
     # Try each PATH dir: create symlink, then verify with which
+    try:
         r = subprocess.run(
             ["ssh", "-o", "BatchMode=yes", "localhost",
              'echo "$PATH"'],
             capture_output=True, timeout=10, text=True)
+    except FileNotFoundError:
+        return
     if r.returncode != 0:
         return
     for d in r.stdout.strip().split(":"):
@@ -639,9 +694,10 @@ def main():
     parser.add_argument("--keep-data", action="store_true")
     parser.add_argument("--unlimited", action="store_true")
     parser.add_argument("--wan", action="store_true")
+    parser.add_argument("--full", action="store_true", help="Run full test suite with network shaping, SSH, rsync benchmarks")
     args = parser.parse_args()
 
-    total_bytes = generate_test_files(args.source_dir)
+    total_bytes = generate_test_files(args.source_dir, full=args.full)
     if os.path.exists(args.dest_dir):
         shutil.rmtree(args.dest_dir)
     os.makedirs(args.dest_dir, exist_ok=True)
@@ -652,12 +708,12 @@ def main():
     elif args.wan:
         profiles.append("WAN")
     else:
-        profiles.append("LAN")
+        profiles.append("LAN" if args.full else "Unlimited")
 
     try:
         all_results = []
         for p in profiles:
-            all_results.extend(run_profile(p, args.source_dir, args.dest_dir))
+            all_results.extend(run_profile(p, args.source_dir, args.dest_dir, full=args.full))
 
         print("\n" + "=" * 130)
         print(f"{'RESULTS':^130}")