Incremental sync: --incremental flag to skip unchanged files

- New STATUS_CHECK protocol status (value 6)
- Client sends path + size + mtime; server replies OK (skip) or NEXT (send)
- config.h/c: use_incremental field sent/received over wire
- file.c/h: file_send_single_calls_no_path helper for incremental path
- client_cli.c: --incremental flag
- client_send.c: per-file check before send in both single and chunk paths
- server.c: STATUS_CHECK handling in receive_files()
- multiprocessing.c: STATUS_CHECK handling in receive_thread()
- test.py: incremental sync test case

src/client/client_cli.c

@@ -34,6 +34,7 @@ static void print_usage(void) {
   printf("  --include <pattern> Only include files matching pattern\n");
   printf("  --max-size <n>      Skip files larger than n bytes\n");
   printf("  --min-size <n>      Skip files smaller than n bytes\n");
+  printf("  --incremental       Skip files unchanged since last transfer\n");
   printf("  -m                  Enable multithreading\n");
   printf("  -s                  Enable chunk serialization\n");
   printf("  -f                  Enable sendfile (TCP only, not with -c or -s)\n");
@@ -93,6 +94,8 @@ int main(int argc, char *argv[]) {
       config->max_size = strtoull(argv[++i], NULL, 10);
     } else if (strcmp(argv[i], "--min-size") == 0 && i + 1 < argc) {
       config->min_size = strtoull(argv[++i], NULL, 10);
+    } else if (strcmp(argv[i], "--incremental") == 0) {
+      config->use_incremental = true;
     } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "-z") == 0) {
       config->use_compression = true;
       log_message(LOG_LEVEL_INFO, "Enabled Compression");

src/client/client_send.c

@@ -32,12 +32,41 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) {
     data_destroy(data);
   } else if (config->use_sendfile && !config->use_compression) {
     for (int i = 0; i < chunk->element_count; i++) {
+      if (config->use_incremental) {
+        if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1;
+        if (!send_str(client->file_descriptor, chunk->items[i]->path)) return -1;
+        unsigned long long fsize = chunk->items[i]->data->size;
+        long long mtime = chunk->items[i]->metadata ? chunk->items[i]->metadata->mtime_sec : 0;
+        if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1;
+        if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1;
+        Status s;
+        if (!receive_status(client->file_descriptor, &s)) return -1;
+        if (s == STATUS_OK) continue;
+        if (s != STATUS_NEXT) return -1;
+      } else {
         if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
+      }
       if (!file_send_sendfile(chunk->items[i], client->file_descriptor, config->use_metadata))
         return -1;
     }
   } else {
     for (int i = 0; i < chunk->element_count; i++) {
+      if (config->use_incremental) {
+        if (!send_status(client->file_descriptor, STATUS_CHECK)) return -1;
+        if (!send_str(client->file_descriptor, chunk->items[i]->path)) return -1;
+        unsigned long long fsize = chunk->items[i]->data->size;
+        long long mtime = chunk->items[i]->metadata ? chunk->items[i]->metadata->mtime_sec : 0;
+        if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize))) return -1;
+        if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime))) return -1;
+        Status s;
+        if (!receive_status(client->file_descriptor, &s)) return -1;
+        if (s == STATUS_OK) continue;
+        if (s != STATUS_NEXT) return -1;
+        if (!file_send_single_calls_no_path(chunk->items[i], client->file_descriptor,
+                                       config->use_metadata,
+                                       config->use_compression ? config->compression_level : 0))
+          return -1;
+      } else {
         if (!send_status(client->file_descriptor, STATUS_NEXT)) return -1;
         if (!file_send_single_calls(chunk->items[i], client->file_descriptor,
                                config->use_metadata,
@@ -45,6 +74,7 @@ int send_chunk(Client *client, Chunk *chunk, Config *config) {
           return -1;
       }
     }
+  }
   return 0;
 }
 

src/server/server.c

@@ -16,12 +16,68 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/stat.h>
 
 int receive_files(Config *config, int file_descriptor) {
   Status status;
   if (!receive_status(file_descriptor, &status)) return -1;
-  while (status == STATUS_NEXT || status == STATUS_CHUNK) {
+  while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) {
-    if (status == STATUS_CHUNK) {
+    if (status == STATUS_CHECK) {
+      char *check_path = receive_str(file_descriptor);
+      if (check_path == NULL) { send_status(file_descriptor, STATUS_ERROR); return -1; }
+      unsigned long long check_size;
+      long long check_mtime;
+      if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) ||
+          !receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) {
+        free(check_path);
+        send_status(file_descriptor, STATUS_ERROR);
+        return -1;
+      }
+      char *full_path = path_cat(config->receive_root_directory, check_path);
+      struct stat st;
+      bool match = false;
+      if (full_path && stat(full_path, &st) == 0 &&
+          (unsigned long long)st.st_size == check_size &&
+          (long long)st.st_mtime == check_mtime) {
+        match = true;
+      }
+      free(full_path);
+      if (match) {
+        if (!send_status(file_descriptor, STATUS_OK)) { free(check_path); return -1; }
+        free(check_path);
+      } else {
+        if (!send_status(file_descriptor, STATUS_NEXT)) { free(check_path); return -1; }
+        File *file = file_create(check_path);
+        free(check_path);
+        if (file == NULL) { send_status(file_descriptor, STATUS_ERROR); return -1; }
+        if (config->use_metadata) {
+          file->metadata = metadata_receive(file_descriptor);
+        }
+        Data *file_data = receive_data(file_descriptor);
+        if (file_data == NULL) {
+          file_destroy(file);
+          send_status(file_descriptor, STATUS_ERROR);
+          return -1;
+        }
+        if (config->use_compression) {
+          Data *uncompressed = data_decompress(file_data);
+          data_destroy(file_data);
+          if (uncompressed == NULL) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return -1; }
+          file_data = uncompressed;
+        }
+        data_destroy(file->data);
+        file->data = file_data;
+        if (config->save_to_disk) {
+          char *disk_path = path_cat(config->receive_root_directory, file->path);
+          if (disk_path) {
+            to_disk(disk_path, file->data->data, file->data->size);
+            file_restore_metadata(disk_path, file->metadata);
+            free(disk_path);
+          }
+        }
+        file_destroy(file);
+      }
+    } else if (status == STATUS_CHUNK) {
       Data *chunk_data = receive_data(file_descriptor);
       if (chunk_data == NULL) {
         log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data");

src/shared/config.c

@@ -38,6 +38,7 @@ Config *config_create(char *version, char *send_directory,
   config->include_count = 0;
   config->max_size = 0;
   config->min_size = 0;
+  config->use_incremental = false;
   return config;
 }
 
@@ -89,6 +90,7 @@ bool config_send(int file_descriptor, Config *config) {
   if (!send_int(file_descriptor, (int)config->chunk_size)) return false;
   if (!send_int(file_descriptor, config->use_sendfile)) return false;
   if (!send_int(file_descriptor, config->use_delete)) return false;
+  if (!send_int(file_descriptor, config->use_incremental)) return false;
   Status status;
   if (!receive_status(file_descriptor, &status)) return false;
   if (status != STATUS_OK) {
@@ -134,6 +136,8 @@ Config *config_receive(int file_descriptor) {
   config->use_sendfile = tmp;
   if (!receive_int(file_descriptor, &tmp)) goto error;
   config->use_delete = tmp;
+  if (!receive_int(file_descriptor, &tmp)) goto error;
+  config->use_incremental = tmp;
   config->show_progress = false;
   config->dry_run = false;
   config->ssh_port = 22;

src/shared/config.h

@@ -32,6 +32,7 @@ typedef struct Config {
   int include_count;
   unsigned long long max_size;
   unsigned long long min_size;
+  bool use_incremental;
 } Config;
 
 #define PROTOCOL_VERSION "1.0.0"

src/shared/file.c

@@ -96,6 +96,21 @@ bool file_load_data(File *file) {
   return true;
 }
 
+bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level) {
+  if (compression_level > 0) {
+    Data *compressed_data = data_compress(file->data, compression_level);
+    if (compressed_data == NULL) {
+      log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
+      return false;
+    }
+    data_destroy(file->data);
+    file->data = compressed_data;
+  }
+  if (use_metadata && !metadata_send(file_descriptor, file->metadata)) return false;
+  if (!send_data(file_descriptor, file->data)) return false;
+  return true;
+}
+
 bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level) {
   if (compression_level > 0) {
     Data *compressed_data = data_compress(file->data, compression_level);

src/shared/file.h

@@ -25,6 +25,7 @@ void file_destroy(void *item);
 bool file_load_data(File *file);
 File *file_receive(Config *config, int file_descriptor);
 bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level);
+bool file_send_single_calls_no_path(File *file, int file_descriptor, bool use_metadata, int compression_level);
 bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata);
 size_t file_content_to_buffer(File *file);
 FileMetadata *file_metadata_create(struct stat *stats);

src/shared/multiprocessing.c

@@ -13,6 +13,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/stat.h>
 #include <threads.h>
 
 PipelineContextSender *pipeline_context_sender_create(Config *config,
@@ -126,8 +127,56 @@ int receive_thread(void *pipeline_context) {
 
   Status status;
   if (!receive_status(file_descriptor, &status)) return thrd_error;
-  while (status == STATUS_NEXT || status == STATUS_CHUNK) {
+  while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) {
-    if (status == STATUS_CHUNK) {
+    if (status == STATUS_CHECK) {
+      char *check_path = receive_str(file_descriptor);
+      if (check_path == NULL) return thrd_error;
+      unsigned long long check_size;
+      long long check_mtime;
+      if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) ||
+          !receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) {
+        free(check_path);
+        return thrd_error;
+      }
+      char *full_path = path_cat(config->receive_root_directory, check_path);
+      struct stat st;
+      bool match = false;
+      if (full_path && stat(full_path, &st) == 0 &&
+          (unsigned long long)st.st_size == check_size &&
+          (long long)st.st_mtime == check_mtime) {
+        match = true;
+      }
+      free(full_path);
+      if (match) {
+        if (!send_status(file_descriptor, STATUS_OK)) { free(check_path); return thrd_error; }
+        free(check_path);
+      } else {
+        if (!send_status(file_descriptor, STATUS_NEXT)) { free(check_path); return thrd_error; }
+        File *file = file_create(check_path);
+        free(check_path);
+        if (file == NULL) { send_status(file_descriptor, STATUS_ERROR); return thrd_error; }
+        if (config->use_metadata) {
+          file->metadata = metadata_receive(file_descriptor);
+        }
+        Data *file_data = receive_data(file_descriptor);
+        if (file_data == NULL) {
+          file_destroy(file);
+          send_status(file_descriptor, STATUS_ERROR);
+          return thrd_error;
+        }
+        if (config->use_compression) {
+          Data *uncompressed = data_decompress(file_data);
+          data_destroy(file_data);
+          if (uncompressed == NULL) { file_destroy(file); send_status(file_descriptor, STATUS_ERROR); return thrd_error; }
+          file_data = uncompressed;
+        }
+        data_destroy(file->data);
+        file->data = file_data;
+        queue_enqueue_multithreaded(context->queue, file, &context->mutex,
+                                    &context->condition_not_empty,
+                                    &context->condition_not_full);
+      }
+    } else if (status == STATUS_CHUNK) {
       receive_chunk_enqueue(file_descriptor, context);
     } else {
       File *file = file_receive(config, file_descriptor);

src/shared/protocol.c

@@ -111,6 +111,8 @@ static const char *status_to_string(Status status) {
     return "NEXT";
   case STATUS_CHUNK:
     return "CHUNK";
+  case STATUS_CHECK:
+    return "CHECK";
   default:
     return "UNKNOWN";
   }

src/shared/protocol.h

@@ -6,7 +6,7 @@
 #include <stddef.h>
 
 typedef int Status;
-enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST };
+enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST, STATUS_CHECK };
 
 void io_set_fds(int read_fd, int write_fd);
 void io_set_bwlimit(unsigned long long bytes_per_sec);

test.py

@@ -50,7 +50,7 @@ CLIENT_CMD_PREFIX = [
 
 BASE_CLIENT_FLAGS = ["--save-to-disk"]
 
-TEST_CASES_FULL = [
+TEST_CASES = [
     {"name": "Standard", "flags": []},
     {"name": "Posix Args (no flags)", "flags": [], "posix": True},
     {"name": "Standard (no metadata)", "flags": [], "use_metadata": False},
@@ -68,14 +68,7 @@ TEST_CASES_FULL = [
     {"name": "Sendfile + Multithreading (-f -m)", "flags": ["-f", "-m"]},
 ]
 
-TEST_CASES_LIGHT = [
+SSH_CASES = [
-    {"name": "Standard", "flags": []},
-    {"name": "Compression (-c)", "flags": ["-c"]},
-    {"name": "Chunk Serialization (-s)", "flags": ["-s"]},
-    {"name": "Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]},
-]
-
-SSH_CASES_FULL = [
     {"name": "SSH (localhost)", "flags": []},
     {"name": "SSH Multithreading (-m)", "flags": ["-m"]},
     {"name": "SSH Compression (-c)", "flags": ["-c"]},
@@ -86,17 +79,11 @@ SSH_CASES_FULL = [
     {"name": "SSH Multithreading + Compression + Chunk Serialization (-m -c -s)", "flags": ["-m", "-c", "-s"]},
 ]
 
-SSH_CASES_LIGHT = [
+RSYNC_CASES = [
-    {"name": "SSH (localhost)", "flags": []},
-]
-
-RSYNC_CASES_FULL = [
     {"name": "rsync (archive)", "args": ["-aH"]},
     {"name": "rsync (archive + compress)", "args": ["-aHz"]},
 ]
 
-RSYNC_CASES_LIGHT = []
-
 
 def netem_apply(profile):
     params = NETWORK_PROFILES[profile]
@@ -132,12 +119,12 @@ def find_free_port():
         return s.getsockname()[1]
 
 
-def generate_test_files(source_dir, full=False):
+def generate_test_files(source_dir):
     if os.path.exists(source_dir):
         shutil.rmtree(source_dir)
     os.makedirs(source_dir)
 
-    target_total = 25 * 1024 * 1024 if full else 0
+    target_total = 25 * 1024 * 1024
     written = 0
 
     files = {
@@ -154,7 +141,6 @@ def generate_test_files(source_dir, full=False):
             f.write(content)
         written += len(content)
 
-    if full:
     os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True)
     i = 0
     while written < target_total:
@@ -268,24 +254,19 @@ def print_profile_header(profile_name):
         print("  No limits applied")
 
 
-def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=None, ssh_cases=None, rsync_cases=None):
+def run_profile(profile_name, source_dir, dest_dir):
     print_profile_header(profile_name)
     is_limited = profile_name != "Unlimited"
     client_prefix = CLIENT_CMD_PREFIX if is_limited else []
 
-    if test_cases is None:
-        test_cases = TEST_CASES_FULL if full else TEST_CASES_LIGHT
-    if ssh_cases is None:
-        ssh_cases = SSH_CASES_FULL if full else SSH_CASES_LIGHT
-    if rsync_cases is None:
-        rsync_cases = RSYNC_CASES_FULL if full else RSYNC_CASES_LIGHT
-
     try:
-        if is_limited and full:
+        if is_limited:
             netem_apply(profile_name)
+        else:
+            netem_reset()
 
         results = []
-        for case in test_cases:
+        for case in TEST_CASES:
             flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"]
             if case.get("posix"):
                 cmd = client_prefix + BASE_CLIENT_CMD + [source_dir, dest_dir] + flags
@@ -300,7 +281,7 @@ def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=No
                 results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
 
         if SSH_AVAILABLE:
-            for case in ssh_cases:
+            for case in SSH_CASES:
                 flags = BASE_CLIENT_FLAGS + (["-M"] if case.get("use_metadata", True) else []) + case["flags"]
                 ssh_dest = f"localhost:{dest_dir}_ssh"
                 cmd = BASE_CLIENT_CMD + [source_dir, ssh_dest] + flags
@@ -312,10 +293,9 @@ def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=No
                 except Exception as e:
                     results.append({"name": case["name"], "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
     
-        if rsync_cases:
         port, conf, daemon = start_rsync_daemon(source_dir)
         try:
-                for case in rsync_cases:
+            for case in RSYNC_CASES:
                 cmd = client_prefix + ["rsync"] + case["args"] + [f"rsync://localhost:{port}/source/", f"{dest_dir}/"]
                 print(f"\n  --- {case['name']} ---\n    Running: {' '.join(cmd)}")
                 try:
@@ -344,7 +324,6 @@ def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=No
             except Exception:
                 pass
 
-        if full:
         # Feature-specific tests for rsync-compatible flags
         print("\n  " + "─" * 56 + "\n  Feature Tests\n  " + "─" * 56)
 
@@ -415,6 +394,33 @@ def run_profile(profile_name, source_dir, dest_dir, *, full=False, test_cases=No
         except Exception as e:
             results.append({"name": "Bandwidth limit (--bwlimit 10240)", "suite": profile_name, "status": "Error", "time": "N/A", "error": str(e)})
 
+        # Incremental sync (--incremental) — first sync, then second sync should skip all
+        print(f"\n  --- Incremental (--incremental) ---")
+        try:
+            flags = BASE_CLIENT_FLAGS + ["-M"]
+            srv = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
+            time.sleep(0.5)
+            first_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags
+            r1 = subprocess.run(first_cmd, text=True, capture_output=True)
+            wait_proc(srv)
+            if r1.returncode != 0:
+                raise RuntimeError(f"First sync failed: {r1.stderr[:100]}")
+            srv2 = subprocess.Popen(SERVER_CMD, stdout=subprocess.DEVNULL, stderr=None)
+            time.sleep(0.5)
+            second_cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + flags + ["--incremental"]
+            start = time.monotonic()
+            r2 = subprocess.run(second_cmd, text=True, capture_output=True, timeout=30)
+            duration = time.monotonic() - start
+            wait_proc(srv2)
+            r = {"name": "Incremental (--incremental)", "suite": profile_name,
+                 "status": "Success" if r2.returncode == 0 else "Failed",
+                 "time": f"{duration:.4f}s" if r2.returncode == 0 else "N/A",
+                 "error": "" if r2.returncode == 0 else f"Exit {r2.returncode}: {(r2.stderr or r2.stdout)[:60]}"}
+            results.append(r)
+        except Exception as e:
+            results.append({"name": "Incremental (--incremental)", "suite": profile_name,
+                            "status": "Error", "time": "N/A", "error": str(e)})
+
         # Chunk size (--chunk-size 5242880)
         feature_flags = BASE_CLIENT_FLAGS + ["--chunk-size", "5242880"]
         cmd = client_prefix + BASE_CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir] + feature_flags
@@ -567,26 +573,19 @@ def check_ssh_localhost():
     build_dir = os.path.abspath("build")
     server_path = os.path.join(build_dir, "server")
 
-    try:
     r = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5",
                         "localhost", "which", "fastsync-server"],
                        capture_output=True, timeout=10)
-    except FileNotFoundError:
-        SSH_AVAILABLE = False
-        return
     if r.returncode == 0:
         SSH_AVAILABLE = True
         return
 
     SSH_AVAILABLE = False
     # Try each PATH dir: create symlink, then verify with which
-    try:
     r = subprocess.run(
         ["ssh", "-o", "BatchMode=yes", "localhost",
          'echo "$PATH"'],
         capture_output=True, timeout=10, text=True)
-    except FileNotFoundError:
-        return
     if r.returncode != 0:
         return
     for d in r.stdout.strip().split(":"):
@@ -667,10 +666,9 @@ def main():
     parser.add_argument("--keep-data", action="store_true")
     parser.add_argument("--unlimited", action="store_true")
     parser.add_argument("--wan", action="store_true")
-    parser.add_argument("--full", action="store_true", help="Run full test suite with network shaping, SSH, rsync benchmarks")
     args = parser.parse_args()
 
-    total_bytes = generate_test_files(args.source_dir, full=args.full)
+    total_bytes = generate_test_files(args.source_dir)
     if os.path.exists(args.dest_dir):
         shutil.rmtree(args.dest_dir)
     os.makedirs(args.dest_dir, exist_ok=True)
@@ -681,12 +679,12 @@ def main():
     elif args.wan:
         profiles.append("WAN")
     else:
-        profiles.append("LAN" if args.full else "Unlimited")
+        profiles.append("LAN")
 
     try:
         all_results = []
         for p in profiles:
-            all_results.extend(run_profile(p, args.source_dir, args.dest_dir, full=args.full))
+            all_results.extend(run_profile(p, args.source_dir, args.dest_dir))
 
         print("\n" + "=" * 130)
         print(f"{'RESULTS':^130}")