fix: scanner mid-directory resume bug, -c -s protocol fix, 50 MB test data

- Fix scanner: when chunk fills up mid-directory, save DIR handle so
  remaining files in that directory are not skipped (pre-existing bug)
- Fix -c -s: add STATUS_CHUNK to protocol, send it before chunk data,
  handle it on the server side for both single and multithreaded paths
- Extract chunk_serialize/chunk_deserialize from chunk_compress/decompress
- Remove dead declarations (file_receive_from_buffer, etc.)
- Fix memory leak in receive_file_receive (free -> data_destroy)
- test.py generates ~50 MB of test data across bulk files
- All 8 integration tests + 7 unit tests pass

src/client/scanner.c

@@ -7,10 +7,14 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
 
 DirectoryScanner *directory_scanner_create(char *root_directory) {
   DirectoryScanner *scanner = malloc(sizeof(DirectoryScanner));
   scanner->directories = queue_create(100, free);
+  scanner->current_dir = NULL;
+  scanner->current_path = NULL;
   queue_enqueue(scanner->directories, str_dup(root_directory));
   return scanner;
 }
@@ -18,11 +22,16 @@ DirectoryScanner *directory_scanner_create(char *root_directory) {
 void directory_scanner_destroy(DirectoryScanner *scanner) {
   if (scanner == NULL)
     return;
+  if (scanner->current_dir) {
+    closedir(scanner->current_dir);
+    scanner->current_dir = NULL;
+  }
+  free(scanner->current_path);
   queue_destroy(scanner->directories);
   free(scanner);
 }
 
-Chunk *chunk_data_to_chunk(ArrayList *chunk_data) {
+static Chunk *chunk_data_to_chunk(ArrayList *chunk_data) {
   void **chunk_items = array_list_to_array(chunk_data);
   Chunk *chunk = chunk_create((File **)chunk_items, chunk_data->size);
   free(chunk_items);
@@ -31,40 +40,66 @@ Chunk *chunk_data_to_chunk(ArrayList *chunk_data) {
   return chunk;
 }
 
+static int open_next_directory(DirectoryScanner *scanner) {
+  if (scanner->current_dir) {
+    closedir(scanner->current_dir);
+    scanner->current_dir = NULL;
+  }
+  free(scanner->current_path);
+
+  if (queue_is_empty(scanner->directories))
+    return 0;
+
+  scanner->current_path = (char *)queue_dequeue(scanner->directories);
+  scanner->current_dir = opendir(scanner->current_path);
+  if (scanner->current_dir == NULL) {
+    perror("Could not open directory!");
+    exit(EXIT_FAILURE);
+  }
+  return 1;
+}
+
 Chunk *directory_scanner_next(DirectoryScanner *scanner) {
   ArrayList *chunk_data = array_list_create(file_destroy);
   unsigned long long chunk_data_size = 0;
 
-  while (!queue_is_empty(scanner->directories)) {
+  while (1) {
-    char *path = (char *)queue_dequeue(scanner->directories);
+    if (scanner->current_dir == NULL) {
-    DIR *dir;
+      if (!open_next_directory(scanner))
-    struct dirent *entry;
+        break;
-    dir = opendir(path);
-    if (dir == NULL) {
-      perror("Could not open directory!");
-      exit(EXIT_FAILURE);
     }
-    while ((entry = readdir(dir)) != NULL) {
+
-      if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) {
+    struct dirent *entry = readdir(scanner->current_dir);
-        continue;
+    if (entry == NULL) {
-      }
+      closedir(scanner->current_dir);
-      char *cur_path = path_cat(path, entry->d_name);
+      scanner->current_dir = NULL;
-      struct stat stats;
+      free(scanner->current_path);
-      stat(cur_path, &stats);
+      scanner->current_path = NULL;
-      if (!S_ISREG(stats.st_mode))
+      continue;
-        queue_enqueue(scanner->directories, (void *)cur_path);
+    }
-      else {
+
-        File *file = file_create(cur_path, &stats);
+    if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
-        array_list_add(chunk_data, file);
+      continue;
-        chunk_data_size += file->stats.st_size;
+
-        if (chunk_data_size > DESIRED_CHUNK_SIZE)
+    char *cur_path = path_cat(scanner->current_path, entry->d_name);
-          return chunk_data_to_chunk(chunk_data);
+    struct stat stats;
-        free(cur_path);
+    if (stat(cur_path, &stats) != 0) {
-      }
+      free(cur_path);
+      continue;
+    }
+
+    if (!S_ISREG(stats.st_mode)) {
+      queue_enqueue(scanner->directories, (void *)cur_path);
+    } else {
+      File *file = file_create(cur_path, &stats);
+      array_list_add(chunk_data, file);
+      chunk_data_size += file->stats.st_size;
+      if (chunk_data_size > DESIRED_CHUNK_SIZE)
+        return chunk_data_to_chunk(chunk_data);
+      free(cur_path);
     }
-    closedir(dir);
-    free(path);
   }
+
   if (chunk_data->size > 0)
     return chunk_data_to_chunk(chunk_data);
   return NULL;

src/client/scanner.h

@@ -3,8 +3,12 @@
 
 #include "chunk.h"
 #include "queue.h"
+#include <dirent.h>
+
 typedef struct {
   Queue *directories;
+  DIR *current_dir;
+  char *current_path;
 } DirectoryScanner;
 
 DirectoryScanner *directory_scanner_create(char *root_directory);

test.py

@@ -56,10 +56,13 @@ def generate_test_files(source_dir):
         shutil.rmtree(source_dir)
     os.makedirs(source_dir)
 
+    target_total = 50 * 1024 * 1024
+    written = 0
+
     files = {
         "small.txt": b"hello world\n",
-        "medium.txt": b"line\n" * 1000,
+        "medium.txt": b"the quick brown fox jumps over the lazy dog\n" * 5000,
-        "binary.bin": bytes(range(256)),
+        "binary.bin": bytes(range(256)) * 1000,
         "nested/subdir/deep.txt": b"deeply nested file\n",
         "nested/another.txt": b"another nested file\n" * 50,
     }
@@ -68,6 +71,21 @@ def generate_test_files(source_dir):
         os.makedirs(os.path.dirname(full_path), exist_ok=True)
         with open(full_path, "wb") as f:
             f.write(content)
+        written += len(content)
+
+    i = 0
+    while written < target_total:
+        chunk_size = min(5 * 1024 * 1024, target_total - written)
+        rel_path = f"bulk/file_{i}.dat"
+        full_path = os.path.join(source_dir, rel_path)
+        os.makedirs(os.path.dirname(full_path), exist_ok=True)
+        with open(full_path, "wb") as f:
+            f.write(b"0" * chunk_size)
+        written += chunk_size
+        i += 1
+
+    total_mb = written / (1024 * 1024)
+    print(f"  Generated {total_mb:.1f} MB of test data in {source_dir}")
 
 
 def verify_transfer(source_dir, dest_dir):

test_data/dest/root/FastSync/test_data/source/nested/another.txt

@@ -0,0 +1,50 @@
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file

test_data/dest/root/FastSync/test_data/source/nested/subdir/deep.txt

@@ -0,0 +1 @@
+deeply nested file

test_data/dest/root/FastSync/test_data/source/small.txt

@@ -0,0 +1 @@
+hello world

test_data/source/nested/another.txt

@@ -0,0 +1,50 @@
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file
+another nested file

test_data/source/nested/subdir/deep.txt

@@ -0,0 +1 @@
+deeply nested file

test_data/source/small.txt

@@ -0,0 +1 @@
+hello world