attempting to fix model caching

Author: ebiser

.env

@@ -50,8 +50,14 @@ BENCHMARK_DURATION=60
 # More workers = more I/O pressure on FlashArray
 BENCHMARK_WORKERS=8
 
+# Number of model copies to create (defeats Linux page cache)
+# Higher = more unique files = less cache hits = true storage I/O
+# Each copy is ~2MB, so 100 copies = ~200MB working set
+BENCHMARK_COPIES=100
+
 # Run inference after each model load (simulates real usage)
-BENCHMARK_INFERENCE=true
+# Set to false for pure I/O stress test
+BENCHMARK_INFERENCE=false
 
 # Batch size for inference (if enabled)
 BENCHMARK_BATCH_SIZE=1000

Makefile

@@ -53,8 +53,9 @@ help:
 	@echo "Benchmark options (FlashArray stress test):"
 	@echo "  BENCHMARK_DURATION=$(BENCHMARK_DURATION)s"
 	@echo "  BENCHMARK_WORKERS=$(BENCHMARK_WORKERS) (concurrent model loaders)"
+	@echo "  BENCHMARK_COPIES=$(BENCHMARK_COPIES) (defeats page cache)"
 	@echo "  BENCHMARK_INFERENCE=$(BENCHMARK_INFERENCE)"
-	@echo "  Example: make benchmark BENCHMARK_DURATION=120 BENCHMARK_WORKERS=32"
+	@echo "  Example: make benchmark BENCHMARK_DURATION=120 BENCHMARK_WORKERS=32 BENCHMARK_COPIES=200"
 
 # Verify environment and paths
 env-check:
@@ -152,7 +153,8 @@ inference:
 # Benchmark settings
 BENCHMARK_DURATION ?= 60
 BENCHMARK_WORKERS ?= 8
-BENCHMARK_INFERENCE ?= true
+BENCHMARK_COPIES ?= 100
+BENCHMARK_INFERENCE ?= false
 BENCHMARK_BATCH_SIZE ?= 1000
 
 # Run FlashArray model reload stress test
@@ -163,6 +165,7 @@ benchmark:
 	@echo "=========================================="
 	@echo "Duration:       $(BENCHMARK_DURATION)s per test"
 	@echo "Workers:        $(BENCHMARK_WORKERS) concurrent"
+	@echo "Model copies:   $(BENCHMARK_COPIES) (defeats page cache)"
 	@echo "Run inference:  $(BENCHMARK_INFERENCE)"
 	@echo ""
 	@if [ ! -d "$(FA_MODEL_REPO)/fraud_xgboost" ] && [ ! -d "$(FA_MODEL_REPO)/fraud_xgboost_gpu" ] && [ ! -d "$(FA_MODEL_REPO)/fraud_xgboost_cpu" ]; then \
@@ -173,26 +176,26 @@ benchmark:
 	fi
 	@ls -d $(FA_MODEL_REPO)/fraud_xgboost* 2>/dev/null | head -1 | xargs -I{} echo "  Found model: {}"
 	@echo ""
-	BENCHMARK_DURATION=$(BENCHMARK_DURATION) BENCHMARK_WORKERS=$(BENCHMARK_WORKERS) BENCHMARK_INFERENCE=$(BENCHMARK_INFERENCE) BENCHMARK_BATCH_SIZE=$(BENCHMARK_BATCH_SIZE) docker compose run --rm benchmark
+	BENCHMARK_DURATION=$(BENCHMARK_DURATION) BENCHMARK_WORKERS=$(BENCHMARK_WORKERS) BENCHMARK_COPIES=$(BENCHMARK_COPIES) BENCHMARK_INFERENCE=$(BENCHMARK_INFERENCE) BENCHMARK_BATCH_SIZE=$(BENCHMARK_BATCH_SIZE) docker compose run --rm benchmark
 	@echo ""
 	@echo "Stress test complete!"
 	@echo "Check Grafana for FlashArray I/O metrics"
 
-# Run FlashArray stress test without inference (pure I/O)
+# Run FlashArray stress test with more copies (aggressive cache defeat)
 benchmark-io:
 	@echo ""
 	@echo "=========================================="
-	@echo "FlashArray Pure I/O Stress Test"
+	@echo "FlashArray Aggressive I/O Stress Test"
 	@echo "=========================================="
 	@echo "Duration:       $(BENCHMARK_DURATION)s per test"
 	@echo "Workers:        $(BENCHMARK_WORKERS) concurrent"
-	@echo "Run inference:  false (pure model load I/O)"
+	@echo "Model copies:   200 (aggressive cache defeat)"
 	@echo ""
 	@if [ ! -d "$(FA_MODEL_REPO)/fraud_xgboost" ] && [ ! -d "$(FA_MODEL_REPO)/fraud_xgboost_gpu" ] && [ ! -d "$(FA_MODEL_REPO)/fraud_xgboost_cpu" ]; then \
 		echo "ERROR: Model not found at $(FA_MODEL_REPO)/"; \
 		exit 1; \
 	fi
-	BENCHMARK_DURATION=$(BENCHMARK_DURATION) BENCHMARK_WORKERS=$(BENCHMARK_WORKERS) BENCHMARK_INFERENCE=false docker compose run --rm benchmark
+	BENCHMARK_DURATION=$(BENCHMARK_DURATION) BENCHMARK_WORKERS=$(BENCHMARK_WORKERS) BENCHMARK_COPIES=200 BENCHMARK_INFERENCE=false docker compose run --rm benchmark
 
 # Test inference
 test:

docker-compose.yaml

@@ -166,15 +166,17 @@ services:
     container_name: fraud-benchmark
     volumes:
       # Model repository on FlashArray (stress test target)
-      - ${FA_MODEL_REPO}:/data/models:ro
+      # NOTE: Must be read-write for creating model copies
+      - ${FA_MODEL_REPO}:/data/models
       # Data from FlashBlade (for test inference)
       - ${FB_DATA}:/data/input:ro
     environment:
       - MODEL_DIR=/data/models
       - DATA_DIR=/data/input
       - DURATION_SECONDS=${BENCHMARK_DURATION:-60}
       - NUM_WORKERS=${BENCHMARK_WORKERS:-8}
-      - RUN_INFERENCE=${BENCHMARK_INFERENCE:-true}
+      - NUM_MODEL_COPIES=${BENCHMARK_COPIES:-100}
+      - RUN_INFERENCE=${BENCHMARK_INFERENCE:-false}
       - INFERENCE_BATCH_SIZE=${BENCHMARK_BATCH_SIZE:-1000}
     networks:
       - fraud-net

pods/benchmark/Dockerfile

@@ -19,7 +19,8 @@ ENV MODEL_DIR=/data/models
 ENV DATA_DIR=/data/input
 ENV DURATION_SECONDS=60
 ENV NUM_WORKERS=8
-ENV RUN_INFERENCE=true
+ENV NUM_MODEL_COPIES=100
+ENV RUN_INFERENCE=false
 ENV INFERENCE_BATCH_SIZE=1000
 ENV PYTHONUNBUFFERED=1