Refresh demo

1. Start using disk cache with prefetching.
2. Change to the standard 4 MiB chunk size.
3. Change demo topic sizes a little.

Author: ivanyu

demo/Makefile

@@ -22,9 +22,9 @@ bootstrap_server_host=localhost:9092
 topic=topic1
 partitions=1
 replication_factor=1
-segment=1000000 # 1MB
-retention_bytes=20000000 # 20MB
-local_retention_bytes=5000000 # 5MB
+segment=1048576 # 1 MiB
+retention_bytes=104857600 # 100 MiB
+local_retention_bytes=1
 retention_ms=360000000 # 100 hours
 local_retention_ms=1000 # 1 second
 

demo/compose-azure-blob-azurite.yml

@@ -59,9 +59,11 @@ services:
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/core/*:/tiered-storage-for-apache-kafka/azure/*"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME: "io.aiven.kafka.tieredstorage.RemoteStorageManager"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_IMPL_PREFIX: "rsm.config."
-      KAFKA_RSM_CONFIG_CHUNK_SIZE: 5242880 # 5MiB
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.InMemoryChunkCache"
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: -1
+      KAFKA_RSM_CONFIG_CHUNK_SIZE: 4194304 # 4 MiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.DiskBasedChunkCache"
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PATH: /home/appuser/kafka-tiered-storage-cache
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: 1073741824 # 1 GiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PREFETCH_MAX_SIZE: 16777216 # 16 MiB
       KAFKA_RSM_CONFIG_CUSTOM_METADATA_FIELDS_INCLUDE: "REMOTE_SIZE"
       # Storage backend
       KAFKA_RSM_CONFIG_KEY_PREFIX: "tiered-storage-demo/"

demo/compose-gcs-fake-gcs-server.yml

@@ -59,9 +59,11 @@ services:
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/core/*:/tiered-storage-for-apache-kafka/gcs/*"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME: "io.aiven.kafka.tieredstorage.RemoteStorageManager"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_IMPL_PREFIX: "rsm.config."
-      KAFKA_RSM_CONFIG_CHUNK_SIZE: 5242880 # 5MiB
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.InMemoryChunkCache"
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: -1
+      KAFKA_RSM_CONFIG_CHUNK_SIZE: 4194304 # 4 MiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.DiskBasedChunkCache"
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PATH: /home/appuser/kafka-tiered-storage-cache
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: 1073741824 # 1 GiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PREFETCH_MAX_SIZE: 16777216 # 16 MiB
       KAFKA_RSM_CONFIG_CUSTOM_METADATA_FIELDS_INCLUDE: "REMOTE_SIZE"
       # Storage backend
       KAFKA_RSM_CONFIG_KEY_PREFIX: "tiered-storage-demo/"

demo/compose-local-fs.yml

@@ -58,9 +58,11 @@ services:
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/core/*"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME: "io.aiven.kafka.tieredstorage.RemoteStorageManager"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_IMPL_PREFIX: "rsm.config."
-      KAFKA_RSM_CONFIG_CHUNK_SIZE: 5242880 # 5MiB
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.InMemoryChunkCache"
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: -1
+      KAFKA_RSM_CONFIG_CHUNK_SIZE: 4194304 # 4 MiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.DiskBasedChunkCache"
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PATH: /home/appuser/kafka-tiered-storage-cache
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: 1073741824 # 1 GiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PREFETCH_MAX_SIZE: 16777216 # 16 MiB
       KAFKA_RSM_CONFIG_CUSTOM_METADATA_FIELDS_INCLUDE: "REMOTE_SIZE"
       # Storage backend
       KAFKA_RSM_CONFIG_KEY_PREFIX: "tiered-storage-demo/"

demo/compose-s3-aws.yml

@@ -58,9 +58,11 @@ services:
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/core/*:/tiered-storage-for-apache-kafka/s3/*"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME: "io.aiven.kafka.tieredstorage.RemoteStorageManager"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_IMPL_PREFIX: "rsm.config."
-      KAFKA_RSM_CONFIG_CHUNK_SIZE: 5242880 # 5MiB
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.InMemoryChunkCache"
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: -1
+      KAFKA_RSM_CONFIG_CHUNK_SIZE: 4194304 # 4 MiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.DiskBasedChunkCache"
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PATH: /home/appuser/kafka-tiered-storage-cache
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: 1073741824 # 1 GiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PREFETCH_MAX_SIZE: 16777216 # 16 MiB
       KAFKA_RSM_CONFIG_CUSTOM_METADATA_FIELDS_INCLUDE: "REMOTE_SIZE"
       # Storage backend
       KAFKA_RSM_CONFIG_KEY_PREFIX: "tiered-storage-demo/"

demo/compose-s3-minio.yml

@@ -59,9 +59,11 @@ services:
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_PATH: "/tiered-storage-for-apache-kafka/core/*:/tiered-storage-for-apache-kafka/s3/*"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_CLASS_NAME: "io.aiven.kafka.tieredstorage.RemoteStorageManager"
       KAFKA_REMOTE_LOG_STORAGE_MANAGER_IMPL_PREFIX: "rsm.config."
-      KAFKA_RSM_CONFIG_CHUNK_SIZE: 5242880 # 5MiB
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.InMemoryChunkCache"
-      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: -1
+      KAFKA_RSM_CONFIG_CHUNK_SIZE: 4194304 # 4 MiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_CLASS: "io.aiven.kafka.tieredstorage.chunkmanager.cache.DiskBasedChunkCache"
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PATH: /home/appuser/kafka-tiered-storage-cache
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_SIZE: 1073741824 # 1 GiB
+      KAFKA_RSM_CONFIG_CHUNK_CACHE_PREFETCH_MAX_SIZE: 16777216 # 16 MiB
       KAFKA_RSM_CONFIG_CUSTOM_METADATA_FIELDS_INCLUDE: "REMOTE_SIZE"
       # Storage backend
       KAFKA_RSM_CONFIG_KEY_PREFIX: "tiered-storage-demo/"