diff --git a/demos/roms-streams/README.md b/demos/roms-streams/README.md new file mode 100644 index 000000000..cf0140b9a --- /dev/null +++ b/demos/roms-streams/README.md @@ -0,0 +1,333 @@ +# Redis Streams Consumer Framework + +This framework provides automatic bean creation for Redis Stream consumers using Spring annotations. + +## Overview + +The framework consists of two main annotations: +- `@EnableRedisStreams`: Enables the automatic scanning and bean creation for Redis Stream consumers +- `@RedisStreamConsumer`: Marks a class as a Redis Stream consumer with specific configuration + +## Quick Start + +### 1. Enable Redis Streams + +Add the `@EnableRedisStreams` annotation to your configuration class: + +```java +@Configuration +@EnableRedisStreams(basePackages = "com.redis.om.streams.consumer") +public class RedisStreamsConfiguration { + // Your configuration +} +``` + +### 2. Create a Consumer + +Create a class that extends `RedisStreamsConsumer` and annotate it with `@RedisStreamConsumer`: + +```java +@RedisStreamConsumer( + topicName = "myTopic", + groupName = "myGroup", + consumerName = "myConsumer", + autoAck = false, + cluster = false +) +public class MyRedisStreamsConsumer extends RedisStreamsConsumer { + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + @Scheduled(fixedDelayString = "${redis.streams.fixed-delay:1000}") + public boolean process() { + TopicEntry topicEntry = consume(); + if (topicEntry != null) { + logger.info("{} processing topic: {}", getClass().getSimpleName(), topicEntry); + return true; + } + return false; + } +} +``` + +## Annotation Configuration + +### @EnableRedisStreams + +| Attribute | Type | Default | Description | +|-----------|------|---------|-------------| +| `basePackages` | String[] | `{"com.redis.om.streams"}` | Base packages to scan for `@RedisStreamConsumer` annotated classes | +| `value` | String[] | `{}` | Alias for `basePackages` | + +### @RedisStreamConsumer + +| Attribute | Type | Default | Description | +|-----------|------|---------|----------------------------------------------------------------| +| `topicName` | String | **Required** | Name of the Redis Stream topic | +| `groupName` | String | **Required** | Name of the consumer group | +| `consumerName` | String | `""` | Name of the consumer within the group | +| `autoAck` | boolean | `false` | Whether the consumer can acknowledge messages | +| `cluster` | boolean | `false` | Whether to use cluster mode | + +## Consumer Types + +The framework automatically creates different types of consumer groups based on the annotation configuration: + +### 1. No Acknowledgment Consumer (default) +```java +@RedisStreamConsumer( + topicName = "topic", + groupName = "group", + autoAck = false, + cluster = false +) +``` +Creates: `NoAckConsumerGroup` + +### 2. Acknowledgment Consumer +```java +@RedisStreamConsumer( + topicName = "topic", + groupName = "group", + autoAck = true, + cluster = false +) +``` +Creates: `ConsumerGroup` + +### 3. Cluster Consumer +```java +@RedisStreamConsumer( + topicName = "topic", + groupName = "group", + autoAck = true, + cluster = true +) +``` +Creates: `SingleClusterPelConsumerGroup` + +## Automatic Bean Creation + +When you use `@EnableRedisStreams`, the framework automatically creates the following beans for each consumer: + +1. **SerialTopicConfig**: Configuration for the topic +2. **TopicManager**: Manages the Redis Stream topic +3. **ConsumerGroup**: The appropriate consumer group based on configuration +4. **Consumer Class**: The annotated consumer class itself + +### Bean Naming Convention + +- `SerialTopicConfig`: `{topicName}SerialTopicConfig` +- `TopicManager`: `{topicName}TopicManager` (unique per topic, shared between consumers of the same topic) +- `ConsumerGroup`: `{groupName}ConsumerGroup` or `{groupName}NoAckConsumerGroup` or `{groupName}SingleClusterPelConsumerGroup` (unique per group, shared between consumers of the same group) +- `Consumer Class`: `{className}` (uncapitalized) + +## Requirements + +### Method Requirements + +Every consumer class must have a `process()` method annotated with `@Scheduled`: + +```java +@Scheduled(fixedDelayString = "${redis.streams.fixed-delay:1000}") +public boolean process() { + // Your processing logic here + TopicEntry topicEntry = consume(); + // Process the message + return acknowledge(topicEntry); // or return true/false +} +``` + +### Dependencies + +Make sure you have the following dependencies in your `pom.xml`: + +```xml + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-web + + + com.redis.om + redis-om-spring + 1.0.0-RC3 + + + org.projectlombok + lombok + + + com.fasterxml.jackson.core + jackson-databind + +``` + +### Configuration + +Ensure that `@EnableRedisStreams` is enabled in your configuration: + +```java +@Configuration +@EnableRedisStreams(basePackages = "com.redis.om.streams.consumer") +public class RedisStreamsConfiguration { + // Your configuration +} +``` + +## Examples + +### Example 1: Basic Consumer with Consumer Name +```java +@RedisStreamConsumer(topicName = "topicFoo", groupName = "groupFoo", consumerName = "Foo") +public class FooRedisStreamsConsumer extends RedisStreamsConsumer { + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + @Scheduled(fixedDelayString = "${redis.streams.fixed-delay:1000}") + public boolean process() { + TopicEntry topicEntry = consume(); + if (topicEntry != null) { + logger.info("{} processing topic: {}", getClass().getSimpleName(), topicEntry); + } + return true; + } +} +``` + +### Example 2: Acknowledgment Consumer +```java +@RedisStreamConsumer( + topicName = "topicFoo", + groupName = "groupFoo", + autoAck = true +) +public class AckRedisStreamsConsumer extends RedisStreamsConsumer { + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + @Scheduled(fixedDelayString = "${redis.streams.fixed-delay:1000}") + public boolean process() { + TopicEntry topicEntry = consume(); + if (topicEntry != null) { + logger.info("{} processing topic: {}", getClass().getSimpleName(), topicEntry); + return acknowledge(topicEntry); + } + return false; + } +} +``` + +### Example 3: No-Ack Consumer (Explicit) +```java +@RedisStreamConsumer( + topicName = "topicFoo", + groupName = "groupFoo", + autoAck = false +) +public class NoAckFooRedisStreamsConsumer extends RedisStreamsConsumer { + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + @Scheduled(fixedDelayString = "${redis.streams.fixed-delay:1000}") + public boolean process() { + TopicEntry topicEntry = consume(); + if (topicEntry != null) { + logger.info("{} processing topic: {}", getClass().getSimpleName(), topicEntry); + } + return true; + } +} +``` + +## Configuration Properties + +Configure your application in `application.properties`: + +```properties +# Server Configuration +server.port=8080 +spring.application.name=redis-om-spring-streams + +# Spring Data Redis Configuration +spring.data.redis.host=localhost +spring.data.redis.port=6379 +spring.data.redis.username= +spring.data.redis.password= + +# Redis Streams Configuration +redis.streams.fixed-delay=5000 +``` + +## Logging + +The framework provides detailed logging for bean creation and consumer operations. You can configure logging levels in your `application.properties`: + +```properties +logging.level.com.redis.om.streams.config=INFO +logging.level.com.redis.om.streams.consumer=INFO +``` + +## Error Handling + +The framework handles various error scenarios: + +- **ClassNotFoundException**: Logs error and continues with other consumers +- **InvalidTopicException**: Throws IllegalStateException during TopicManager creation +- **TopicNotFoundException**: Handled by individual consumers +- **InvalidMessageException**: Handled by producers +- **ProducerTimeoutException**: Handled by producers + +## Best Practices + +1. **Package Organization**: Keep your consumers in dedicated packages for better organization +2. **Bean Naming**: Use descriptive topic and group names to avoid conflicts +3. **Error Handling**: Implement proper error handling in your `process()` methods +4. **Logging**: Use appropriate log levels for debugging and monitoring +5. **Configuration**: Use environment-specific configurations for different deployment environments +6. **Scheduling**: Use configurable delays with `fixedDelayString` for easy tuning + +## Troubleshooting + +### Common Issues + +1. **No beans created**: Check that `@EnableRedisStreams` is properly configured and base packages are correct +2. **Scheduling not working**: Ensure `@EnableScheduling` is enabled in your configuration +3. **Redis connection issues**: Verify Redis connection configuration in `application.properties` +4. **Bean conflicts**: Check for duplicate bean names, especially with topic configurations +5. **Message production issues**: Verify that the `Producer` bean is properly configured + +### Debug Mode + +Enable debug logging to see detailed bean creation information: + +```properties +logging.level.com.redis.om.streams=DEBUG +``` + +## Sample project structure for this demo + +``` +src/main/java/com/redis/om/streams/ +├── config/ +│ └── RedisStreamsConfiguration.java +├── consumer/ +│ ├── AckRedisStreamsConsumer.java +│ ├── FooRedisStreamsConsumer.java +│ └── NoAckFooRedisStreamsConsumer.java +├── controller/ +│ └── StreamsController.java +├── model/ +│ └── TextData.java +└── DemoApplication.java +``` + +## Running the Application + +1. Start Redis server +2. Run the Spring Boot application +3. Use the REST endpoints to produce messages +4. Watch the consumer logs to see message processing \ No newline at end of file diff --git a/demos/roms-streams/build.gradle b/demos/roms-streams/build.gradle new file mode 100644 index 000000000..eb2eec42c --- /dev/null +++ b/demos/roms-streams/build.gradle @@ -0,0 +1,70 @@ +plugins { + id 'org.springframework.boot' version '3.4.5' + // id 'io.spring.dependency-management' version '1.1.7' + id 'java' +} + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(21) + } +} + +compileJava.options.encoding = 'UTF-8' +compileTestJava.options.encoding = 'UTF-8' + +// Optional: Skip deploy if using a publishing step +tasks.withType(PublishToMavenRepository).configureEach { + enabled = false +} + +repositories { + mavenLocal() + mavenCentral() + maven { + name = 'Spring Milestones' + url = 'https://repo.spring.io/milestone' + } + maven { + name = 'Spring Snapshots' + url = 'https://repo.spring.io/snapshot' + } +} + +dependencies { + // Spring + implementation 'org.springframework.boot:spring-boot-starter-web' + + // Redis OM for Spring (RC version) + implementation project(':redis-om-spring') + + // Jedis client + implementation 'redis.clients:jedis:5.2.0' + + // Jackson + implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.3' + + // Faker + implementation('com.github.javafaker:javafaker:1.0.2') { + exclude group: 'org.yaml', module: 'snakeyaml' + } + + // Lombok + compileOnly 'org.projectlombok:lombok:1.18.36' + annotationProcessor 'org.projectlombok:lombok:1.18.36' + + // Test + testImplementation 'org.springframework.boot:spring-boot-starter-test:3.4.5' + testImplementation 'org.mockito:mockito-core:5.14.2' + testImplementation 'org.testcontainers:testcontainers:1.20.4' + testImplementation 'org.testcontainers:junit-jupiter:1.20.4' + testImplementation 'com.redis:testcontainers-redis:2.2.4' +} + +tasks.withType(JavaCompile).configureEach { + options.encoding = 'UTF-8' +} + +test { + useJUnitPlatform() +} diff --git a/demos/roms-streams/src/main/java/com/redis/om/streams/DemoApplication.java b/demos/roms-streams/src/main/java/com/redis/om/streams/DemoApplication.java new file mode 100644 index 000000000..7456c1d50 --- /dev/null +++ b/demos/roms-streams/src/main/java/com/redis/om/streams/DemoApplication.java @@ -0,0 +1,13 @@ +package com.redis.om.streams; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class DemoApplication { + + public static void main(String[] args) { + SpringApplication.run(DemoApplication.class, args); + } + +} diff --git a/demos/roms-streams/src/main/java/com/redis/om/streams/config/RedisStreamsConfiguration.java b/demos/roms-streams/src/main/java/com/redis/om/streams/config/RedisStreamsConfiguration.java new file mode 100644 index 000000000..066880aee --- /dev/null +++ b/demos/roms-streams/src/main/java/com/redis/om/streams/config/RedisStreamsConfiguration.java @@ -0,0 +1,72 @@ +package com.redis.om.streams.config; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.connection.RedisStandaloneConfiguration; +import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; + +import com.redis.om.streams.Producer; +import com.redis.om.streams.annotation.EnableRedisStreams; +import com.redis.om.streams.command.serial.TopicProducer; + +import jakarta.annotation.PostConstruct; +import redis.clients.jedis.JedisPooled; + +@Configuration +@EnableRedisStreams( + basePackages = "com.redis.om.streams.consumer" +) +public class RedisStreamsConfiguration { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + @Value( + "${spring.data.redis.host}" + ) + private String host; + @Value( + "${spring.data.redis.port}" + ) + private int port; + @Value( + "${spring.data.redis.username}" + ) + private String username; + @Value( + "${spring.data.redis.password}" + ) + private String password; + + @PostConstruct + private void init() { + logger.info("{} init", getClass().getSimpleName()); + } + + @Bean + public JedisConnectionFactory redisConnectionFactory() { + RedisStandaloneConfiguration redisStandaloneConfiguration = new RedisStandaloneConfiguration(); + redisStandaloneConfiguration.setHostName(host); + redisStandaloneConfiguration.setPort(port); + redisStandaloneConfiguration.setUsername(username); + redisStandaloneConfiguration.setPassword(password); + JedisConnectionFactory jediConnectionFactory = new JedisConnectionFactory(redisStandaloneConfiguration); + jediConnectionFactory.setConvertPipelineAndTxResults(false); + return jediConnectionFactory; + } + + @Bean + public JedisPooled jedisPooled() { + logger.info("Creating JedisPooled"); + return new JedisPooled(host, port); + } + + @Bean + public Producer topicProducer(JedisPooled jedisPooled) { + logger.info("Creating TopicProducer"); + return new TopicProducer(jedisPooled, "topicFoo"); + } + +} \ No newline at end of file diff --git a/demos/roms-streams/src/main/java/com/redis/om/streams/consumer/AckRedisStreamsConsumer.java b/demos/roms-streams/src/main/java/com/redis/om/streams/consumer/AckRedisStreamsConsumer.java new file mode 100644 index 000000000..60aac6e5e --- /dev/null +++ b/demos/roms-streams/src/main/java/com/redis/om/streams/consumer/AckRedisStreamsConsumer.java @@ -0,0 +1,35 @@ +package com.redis.om.streams.consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; + +import com.redis.om.streams.TopicEntry; +import com.redis.om.streams.annotation.RedisStreamConsumer; + +import jakarta.annotation.PostConstruct; + +@RedisStreamConsumer( + topicName = "topicFoo", groupName = "groupFoo", autoAck = true +) +public class AckRedisStreamsConsumer extends RedisStreamsConsumer { + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + @PostConstruct + private void init() { + logger.info("{} init", getClass().getSimpleName()); + } + + @Scheduled( + fixedDelayString = "${redis.streams.fixed-delay:1000}" + ) + public boolean process() { + TopicEntry topicEntry = consume(); + if (topicEntry != null) { + logger.info("{} processing topic: {}", getClass().getSimpleName(), topicEntry); + return acknowledge(topicEntry); + } + return false; + } +} diff --git a/demos/roms-streams/src/main/java/com/redis/om/streams/consumer/FooRedisStreamsConsumer.java b/demos/roms-streams/src/main/java/com/redis/om/streams/consumer/FooRedisStreamsConsumer.java new file mode 100644 index 000000000..d8376392e --- /dev/null +++ b/demos/roms-streams/src/main/java/com/redis/om/streams/consumer/FooRedisStreamsConsumer.java @@ -0,0 +1,34 @@ +package com.redis.om.streams.consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; + +import com.redis.om.streams.TopicEntry; +import com.redis.om.streams.annotation.RedisStreamConsumer; + +import jakarta.annotation.PostConstruct; + +@RedisStreamConsumer( + topicName = "topicFoo", groupName = "groupFoo", consumerName = "Foo" +) +public class FooRedisStreamsConsumer extends RedisStreamsConsumer { + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + @PostConstruct + private void init() { + logger.info("{} init", getClass().getSimpleName()); + } + + @Scheduled( + fixedDelayString = "${redis.streams.fixed-delay:1000}" + ) + public boolean process() { + TopicEntry topicEntry = consume(); + if (topicEntry != null) { + logger.info("{} processing topic: {}", getClass().getSimpleName(), topicEntry); + } + return true; + } +} diff --git a/demos/roms-streams/src/main/java/com/redis/om/streams/consumer/NoAckFooRedisStreamsConsumer.java b/demos/roms-streams/src/main/java/com/redis/om/streams/consumer/NoAckFooRedisStreamsConsumer.java new file mode 100644 index 000000000..731ef191e --- /dev/null +++ b/demos/roms-streams/src/main/java/com/redis/om/streams/consumer/NoAckFooRedisStreamsConsumer.java @@ -0,0 +1,34 @@ +package com.redis.om.streams.consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; + +import com.redis.om.streams.TopicEntry; +import com.redis.om.streams.annotation.RedisStreamConsumer; + +import jakarta.annotation.PostConstruct; + +@RedisStreamConsumer( + topicName = "topicFoo", groupName = "groupFoo", autoAck = false +) +public class NoAckFooRedisStreamsConsumer extends RedisStreamsConsumer { + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + @PostConstruct + private void init() { + logger.info("{} init", getClass().getSimpleName()); + } + + @Scheduled( + fixedDelayString = "${redis.streams.fixed-delay:1000}" + ) + public boolean process() { + TopicEntry topicEntry = consume(); + if (topicEntry != null) { + logger.info("{} processing topic: {}", getClass().getSimpleName(), topicEntry); + } + return true; + } +} diff --git a/demos/roms-streams/src/main/java/com/redis/om/streams/controller/StreamsController.java b/demos/roms-streams/src/main/java/com/redis/om/streams/controller/StreamsController.java new file mode 100644 index 000000000..7e7465a67 --- /dev/null +++ b/demos/roms-streams/src/main/java/com/redis/om/streams/controller/StreamsController.java @@ -0,0 +1,116 @@ +package com.redis.om.streams.controller; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.javafaker.Faker; +import com.redis.om.streams.Producer; +import com.redis.om.streams.exception.InvalidMessageException; +import com.redis.om.streams.exception.ProducerTimeoutException; +import com.redis.om.streams.exception.TopicNotFoundException; +import com.redis.om.streams.model.TextData; + +@RestController +@RequestMapping( + "/api/streams" +) +public class StreamsController { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + public static boolean stopLoading = Boolean.FALSE; + + private final Producer producer; + private final ObjectMapper objectMapper; + + public StreamsController(Producer producer, ObjectMapper objectMapper) { + this.producer = producer; + this.objectMapper = objectMapper; + } + + private void create(TextData textData) { + try { + producer.produce(objectMapper.convertValue(textData, new TypeReference<>() { + })); + } catch (TopicNotFoundException | InvalidMessageException | ProducerTimeoutException e) { + logger.error(e.getMessage(), e); + } + } + + @GetMapping( + path = "/start-load" + ) + public ResponseEntity startLoading() { + Faker faker = new Faker(); + StreamsController.stopLoading = Boolean.FALSE; + AtomicInteger created = new AtomicInteger(); + while (!StreamsController.stopLoading) { + TextData textData = TextData.of(); + try { + textData.setId(created.getAndIncrement()); + textData.setName(faker.dune().character()); + textData.setDescription(faker.dune().quote()); + create(textData); + showSpinner(created.get()); + } catch (Exception e) { + logger.error("Error while creating new TextData: {}", textData, e); + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build(); + } + } + System.out.print("\r "); + System.out.print("\r"); + return ResponseEntity.ok(created.get()); + } + + @GetMapping( + path = "/start-load/{count}" + ) + public ResponseEntity startLoading(@PathVariable int count) { + Faker faker = new Faker(); + StreamsController.stopLoading = Boolean.FALSE; + AtomicInteger created = new AtomicInteger(); + while (created.get() < count) { + TextData textData = TextData.of(); + try { + textData.setId(created.getAndIncrement()); + textData.setName(faker.dune().character()); + textData.setDescription(faker.dune().quote()); + create(textData); + showSpinner(created.get()); + } catch (Exception e) { + logger.error("Error while creating new TextData: {}", textData, e); + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).build(); + } + } + System.out.print("\r "); + System.out.print("\r"); + return ResponseEntity.ok(created.get()); + } + + @GetMapping( + path = "/stop-load" + ) + public ResponseEntity stopLoading() { + StreamsController.stopLoading = true; + return ResponseEntity.noContent().build(); + } + + // private final List wheel = List.of("|", "/", "-", "\\", "|", "/", "-", "\\"); + // private final String wheel = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"; + // private final String wheel = "⠹⠸⠼⠧⠇⠏"; + private void showSpinner(int count) { + String s = String.format("%,d", count); + String wheel = "⠁ ⠃ ⠇ ⠧⠷⠿⡿⣟⣯⣷"; + System.out.printf("\rProgress: " + s + " -> " + wheel.charAt(count % wheel.length()) + " "); + } +} diff --git a/demos/roms-streams/src/main/java/com/redis/om/streams/model/TextData.java b/demos/roms-streams/src/main/java/com/redis/om/streams/model/TextData.java new file mode 100644 index 000000000..153c3b20c --- /dev/null +++ b/demos/roms-streams/src/main/java/com/redis/om/streams/model/TextData.java @@ -0,0 +1,14 @@ +package com.redis.om.streams.model; + +import lombok.Data; +import lombok.RequiredArgsConstructor; + +@Data +@RequiredArgsConstructor( + staticName = "of" +) +public class TextData { + private int id; + private String name; + private String description; +} diff --git a/demos/roms-streams/src/main/resources/application.properties b/demos/roms-streams/src/main/resources/application.properties new file mode 100644 index 000000000..dd0628bae --- /dev/null +++ b/demos/roms-streams/src/main/resources/application.properties @@ -0,0 +1,10 @@ +server.port=8080 +spring.application.name=redis-om-spring-streams + +# Spring Data Redis Configuration +spring.data.redis.host=localhost +spring.data.redis.port=6379 +spring.data.redis.username= +spring.data.redis.password= + +redis.streams.fixed-delay=5000 diff --git a/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/RedisStreamsConsumerTest.java b/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/RedisStreamsConsumerTest.java new file mode 100644 index 000000000..1bc1994fd --- /dev/null +++ b/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/RedisStreamsConsumerTest.java @@ -0,0 +1,193 @@ +package com.redis.om.streams.consumer; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.redis.om.streams.Producer; +import com.redis.om.streams.TopicEntry; +import com.redis.om.streams.TopicEntryId; +import com.redis.om.streams.annotation.RedisStreamConsumer; +import com.redis.om.streams.command.serial.TopicProducer; +import com.redis.om.streams.model.TextData; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; +import redis.clients.jedis.JedisPooled; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +@SpringBootTest +@Testcontainers +class RedisStreamsConsumerTest { + + @Container + static GenericContainer redis = new GenericContainer<>(DockerImageName.parse("redis:8.0.2-alpine")) + .withExposedPorts(6379); + + @DynamicPropertySource + static void redisProperties(DynamicPropertyRegistry registry) { + registry.add("spring.data.redis.host", redis::getHost); + registry.add("spring.data.redis.port", redis::getFirstMappedPort); + registry.add("spring.data.redis.username", () -> ""); + registry.add("spring.data.redis.password", () -> ""); + registry.add("redis.streams.fixed-delay", () -> "1000"); + } + + @Autowired + private Test_tFoo_gBar_Ack_NoCluster test_tFoo_gBar_ack_noCluster; + @Autowired + private Test_tFoo_gBar_NoAck_NoCluster test_tFoo_gBar_noAck_noCluster; + @Autowired + private Test_tFoo_gFoo_Ack_NoCluster test_tFoo_gFoo_ack_noCluster; + @Autowired + private Test_tFoo_gFoo_NoAck_NoCluster test_tFoo_gFoo_noAck_noCluster; + + @Autowired + private JedisPooled jedisPooled; + + private Producer producerFoo; + private Producer producerBar; + private ObjectMapper objectMapper; + + @BeforeEach + void setUp() { + producerFoo = new TopicProducer(jedisPooled, "topicFoo"); + producerBar = new TopicProducer(jedisPooled, "topicFoo"); + objectMapper = new ObjectMapper(); + } + + @Test + void testConsumerInitialization() { + assertNotNull(test_tFoo_gBar_ack_noCluster, "Consumer should be initialized"); + assertNotNull(test_tFoo_gBar_noAck_noCluster, "Consumer should be initialized"); + assertNotNull(test_tFoo_gFoo_ack_noCluster, "Consumer should be initialized"); + assertNotNull(test_tFoo_gFoo_noAck_noCluster, "Consumer should be initialized"); + } + + @Test + void testProduce_tFoo_gFoo_noAck_noCluster() throws Exception { + TextData textData = TextData.of(); + textData.setId(1); + textData.setName("Test Name 1"); + textData.setDescription("Test Description 1"); + Map payload = objectMapper.convertValue(textData, new TypeReference<>() {}); + TopicEntryId topicEntryId = producerFoo.produce(payload); + System.out.println(topicEntryId); + assertNotNull(topicEntryId, "TopicEntryId should not be null"); + + TopicEntry topicEntry = test_tFoo_gFoo_noAck_noCluster.consume(); + assertNotNull(topicEntry, "TopicEntry should not be null"); + System.out.println(topicEntry); + assert topicEntry.getId().equals(topicEntryId); + + boolean ack = test_tFoo_gFoo_noAck_noCluster.acknowledge(topicEntry); + assertFalse(ack, "TopicEntry should not be acknowledged"); + } + + @Test + void testProduce_tFoo_gFoo_ack_noCluster() throws Exception { + TextData textData = TextData.of(); + textData.setId(1); + textData.setName("Test Name 1"); + textData.setDescription("Test Description 1"); + Map payload = objectMapper.convertValue(textData, new TypeReference<>() {}); + TopicEntryId topicEntryId = producerFoo.produce(payload); + System.out.println(topicEntryId); + assertNotNull(topicEntryId, "TopicEntryId should not be null"); + + TopicEntry topicEntry = test_tFoo_gFoo_ack_noCluster.consume(); + assertNotNull(topicEntry, "TopicEntry should not be null"); + System.out.println(topicEntry); + assert topicEntry.getId().equals(topicEntryId); + + boolean ack = test_tFoo_gFoo_ack_noCluster.acknowledge(topicEntry); + assertTrue(ack, "TopicEntry should be acknowledged"); + } + + /** + * FIXME: + * I produce a message on a topic named topicFoo. + * Then I have two consumers of type {@link com.redis.om.streams.command.serial.ConsumerGroup}, + * one belonging to groupFoo, and the other one belonging to groupBar. + * Both consume the exact same message from the exact same topic: + *
+     * Consumer for Group Foo:TopicEntry(streamName=__rsj:topic:stream:topicFoo:0, groupName=groupFoo, id=1750954394888-0-0, message={name=Test Name 1, description=Test Description 1, id=1})
+     * Consumer for Group Bar:TopicEntry(streamName=__rsj:topic:stream:topicFoo:0, groupName=groupBar, id=1750954394888-0-0, message={name=Test Name 1, description=Test Description 1, id=1})
+     * 
+ * + * Now, if I acknowledge the TopicEntry coming from the consumer of groupFoo + * with the consumer of groupBar, I would expect an error, an exception, something... + * Instead, the TopicEntry gets acknowledged despite the mismatch on the consumer group paternity. + */ +// @Test + void testAckWrongGroups() throws Exception { + TextData textData = TextData.of(); + textData.setId(1); + textData.setName("Test Name 1"); + textData.setDescription("Test Description 1"); + Map payload = objectMapper.convertValue(textData, new TypeReference<>() {}); + TopicEntryId topicEntryId = producerFoo.produce(payload); + System.out.println(topicEntryId); + assertNotNull(topicEntryId, "TopicEntryId should not be null"); + + TopicEntry topicEntryGroupFoo; + TopicEntry topicEntryGroupBar; + boolean ack; + topicEntryGroupFoo = test_tFoo_gFoo_ack_noCluster.consume(); + System.out.println("Consumer for Group Foo:" + topicEntryGroupFoo); + assertNotNull(topicEntryGroupFoo, "TopicEntry should not be null"); + assert topicEntryGroupFoo.getId().equals(topicEntryId); + + topicEntryGroupBar = test_tFoo_gBar_ack_noCluster.consume(); + System.out.println("Consumer for Group Bar:" + topicEntryGroupBar); + assertNotNull(topicEntryGroupBar, "TopicEntry should not be null"); + assert topicEntryGroupBar.getId().equals(topicEntryId); + + ack = test_tFoo_gFoo_ack_noCluster.acknowledge(topicEntryGroupBar); + assertFalse(ack, "TopicEntry should not be acknowledged because of consumer wrong group"); + ack = test_tFoo_gBar_ack_noCluster.acknowledge(topicEntryGroupFoo); + assertFalse(ack, "TopicEntry should not be acknowledged because of consumer wrong group"); + + ack = test_tFoo_gFoo_ack_noCluster.acknowledge(topicEntryGroupBar); + assertTrue(ack, "TopicEntry should be acknowledged because of consumer right group"); + ack = test_tFoo_gBar_ack_noCluster.acknowledge(topicEntryGroupFoo); + assertTrue(ack, "TopicEntry should be acknowledged because of consumer right group"); + } + + @Test + void testProduce_tFoo_gFoo_gFoo_ack_noAck_noCluster() throws Exception { + TextData textData = TextData.of(); + textData.setId(1); + textData.setName("Test Name 1"); + textData.setDescription("Test Description 1"); + Map payload = objectMapper.convertValue(textData, new TypeReference<>() {}); + TopicEntryId topicEntryId = producerFoo.produce(payload); + System.out.println(topicEntryId); + assertNotNull(topicEntryId, "TopicEntryId should not be null"); + + TopicEntry topicEntry; + boolean ack; + + topicEntry = test_tFoo_gFoo_ack_noCluster.consume(); + System.out.println("Consumer for Group Foo:" + topicEntry); + assertNotNull(topicEntry, "TopicEntry should not be null"); + ack = test_tFoo_gFoo_ack_noCluster.acknowledge(topicEntry); + assertTrue(ack, "TopicEntry should be acknowledged"); + + topicEntry = test_tFoo_gFoo_noAck_noCluster.consume(); + System.out.println("Consumer for Group Foo:" + topicEntry); + assertNull(topicEntry, "TopicEntry should be null"); + ack = test_tFoo_gFoo_noAck_noCluster.acknowledge(topicEntry); + assertFalse(ack, "TopicEntry should not be acknowledged"); + + } + +} \ No newline at end of file diff --git a/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/Test_tFoo_gBar_Ack_NoCluster.java b/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/Test_tFoo_gBar_Ack_NoCluster.java new file mode 100644 index 000000000..1ce3d33a6 --- /dev/null +++ b/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/Test_tFoo_gBar_Ack_NoCluster.java @@ -0,0 +1,6 @@ +package com.redis.om.streams.consumer; + +import com.redis.om.streams.annotation.RedisStreamConsumer; + +@RedisStreamConsumer(topicName = "topicFoo", groupName = "groupBar", autoAck = false, consumerName = "", cluster = false) +public class Test_tFoo_gBar_Ack_NoCluster extends RedisStreamsConsumer {} \ No newline at end of file diff --git a/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/Test_tFoo_gBar_NoAck_NoCluster.java b/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/Test_tFoo_gBar_NoAck_NoCluster.java new file mode 100644 index 000000000..12cc0e988 --- /dev/null +++ b/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/Test_tFoo_gBar_NoAck_NoCluster.java @@ -0,0 +1,6 @@ +package com.redis.om.streams.consumer; + +import com.redis.om.streams.annotation.RedisStreamConsumer; + +@RedisStreamConsumer(topicName = "topicFoo", groupName = "groupBar", autoAck = false, consumerName = "", cluster = false) +public class Test_tFoo_gBar_NoAck_NoCluster extends RedisStreamsConsumer {} \ No newline at end of file diff --git a/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/Test_tFoo_gFoo_Ack_NoCluster.java b/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/Test_tFoo_gFoo_Ack_NoCluster.java new file mode 100644 index 000000000..daa971764 --- /dev/null +++ b/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/Test_tFoo_gFoo_Ack_NoCluster.java @@ -0,0 +1,6 @@ +package com.redis.om.streams.consumer; + +import com.redis.om.streams.annotation.RedisStreamConsumer; + +@RedisStreamConsumer(topicName = "topicFoo", groupName = "groupFoo", autoAck = true, consumerName = "", cluster = false) +public class Test_tFoo_gFoo_Ack_NoCluster extends RedisStreamsConsumer {} \ No newline at end of file diff --git a/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/Test_tFoo_gFoo_NoAck_NoCluster.java b/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/Test_tFoo_gFoo_NoAck_NoCluster.java new file mode 100644 index 000000000..0714948e5 --- /dev/null +++ b/demos/roms-streams/src/test/java/com/redis/om/streams/consumer/Test_tFoo_gFoo_NoAck_NoCluster.java @@ -0,0 +1,6 @@ +package com.redis.om.streams.consumer; + +import com.redis.om.streams.annotation.RedisStreamConsumer; + +@RedisStreamConsumer(topicName = "topicFoo", groupName = "groupFoo", autoAck = false, consumerName = "", cluster = false) +public class Test_tFoo_gFoo_NoAck_NoCluster extends RedisStreamsConsumer {} \ No newline at end of file diff --git a/demos/roms-streams/src/test/resources/application-test.properties b/demos/roms-streams/src/test/resources/application-test.properties new file mode 100644 index 000000000..8d39c43cd --- /dev/null +++ b/demos/roms-streams/src/test/resources/application-test.properties @@ -0,0 +1,14 @@ +# Test Configuration +spring.application.name=redis-om-spring-streams-test + +# Redis Streams Configuration for Tests +redis.streams.fixed-delay=1000 + +# Logging Configuration for Tests +logging.level.com.redis.om.streams=DEBUG +logging.level.com.redis.om.streams.consumer=DEBUG +logging.level.com.redis.om.streams.config=DEBUG + +# Disable scheduling for tests to avoid interference +spring.task.scheduling.enabled=false +spring.main.allow-bean-definition-overriding=true \ No newline at end of file diff --git a/demos/roms-streams/src/test/resources/logback-test.xml b/demos/roms-streams/src/test/resources/logback-test.xml new file mode 100644 index 000000000..e389f02da --- /dev/null +++ b/demos/roms-streams/src/test/resources/logback-test.xml @@ -0,0 +1,11 @@ + + + + + %d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX} %-5level %logger{36} - [%thread] - %method - %msg%n + + + + + + \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 588d3b318..e3ab146d0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -20,3 +20,7 @@ djlStarterVersion = 0.26 djlVersion = 0.30.0 springAiVersion = 1.0.0 azureIdentityVersion = 1.15.4 + +lettuceCoreVersion = 6.7.1.RELEASE +lettucemodVersion = 4.2.1 +globVersion = 0.9.0 \ No newline at end of file diff --git a/redis-om-spring-ai/src/main/java/com/redis/om/spring/AIRedisConfiguration.java b/redis-om-spring-ai/src/main/java/com/redis/om/spring/AIRedisConfiguration.java index d70f34825..165d69491 100644 --- a/redis-om-spring-ai/src/main/java/com/redis/om/spring/AIRedisConfiguration.java +++ b/redis-om-spring-ai/src/main/java/com/redis/om/spring/AIRedisConfiguration.java @@ -404,8 +404,8 @@ public HuggingFaceTokenizer sentenceTokenizer(AIRedisOMProperties properties) { //noinspection ResultOfMethodCallIgnored InetAddress.getByName("www.huggingface.co").isReachable(5000); return HuggingFaceTokenizer.newInstance(properties.getDjl().getSentenceTokenizerModel(), options); - } catch (IOException ioe) { - logger.warn("Error retrieving default DJL sentence tokenizer"); + } catch (IOException | RuntimeException ioe) { + logger.warn("Error retrieving default DJL sentence tokenizer: " + ioe.getMessage()); return null; } } diff --git a/redis-om-spring-ai/src/main/java/com/redis/om/spring/AIRedisOMProperties.java b/redis-om-spring-ai/src/main/java/com/redis/om/spring/AIRedisOMProperties.java index e096347e1..456b2dfd0 100644 --- a/redis-om-spring-ai/src/main/java/com/redis/om/spring/AIRedisOMProperties.java +++ b/redis-om-spring-ai/src/main/java/com/redis/om/spring/AIRedisOMProperties.java @@ -232,10 +232,10 @@ public static class Djl { /** * Model identifier for sentence embeddings. - * Default is "sentence-transformers/msmarco-distilbert-dot-v5". + * Default is "https://huggingface.co/sentence-transformers/msmarco-distilbert-dot-v5". */ @NotNull - private String sentenceTokenizerModel = "sentence-transformers/msmarco-distilbert-dot-v5"; + private String sentenceTokenizerModel = "https://huggingface.co/sentence-transformers/msmarco-distilbert-dot-v5"; // face detection /** diff --git a/redis-om-spring/build.gradle b/redis-om-spring/build.gradle index db58f3bfd..337348e04 100644 --- a/redis-om-spring/build.gradle +++ b/redis-om-spring/build.gradle @@ -19,4 +19,16 @@ dependencies { compileOnly "javax.enterprise:cdi-api:${cdi}" implementation "com.google.auto.service:auto-service:${autoServiceVersion}" annotationProcessor "com.google.auto.service:auto-service:${autoServiceVersion}" + + + compileOnly 'org.projectlombok:lombok:1.18.38' + + annotationProcessor 'org.projectlombok:lombok:1.18.38' + + implementation 'org.springframework.session:spring-session-core' + implementation "io.lettuce:lettuce-core:$lettuceCoreVersion" + implementation "com.redis:lettucemod:$lettucemodVersion" + implementation "com.hrakaroo:glob:$globVersion" + implementation 'io.micrometer:micrometer-core:1.15.0' + implementation 'jakarta.annotation:jakarta.annotation-api:2.1.1' } diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/AbstractRedisCacheAccessor.java b/redis-om-spring/src/main/java/com/redis/om/cache/AbstractRedisCacheAccessor.java new file mode 100644 index 000000000..2d4ad8ff6 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/AbstractRedisCacheAccessor.java @@ -0,0 +1,177 @@ +package com.redis.om.cache; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; + +import com.redis.lettucemod.api.StatefulRedisModulesConnection; + +import io.lettuce.core.ScanArgs; +import io.lettuce.core.ScanIterator; + +/** + * Abstract base class for Redis cache accessors that provides common functionality + * for interacting with Redis as a cache. + */ +abstract class AbstractRedisCacheAccessor implements CacheAccessor { + + /** + * Default number of elements to scan in each iteration when cleaning cache entries. + */ + public static final long DEFAULT_SCAN_COUNT = 100; + + /** + * The Redis connection used for cache operations. + */ + protected final StatefulRedisModulesConnection connection; + + private long scanCount = DEFAULT_SCAN_COUNT; + + /** + * Creates a new {@link AbstractRedisCacheAccessor} with the given connection. + * + * @param connection the Redis connection, must not be {@literal null}. + */ + AbstractRedisCacheAccessor(StatefulRedisModulesConnection connection) { + Assert.notNull(connection, "Connection must not be null"); + this.connection = connection; + + } + + /** + * Converts a String key to a byte array using UTF-8 encoding. + * + * @param key the key to convert, never {@literal null}. + * @return the key as a byte array. + */ + private byte[] convertKey(String key) { + return key.getBytes(StandardCharsets.UTF_8); + } + + @Override + public Object get(String key, Duration ttl) { + return get(convertKey(key), ttl); + } + + /** + * Retrieves a cached object for the given key, optionally extending its TTL. + * + * @param key the cache key as byte array, never {@literal null}. + * @param ttl the time-to-live to set if the key exists, can be {@literal null}. + * @return the cached object or {@literal null} if not found. + */ + protected abstract Object get(byte[] key, Duration ttl); + + @Override + public void put(String key, Object value, Duration ttl) { + put(convertKey(key), value, ttl); + } + + /** + * Stores an object in the cache with the given key and TTL. + * + * @param key the cache key as byte array, never {@literal null}. + * @param value the object to cache, can be {@literal null}. + * @param ttl the time-to-live for the cached entry, can be {@literal null}. + */ + protected abstract void put(byte[] key, Object value, Duration ttl); + + @Override + public Object putIfAbsent(String key, Object value, Duration ttl) { + return putIfAbsent(convertKey(key), value, ttl); + } + + /** + * Stores an object in the cache only if the key does not already exist. + * + * @param key the cache key as byte array, never {@literal null}. + * @param value the object to cache, can be {@literal null}. + * @param ttl the time-to-live for the cached entry, can be {@literal null}. + * @return the previous value associated with the key, or {@literal null} if there was no value. + */ + protected abstract Object putIfAbsent(byte[] key, Object value, Duration ttl); + + /** + * Sets the number of elements to scan in each iteration when cleaning cache entries. + * + * @param count the number of elements to scan in each iteration + */ + public void setScanCount(long count) { + this.scanCount = count; + } + + @Override + public void remove(String key) { + Assert.notNull(key, "Key must not be null"); + delete(convertKey(key)); + } + + @Override + public long clean(String pattern) { + Assert.notNull(pattern, "Pattern must not be null"); + ScanArgs args = new ScanArgs(); + args.match(pattern); + args.limit(scanCount); + ScanIterator scanIterator = ScanIterator.scan(connection.sync(), args); + List keys = new ArrayList<>(); + long count = 0; + while (scanIterator.hasNext()) { + keys.add(scanIterator.next()); + if (keys.size() >= scanCount) { + count += delete(keys); + keys.clear(); + } + } + count += delete(keys); + return count; + } + + /** + * Checks if a key exists in Redis. + * + * @param key the key to check, never {@literal null}. + * @return {@literal true} if the key exists, {@literal false} otherwise. + */ + protected boolean exists(byte[] key) { + return connection.sync().exists(key) > 0; + } + + /** + * Deletes multiple keys from Redis. + * + * @param keys the list of keys to delete, can be empty. + * @return the number of keys that were deleted. + */ + private long delete(List keys) { + if (CollectionUtils.isEmpty(keys)) { + return 0; + } + return delete(keys.toArray(new byte[0][])); + } + + /** + * Deletes multiple keys from Redis. + * + * @param keys the array of keys to delete. + * @return the number of keys that were deleted. + */ + private long delete(byte[]... keys) { + return connection.sync().del(keys); + } + + /** + * Determines if the given TTL duration should be applied as an expiration. + * + * @param ttl the time-to-live duration, can be {@literal null} + * @return {@literal true} if the TTL is not null, not zero, and not negative + */ + protected boolean shouldExpireWithin(@Nullable Duration ttl) { + return ttl != null && !ttl.isZero() && !ttl.isNegative(); + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/CacheAccessor.java b/redis-om-spring/src/main/java/com/redis/om/cache/CacheAccessor.java new file mode 100644 index 000000000..b139ef56c --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/CacheAccessor.java @@ -0,0 +1,67 @@ +package com.redis.om.cache; + +import java.time.Duration; + +import org.springframework.lang.Nullable; + +/** + * {@link CacheAccessor} provides low-level access to Redis commands + * ({@code HSET, HGETALL, EXPIRE,...}) used for caching. + *

+ * The {@link CacheAccessor} may be shared by multiple cache implementations and + * is responsible for reading/writing binary data from/to Redis. The + * implementation honors potential cache lock flags that might be set. + */ +public interface CacheAccessor { + + /** + * Get the binary value representation from Redis stored for the given key and + * set the given {@link Duration TTL expiration} for the cache entry. + * + * @param key must not be {@literal null}. + * @param ttl {@link Duration} specifying the {@literal expiration timeout} for + * the cache entry. + * @return {@literal null} if key does not exist or has {@literal expired}. + */ + @Nullable + Object get(String key, @Nullable Duration ttl); + + /** + * Write the given key/value pair to Redis and set the expiration time if + * defined. + * + * @param key The key for the cache entry. Must not be {@literal null}. + * @param value The value stored for the key. Must not be {@literal null}. + * @param ttl Optional expiration time. Can be {@literal null}. + */ + void put(String key, Object value, @Nullable Duration ttl); + + /** + * Write the given value to Redis if the key does not already exist. + * + * @param key The key for the cache entry. Must not be {@literal null}. + * @param value The value stored for the key. Must not be {@literal null}. + * @param ttl Optional expiration time. Can be {@literal null}. + * @return {@literal null} if the value has been written, the value stored for + * the key if it already exists. + */ + @Nullable + Object putIfAbsent(String key, Object value, @Nullable Duration ttl); + + /** + * Remove the given key from Redis. + * + * @param key The key for the cache entry. Must not be {@literal null}. + */ + void remove(String key); + + /** + * Remove all keys following the given pattern. + * + * @param pattern The pattern for the keys to remove. Must not be + * {@literal null}. + * @return number of keys deleted + */ + long clean(String pattern); + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/KeyFunction.java b/redis-om-spring/src/main/java/com/redis/om/cache/KeyFunction.java new file mode 100644 index 000000000..c4237c0f4 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/KeyFunction.java @@ -0,0 +1,66 @@ +package com.redis.om.cache; + +import org.springframework.util.Assert; + +/** + * {@link KeyFunction} is a function for creating custom prefixes prepended to + * the actual {@literal key} stored in Redis. + * + */ +@FunctionalInterface +public interface KeyFunction { + + /** + * Default separator. + * + */ + String SEPARATOR = ":"; + + /** + * A pass-through implementation that returns the key unchanged without any prefix. + * This can be used when no key transformation is needed. + */ + KeyFunction PASSTHROUGH = (cache, key) -> key; + + /** + * Default {@link KeyFunction} scheme that prefixes cache keys with + * the {@link String name} of the cache followed by double colons. + * + * For example, a cache named {@literal myCache} will prefix all cache keys with + * {@literal myCache::}. + * + */ + KeyFunction SIMPLE = (cache, key) -> cache + SEPARATOR + key; + + /** + * Compute the {@link String prefix} for the actual {@literal cache key} stored + * in Redis. + * + * @param cache {@link String name} of the cache in which the key is stored; + * will never be {@literal null}. + * @param key the cache key to be processed; will never be {@literal null}. + * @return the computed {@literal cache key} stored in Redis; never + * {@literal null}. + */ + String compute(String cache, String key); + + /** + * Creates a {@link KeyFunction} scheme that prefixes cache keys with the given + * {@link String prefix}. + * + * The {@link String prefix} is prepended to the {@link String cacheName} + * followed by double colons. + * + * For example, a prefix {@literal redis-} with a cache named + * {@literal myCache} results in {@literal redis-myCache::}. + * + * @param prefix must not be {@literal null}. + * @return the default {@link KeyFunction} scheme. + * @since 2.3 + */ + static KeyFunction prefixed(String prefix) { + Assert.notNull(prefix, "Prefix must not be null"); + return (name, key) -> prefix + name + SEPARATOR + key; + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/LocalCacheAccessor.java b/redis-om-spring/src/main/java/com/redis/om/cache/LocalCacheAccessor.java new file mode 100644 index 000000000..58b3fcbbe --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/LocalCacheAccessor.java @@ -0,0 +1,128 @@ +package com.redis.om.cache; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.hrakaroo.glob.GlobPattern; +import com.hrakaroo.glob.MatchingEngine; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; + +/** + * A {@link CacheAccessor} implementation that provides a local in-memory cache in front of another + * {@link CacheAccessor} delegate. This implementation helps reduce network calls by caching values locally. + * It also provides metrics for cache hits, misses, and evictions. + */ +public class LocalCacheAccessor implements CacheAccessor { + + private static final String DESCRIPTION_GETS = "The number of times local cache lookup methods have returned a cached (hit) or uncached (newly loaded or null) value (miss)."; + private static final String DESCRIPTION_EVICTIONS = "The number of times the local cache was evicted"; + + private final Map map; + private final CacheAccessor delegate; + private final Counter hits; + private final Counter misses; + private final Counter evictions; + + /** + * Creates a new LocalCacheAccessor with the specified cache map, delegate, name, and registry. + * + * @param cache the map to use for local caching + * @param delegate the underlying CacheAccessor to delegate calls to + * @param name the name of the cache, used for metrics + * @param registry the meter registry for recording metrics + */ + public LocalCacheAccessor(Map cache, CacheAccessor delegate, String name, MeterRegistry registry) { + this.map = cache; + this.delegate = delegate; + this.hits = Counter.builder("cache.local.gets").tags("name", name).tag("result", "hit").description( + DESCRIPTION_GETS).register(registry); + this.misses = Counter.builder("cache.local.gets").tag("name", name).tag("result", "miss").description( + DESCRIPTION_GETS).register(registry); + this.evictions = Counter.builder("cache.local.evictions").tag("name", name).description(DESCRIPTION_EVICTIONS) + .register(registry); + } + + /** + * Returns the map used for local caching. + * + * @return the map containing locally cached values + */ + public Map getMap() { + return map; + } + + /** + * Returns the delegate CacheAccessor that this LocalCacheAccessor wraps. + * + * @return the underlying CacheAccessor delegate + */ + public CacheAccessor getDelegate() { + return delegate; + } + + @Override + public Object get(String key, Duration ttl) { + Object value = map.get(key); + if (value == null) { + misses.increment(); + value = delegate.get(key, ttl); + if (value != null) { + map.put(key, value); + } + } else { + hits.increment(); + } + return value; + } + + @Override + public void put(String key, Object value, Duration ttl) { + map.put(key, value); + delegate.put(key, value, ttl); + // Register interest in key + delegate.get(key, ttl); + } + + @Override + public Object putIfAbsent(String key, Object value, Duration ttl) { + if (!map.containsKey(key)) { + map.put(key, value); + } + Object result = delegate.putIfAbsent(key, value, ttl); + // Register interest in key + delegate.get(key, ttl); + return result; + } + + @Override + public void remove(String key) { + delegate.remove(key); + map.remove(key); + evictions.increment(); + } + + /** + * Removes an entry from the local cache without affecting the delegate cache. + * This is useful for selectively invalidating local cache entries. + * + * @param key the key to remove from the local cache + */ + public void evictLocal(String key) { + map.remove(key); + evictions.increment(); + } + + @Override + public long clean(String pattern) { + MatchingEngine engine = GlobPattern.compile(pattern); + List keys = map.keySet().stream().filter(engine::matches).collect(Collectors.toList()); + keys.forEach(map::remove); + evictions.increment(keys.size()); + return delegate.clean(pattern); + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/RedisCache.java b/redis-om-spring/src/main/java/com/redis/om/cache/RedisCache.java new file mode 100644 index 000000000..5249538b2 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/RedisCache.java @@ -0,0 +1,497 @@ +package com.redis.om.cache; + +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; + +import org.springframework.cache.Cache; +import org.springframework.cache.support.AbstractValueAdaptingCache; +import org.springframework.cache.support.NullValue; +import org.springframework.cache.support.SimpleValueWrapper; +import org.springframework.core.convert.ConversionFailedException; +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import org.springframework.util.ObjectUtils; +import org.springframework.util.ReflectionUtils; +import org.springframework.util.StringUtils; + +import com.redis.lettucemod.RedisModulesUtils; +import com.redis.lettucemod.api.StatefulRedisModulesConnection; +import com.redis.lettucemod.search.CreateOptions; +import com.redis.lettucemod.search.CreateOptions.DataType; +import com.redis.lettucemod.search.Field; +import com.redis.lettucemod.search.IndexInfo; + +import io.lettuce.core.AbstractRedisClient; +import io.lettuce.core.RedisCommandExecutionException; +import io.lettuce.core.TrackingArgs; +import io.lettuce.core.codec.ByteArrayCodec; +import io.lettuce.core.codec.StringCodec; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Timer; + +/** + * {@link AbstractValueAdaptingCache Cache} implementation using Redis as the underlying store for cache data. + *

+ * Use {@link RedisCacheManager} to create {@link RedisCache} instances. + */ +public class RedisCache extends AbstractValueAdaptingCache implements AutoCloseable { + + private static final String DESCRIPTION_GETS = "The number of times cache lookup methods have returned a cached (hit) or uncached (miss) value."; + + private static final String DESCRIPTION_PUTS = "The number of entries added to the cache."; + + private static final String DESCRIPTION_EVICTIONS = "The number of times the cache was evicted"; + + static final String CACHE_RETRIEVAL_UNSUPPORTED_OPERATION_EXCEPTION_MESSAGE = "The Redis driver configured with RedisCache through RedisCacheWriter does not support CompletableFuture-based retrieval"; + + private final String name; + + private final AbstractRedisClient redisClient; + + private final StatefulRedisModulesConnection connection; + + private final CacheAccessor accessor; + + private final RedisCacheConfiguration configuration; + + private final Counter hits; + + private final Counter misses; + + private final Counter puts; + + private final Counter evictions; + + private final Timer getLatency; + + private final Timer putLatency; + + private final Timer evictionLatency; + + /** + * Create a new {@link RedisCache} with the given {@link String name} and {@link RedisCacheConfiguration}, using the + * {@link CacheAccessor} to execute Redis commands supporting the cache operations. + * + * @param name {@link String name} for this {@link Cache}; must not be {@literal null}. + * @param client Default {@link AbstractRedisClient} to use if none specified in given RedisCacheConfiguration. + * @param configuration {@link RedisCacheConfiguration} applied to this {@link RedisCache} on creation; must not be + * {@literal null}. + * @throws IllegalArgumentException if either the given {@link CacheAccessor} or {@link RedisCacheConfiguration} are + * {@literal null} or the given {@link String} name for this {@link RedisCache} is + * {@literal null}. + */ + public RedisCache(String name, AbstractRedisClient client, RedisCacheConfiguration configuration) { + super(false); + Assert.notNull(name, "Name must not be null"); + this.name = name; + this.redisClient = configuration.getClient() == null ? client : configuration.getClient(); + this.connection = RedisModulesUtils.connection(redisClient, ByteArrayCodec.INSTANCE); + this.configuration = configuration; + this.accessor = accessor(); + this.hits = Counter.builder("cache.gets").tags("name", name).tag("result", "hit").description(DESCRIPTION_GETS) + .register(configuration.getMeterRegistry()); + this.misses = Counter.builder("cache.gets").tag("name", name).tag("result", "miss").description(DESCRIPTION_GETS) + .register(configuration.getMeterRegistry()); + this.puts = Counter.builder("cache.puts").tag("name", name).description(DESCRIPTION_PUTS).register(configuration + .getMeterRegistry()); + this.evictions = Counter.builder("cache.evictions").tag("name", name).description(DESCRIPTION_EVICTIONS).register( + configuration.getMeterRegistry()); + this.getLatency = Timer.builder("cache.gets.latency").tag("name", name).description("Cache gets").register( + configuration.getMeterRegistry()); + this.putLatency = Timer.builder("cache.puts.latency").tag("name", name).description("Cache puts").register( + configuration.getMeterRegistry()); + this.evictionLatency = Timer.builder("cache.evictions.latency").tag("name", name).description("Cache evictions") + .register(configuration.getMeterRegistry()); + if (configuration.isIndexEnabled()) { + createIndex(); + } + } + + @SuppressWarnings( + "unchecked" + ) + private void createIndex() { + CreateOptions.Builder createOptions = CreateOptions.builder(); + createOptions.on(indexType()); + createOptions.prefix(getName() + KeyFunction.SEPARATOR); + createOptions.noFields(); // Disable storing attribute bits for each term. It saves memory, but it does not allow + // filtering by specific attributes. + try (StatefulRedisModulesConnection connection = connection()) { + String indexName = indexName(); + try { + connection.sync().ftDropindex(indexName); + } catch (RedisCommandExecutionException e) { + // ignore as index might not exist + } + connection.sync().ftCreate(indexName, createOptions.build(), indexField()); + } + } + + private DataType indexType() { + switch (configuration.getRedisType()) { + case HASH: + return DataType.HASH; + case JSON: + return DataType.JSON; + default: + throw new IllegalArgumentException(String.format("Redis type %s not indexable", configuration.getRedisType())); + } + } + + private StatefulRedisModulesConnection connection() { + return RedisModulesUtils.connection(redisClient); + } + + /** + * Returns the number of documents in the index if enabled. + * + * @return the number of documents in the index or -1 if cache is not indexed + */ + public long getCount() { + if (configuration.isIndexEnabled()) { + try (StatefulRedisModulesConnection connection = connection()) { + IndexInfo cacheInfo = RedisModulesUtils.indexInfo(connection.sync().ftInfo(indexName())); + Double numDocs = cacheInfo.getNumDocs(); + if (numDocs != null) { + return numDocs.longValue(); + } + } + } + return -1; + } + + private String indexName() { + if (StringUtils.hasLength(configuration.getIndexName())) { + return configuration.getIndexName(); + } + return name + "Idx"; + } + + private Field indexField() { + if (configuration.getRedisType() == RedisType.JSON) { + return Field.tag("$._class").as("_class").build(); + } + return Field.tag("_class").build(); + } + + @Override + public void close() { + connection.close(); + } + + @SuppressWarnings( + "unchecked" + ) + private CacheAccessor accessor() { + CacheAccessor redisCacheAccessor = redisCacheAccessor(); + if (configuration.getLocalCache().isPresent()) { + connection.sync().clientTracking(TrackingArgs.Builder.enabled()); + LocalCacheAccessor localCacheAccessor = new LocalCacheAccessor(configuration.getLocalCache().get(), + redisCacheAccessor, name, configuration.getMeterRegistry()); + connection.addListener(msg -> { + if (msg.getType().equals("invalidate")) { + List content = msg.getContent(StringCodec.UTF8::decodeKey); + List keys = (List) content.get(1); + keys.forEach(localCacheAccessor::evictLocal); + } + }); + return localCacheAccessor; + } + return redisCacheAccessor; + } + + private CacheAccessor redisCacheAccessor() { + switch (configuration.getRedisType()) { + case JSON: + return new RedisJsonCacheAccessor(connection, configuration.getJsonMapper()); + case STRING: + return new RedisStringCacheAccessor(connection, configuration.getStringMapper()); + default: + return new RedisHashCacheAccessor(connection, configuration.getHashMapper()); + } + } + + @Override + public String getName() { + return this.name; + } + + @Override + public CacheAccessor getNativeCache() { + return accessor; + } + + @Override + @SuppressWarnings( + "unchecked" + ) + public V get(Object key, Callable valueLoader) { + ValueWrapper result = get(key); + if (result == null) { + return loadCacheValue(key, valueLoader); + } + return (V) result.get(); + } + + /** + * Loads the {@link Object} using the given {@link Callable valueLoader} and {@link #put(Object, Object) puts} the + * {@link Object loaded value} in the cache. + * + * @param {@link Class type} of the loaded {@link Object cache value}. + * @param key {@link Object key} mapped to the loaded {@link Object cache value}. + * @param valueLoader {@link Callable} object used to load the {@link Object value} for the given {@link Object key}. + * @return the loaded {@link Object value}. + */ + protected V loadCacheValue(Object key, Callable valueLoader) { + V value; + try { + value = valueLoader.call(); + } catch (Exception ex) { + throw new ValueRetrievalException(key, valueLoader, ex); + } + put(key, value); + return value; + } + + @Override + protected Object lookup(Object key) { + return getLatency.record(() -> { + Object result = accessor.get(cacheKey(key), lookupTtl(key)); + if (result == null) { + misses.increment(); + } else { + hits.increment(); + } + return result; + }); + } + + private Duration lookupTtl(Object key) { + if (configuration.isExpireOnGet()) { + return ttl(key); + } + return TtlFunction.NO_EXPIRATION; + } + + /** + * Returns the configuration used by this cache instance. + * + * @return the RedisCacheConfiguration instance that defines the behavior of this cache + */ + public RedisCacheConfiguration getConfiguration() { + return configuration; + } + + private Duration ttl(Object key) { + return ttl(key, null); + } + + private Duration ttl(Object key, @Nullable Object value) { + return configuration.getTtlFunction().getTtl(key, value); + } + + @Override + public void put(Object key, @Nullable Object value) { + putLatency.record(() -> { + Object cacheValue = processAndCheckValue(value); + String cacheKey = cacheKey(key); + accessor.put(cacheKey, cacheValue, ttl(key, value)); + puts.increment(); + }); + } + + @Override + public ValueWrapper putIfAbsent(Object key, @Nullable Object value) { + return putLatency.record(() -> { + Object cacheValue = preProcessCacheValue(value); + if (nullCacheValueIsNotAllowed(cacheValue)) { + return get(key); + } + Duration ttl = ttl(key, value); + String cacheKey = cacheKey(key); + Object result = accessor.putIfAbsent(cacheKey, cacheValue, ttl); + if (result == null) { + puts.increment(); + return null; + } + return new SimpleValueWrapper(fromStoreValue(result)); + }); + } + + @Override + public void clear() { + clear("*"); + } + + /** + * Clear keys that match the given {@link String keyPattern}. + *

+ * Useful when cache keys are formatted in a style where Redis patterns can be used for matching these. + * + * @param keyPattern {@link String pattern} used to match Redis keys to clear. + */ + public void clear(String keyPattern) { + long count = accessor.clean(cacheKey(keyPattern)); + evictions.increment(count); + } + + @Override + public void evict(Object key) { + evictionLatency.record(() -> { + accessor.remove(cacheKey(key)); + evictions.increment(); + }); + } + + @Override + public CompletableFuture retrieve(Object key) { + throw new UnsupportedOperationException(CACHE_RETRIEVAL_UNSUPPORTED_OPERATION_EXCEPTION_MESSAGE); + } + + @Override + public CompletableFuture retrieve(Object key, Supplier> valueLoader) { + throw new UnsupportedOperationException(CACHE_RETRIEVAL_UNSUPPORTED_OPERATION_EXCEPTION_MESSAGE); + } + + private Object processAndCheckValue(@Nullable Object value) { + Object cacheValue = preProcessCacheValue(value); + if (nullCacheValueIsNotAllowed(cacheValue)) { + String message = String.format( + "Cache '%s' does not allow 'null' values; Avoid storing null" + " via '@Cacheable(unless=\"#result == null\")' or configure RedisCache to allow 'null'" + " via RedisCacheConfiguration", + getName()); + throw new IllegalArgumentException(message); + } + return cacheValue; + } + + /** + * Customization hook called before serializing object. + * + * @param value can be {@literal null}. + * @return preprocessed value. Can be {@literal null}. + */ + @Nullable + protected Object preProcessCacheValue(@Nullable Object value) { + return value != null ? value : isAllowNullValues() ? NullValue.INSTANCE : null; + } + + /** + * Customization hook for creating cache key before it gets serialized. + * + * @param key will never be {@literal null}. + * @return never {@literal null}. + */ + protected String cacheKey(Object key) { + String convertedKey = convertKey(key); + return configuration.getKeyFunction().compute(getName(), convertedKey); + } + + /** + * Convert {@code key} to a {@link String} used in cache key creation. + * + * @param key will never be {@literal null}. + * @return never {@literal null}. + * @throws IllegalStateException if {@code key} cannot be converted to {@link String}. + */ + protected String convertKey(Object key) { + + if (key instanceof String stringKey) { + return stringKey; + } + + TypeDescriptor source = TypeDescriptor.forObject(key); + + ConversionService conversionService = configuration.getConversionService(); + + if (conversionService.canConvert(source, TypeDescriptor.valueOf(String.class))) { + try { + return conversionService.convert(key, String.class); + } catch (ConversionFailedException ex) { + + // May fail if the given key is a collection + if (isCollectionLikeOrMap(source)) { + return convertCollectionLikeOrMapKey(key, source); + } + + throw ex; + } + } + + if (hasToStringMethod(key)) { + return key.toString(); + } + + String message = String.format( + "Cannot convert cache key %s to String; Please register a suitable Converter" + " via 'RedisCacheConfiguration.configureKeyConverters(...)' or override '%s.toString()'", + source, key.getClass().getName()); + + throw new IllegalStateException(message); + } + + @Nullable + private Object nullSafeDeserializedStoreValue(@Nullable Object value) { + return value != null ? fromStoreValue(value) : null; + } + + private boolean hasToStringMethod(Object target) { + return hasToStringMethod(target.getClass()); + } + + private boolean hasToStringMethod(Class type) { + + Method toString = ReflectionUtils.findMethod(type, "toString"); + + return toString != null && !Object.class.equals(toString.getDeclaringClass()); + } + + private boolean isCollectionLikeOrMap(TypeDescriptor source) { + return source.isArray() || source.isCollection() || source.isMap(); + } + + private String convertCollectionLikeOrMapKey(Object key, TypeDescriptor source) { + + if (source.isMap()) { + + int count = 0; + + StringBuilder target = new StringBuilder("{"); + + for (Entry entry : ((Map) key).entrySet()) { + target.append(convertKey(entry.getKey())).append("=").append(convertKey(entry.getValue())); + target.append(++count > 1 ? ", " : ""); + } + + target.append("}"); + + return target.toString(); + + } else if (source.isCollection() || source.isArray()) { + + StringJoiner stringJoiner = new StringJoiner(","); + + Collection collection = source.isCollection() ? + (Collection) key : + Arrays.asList(ObjectUtils.toObjectArray(key)); + + for (Object collectedKey : collection) { + stringJoiner.add(convertKey(collectedKey)); + } + + return "[" + stringJoiner + "]"; + } + + throw new IllegalArgumentException(String.format("Cannot convert cache key [%s] to String", key)); + } + + private boolean nullCacheValueIsNotAllowed(@Nullable Object cacheValue) { + return cacheValue == null && !isAllowNullValues(); + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/RedisCacheConfiguration.java b/redis-om-spring/src/main/java/com/redis/om/cache/RedisCacheConfiguration.java new file mode 100644 index 000000000..853f52bd3 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/RedisCacheConfiguration.java @@ -0,0 +1,540 @@ +package com.redis.om.cache; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Consumer; + +import org.springframework.cache.Cache; +import org.springframework.cache.interceptor.SimpleKey; +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.converter.Converter; +import org.springframework.core.convert.converter.ConverterRegistry; +import org.springframework.format.support.DefaultFormattingConversionService; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +import com.redis.om.cache.common.RedisHashMapper; +import com.redis.om.cache.common.RedisStringMapper; +import com.redis.om.cache.common.mapping.GenericJackson2JsonMapper; +import com.redis.om.cache.common.mapping.JdkSerializationStringMapper; +import com.redis.om.cache.common.mapping.ObjectHashMapper; + +import io.lettuce.core.AbstractRedisClient; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Metrics; + +/** + * Configuration class for Redis Cache settings. + * This class provides a fluent API for configuring various aspects of Redis caching, + * including key generation, TTL, serialization, and Redis data types. + */ +public class RedisCacheConfiguration implements Cloneable { + + /** + * Default TTL function that creates persistent entries (no expiration). + */ + public static final TtlFunction DEFAULT_TTL_FUNCTION = TtlFunction.PERSISTENT; + + /** + * Default key function that uses a simple key generation strategy. + */ + public static final KeyFunction DEFAULT_KEY_FUNCTION = KeyFunction.SIMPLE; + + /** + * Default Redis data type (HASH) used for storing cache entries. + */ + public static final RedisType DEFAULT_REDIS_TYPE = RedisType.HASH; + + /** + * Default mapper for converting objects to JSON format. + */ + public static final RedisStringMapper DEFAULT_JSON_MAPPER = jsonStringMapper(); + + /** + * Default mapper for converting objects to String format using Java serialization. + */ + public static final RedisStringMapper DEFAULT_STRING_MAPPER = javaStringMapper(); + + /** + * Default mapper for converting objects to Redis Hash entries. + */ + public static final RedisHashMapper DEFAULT_HASH_MAPPER = new ObjectHashMapper(); + + /** + * Default expireOnGet value + */ + public static final boolean DEFAULT_EXPIRE_ON_GET = true; + + private AbstractRedisClient client; + + private RedisType redisType = DEFAULT_REDIS_TYPE; + + private ConversionService conversionService = defaultConversionService(); + + private KeyFunction keyFunction = DEFAULT_KEY_FUNCTION; + + private TtlFunction ttlFunction = DEFAULT_TTL_FUNCTION; + + private boolean expireOnGet = DEFAULT_EXPIRE_ON_GET; + + private RedisHashMapper hashMapper = DEFAULT_HASH_MAPPER; + + private RedisStringMapper stringMapper = DEFAULT_STRING_MAPPER; + + private RedisStringMapper jsonMapper = DEFAULT_JSON_MAPPER; + + private Optional> localCache = Optional.empty(); + + private MeterRegistry meterRegistry = Metrics.globalRegistry; + + private boolean indexEnabled; + + private String indexName; + + static JdkSerializationStringMapper javaStringMapper() { + return java(null); + } + + static JdkSerializationStringMapper java(@Nullable ClassLoader classLoader) { + return new JdkSerializationStringMapper(classLoader); + } + + static GenericJackson2JsonMapper jsonStringMapper() { + return new GenericJackson2JsonMapper(); + } + + /** + * Creates a default configuration for Redis caching. + * + * @return a new instance of {@link RedisCacheConfiguration} with default settings + */ + public static RedisCacheConfiguration defaultConfig() { + return new RedisCacheConfiguration(); + } + + /** + * Prefix the {@link RedisCache#getName() cache name} with the given value.
+ * The generated cache key will be: {@code prefix + cache name + "::" + cache entry key}. + * + * @param prefix the prefix to prepend to the cache name. + * @return new {@link RedisCacheConfiguration}. + * @see KeyFunction#prefixed(String) + */ + public RedisCacheConfiguration keyPrefix(String prefix) { + return keyFunction(KeyFunction.prefixed(prefix)); + } + + /** + * Use the given {@link KeyFunction} to compute the Redis {@literal key} given the {@literal cache name} and + * {@literal key} + * as function inputs. + * + * @param function must not be {@literal null}. + * @return new {@link RedisCacheConfiguration}. + * @see KeyFunction + */ + RedisCacheConfiguration keyFunction(KeyFunction function) { + Assert.notNull(keyFunction, "Function used to compute cache key must not be null"); + return clone(config -> config.keyFunction = function); + } + + /** + * Configure the Redis data type to use for the cache. + * + * @param type the Redis data type to use + * @return new {@link RedisCacheConfiguration} with the specified Redis data type + */ + public RedisCacheConfiguration redisType(RedisType type) { + return clone(c -> c.redisType = type); + } + + /** + * Configure the cache to use Redis JSON data type. + * + * @return new {@link RedisCacheConfiguration} with Redis JSON data type + */ + public RedisCacheConfiguration json() { + return redisType(RedisType.JSON); + } + + /** + * Configure the cache to use Redis Hash data type. + * + * @return new {@link RedisCacheConfiguration} with Redis Hash data type + */ + public RedisCacheConfiguration hash() { + return redisType(RedisType.HASH); + } + + /** + * Configure the cache to use Redis String data type. + * + * @return new {@link RedisCacheConfiguration} with Redis String data type + */ + public RedisCacheConfiguration string() { + return redisType(RedisType.STRING); + } + + @Override + public int hashCode() { + return Objects.hash(client, conversionService, hashMapper, jsonMapper, keyFunction, localCache, meterRegistry, + redisType, stringMapper, ttlFunction, expireOnGet); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RedisCacheConfiguration other = (RedisCacheConfiguration) obj; + return Objects.equals(client, other.client) && Objects.equals(conversionService, other.conversionService) && Objects + .equals(hashMapper, other.hashMapper) && Objects.equals(jsonMapper, other.jsonMapper) && Objects.equals( + keyFunction, other.keyFunction) && Objects.equals(localCache, other.localCache) && Objects.equals( + meterRegistry, other.meterRegistry) && redisType == other.redisType && Objects.equals(stringMapper, + other.stringMapper) && Objects.equals(ttlFunction, + other.ttlFunction) && expireOnGet == other.expireOnGet; + } + + /** + * Disable using cache key prefixes.
+ * NOTE: {@link Cache#clear()} might result in unintended removal of {@literal key}s in Redis. Make + * sure to + * use a dedicated Redis instance when disabling prefixes. + * + * @return new {@link RedisCacheConfiguration}. + */ + public RedisCacheConfiguration disableKeyPrefix() { + return clone(config -> config.keyFunction = KeyFunction.PASSTHROUGH); + } + + private RedisCacheConfiguration clone(Consumer consumer) { + RedisCacheConfiguration config = clone(); + consumer.accept(config); + return config; + } + + @Override + public RedisCacheConfiguration clone() { + RedisCacheConfiguration config = new RedisCacheConfiguration(); + config.client = this.client; + config.conversionService = this.conversionService; + config.hashMapper = this.hashMapper; + config.jsonMapper = this.jsonMapper; + config.keyFunction = this.keyFunction; + config.localCache = this.localCache; + config.meterRegistry = this.meterRegistry; + config.redisType = this.redisType; + config.stringMapper = this.stringMapper; + config.ttlFunction = this.ttlFunction; + config.expireOnGet = this.expireOnGet; + config.indexEnabled = this.indexEnabled; + config.indexName = this.indexName; + return config; + } + + /** + * Configure the meter registry to use for metrics collection. + * + * @param registry the meter registry to use + * @return new {@link RedisCacheConfiguration} with the configured meter registry + */ + public RedisCacheConfiguration meterRegistry(MeterRegistry registry) { + return clone(config -> config.meterRegistry = registry); + } + + /** + * Enable or disable indexing for the cache. + * + * @param enable whether to enable indexing + * @return new {@link RedisCacheConfiguration} with indexing enabled or disabled + */ + public RedisCacheConfiguration indexEnabled(boolean enable) { + return clone(config -> config.indexEnabled = enable); + } + + /** + * Configure the name of the index to use. + * + * @param index the name of the index + * @return new {@link RedisCacheConfiguration} with the configured index name + */ + public RedisCacheConfiguration indexName(String index) { + return clone(config -> config.indexName = index); + } + + /** + * Set the ttl to apply for cache entries. Use {@link Duration#ZERO} to declare an eternal cache. + * + * @param ttl must not be {@literal null}. + * @return new {@link RedisCacheConfiguration}. + */ + public RedisCacheConfiguration entryTtl(Duration ttl) { + Assert.notNull(ttl, "TTL duration must not be null"); + return entryTtl(TtlFunction.just(ttl)); + } + + /** + * + * @param enable if true, will set expiration timeout when key is read + * @return new {@link RedisCacheConfiguration} with the configured expireOnGet + */ + public RedisCacheConfiguration expireOnGet(boolean enable) { + return clone(config -> config.expireOnGet = enable); + } + + /** + * Configure a local cache to use alongside Redis. + * + * @param cache the map to use as local cache + * @return new {@link RedisCacheConfiguration} with the configured local cache + */ + public RedisCacheConfiguration localCache(Map cache) { + return clone(config -> config.localCache = Optional.of(cache)); + } + + /** + * Set the {@link TtlFunction TTL function} to compute the time to live for cache entries. + * + * @param function the {@link TtlFunction} to compute the time to live for cache entries, must not be {@literal null}. + * @return new {@link RedisCacheConfiguration}. + */ + public RedisCacheConfiguration entryTtl(TtlFunction function) { + Assert.notNull(function, "TtlFunction must not be null"); + return clone(config -> config.ttlFunction = function); + } + + /** + * Define the {@link ConversionService} used for cache key to {@link String} conversion. + * + * @param conversionService must not be {@literal null}. + * @return new {@link RedisCacheConfiguration}. + */ + public RedisCacheConfiguration conversionService(ConversionService conversionService) { + Assert.notNull(conversionService, "ConversionService must not be null"); + return clone(config -> config.conversionService = conversionService); + } + + /** + * Configure the Redis client to use for cache operations. + * + * @param client the Redis client to use + * @return new {@link RedisCacheConfiguration} with the configured Redis client + */ + public RedisCacheConfiguration client(AbstractRedisClient client) { + return clone(config -> config.client = client); + } + + /** + * Configure the mapper to use for converting objects to Redis Hash values. + * + * @param mapper the mapper to use for converting objects to Redis Hash values + * @return new {@link RedisCacheConfiguration} with the configured hash mapper + */ + public RedisCacheConfiguration hashMapper(RedisHashMapper mapper) { + return clone(config -> config.hashMapper = mapper); + } + + /** + * Configure the mapper to use for converting objects to Redis String values. + * + * @param mapper the mapper to use for converting objects to Redis String values + * @return new {@link RedisCacheConfiguration} with the configured string mapper + */ + public RedisCacheConfiguration stringMapper(RedisStringMapper mapper) { + return clone(config -> config.stringMapper = mapper); + } + + /** + * Configure the mapper to use for converting objects to JSON values. + * + * @param mapper the mapper to use for converting objects to JSON values + * @return new {@link RedisCacheConfiguration} with the configured JSON mapper + */ + public RedisCacheConfiguration jsonMapper(RedisStringMapper mapper) { + return clone(config -> config.jsonMapper = mapper); + } + + /** + * Returns the Redis data type configured for this cache. + * + * @return the configured {@link RedisType} + */ + public RedisType getRedisType() { + return redisType; + } + + /** + * Returns the local cache configuration if one is configured. + * + * @return an {@link Optional} containing the local cache map, or empty if no local cache is configured + */ + public Optional> getLocalCache() { + return localCache; + } + + /** + * @return The {@link ConversionService} used for cache key to {@link String} conversion. Never {@literal null}. + */ + public ConversionService getConversionService() { + return this.conversionService; + } + + /** + * Returns the Redis client configured for this cache. + * + * @return the configured Redis client + */ + public AbstractRedisClient getClient() { + return client; + } + + /** + * Returns the mapper used for converting objects to Redis Hash entries. + * + * @return the configured {@link RedisHashMapper} + */ + public RedisHashMapper getHashMapper() { + return hashMapper; + } + + /** + * Returns the mapper used for converting objects to JSON format. + * + * @return the configured JSON {@link RedisStringMapper} + */ + public RedisStringMapper getJsonMapper() { + return jsonMapper; + } + + /** + * Returns the mapper used for converting objects to String format. + * + * @return the configured {@link RedisStringMapper} for String serialization + */ + public RedisStringMapper getStringMapper() { + return stringMapper; + } + + /** + * Gets the {@link TtlFunction} used to compute a cache key {@literal time-to-live (TTL) expiration}. + * + * @return the {@link TtlFunction} used to compute expiration time (TTL) for cache entries; never {@literal null}. + */ + public TtlFunction getTtlFunction() { + return this.ttlFunction; + } + + /** + * + * @return true if expireOnGet is enabled, false otherwise + */ + public boolean isExpireOnGet() { + return expireOnGet; + } + + /** + * Returns the {@link KeyFunction} used to compute a cache key. + * + * @return the {@link KeyFunction} used to compute keys for cache entries; never {@literal null}. + */ + public KeyFunction getKeyFunction() { + return keyFunction; + } + + /** + * Returns the meter registry used for metrics collection. + * + * @return the configured {@link MeterRegistry} + */ + public MeterRegistry getMeterRegistry() { + return meterRegistry; + } + + /** + * Returns whether indexing is enabled for this cache. + * + * @return {@code true} if indexing is enabled, {@code false} otherwise + */ + public boolean isIndexEnabled() { + return indexEnabled; + } + + /** + * Returns the name of the index configured for this cache. + * + * @return the configured index name + */ + public String getIndexName() { + return indexName; + } + + /** + * Adds a {@link Converter} to extract the {@link String} representation of a {@literal cache key} if no suitable + * {@link Object#toString()} method is present. + * + * @param cacheKeyConverter {@link Converter} used to convert a {@literal cache key} into a {@link String}. + * @throws IllegalStateException if {@link #getConversionService()} does not allow {@link Converter} registration. + * @see Converter + */ + public void addCacheKeyConverter(Converter cacheKeyConverter) { + configureKeyConverters(it -> it.addConverter(cacheKeyConverter)); + } + + /** + * Configure the underlying {@link ConversionService} used to extract the {@literal cache key}. + * + * @param registryConsumer {@link Consumer} used to register a {@link Converter} with the configured + * {@link ConverterRegistry}; never {@literal null}. + * @throws IllegalStateException if {@link #getConversionService()} does not allow {@link Converter} registration. + * @see ConverterRegistry + */ + public void configureKeyConverters(Consumer registryConsumer) { + if (!(getConversionService() instanceof ConverterRegistry)) { + + String message = "'%s' returned by getConversionService() does not allow Converter registration;" + " Please make sure to provide a ConversionService that implements ConverterRegistry"; + + throw new IllegalStateException(String.format(message, getConversionService().getClass().getName())); + } + registryConsumer.accept((ConverterRegistry) getConversionService()); + } + + /** + * Registers default cache {@link Converter key converters}. + * + * The following converters get registered: + * + *

    + *
  • {@link String} to byte[] using UTF-8 encoding.
  • + *
  • {@link SimpleKey} to {@link String}
  • + *
+ * + * @param registry {@link ConverterRegistry} in which the {@link Converter key converters} are registered; must not be + * {@literal null}. + * @see ConverterRegistry + */ + public static void registerDefaultConverters(ConverterRegistry registry) { + + Assert.notNull(registry, "ConverterRegistry must not be null"); + + registry.addConverter(String.class, byte[].class, source -> source.getBytes(StandardCharsets.UTF_8)); + registry.addConverter(SimpleKey.class, String.class, SimpleKey::toString); + } + + /** + * Creates a default conversion service with the standard converters registered. + * + * @return a new {@link ConversionService} with default converters + */ + public static ConversionService defaultConversionService() { + DefaultFormattingConversionService conversionService = new DefaultFormattingConversionService(); + registerDefaultConverters(conversionService); + return conversionService; + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/RedisCacheManager.java b/redis-om-spring/src/main/java/com/redis/om/cache/RedisCacheManager.java new file mode 100644 index 000000000..06d4138ab --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/RedisCacheManager.java @@ -0,0 +1,554 @@ +package com.redis.om.cache; + +import java.util.*; + +import org.springframework.cache.Cache; +import org.springframework.cache.CacheManager; +import org.springframework.cache.transaction.AbstractTransactionSupportingCacheManager; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +import io.lettuce.core.AbstractRedisClient; + +/** + * {@link CacheManager} implementation for Redis backed by {@link RedisCache}. + *

+ * This {@link CacheManager} creates {@link Cache caches} on first write, by default. Empty {@link Cache caches} are not + * visible + * in Redis due to how Redis represents empty data structures. + *

+ * {@link Cache Caches} requiring a different {@link RedisCacheConfiguration cache configuration} than the default cache + * configuration} can be specified via {@link RedisCacheManagerBuilder#initialConfigurations(Map)} or individually using + * {@link RedisCacheManagerBuilder#configuration(String, RedisCacheConfiguration)}. + * + * @see AbstractTransactionSupportingCacheManager + */ +public class RedisCacheManager extends AbstractTransactionSupportingCacheManager { + + /** + * Default setting for allowing runtime cache creation. + */ + protected static final boolean DEFAULT_ALLOW_RUNTIME_CACHE_CREATION = true; + + private final boolean allowRuntimeCacheCreation; + + private final RedisCacheConfiguration defaultCacheConfiguration; + + private final AbstractRedisClient client; + + private final Map initialCacheConfiguration; + + /** + * Creates a new {@link RedisCacheManager} initialized with the given {@link AbstractRedisClient} and default + * {@link RedisCacheConfiguration}. + *

+ * Allows {@link RedisCache cache} creation at runtime. + * + * @param client {@link AbstractRedisClient} used to perform {@link RedisCache} operations by + * executing appropriate Redis + * commands; must not be {@literal null}. + * @param defaultCacheConfiguration {@link RedisCacheConfiguration} applied to new {@link RedisCache Redis caches} by + * default when no cache-specific {@link RedisCacheConfiguration} is provided; must + * not be {@literal null}. + * @throws IllegalArgumentException if either the given {@link AbstractRedisClient} or {@link RedisCacheConfiguration} + * are + * {@literal null}. + */ + public RedisCacheManager(AbstractRedisClient client, RedisCacheConfiguration defaultCacheConfiguration) { + this(client, defaultCacheConfiguration, DEFAULT_ALLOW_RUNTIME_CACHE_CREATION); + } + + /** + * Creates a new {@link RedisCacheManager} initialized with the given {@link AbstractRedisClient} and default + * {@link RedisCacheConfiguration} along with whether to allow cache creation at runtime. + * + * @param client {@link AbstractRedisClient} used to perform {@link RedisCache} operations by + * executing appropriate Redis + * commands; must not be {@literal null}. + * @param defaultCacheConfiguration {@link RedisCacheConfiguration} applied to new {@link RedisCache Redis caches} by + * default when no cache-specific {@link RedisCacheConfiguration} is provided; must + * not be {@literal null}. + * @param allowRuntimeCacheCreation boolean specifying whether to allow creation of undeclared caches at runtime; + * {@literal true} by default. + * @throws IllegalArgumentException if either the given {@link AbstractRedisClient} or {@link RedisCacheConfiguration} + * are + * {@literal null}. + */ + private RedisCacheManager(AbstractRedisClient client, RedisCacheConfiguration defaultCacheConfiguration, + boolean allowRuntimeCacheCreation) { + Assert.notNull(defaultCacheConfiguration, "DefaultCacheConfiguration must not be null"); + this.defaultCacheConfiguration = defaultCacheConfiguration; + Assert.notNull(client, "Client must not be null"); + this.client = client; + this.initialCacheConfiguration = new LinkedHashMap<>(); + this.allowRuntimeCacheCreation = allowRuntimeCacheCreation; + } + + /** + * Creates a new {@link RedisCacheManager} initialized with the given {@link AbstractRedisClient} and a default + * {@link RedisCacheConfiguration} along with an optional, initial set of {@link String cache names} used to create + * {@link RedisCache Redis caches} on startup. + *

+ * Allows {@link RedisCache cache} creation at runtime. + * + * @param client {@link AbstractRedisClient} used to perform {@link RedisCache} operations by + * executing appropriate Redis + * commands; must not be {@literal null}. + * @param defaultCacheConfiguration {@link RedisCacheConfiguration} applied to new {@link RedisCache Redis caches} by + * default when no cache-specific {@link RedisCacheConfiguration} is provided; must + * not be {@literal null}. + * @param initialCacheNames optional set of {@link String cache names} used to create {@link RedisCache Redis + * caches} on + * startup. The default {@link RedisCacheConfiguration} will be applied to each + * cache. + * @throws IllegalArgumentException if either the given {@link AbstractRedisClient} or {@link RedisCacheConfiguration} + * are + * {@literal null}. + */ + public RedisCacheManager(AbstractRedisClient client, RedisCacheConfiguration defaultCacheConfiguration, + String... initialCacheNames) { + + this(client, defaultCacheConfiguration, DEFAULT_ALLOW_RUNTIME_CACHE_CREATION, initialCacheNames); + } + + /** + * Creates a new {@link RedisCacheManager} initialized with the given {@link AbstractRedisClient} and default + * {@link RedisCacheConfiguration} along with whether to allow cache creation at runtime. + *

+ * Additionally, the optional, initial set of {@link String cache names} will be used to create {@link RedisCache + * Redis + * caches} on startup. + * + * @param client {@link AbstractRedisClient} used to perform {@link RedisCache} operations by + * executing appropriate Redis + * commands; must not be {@literal null}. + * @param defaultCacheConfiguration {@link RedisCacheConfiguration} applied to new {@link RedisCache Redis caches} by + * default when no cache-specific {@link RedisCacheConfiguration} is provided; must + * not be {@literal null}. + * @param allowRuntimeCacheCreation boolean specifying whether to allow creation of undeclared caches at runtime; + * {@literal true} by default. + * @param initialCacheNames optional set of {@link String cache names} used to create {@link RedisCache Redis + * caches} on + * startup. The default {@link RedisCacheConfiguration} will be applied to each + * cache. + * @throws IllegalArgumentException if either the given {@link AbstractRedisClient} or {@link RedisCacheConfiguration} + * are + * {@literal null}. + */ + public RedisCacheManager(AbstractRedisClient client, RedisCacheConfiguration defaultCacheConfiguration, + boolean allowRuntimeCacheCreation, String... initialCacheNames) { + + this(client, defaultCacheConfiguration, allowRuntimeCacheCreation); + + for (String cacheName : initialCacheNames) { + this.initialCacheConfiguration.put(cacheName, defaultCacheConfiguration); + } + } + + /** + * Creates new {@link RedisCacheManager} using given {@link AbstractRedisClient} and default + * {@link RedisCacheConfiguration}. + *

+ * Additionally, an initial {@link RedisCache} will be created and configured using the associated + * {@link RedisCacheConfiguration} for each {@link String named} {@link RedisCache} in the given {@link Map}. + *

+ * Allows {@link RedisCache cache} creation at runtime. + * + * @param client {@link AbstractRedisClient} used to perform {@link RedisCache} operations by + * executing appropriate Redis + * commands; must not be {@literal null}. + * @param defaultCacheConfiguration {@link RedisCacheConfiguration} applied to new {@link RedisCache Redis caches} by + * default when no cache-specific {@link RedisCacheConfiguration} is provided; must + * not be {@literal null}. + * @param initialCacheConfigurations {@link Map} of declared, known {@link String cache names} along with associated + * {@link RedisCacheConfiguration} used to create and configure {@link RedisCache + * Reds caches} on startup; must not + * be {@literal null}. + * @throws IllegalArgumentException if either the given {@link AbstractRedisClient} or {@link RedisCacheConfiguration} + * are + * {@literal null}. + */ + public RedisCacheManager(AbstractRedisClient client, RedisCacheConfiguration defaultCacheConfiguration, + Map initialCacheConfigurations) { + + this(client, defaultCacheConfiguration, DEFAULT_ALLOW_RUNTIME_CACHE_CREATION, initialCacheConfigurations); + } + + /** + * Creates a new {@link RedisCacheManager} initialized with the given {@link AbstractRedisClient} and a default + * {@link RedisCacheConfiguration}, and whether to allow {@link RedisCache} creation at runtime. + *

+ * Additionally, an initial {@link RedisCache} will be created and configured using the associated + * {@link RedisCacheConfiguration} for each {@link String named} {@link RedisCache} in the given {@link Map}. + * + * @param client {@link AbstractRedisClient} used to perform {@link RedisCache} operations by + * executing appropriate Redis + * commands; must not be {@literal null}. + * @param defaultCacheConfiguration {@link RedisCacheConfiguration} applied to new {@link RedisCache Redis caches} by + * default when no cache-specific {@link RedisCacheConfiguration} is provided; must + * not be {@literal null}. + * @param allowRuntimeCacheCreation boolean specifying whether to allow creation of undeclared caches at runtime; + * {@literal true} by default. + * @param initialCacheConfigurations {@link Map} of declared, known {@link String cache names} along with the + * associated + * {@link RedisCacheConfiguration} used to create and configure {@link RedisCache + * Redis caches} on startup; must not + * be {@literal null}. + * @throws IllegalArgumentException if either the given {@link AbstractRedisClient} or {@link RedisCacheConfiguration} + * are + * {@literal null}. + */ + public RedisCacheManager(AbstractRedisClient client, RedisCacheConfiguration defaultCacheConfiguration, + boolean allowRuntimeCacheCreation, Map initialCacheConfigurations) { + + this(client, defaultCacheConfiguration, allowRuntimeCacheCreation); + + Assert.notNull(initialCacheConfigurations, "InitialCacheConfigurations must not be null"); + + this.initialCacheConfiguration.putAll(initialCacheConfigurations); + } + + /** + * Factory method returning a {@literal Builder} used to construct and configure a {@link RedisCacheManager}. + * + * @return new {@link RedisCacheManagerBuilder}. + */ + public static RedisCacheManagerBuilder builder() { + return new RedisCacheManagerBuilder(); + } + + /** + * Factory method returning a {@literal Builder} used to construct and configure a {@link RedisCacheManager} + * initialized + * with the given {@link AbstractRedisClient}. + * + * @param client {@link AbstractRedisClient} used by the {@link RedisCacheManager} to acquire connections to Redis + * when + * performing {@link RedisCache} operations; must not be {@literal null}. + * @return new {@link RedisCacheManagerBuilder}. + * @throws IllegalArgumentException if the given {@link AbstractRedisClient} is {@literal null}. + */ + public static RedisCacheManagerBuilder builder(AbstractRedisClient client) { + + Assert.notNull(client, "Client must not be null"); + + return RedisCacheManagerBuilder.fromClient(client); + } + + /** + * Factory method used to construct a new {@link RedisCacheManager} initialized with the given + * {@link AbstractRedisClient} + * and using defaults for caching. + *

+ *
initial caches
+ *
none
+ *
in-flight cache creation
+ *
enabled
+ *
+ * + * @param client {@link AbstractRedisClient} used by the {@link RedisCacheManager} to acquire connections to Redis + * when + * performing {@link RedisCache} operations; must not be {@literal null}. + * @return new {@link RedisCacheManager}. + * @throws IllegalArgumentException if the given {@link AbstractRedisClient} is {@literal null}. + */ + public static RedisCacheManager create(AbstractRedisClient client) { + Assert.notNull(client, "Client must not be null"); + return new RedisCacheManager(client, new RedisCacheConfiguration()); + } + + /** + * Determines whether {@link RedisCache Redis caches} are allowed to be created at runtime. + * + * @return a boolean value indicating whether {@link RedisCache Redis caches} are allowed to be created at runtime. + */ + public boolean isAllowRuntimeCacheCreation() { + return this.allowRuntimeCacheCreation; + } + + /** + * Return an {@link Collections#unmodifiableMap(Map) unmodifiable Map} containing {@link String caches name} mapped to + * the + * {@link RedisCache} {@link RedisCacheConfiguration configuration}. + * + * @return unmodifiable {@link Map} containing {@link String cache name} / {@link RedisCacheConfiguration + * configuration} + * pairs. + */ + public Map getCacheConfigurations() { + + Map cacheConfigurationMap = new HashMap<>(getCacheNames().size()); + + getCacheNames().forEach(cacheName -> { + RedisCache cache = (RedisCache) lookupCache(cacheName); + RedisCacheConfiguration cacheConfiguration = cache != null ? cache.getConfiguration() : null; + cacheConfigurationMap.put(cacheName, cacheConfiguration); + }); + + return Collections.unmodifiableMap(cacheConfigurationMap); + } + + /** + * Gets the default {@link RedisCacheConfiguration} applied to new {@link RedisCache} instances on creation when + * custom, + * non-specific {@link RedisCacheConfiguration} was not provided. + * + * @return the default {@link RedisCacheConfiguration}. + */ + protected RedisCacheConfiguration getDefaultCacheConfiguration() { + return this.defaultCacheConfiguration; + } + + /** + * Gets a {@link Map} of {@link String cache names} to {@link RedisCacheConfiguration} objects as the initial set of + * {@link RedisCache Redis caches} to create on startup. + * + * @return a {@link Map} of {@link String cache names} to {@link RedisCacheConfiguration} objects. + */ + protected Map getInitialCacheConfiguration() { + return Collections.unmodifiableMap(this.initialCacheConfiguration); + } + + /** + * Adds a cache configuration to this cache manager. + * + * @param name the name of the cache + * @param configuration the cache configuration to use + */ + public void addCacheConfiguration(String name, RedisCacheConfiguration configuration) { + this.initialCacheConfiguration.put(name, configuration); + } + + @Override + public RedisCache getMissingCache(String name) { + return isAllowRuntimeCacheCreation() ? createRedisCache(name, getDefaultCacheConfiguration()) : null; + } + + /** + * Creates a new {@link RedisCache} with given {@link String name} and {@link RedisCacheConfiguration}. + * + * @param name {@link String name} for the {@link RedisCache}; must not be {@literal null}. + * @param cacheConfiguration {@link RedisCacheConfiguration} used to configure the {@link RedisCache}; resolves to the + * {@link #getDefaultCacheConfiguration()} if {@literal null}. + * @return a new {@link RedisCache} instance; never {@literal null}. + */ + protected RedisCache createRedisCache(String name, @Nullable RedisCacheConfiguration cacheConfiguration) { + return new RedisCache(name, client, resolveCacheConfiguration(cacheConfiguration)); + } + + @Override + protected Collection loadCaches() { + + return getInitialCacheConfiguration().entrySet().stream().map(entry -> createRedisCache(entry.getKey(), entry + .getValue())).toList(); + } + + private RedisCacheConfiguration resolveCacheConfiguration(@Nullable RedisCacheConfiguration cacheConfiguration) { + return cacheConfiguration != null ? cacheConfiguration : getDefaultCacheConfiguration(); + } + + /** + * {@literal Builder} for creating a {@link RedisCacheManager}. + * + */ + public static class RedisCacheManagerBuilder { + + /** + * Factory method returning a new {@literal Builder} used to create and configure a {@link RedisCacheManager} using + * the + * given {@link AbstractRedisClient}. + * + * @param client {@link AbstractRedisClient} used by the {@link RedisCacheManager} to acquire connections to Redis + * when + * performing {@link RedisCache} operations; must not be {@literal null}. + * @return new {@link RedisCacheManagerBuilder}. + * @throws IllegalArgumentException if the given {@link AbstractRedisClient} is {@literal null}. + */ + public static RedisCacheManagerBuilder fromClient(AbstractRedisClient client) { + + Assert.notNull(client, "Client must not be null"); + + return new RedisCacheManagerBuilder(client); + } + + private boolean allowRuntimeCacheCreation = true; + + private final Map initialCaches = new LinkedHashMap<>(); + + private RedisCacheConfiguration defaultCacheConfiguration = new RedisCacheConfiguration(); + + private @Nullable AbstractRedisClient client; + + private RedisCacheManagerBuilder() { + } + + private RedisCacheManagerBuilder(AbstractRedisClient client) { + this.client = client; + ; + } + + /** + * Configure whether to allow cache creation at runtime. + * + * @param allowRuntimeCacheCreation boolean to allow creation of undeclared caches at runtime; {@literal true} by + * default. + * @return this {@link RedisCacheManagerBuilder}. + */ + public RedisCacheManagerBuilder allowCreateOnMissingCache(boolean allowRuntimeCacheCreation) { + this.allowRuntimeCacheCreation = allowRuntimeCacheCreation; + return this; + } + + /** + * Disable {@link RedisCache} creation at runtime for non-configured, undeclared caches. + *

+ * {@link RedisCacheManager#getMissingCache(String)} returns {@literal null} for any non-configured, undeclared + * {@link Cache} instead of a new {@link RedisCache} instance. This allows the + * {@link org.springframework.cache.support.CompositeCacheManager} to participate. + * + * @return this {@link RedisCacheManagerBuilder}. + * @see #allowCreateOnMissingCache(boolean) + * @see #enableCreateOnMissingCache() + */ + public RedisCacheManagerBuilder disableCreateOnMissingCache() { + return allowCreateOnMissingCache(false); + } + + /** + * Enables {@link RedisCache} creation at runtime for unconfigured, undeclared caches. + * + * @return this {@link RedisCacheManagerBuilder}. + * @see #allowCreateOnMissingCache(boolean) + * @see #disableCreateOnMissingCache() + */ + public RedisCacheManagerBuilder enableCreateOnMissingCache() { + return allowCreateOnMissingCache(true); + } + + /** + * Returns the default {@link RedisCacheConfiguration}. + * + * @return the default {@link RedisCacheConfiguration}. + */ + public RedisCacheConfiguration defaults() { + return this.defaultCacheConfiguration; + } + + /** + * Define a default {@link RedisCacheConfiguration} applied to dynamically created {@link RedisCache}s. + * + * @param configuration must not be {@literal null}. + * @return this {@link RedisCacheManagerBuilder}. + */ + public RedisCacheManagerBuilder defaults(RedisCacheConfiguration configuration) { + + Assert.notNull(configuration, "DefaultCacheConfiguration must not be null"); + + this.defaultCacheConfiguration = configuration; + + return this; + } + + /** + * Configure a {@link AbstractRedisClient}. + * + * @param client must not be {@literal null}. + * @return this {@link RedisCacheManagerBuilder}. + */ + public RedisCacheManagerBuilder client(AbstractRedisClient client) { + Assert.notNull(client, "Client must not be null"); + this.client = client; + return this; + } + + /** + * Append a {@link Set} of cache names to be pre initialized with current {@link RedisCacheConfiguration}. + * NOTE: This calls depends on {@link #defaults(RedisCacheConfiguration)} using whatever default + * {@link RedisCacheConfiguration} is present at the time of invoking this method. + * + * @param cacheNames must not be {@literal null}. + * @return this {@link RedisCacheManagerBuilder}. + */ + public RedisCacheManagerBuilder initialCacheNames(Set cacheNames) { + + Assert.notNull(cacheNames, "CacheNames must not be null"); + cacheNames.forEach(it -> configuration(it, defaultCacheConfiguration)); + + return this; + } + + /** + * Registers the given {@link String cache name} and {@link RedisCacheConfiguration} used to create and configure a + * {@link RedisCache} on startup. + * + * @param cacheName {@link String name} of the cache to register for creation on startup. + * @param cacheConfiguration {@link RedisCacheConfiguration} used to configure the new cache on startup. + * @return this {@link RedisCacheManagerBuilder}. + */ + public RedisCacheManagerBuilder configuration(String cacheName, RedisCacheConfiguration cacheConfiguration) { + + Assert.notNull(cacheName, "CacheName must not be null"); + Assert.notNull(cacheConfiguration, "CacheConfiguration must not be null"); + + this.initialCaches.put(cacheName, cacheConfiguration); + + return this; + } + + /** + * Append a {@link Map} of cache name/{@link RedisCacheConfiguration} pairs to be pre initialized. + * + * @param configs must not be {@literal null}. + * @return this {@link RedisCacheManagerBuilder}. + */ + public RedisCacheManagerBuilder initialConfigurations(Map configs) { + + Assert.notNull(configs, "CacheConfigurations must not be null"); + configs.forEach((cacheName, cacheConfiguration) -> Assert.notNull(cacheConfiguration, String.format( + "RedisCacheConfiguration for cache [%s] must not be null", cacheName))); + + this.initialCaches.putAll(configs); + + return this; + } + + /** + * Get the {@link RedisCacheConfiguration} for a given cache by its name. + * + * @param cacheName must not be {@literal null}. + * @return {@link Optional#empty()} if no {@link RedisCacheConfiguration} set for the given cache name. + */ + public Optional getCacheConfigurationFor(String cacheName) { + return Optional.ofNullable(this.initialCaches.get(cacheName)); + } + + /** + * Get the {@link Set} of cache names for which the builder holds {@link RedisCacheConfiguration configuration}. + * + * @return an unmodifiable {@link Set} holding the name of caches for which a {@link RedisCacheConfiguration + * configuration} has been set. + */ + public Set getConfiguredCaches() { + return Collections.unmodifiableSet(this.initialCaches.keySet()); + } + + /** + * Create new instance of {@link RedisCacheManager} with configuration options applied. + * + * @return new instance of {@link RedisCacheManager}. + */ + public RedisCacheManager build() { + + Assert.state(client != null, + "Client must not be null;" + " You can provide one via 'RedisCacheManagerBuilder#cacheWriter(RedisCacheWriter)'"); + + return newRedisCacheManager(client); + } + + private RedisCacheManager newRedisCacheManager(AbstractRedisClient client) { + return new RedisCacheManager(client, defaults(), this.allowRuntimeCacheCreation, this.initialCaches); + } + + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/RedisHashCacheAccessor.java b/redis-om-spring/src/main/java/com/redis/om/cache/RedisHashCacheAccessor.java new file mode 100644 index 000000000..182ea13f7 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/RedisHashCacheAccessor.java @@ -0,0 +1,59 @@ +package com.redis.om.cache; + +import java.time.Duration; + +import org.springframework.lang.Nullable; + +import com.redis.lettucemod.api.StatefulRedisModulesConnection; +import com.redis.om.cache.common.RedisHashMapper; + +/** + * Implementation of {@link CacheAccessor} that stores objects as Redis hashes. + * Uses a {@link RedisHashMapper} to convert objects to and from Redis hash entries. + */ +public class RedisHashCacheAccessor extends AbstractRedisCacheAccessor { + + private final RedisHashMapper mapper; + + /** + * Creates a new {@link RedisHashCacheAccessor} with the given connection and mapper. + * + * @param connection must not be {@literal null}. + * @param mapper the mapper used to convert objects to and from Redis hash entries, must not be {@literal null}. + */ + public RedisHashCacheAccessor(StatefulRedisModulesConnection connection, RedisHashMapper mapper) { + super(connection); + this.mapper = mapper; + } + + @Override + protected Object get(byte[] key, Duration ttl) { + Object value = mapper.fromHash(connection.sync().hgetall(key)); + if (shouldExpireWithin(ttl)) { + connection.sync().pexpire(key, ttl.toMillis()); + } + return value; + } + + @Override + protected void put(byte[] key, Object value, @Nullable Duration ttl) { + doPut(key, value, ttl); + } + + private void doPut(byte[] key, Object value, Duration ttl) { + connection.sync().hset(key, mapper.toHash(value)); + if (shouldExpireWithin(ttl)) { + connection.sync().pexpire(key, ttl.toMillis()); + } + } + + @Override + protected Object putIfAbsent(byte[] key, Object value, Duration ttl) { + if (exists(key)) { + return get(key, null); + } + doPut(key, value, ttl); + return null; + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/RedisJsonCacheAccessor.java b/redis-om-spring/src/main/java/com/redis/om/cache/RedisJsonCacheAccessor.java new file mode 100644 index 000000000..ba0744c49 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/RedisJsonCacheAccessor.java @@ -0,0 +1,82 @@ +package com.redis.om.cache; + +import java.nio.ByteBuffer; +import java.time.Duration; +import java.util.List; + +import org.springframework.util.CollectionUtils; + +import com.redis.lettucemod.api.StatefulRedisModulesConnection; +import com.redis.om.cache.common.RedisStringMapper; + +import io.lettuce.core.json.JsonPath; +import io.lettuce.core.json.JsonValue; + +/** + * Implementation of {@link AbstractRedisCacheAccessor} that uses Redis JSON to store and retrieve cache values. + * This accessor uses a {@link RedisStringMapper} to convert objects to and from JSON strings. + */ +public class RedisJsonCacheAccessor extends AbstractRedisCacheAccessor { + + private final RedisStringMapper mapper; + + /** + * @param connection must not be {@literal null}. + * @param mapper the mapper used to convert objects to and from JSON strings + */ + public RedisJsonCacheAccessor(StatefulRedisModulesConnection connection, RedisStringMapper mapper) { + super(connection); + this.mapper = mapper; + } + + @Override + public Object get(byte[] key, Duration ttl) { + List jsonValues = null; + try { + jsonValues = connection.sync().jsonGet(key); + } catch (NullPointerException e) { + // Workaround for Lettuce 6.5 bug: + // https://github.com/redis/lettuce/commit/341cdadc987e2866432dc6700b34b0f869134ae6 + // TODO: Remove with Lettuce 6.5.6+ + } + if (CollectionUtils.isEmpty(jsonValues)) { + return null; + } + JsonValue jsonValue = jsonValues.get(0); + if (jsonValue == null) { + return null; + } + ByteBuffer byteBuffer = jsonValue.asByteBuffer(); + if (byteBuffer == null) { + return null; + } + Object value = mapper.fromString(byteBuffer.array()); + if (shouldExpireWithin(ttl)) { + connection.sync().pexpire(key, ttl.toMillis()); + } + return value; + } + + @Override + public void put(byte[] key, Object value, Duration ttl) { + doPut(key, value, ttl); + } + + private void doPut(byte[] key, Object value, Duration ttl) { + connection.sync().jsonSet(key, JsonPath.ROOT_PATH, connection.sync().getJsonParser().createJsonValue(ByteBuffer + .wrap(mapper.toString(value)))); + if (shouldExpireWithin(ttl)) { + connection.sync().pexpire(key, ttl.toMillis()); + } + } + + @Override + public Object putIfAbsent(byte[] key, Object value, Duration ttl) { + if (exists(key)) { + return get(key, null); + } + doPut(key, value, ttl); + return null; + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/RedisStringCacheAccessor.java b/redis-om-spring/src/main/java/com/redis/om/cache/RedisStringCacheAccessor.java new file mode 100644 index 000000000..5cf4b1a00 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/RedisStringCacheAccessor.java @@ -0,0 +1,65 @@ +package com.redis.om.cache; + +import java.time.Duration; + +import org.springframework.lang.Nullable; + +import com.redis.lettucemod.api.StatefulRedisModulesConnection; +import com.redis.om.cache.common.RedisStringMapper; + +import io.lettuce.core.GetExArgs; +import io.lettuce.core.SetArgs; + +/** + * Implementation of {@link AbstractRedisCacheAccessor} that uses Redis String data structures + * for caching. This accessor uses a {@link RedisStringMapper} to convert between objects and + * their string representation in Redis. + */ +public class RedisStringCacheAccessor extends AbstractRedisCacheAccessor { + + private final RedisStringMapper mapper; + + /** + * @param connection must not be {@literal null}. + * @param mapper the mapper used to convert between objects and their string representation + */ + public RedisStringCacheAccessor(StatefulRedisModulesConnection connection, RedisStringMapper mapper) { + super(connection); + this.mapper = mapper; + } + + @Override + public Object get(byte[] key, @Nullable Duration ttl) { + if (shouldExpireWithin(ttl)) { + return mapper.fromString(connection.sync().getex(key, GetExArgs.Builder.ex(ttl))); + } + return mapper.fromString(connection.sync().get(key)); + } + + @Override + public void put(byte[] key, Object value, Duration ttl) { + doPut(key, value, ttl); + } + + private void doPut(byte[] key, Object value, Duration ttl) { + if (shouldExpireWithin(ttl)) { + connection.sync().psetex(key, ttl.toMillis(), mapper.toString(value)); + } else { + connection.sync().set(key, mapper.toString(value)); + } + } + + @Override + public Object putIfAbsent(byte[] key, Object value, Duration ttl) { + SetArgs args = SetArgs.Builder.nx(); + if (shouldExpireWithin(ttl)) { + args.ex(ttl); + } + boolean put = "OK".equalsIgnoreCase(connection.sync().set(key, mapper.toString(value), args)); + if (put) { + return null; + } + return mapper.fromString(connection.sync().get(key)); + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/RedisType.java b/redis-om-spring/src/main/java/com/redis/om/cache/RedisType.java new file mode 100644 index 000000000..ea8ece199 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/RedisType.java @@ -0,0 +1,23 @@ +package com.redis.om.cache; + +/** + * Enum representing the different Redis data structure types that can be used for caching. + */ +public enum RedisType { + + /** + * Redis Hash data structure, storing field-value pairs. + */ + HASH, + + /** + * Redis String data structure, storing simple string values. + */ + STRING, + + /** + * Redis JSON data structure, storing JSON documents. + */ + JSON + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/TtlFunction.java b/redis-om-spring/src/main/java/com/redis/om/cache/TtlFunction.java new file mode 100644 index 000000000..c89a09508 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/TtlFunction.java @@ -0,0 +1,82 @@ +package com.redis.om.cache; + +import java.time.Duration; + +import org.springframework.util.Assert; + +/** + * Interface defining functions to determine the time-to-live (TTL) for cache entries. + * Implementations of this interface provide strategies for calculating expiration durations + * for cache entries based on their keys and values. + */ +public interface TtlFunction { + + /** + * Constant representing no expiration (persistent entries). + */ + Duration NO_EXPIRATION = Duration.ZERO; + + /** + * {@link TtlFunction} to create persistent entires that do not expire. + */ + TtlFunction PERSISTENT = just(NO_EXPIRATION); + + /** + * {@link TtlFunction} implementation returning the given, predetermined + * {@link Duration} used for per cache entry + * {@literal time-to-live (TTL) expiration}. + * + */ + public static class FixedDurationTtlFunction implements TtlFunction { + + private final Duration duration; + + /** + * Creates a new FixedDurationTtlFunction with the specified duration. + * + * @param duration the fixed duration to use for all cache entries + */ + public FixedDurationTtlFunction(Duration duration) { + this.duration = duration; + } + + @Override + public Duration getTtl(Object key, Object value) { + return this.duration; + } + } + + /** + * Creates a {@literal Singleton} {@link TtlFunction} using the given + * {@link Duration}. + * + * @param duration the time to live. Can be {@link Duration#ZERO} for persistent + * values (i.e. cache entry does not expire). + * @return a singleton {@link TtlFunction} using {@link Duration}. + */ + static TtlFunction just(Duration duration) { + Assert.notNull(duration, "TTL Duration must not be null"); + return new FixedDurationTtlFunction(duration); + } + + /** + * Compute a {@link Duration time-to-live (TTL)} using the cache {@code key} and + * {@code value}. + *

+ * The {@link Duration time-to-live (TTL)} is computed on each write operation. + * Redis uses millisecond granularity for timeouts. Any more granular values + * (e.g. micros or nanos) are not considered and will be truncated due to + * rounding. Returning {@link Duration#ZERO}, or a value less than + * {@code Duration.ofMillis(1)}, results in a persistent value that does not + * expire. + * + * @param key the cache key. + * @param value the cache value. Can be {@code null} if the cache supports + * {@code null} value caching. + * @return the computed {@link Duration time-to-live (TTL)}. Can be + * {@link Duration#ZERO} for persistent values (i.e. cache entry does + * not expire). + */ + Duration getTtl(Object key, Object value); + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/RedisHashMapper.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/RedisHashMapper.java new file mode 100644 index 000000000..d149dacbf --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/RedisHashMapper.java @@ -0,0 +1,28 @@ +package com.redis.om.cache.common; + +import java.util.Map; + +/** + * Interface for mapping between Java objects and Redis hash structures. + * Implementations of this interface handle the conversion of Java objects to Redis hash maps + * and vice versa. + */ +public interface RedisHashMapper { + + /** + * Converts a Java object to a Redis hash representation. + * + * @param value the Java object to convert + * @return a map of byte arrays representing the Redis hash + */ + Map toHash(Object value); + + /** + * Converts a Redis hash representation back to a Java object. + * + * @param hash the Redis hash as a map of byte arrays + * @return the reconstructed Java object + */ + Object fromHash(Map hash); + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/RedisStringMapper.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/RedisStringMapper.java new file mode 100644 index 000000000..4428cb0df --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/RedisStringMapper.java @@ -0,0 +1,26 @@ +package com.redis.om.cache.common; + +/** + * Interface for mapping between Java objects and Redis string structures. + * Implementations of this interface handle the conversion of Java objects to Redis string values + * and vice versa. + */ +public interface RedisStringMapper { + + /** + * Converts a Java object to a Redis string representation. + * + * @param value the Java object to convert + * @return a byte array representing the Redis string + */ + byte[] toString(Object value); + + /** + * Converts a Redis string representation back to a Java object. + * + * @param bytes the Redis string as a byte array + * @return the reconstructed Java object + */ + Object fromString(byte[] bytes); + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/SerializationException.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/SerializationException.java new file mode 100644 index 000000000..a8f1c9a2e --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/SerializationException.java @@ -0,0 +1,31 @@ +package com.redis.om.cache.common; + +import org.springframework.core.NestedRuntimeException; + +/** + * Exception thrown when an error occurs during serialization or deserialization operations. + * This exception is typically thrown by implementations of {@link RedisStringMapper} and + * {@link RedisHashMapper} when they encounter issues converting between Java objects and + * Redis data structures. + */ +public class SerializationException extends NestedRuntimeException { + + /** + * Constructs a new {@link SerializationException} instance. + * + * @param msg the detail message describing the error + */ + public SerializationException(String msg) { + super(msg); + } + + /** + * Constructs a new {@link SerializationException} instance. + * + * @param msg the detail message describing the error + * @param cause the nested exception that caused this exception + */ + public SerializationException(String msg, Throwable cause) { + super(msg, cause); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/BasicRedisPersistentEntity.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/BasicRedisPersistentEntity.java new file mode 100644 index 000000000..7a345dab5 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/BasicRedisPersistentEntity.java @@ -0,0 +1,68 @@ +package com.redis.om.cache.common.convert; + +import org.springframework.data.annotation.Id; +import org.springframework.data.keyvalue.core.mapping.BasicKeyValuePersistentEntity; +import org.springframework.data.keyvalue.core.mapping.KeySpaceResolver; +import org.springframework.data.mapping.MappingException; +import org.springframework.data.util.TypeInformation; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * {@link RedisPersistentEntity} implementation. + * + * @param the type of the entity + */ +public class BasicRedisPersistentEntity extends BasicKeyValuePersistentEntity implements + RedisPersistentEntity { + + /** + * Creates new {@link BasicRedisPersistentEntity}. + * + * @param information must not be {@literal null}. + * @param keySpaceResolver can be {@literal null}. + */ + public BasicRedisPersistentEntity(TypeInformation information, @Nullable KeySpaceResolver keySpaceResolver) { + super(information, keySpaceResolver); + + } + + @Override + @Nullable + protected RedisPersistentProperty returnPropertyIfBetterIdPropertyCandidateOrNull(RedisPersistentProperty property) { + + Assert.notNull(property, "Property must not be null"); + + if (!property.isIdProperty()) { + return null; + } + + RedisPersistentProperty currentIdProperty = getIdProperty(); + boolean currentIdPropertyIsSet = currentIdProperty != null; + + if (!currentIdPropertyIsSet) { + return property; + } + + boolean currentIdPropertyIsExplicit = currentIdProperty.isAnnotationPresent(Id.class); + boolean newIdPropertyIsExplicit = property.isAnnotationPresent(Id.class); + + if (currentIdPropertyIsExplicit && newIdPropertyIsExplicit) { + throw new MappingException(String.format( + "Attempt to add explicit id property %s but already have an property %s registered " + "as explicit id; Check your mapping configuration", + property.getField(), currentIdProperty.getField())); + } + + if (!currentIdPropertyIsExplicit && !newIdPropertyIsExplicit) { + throw new MappingException(String.format( + "Attempt to add id property %s but already have an property %s registered " + "as id; Check your mapping configuration", + property.getField(), currentIdProperty.getField())); + } + + if (newIdPropertyIsExplicit) { + return property; + } + + return null; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/BinaryConverters.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/BinaryConverters.java new file mode 100644 index 000000000..5a795b597 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/BinaryConverters.java @@ -0,0 +1,249 @@ +package com.redis.om.cache.common.convert; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.text.DateFormat; +import java.text.ParseException; +import java.util.*; + +import org.springframework.core.convert.converter.Converter; +import org.springframework.core.convert.converter.ConverterFactory; +import org.springframework.data.convert.ReadingConverter; +import org.springframework.data.convert.WritingConverter; +import org.springframework.util.NumberUtils; +import org.springframework.util.ObjectUtils; + +/** + * Set of {@link ReadingConverter} and {@link WritingConverter} used to convert Objects into binary format. + * + */ +final class BinaryConverters { + + /** + * Use {@literal UTF-8} as default charset. + */ + public static final Charset CHARSET = StandardCharsets.UTF_8; + + private BinaryConverters() { + } + + static Collection getConvertersToRegister() { + + List converters = new ArrayList<>(12); + + converters.add(new StringToBytesConverter()); + converters.add(new BytesToStringConverter()); + converters.add(new NumberToBytesConverter()); + converters.add(new BytesToNumberConverterFactory()); + converters.add(new EnumToBytesConverter()); + converters.add(new BytesToEnumConverterFactory()); + converters.add(new BooleanToBytesConverter()); + converters.add(new BytesToBooleanConverter()); + converters.add(new DateToBytesConverter()); + converters.add(new BytesToDateConverter()); + converters.add(new UuidToBytesConverter()); + converters.add(new BytesToUuidConverter()); + + return converters; + } + + static class StringBasedConverter { + + byte[] fromString(String source) { + return source.getBytes(CHARSET); + } + + String toString(byte[] source) { + return new String(source, CHARSET); + } + } + + @WritingConverter + static class StringToBytesConverter extends StringBasedConverter implements Converter { + + @Override + public byte[] convert(String source) { + return fromString(source); + } + } + + @ReadingConverter + static class BytesToStringConverter extends StringBasedConverter implements Converter { + + @Override + public String convert(byte[] source) { + return toString(source); + } + + } + + @WritingConverter + static class NumberToBytesConverter extends StringBasedConverter implements Converter { + + @Override + public byte[] convert(Number source) { + return fromString(source.toString()); + } + } + + @WritingConverter + static class EnumToBytesConverter extends StringBasedConverter implements Converter, byte[]> { + + @Override + public byte[] convert(Enum source) { + return fromString(source.name()); + } + } + + @ReadingConverter + static final class BytesToEnumConverterFactory implements ConverterFactory> { + + @Override + @SuppressWarnings( + { "unchecked", "rawtypes" } + ) + public > Converter getConverter(Class targetType) { + + Class enumType = targetType; + while (enumType != null && !enumType.isEnum()) { + enumType = enumType.getSuperclass(); + } + if (enumType == null) { + throw new IllegalArgumentException("The target type " + targetType.getName() + " does not refer to an enum"); + } + return new BytesToEnum(enumType); + } + + private class BytesToEnum> extends StringBasedConverter implements Converter { + + private final Class enumType; + + public BytesToEnum(Class enumType) { + this.enumType = enumType; + } + + @Override + public T convert(byte[] source) { + + if (ObjectUtils.isEmpty(source)) { + return null; + } + + return Enum.valueOf(this.enumType, toString(source).trim()); + } + } + } + + @ReadingConverter + static class BytesToNumberConverterFactory implements ConverterFactory { + + @Override + public Converter getConverter(Class targetType) { + return new BytesToNumberConverter<>(targetType); + } + + private static final class BytesToNumberConverter extends StringBasedConverter implements + Converter { + + private final Class targetType; + + public BytesToNumberConverter(Class targetType) { + this.targetType = targetType; + } + + @Override + public T convert(byte[] source) { + + if (ObjectUtils.isEmpty(source)) { + return null; + } + + return NumberUtils.parseNumber(toString(source), targetType); + } + } + } + + @WritingConverter + static class BooleanToBytesConverter extends StringBasedConverter implements Converter { + + byte[] _true = fromString("1"); + byte[] _false = fromString("0"); + + @Override + public byte[] convert(Boolean source) { + return source.booleanValue() ? _true : _false; + } + } + + @ReadingConverter + static class BytesToBooleanConverter extends StringBasedConverter implements Converter { + + @Override + public Boolean convert(byte[] source) { + + if (ObjectUtils.isEmpty(source)) { + return null; + } + + String value = toString(source); + return ("1".equals(value) || "true".equalsIgnoreCase(value)) ? Boolean.TRUE : Boolean.FALSE; + } + } + + @WritingConverter + static class DateToBytesConverter extends StringBasedConverter implements Converter { + + @Override + public byte[] convert(Date source) { + return fromString(Long.toString(source.getTime())); + } + } + + @ReadingConverter + static class BytesToDateConverter extends StringBasedConverter implements Converter { + + @Override + public Date convert(byte[] source) { + + if (ObjectUtils.isEmpty(source)) { + return null; + } + + String value = toString(source); + try { + return new Date(NumberUtils.parseNumber(value, Long.class)); + } catch (NumberFormatException ignore) { + } + + try { + return DateFormat.getInstance().parse(value); + } catch (ParseException ignore) { + } + + throw new IllegalArgumentException(String.format("Cannot parse date out of %s", Arrays.toString(source))); + } + } + + @WritingConverter + static class UuidToBytesConverter extends StringBasedConverter implements Converter { + + @Override + public byte[] convert(UUID source) { + return fromString(source.toString()); + } + } + + @ReadingConverter + static class BytesToUuidConverter extends StringBasedConverter implements Converter { + + @Override + public UUID convert(byte[] source) { + + if (ObjectUtils.isEmpty(source)) { + return null; + } + + return UUID.fromString(toString(source)); + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/Bucket.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/Bucket.java new file mode 100644 index 000000000..aff94d037 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/Bucket.java @@ -0,0 +1,383 @@ +package com.redis.om.cache.common.convert; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; +import org.springframework.util.comparator.NullSafeComparator; + +/** + * Bucket is the data bag for Redis hash structures to be used with + * {@link RedisData}. It provides a container for storing and manipulating + * key-value pairs that will be persisted as Redis hash structures. + * + */ +@SuppressWarnings( + "deprecation" +) +public class Bucket { + + /** + * Encoding used for converting {@link Byte} to and from {@link String}. + */ + public static final Charset CHARSET = StandardCharsets.UTF_8; + + private static final Comparator COMPARATOR = new NullSafeComparator<>(Comparator.naturalOrder(), + true); + + /** + * The Redis data as {@link Map} sorted by the keys. + */ + private final NavigableMap data = new TreeMap<>(COMPARATOR); + + /** + * Creates a new empty bucket. + */ + public Bucket() { + } + + Bucket(Map data) { + + Assert.notNull(data, "Initial data must not be null"); + this.data.putAll(data); + } + + /** + * Add {@link String} representation of property dot path with given value. + * + * @param path must not be {@literal null} or {@link String#isEmpty()}. + * @param value can be {@literal null}. + */ + public void put(String path, @Nullable byte[] value) { + + Assert.hasText(path, "Path to property must not be null or empty"); + data.put(path, value); + } + + /** + * Remove the property at property dot {@code path}. + * + * @param path must not be {@literal null} or {@link String#isEmpty()}. + */ + public void remove(String path) { + + Assert.hasText(path, "Path to property must not be null or empty"); + data.remove(path); + } + + /** + * Get value assigned with path. + * + * @param path must not be {@literal null} or {@link String#isEmpty()}. + * @return {@literal null} if not set. + */ + @Nullable + public byte[] get(String path) { + + Assert.hasText(path, "Path to property must not be null or empty"); + return data.get(path); + } + + /** + * Return whether {@code path} is associated with a non-{@code null} value. + * + * @param path must not be {@literal null} or {@link String#isEmpty()}. + * @return {@literal true} if the {@code path} is associated with a + * non-{@code null} value. + * @since 2.5 + */ + public boolean hasValue(String path) { + return get(path) != null; + } + + /** + * A set view of the mappings contained in this bucket. + * + * @return never {@literal null}. + */ + public Set> entrySet() { + return data.entrySet(); + } + + /** + * @return {@literal true} when no data present in {@link Bucket}. + */ + public boolean isEmpty() { + return data.isEmpty(); + } + + /** + * @return the number of key-value mappings of the {@link Bucket}. + */ + public int size() { + return data.size(); + } + + /** + * @return never {@literal null}. + */ + public Collection values() { + return data.values(); + } + + /** + * @return never {@literal null}. + */ + public Set keySet() { + return data.keySet(); + } + + /** + * Key/value pairs contained in the {@link Bucket}. + * + * @return never {@literal null}. + */ + public Map asMap() { + return Collections.unmodifiableMap(this.data); + } + + /** + * Extracts a bucket containing key/value pairs with the {@code prefix}. + * + * @param prefix the prefix to filter keys by + * @return a new bucket containing only the key/value pairs with the specified prefix + */ + public Bucket extract(String prefix) { + return new Bucket(data.subMap(prefix, prefix + Character.MAX_VALUE)); + } + + /** + * Get all the keys matching a given path. + * + * @param path the path to look for. Can be {@literal null}. + * @return all keys if path is null or empty. + */ + public Set extractAllKeysFor(String path) { + + if (!StringUtils.hasText(path)) { + return keySet(); + } + + Pattern pattern = Pattern.compile("^(" + Pattern.quote(path) + ")\\.\\[.*?\\]"); + + Set keys = new LinkedHashSet<>(); + for (Entry entry : data.entrySet()) { + + Matcher matcher = pattern.matcher(entry.getKey()); + if (matcher.find()) { + keys.add(matcher.group()); + } + } + + return keys; + } + + /** + * Get keys and values in binary format. + * + * @return never {@literal null}. + */ + public Map rawMap() { + + Map raw = new LinkedHashMap<>(data.size()); + for (Entry entry : data.entrySet()) { + if (entry.getValue() != null) { + raw.put(entry.getKey().getBytes(CHARSET), entry.getValue()); + } + } + return raw; + } + + /** + * Get the {@link BucketPropertyPath} leading to the current {@link Bucket}. + * + * @return new instance of {@link BucketPropertyPath}. + * @since 2.1 + */ + public BucketPropertyPath getPath() { + return BucketPropertyPath.from(this); + } + + /** + * Get the {@link BucketPropertyPath} for a given property within the current + * {@link Bucket}. + * + * @param property the property to look up. + * @return new instance of {@link BucketPropertyPath}. + * @since 2.1 + */ + public BucketPropertyPath getPropertyPath(String property) { + return BucketPropertyPath.from(this, property); + } + + /** + * Creates a new Bucket from a given raw map. + * + * @param source can be {@literal null}. + * @return never {@literal null}. + */ + public static Bucket newBucketFromRawMap(Map source) { + + Bucket bucket = new Bucket(); + + for (Entry entry : source.entrySet()) { + bucket.put(new String(entry.getKey(), CHARSET), entry.getValue()); + } + return bucket; + } + + /** + * Creates a new Bucket from a given {@link String} map. + * + * @param source can be {@literal null}. + * @return never {@literal null}. + */ + public static Bucket newBucketFromStringMap(Map source) { + + Bucket bucket = new Bucket(); + + for (Entry entry : source.entrySet()) { + bucket.put(entry.getKey(), StringUtils.hasText(entry.getValue()) ? + entry.getValue().getBytes(CHARSET) : + new byte[] {}); + } + return bucket; + } + + @Override + public String toString() { + return "Bucket [data=" + safeToString() + "]"; + } + + private String safeToString() { + + if (data.isEmpty()) { + return "{}"; + } + + StringBuilder sb = new StringBuilder(); + sb.append('{'); + + Iterator> iterator = data.entrySet().iterator(); + + for (;;) { + + Entry e = iterator.next(); + sb.append(e.getKey()); + sb.append('='); + sb.append(toUtf8String(e.getValue())); + + if (!iterator.hasNext()) { + return sb.append('}').toString(); + } + + sb.append(',').append(' '); + } + } + + @Nullable + private static String toUtf8String(byte[] raw) { + + try { + return new String(raw, CHARSET); + } catch (Exception ignore) { + } + + return null; + } + + /** + * Value object representing a path within a {@link Bucket}. Paths can be either + * top-level (if the {@code prefix} is {@literal null} or empty) or nested with + * a given {@code prefix}. + * + */ + public static class BucketPropertyPath { + + private final Bucket bucket; + private final @Nullable String prefix; + + private BucketPropertyPath(Bucket bucket, String prefix) { + + Assert.notNull(bucket, "Bucket must not be null"); + + this.bucket = bucket; + this.prefix = prefix; + } + + /** + * Creates a top-level {@link BucketPropertyPath} given {@link Bucket}. + * + * @param bucket the bucket, must not be {@literal null}. + * @return {@link BucketPropertyPath} within the given {@link Bucket}. + */ + public static BucketPropertyPath from(Bucket bucket) { + return new BucketPropertyPath(bucket, null); + } + + /** + * Creates a {@link BucketPropertyPath} given {@link Bucket} and {@code prefix}. + * The resulting path is top-level if {@code prefix} is empty or nested, if + * {@code prefix} is not empty. + * + * @param bucket the bucket, must not be {@literal null}. + * @param prefix the prefix. Property path is top-level if {@code prefix} is + * {@literal null} or empty. + * @return {@link BucketPropertyPath} within the given {@link Bucket} using + * {@code prefix}. + */ + public static BucketPropertyPath from(Bucket bucket, @Nullable String prefix) { + return new BucketPropertyPath(bucket, prefix); + } + + /** + * Retrieve a value at {@code key} considering top-level/nesting. + * + * @param key must not be {@literal null} or empty. + * @return the resulting value, may be {@literal null}. + */ + @Nullable + public byte[] get(String key) { + return bucket.get(getPath(key)); + } + + /** + * Write a {@code value} at {@code key} considering top-level/nesting. + * + * @param key must not be {@literal null} or empty. + * @param value the value. + */ + public void put(String key, byte[] value) { + bucket.put(getPath(key), value); + } + + private String getPath(String key) { + return StringUtils.hasText(prefix) ? prefix + "." + key : key; + } + + /** + * Gets the bucket associated with this property path. + * + * @return the bucket instance + */ + public Bucket getBucket() { + return this.bucket; + } + + /** + * Gets the prefix for this property path. + * + * @return the prefix string, may be {@literal null} for top-level paths + */ + @Nullable + public String getPrefix() { + return this.prefix; + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/ByteUtils.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/ByteUtils.java new file mode 100644 index 000000000..976487601 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/ByteUtils.java @@ -0,0 +1,244 @@ +package com.redis.om.cache.common.convert; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.springframework.util.Assert; +import org.springframework.util.ObjectUtils; + +/** + * Some handy methods for dealing with {@code byte} arrays. + * + */ +public final class ByteUtils { + + private ByteUtils() { + } + + /** + * Concatenate the given {@code byte} arrays into one. + *

+ * The order of elements in the original arrays is preserved. + * + * @param array1 the first array. + * @param array2 the second array. + * @return the new array. + */ + public static byte[] concat(byte[] array1, byte[] array2) { + return concatAll(array1, array2); + } + + /** + * Concatenate the given {@code byte} arrays into one. Returns a new, empty array if {@code arrays} was empty and + * returns the first array if {@code arrays} contains only a single array. + *

+ * The order of elements in the original arrays is preserved. + * + * @param arrays the arrays. + * @return the new array. + */ + public static byte[] concatAll(byte[]... arrays) { + + if (arrays.length == 0) { + return new byte[0]; + } + + if (arrays.length == 1) { + return arrays[0]; + } + + long totalArraySize = 0; + for (byte[] array : arrays) { + totalArraySize += array.length; + } + + if (totalArraySize == 0) { + return new byte[0]; + } + + byte[] result = new byte[Math.toIntExact(totalArraySize)]; + int copied = 0; + for (byte[] array : arrays) { + System.arraycopy(array, 0, result, copied, array.length); + copied += array.length; + } + + return result; + } + + /** + * Merge multiple {@code byte} arrays into one array + * + * @param firstArray must not be {@literal null} + * @param additionalArrays must not be {@literal null} + * @return the merged array. + */ + public static byte[][] mergeArrays(byte[] firstArray, byte[]... additionalArrays) { + + Assert.notNull(firstArray, "first array must not be null"); + Assert.notNull(additionalArrays, "additional arrays must not be null"); + + byte[][] result = new byte[additionalArrays.length + 1][]; + result[0] = firstArray; + System.arraycopy(additionalArrays, 0, result, 1, additionalArrays.length); + + return result; + } + + /** + * Split {@code source} into partitioned arrays using delimiter {@code c}. + * + * @param source the source array. + * @param c delimiter. + * @return the partitioned arrays. + */ + public static byte[][] split(byte[] source, int c) { + + if (ObjectUtils.isEmpty(source)) { + return new byte[][] {}; + } + + List bytes = new ArrayList<>(); + int offset = 0; + for (int i = 0; i <= source.length; i++) { + + if (i == source.length) { + + bytes.add(Arrays.copyOfRange(source, offset, i)); + break; + } + + if (source[i] == c) { + bytes.add(Arrays.copyOfRange(source, offset, i)); + offset = i + 1; + } + } + return bytes.toArray(new byte[bytes.size()][]); + } + + /** + * Extract a byte array from {@link ByteBuffer} without consuming it. The resulting {@code byte[]} is a copy of the + * buffer's contents and not updated upon changes within the buffer. + * + * @param byteBuffer must not be {@literal null}. + * @return a byte array containing the data from the ByteBuffer + * @since 2.0 + */ + public static byte[] getBytes(ByteBuffer byteBuffer) { + + Assert.notNull(byteBuffer, "ByteBuffer must not be null"); + + ByteBuffer duplicate = byteBuffer.duplicate(); + byte[] bytes = new byte[duplicate.remaining()]; + duplicate.get(bytes); + return bytes; + } + + /** + * Tests if the {@code haystack} starts with the given {@code prefix}. + * + * @param haystack the source to scan. + * @param prefix the prefix to find. + * @return {@literal true} if {@code haystack} at position {@code offset} starts with {@code prefix}. + * @since 1.8.10 + * @see #startsWith(byte[], byte[], int) + */ + public static boolean startsWith(byte[] haystack, byte[] prefix) { + return startsWith(haystack, prefix, 0); + } + + /** + * Tests if the {@code haystack} beginning at the specified {@code offset} starts with the given {@code prefix}. + * + * @param haystack the source to scan. + * @param prefix the prefix to find. + * @param offset the offset to start at. + * @return {@literal true} if {@code haystack} at position {@code offset} starts with {@code prefix}. + * @since 1.8.10 + */ + public static boolean startsWith(byte[] haystack, byte[] prefix, int offset) { + + int to = offset; + int prefixOffset = 0; + int prefixLength = prefix.length; + + if ((offset < 0) || (offset > haystack.length - prefixLength)) { + return false; + } + + while (--prefixLength >= 0) { + if (haystack[to++] != prefix[prefixOffset++]) { + return false; + } + } + + return true; + } + + /** + * Searches the specified array of bytes for the specified value. Returns the index of the first matching value in the + * {@code haystack}s natural order or {@code -1} of {@code needle} could not be found. + * + * @param haystack the source to scan. + * @param needle the value to scan for. + * @return index of first appearance, or -1 if not found. + * @since 1.8.10 + */ + public static int indexOf(byte[] haystack, byte needle) { + + for (int i = 0; i < haystack.length; i++) { + if (haystack[i] == needle) { + return i; + } + } + + return -1; + } + + /** + * Convert a {@link String} into a {@link ByteBuffer} using {@link StandardCharsets#UTF_8}. + * + * @param theString must not be {@literal null}. + * @return a ByteBuffer containing the UTF-8 encoded string + * @since 2.1 + */ + public static ByteBuffer getByteBuffer(String theString) { + return getByteBuffer(theString, StandardCharsets.UTF_8); + } + + /** + * Convert a {@link String} into a {@link ByteBuffer} using the given {@link Charset}. + * + * @param theString must not be {@literal null}. + * @param charset must not be {@literal null}. + * @return a ByteBuffer containing the string encoded with the specified charset + * @since 2.1 + */ + public static ByteBuffer getByteBuffer(String theString, Charset charset) { + + Assert.notNull(theString, "The String must not be null"); + Assert.notNull(charset, "The String must not be null"); + + return charset.encode(theString); + } + + /** + * Extract/Transfer bytes from the given {@link ByteBuffer} into an array by duplicating the buffer and fetching its + * content. + * + * @param buffer must not be {@literal null}. + * @return the extracted bytes. + * @since 2.1 + * @deprecated Since 3.2. Use {@link #getBytes(ByteBuffer)} instead. + */ + @Deprecated( + since = "3.2" + ) + public static byte[] extractBytes(ByteBuffer buffer) { + return getBytes(buffer); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/CompositeIndexResolver.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/CompositeIndexResolver.java new file mode 100644 index 000000000..92ee27ea6 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/CompositeIndexResolver.java @@ -0,0 +1,61 @@ +package com.redis.om.cache.common.convert; + +import java.util.*; + +import org.springframework.data.util.TypeInformation; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; + +/** + * Composite {@link IndexResolver} implementation that iterates over a given collection of delegate + * {@link IndexResolver} instances.
+ *
+ * NOTE {@link IndexedData} created by an {@link IndexResolver} can be overwritten by subsequent + * {@link IndexResolver}. + * + */ +public class CompositeIndexResolver implements IndexResolver { + + private final List resolvers; + + /** + * Create new {@link CompositeIndexResolver}. + * + * @param resolvers must not be {@literal null}. + */ + public CompositeIndexResolver(Collection resolvers) { + + Assert.notNull(resolvers, "Resolvers must not be null"); + if (CollectionUtils.contains(resolvers.iterator(), null)) { + throw new IllegalArgumentException("Resolvers must no contain null values"); + } + this.resolvers = new ArrayList<>(resolvers); + } + + @Override + public Set resolveIndexesFor(TypeInformation typeInformation, @Nullable Object value) { + + if (resolvers.isEmpty()) { + return Collections.emptySet(); + } + + Set data = new LinkedHashSet<>(); + for (IndexResolver resolver : resolvers) { + data.addAll(resolver.resolveIndexesFor(typeInformation, value)); + } + return data; + } + + @Override + public Set resolveIndexesFor(String keyspace, String path, TypeInformation typeInformation, + @Nullable Object value) { + + Set data = new LinkedHashSet<>(); + for (IndexResolver resolver : resolvers) { + data.addAll(resolver.resolveIndexesFor(keyspace, path, typeInformation, value)); + } + return data; + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/DefaultRedisTypeMapper.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/DefaultRedisTypeMapper.java new file mode 100644 index 000000000..2a396aad1 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/DefaultRedisTypeMapper.java @@ -0,0 +1,145 @@ +package com.redis.om.cache.common.convert; + +import java.util.Collections; +import java.util.List; + +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.support.GenericConversionService; +import org.springframework.data.convert.DefaultTypeMapper; +import org.springframework.data.convert.SimpleTypeInformationMapper; +import org.springframework.data.convert.TypeAliasAccessor; +import org.springframework.data.convert.TypeInformationMapper; +import org.springframework.data.mapping.Alias; +import org.springframework.data.mapping.PersistentEntity; +import org.springframework.data.mapping.context.MappingContext; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +import com.redis.om.cache.common.convert.Bucket.BucketPropertyPath; + +/** + * Default implementation of {@link RedisTypeMapper} allowing configuration of the key to lookup and store type + * information via {@link BucketPropertyPath} in buckets. The key defaults to {@link #DEFAULT_TYPE_KEY}. Actual + * type-to-{@code byte[]} conversion and back is done in {@link BucketTypeAliasAccessor}. + * + */ +public class DefaultRedisTypeMapper extends DefaultTypeMapper implements RedisTypeMapper { + + /** + * Default type key used for storing type information. + */ + public static final String DEFAULT_TYPE_KEY = "_class"; + + private final @Nullable String typeKey; + + /** + * Create a new {@link DefaultRedisTypeMapper} using {@link #DEFAULT_TYPE_KEY} to exchange type hints. + */ + public DefaultRedisTypeMapper() { + this(DEFAULT_TYPE_KEY); + } + + /** + * Create a new {@link DefaultRedisTypeMapper} given {@code typeKey} to exchange type hints. Does not consider type + * hints if {@code typeKey} is {@literal null}. + * + * @param typeKey the type key can be {@literal null} to skip type hinting. + */ + public DefaultRedisTypeMapper(@Nullable String typeKey) { + this(typeKey, Collections.singletonList(new SimpleTypeInformationMapper())); + } + + /** + * Create a new {@link DefaultRedisTypeMapper} given {@code typeKey} to exchange type hints and + * {@link MappingContext}. Does not consider type hints if {@code typeKey} is {@literal null}. {@link MappingContext} + * is used to obtain entity-based aliases + * + * @param typeKey the type key can be {@literal null} to skip type hinting. + * @param mappingContext must not be {@literal null}. + * @see org.springframework.data.annotation.TypeAlias + */ + public DefaultRedisTypeMapper(@Nullable String typeKey, + MappingContext, ?> mappingContext) { + this(typeKey, new BucketTypeAliasAccessor(typeKey, getConversionService()), mappingContext, Collections + .singletonList(new SimpleTypeInformationMapper())); + } + + /** + * Create a new {@link DefaultRedisTypeMapper} given {@code typeKey} to exchange type hints and {@link List} of + * {@link TypeInformationMapper}. Does not consider type hints if {@code typeKey} is {@literal null}. + * {@link MappingContext} is used to obtain entity-based aliases + * + * @param typeKey the type key can be {@literal null} to skip type hinting. + * @param mappers must not be {@literal null}. + */ + public DefaultRedisTypeMapper(@Nullable String typeKey, List mappers) { + this(typeKey, new BucketTypeAliasAccessor(typeKey, getConversionService()), null, mappers); + } + + private DefaultRedisTypeMapper(@Nullable String typeKey, TypeAliasAccessor accessor, + @Nullable MappingContext, ?> mappingContext, + List mappers) { + + super(accessor, mappingContext, mappers); + + this.typeKey = typeKey; + } + + private static GenericConversionService getConversionService() { + + GenericConversionService conversionService = new GenericConversionService(); + new RedisCustomConversions().registerConvertersIn(conversionService); + + return conversionService; + } + + public boolean isTypeKey(@Nullable String key) { + return key != null && typeKey != null && key.endsWith(typeKey); + } + + /** + * {@link TypeAliasAccessor} to store aliases in a {@link Bucket}. + * + */ + static final class BucketTypeAliasAccessor implements TypeAliasAccessor { + + private final @Nullable String typeKey; + + private final ConversionService conversionService; + + BucketTypeAliasAccessor(@Nullable String typeKey, ConversionService conversionService) { + + Assert.notNull(conversionService, "ConversionService must not be null"); + + this.typeKey = typeKey; + this.conversionService = conversionService; + } + + public Alias readAliasFrom(BucketPropertyPath source) { + + if (typeKey == null || source instanceof List) { + return Alias.NONE; + } + + byte[] bytes = source.get(typeKey); + + if (bytes != null) { + return Alias.ofNullable(conversionService.convert(bytes, String.class)); + } + + return Alias.NONE; + } + + public void writeTypeTo(BucketPropertyPath sink, Object alias) { + + if (typeKey != null) { + + if (alias instanceof byte[] aliasBytes) { + sink.put(typeKey, aliasBytes); + } else { + sink.put(typeKey, conversionService.convert(alias, byte[].class)); + } + } + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/IndexResolver.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/IndexResolver.java new file mode 100644 index 000000000..78a3037cb --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/IndexResolver.java @@ -0,0 +1,37 @@ +package com.redis.om.cache.common.convert; + +import java.util.Set; + +import org.springframework.data.mapping.PersistentProperty; +import org.springframework.data.util.TypeInformation; +import org.springframework.lang.Nullable; + +/** + * {@link IndexResolver} extracts secondary index structures to be applied on a given path, {@link PersistentProperty} + * and value. + * + */ +public interface IndexResolver { + + /** + * Resolves all indexes for given type information / value combination. + * + * @param typeInformation must not be {@literal null}. + * @param value the actual value. Can be {@literal null}. + * @return never {@literal null}. + */ + Set resolveIndexesFor(TypeInformation typeInformation, @Nullable Object value); + + /** + * Resolves all indexes for given type information / value combination. + * + * @param keyspace must not be {@literal null}. + * @param path must not be {@literal null}. + * @param typeInformation must not be {@literal null}. + * @param value the actual value. Can be {@literal null}. + * @return never {@literal null}. + */ + Set resolveIndexesFor(String keyspace, String path, TypeInformation typeInformation, + @Nullable Object value); + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/IndexedData.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/IndexedData.java new file mode 100644 index 000000000..f20bf48b8 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/IndexedData.java @@ -0,0 +1,23 @@ +package com.redis.om.cache.common.convert; + +/** + * {@link IndexedData} represents a secondary index for a property path in a given keyspace. + * + */ +public interface IndexedData { + + /** + * Get the {@link String} representation of the index name. + * + * @return never {@literal null}. + */ + String getIndexName(); + + /** + * Get the associated keyspace the index resides in. + * + * @return the keyspace name, never {@literal null}. + */ + String getKeyspace(); + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/Jsr310Converters.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/Jsr310Converters.java new file mode 100644 index 000000000..942503d43 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/Jsr310Converters.java @@ -0,0 +1,226 @@ +package com.redis.om.cache.common.convert; + +import java.time.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import org.springframework.core.convert.converter.Converter; + +import com.redis.om.cache.common.convert.BinaryConverters.StringBasedConverter; + +/** + * Helper class to register JSR-310 specific {@link Converter} implementations. + * + */ +public abstract class Jsr310Converters { + + /** + * Returns the {@link Converter Converters} to be registered. + * + * @return the {@link Converter Converters} to be registered. + */ + public static Collection> getConvertersToRegister() { + + List> converters = new ArrayList<>(20); + + converters.add(new LocalDateTimeToBytesConverter()); + converters.add(new BytesToLocalDateTimeConverter()); + converters.add(new LocalDateToBytesConverter()); + converters.add(new BytesToLocalDateConverter()); + converters.add(new LocalTimeToBytesConverter()); + converters.add(new BytesToLocalTimeConverter()); + converters.add(new ZonedDateTimeToBytesConverter()); + converters.add(new BytesToZonedDateTimeConverter()); + converters.add(new InstantToBytesConverter()); + converters.add(new BytesToInstantConverter()); + converters.add(new ZoneIdToBytesConverter()); + converters.add(new BytesToZoneIdConverter()); + converters.add(new PeriodToBytesConverter()); + converters.add(new BytesToPeriodConverter()); + converters.add(new DurationToBytesConverter()); + converters.add(new BytesToDurationConverter()); + converters.add(new OffsetDateTimeToBytesConverter()); + converters.add(new BytesToOffsetDateTimeConverter()); + converters.add(new OffsetTimeToBytesConverter()); + converters.add(new BytesToOffsetTimeConverter()); + + return converters; + } + + /** + * Checks if the given type is supported by the converters in this class. + * + * @param type the class to check for support. + * @return {@literal true} if the type is supported, {@literal false} otherwise. + */ + public static boolean supports(Class type) { + + return Arrays.>asList(LocalDateTime.class, LocalDate.class, LocalTime.class, Instant.class, + ZonedDateTime.class, ZoneId.class, Period.class, Duration.class, OffsetDateTime.class, OffsetTime.class) + .contains(type); + } + + static class LocalDateTimeToBytesConverter extends StringBasedConverter implements Converter { + + @Override + public byte[] convert(LocalDateTime source) { + return fromString(source.toString()); + } + } + + static class BytesToLocalDateTimeConverter extends StringBasedConverter implements Converter { + + @Override + public LocalDateTime convert(byte[] source) { + return LocalDateTime.parse(toString(source)); + } + } + + static class LocalDateToBytesConverter extends StringBasedConverter implements Converter { + + @Override + public byte[] convert(LocalDate source) { + return fromString(source.toString()); + } + } + + static class BytesToLocalDateConverter extends StringBasedConverter implements Converter { + + @Override + public LocalDate convert(byte[] source) { + return LocalDate.parse(toString(source)); + } + } + + static class LocalTimeToBytesConverter extends StringBasedConverter implements Converter { + + @Override + public byte[] convert(LocalTime source) { + return fromString(source.toString()); + } + } + + static class BytesToLocalTimeConverter extends StringBasedConverter implements Converter { + + @Override + public LocalTime convert(byte[] source) { + return LocalTime.parse(toString(source)); + } + } + + static class ZonedDateTimeToBytesConverter extends StringBasedConverter implements Converter { + + @Override + public byte[] convert(ZonedDateTime source) { + return fromString(source.toString()); + } + } + + static class BytesToZonedDateTimeConverter extends StringBasedConverter implements Converter { + + @Override + public ZonedDateTime convert(byte[] source) { + return ZonedDateTime.parse(toString(source)); + } + } + + static class InstantToBytesConverter extends StringBasedConverter implements Converter { + + @Override + public byte[] convert(Instant source) { + return fromString(source.toString()); + } + } + + static class BytesToInstantConverter extends StringBasedConverter implements Converter { + + @Override + public Instant convert(byte[] source) { + return Instant.parse(toString(source)); + } + } + + static class ZoneIdToBytesConverter extends StringBasedConverter implements Converter { + + @Override + public byte[] convert(ZoneId source) { + return fromString(source.toString()); + } + } + + static class BytesToZoneIdConverter extends StringBasedConverter implements Converter { + + @Override + public ZoneId convert(byte[] source) { + return ZoneId.of(toString(source)); + } + } + + static class PeriodToBytesConverter extends StringBasedConverter implements Converter { + + @Override + public byte[] convert(Period source) { + return fromString(source.toString()); + } + } + + static class BytesToPeriodConverter extends StringBasedConverter implements Converter { + + @Override + public Period convert(byte[] source) { + return Period.parse(toString(source)); + } + } + + static class DurationToBytesConverter extends StringBasedConverter implements Converter { + + @Override + public byte[] convert(Duration source) { + return fromString(source.toString()); + } + } + + static class BytesToDurationConverter extends StringBasedConverter implements Converter { + + @Override + public Duration convert(byte[] source) { + return Duration.parse(toString(source)); + } + } + + static class OffsetDateTimeToBytesConverter extends StringBasedConverter implements + Converter { + + @Override + public byte[] convert(OffsetDateTime source) { + return fromString(source.toString()); + } + } + + static class BytesToOffsetDateTimeConverter extends StringBasedConverter implements + Converter { + + @Override + public OffsetDateTime convert(byte[] source) { + return OffsetDateTime.parse(toString(source)); + } + } + + static class OffsetTimeToBytesConverter extends StringBasedConverter implements Converter { + + @Override + public byte[] convert(OffsetTime source) { + return fromString(source.toString()); + } + } + + static class BytesToOffsetTimeConverter extends StringBasedConverter implements Converter { + + @Override + public OffsetTime convert(byte[] source) { + return OffsetTime.parse(toString(source)); + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/KeyspaceConfiguration.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/KeyspaceConfiguration.java new file mode 100644 index 000000000..2d50fb672 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/KeyspaceConfiguration.java @@ -0,0 +1,175 @@ +package com.redis.om.cache.common.convert; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.springframework.util.Assert; +import org.springframework.util.ClassUtils; + +/** + * {@link KeyspaceConfiguration} allows programmatic setup of keyspaces and time + * to live options for certain types. This is suitable for cases where there is + * no option to use the equivalent RedisHash annotations. + * + */ +public class KeyspaceConfiguration { + + private Map, KeyspaceSettings> settingsMap; + + /** + * Creates a new {@link KeyspaceConfiguration} with initial settings from {@link #initialConfiguration()}. + */ + public KeyspaceConfiguration() { + + this.settingsMap = new ConcurrentHashMap<>(); + for (KeyspaceSettings initial : initialConfiguration()) { + settingsMap.put(initial.type, initial); + } + } + + /** + * Check if specific {@link KeyspaceSettings} are available for given type. + * + * @param type must not be {@literal null}. + * @return true if settings exist. + */ + public boolean hasSettingsFor(Class type) { + + Assert.notNull(type, "Type to lookup must not be null"); + + if (settingsMap.containsKey(type)) { + + if (settingsMap.get(type) instanceof DefaultKeyspaceSetting) { + return false; + } + + return true; + } + + for (KeyspaceSettings assignment : settingsMap.values()) { + if (assignment.inherit) { + if (ClassUtils.isAssignable(assignment.type, type)) { + settingsMap.put(type, assignment.cloneFor(type)); + return true; + } + } + } + + settingsMap.put(type, new DefaultKeyspaceSetting(type)); + return false; + } + + /** + * Get the {@link KeyspaceSettings} for given type. + * + * @param type must not be {@literal null} + * @return {@literal null} if no settings configured. + */ + public KeyspaceSettings getKeyspaceSettings(Class type) { + + if (!hasSettingsFor(type)) { + return null; + } + + KeyspaceSettings settings = settingsMap.get(type); + if (settings == null || settings instanceof DefaultKeyspaceSetting) { + return null; + } + + return settings; + } + + /** + * Customization hook. + * + * @return must not return {@literal null}. + */ + protected Iterable initialConfiguration() { + return Collections.emptySet(); + } + + /** + * Add {@link KeyspaceSettings} for type. + * + * @param keyspaceSettings must not be {@literal null}. + */ + public void addKeyspaceSettings(KeyspaceSettings keyspaceSettings) { + + Assert.notNull(keyspaceSettings, "KeyspaceSettings must not be null"); + this.settingsMap.put(keyspaceSettings.getType(), keyspaceSettings); + } + + /** + * Settings class that holds keyspace configuration for a specific type. + */ + public static class KeyspaceSettings { + + private final String keyspace; + private final Class type; + private final boolean inherit; + + /** + * Creates a new {@link KeyspaceSettings} for the given type and keyspace with inheritance enabled. + * + * @param type the type to configure the keyspace for + * @param keyspace the keyspace to use + */ + public KeyspaceSettings(Class type, String keyspace) { + this(type, keyspace, true); + } + + /** + * Creates a new {@link KeyspaceSettings} for the given type and keyspace. + * + * @param type the type to configure the keyspace for + * @param keyspace the keyspace to use + * @param inherit whether the settings should be inherited by subtypes + */ + public KeyspaceSettings(Class type, String keyspace, boolean inherit) { + + this.type = type; + this.keyspace = keyspace; + this.inherit = inherit; + } + + /** + * Creates a clone of this {@link KeyspaceSettings} for the given type with inheritance disabled. + * + * @param type the type to create the clone for + * @return a new {@link KeyspaceSettings} instance + */ + KeyspaceSettings cloneFor(Class type) { + return new KeyspaceSettings(type, this.keyspace, false); + } + + /** + * Returns the configured keyspace. + * + * @return the keyspace + */ + public String getKeyspace() { + return keyspace; + } + + /** + * Returns the type these settings are for. + * + * @return the type + */ + public Class getType() { + return type; + } + + } + + /** + * Marker class indicating no settings defined. + */ + private static class DefaultKeyspaceSetting extends KeyspaceSettings { + + public DefaultKeyspaceSetting(Class type) { + super(type, "#default#", false); + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/MappingConfiguration.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/MappingConfiguration.java new file mode 100644 index 000000000..37793a45c --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/MappingConfiguration.java @@ -0,0 +1,28 @@ +package com.redis.om.cache.common.convert; + +/** + * {@link MappingConfiguration} is used for programmatic configuration of key + * prefixes. + * + */ +public class MappingConfiguration { + + private final KeyspaceConfiguration keyspaceConfiguration; + + /** + * Creates new {@link MappingConfiguration}. + * + * @param keyspaceConfiguration must not be {@literal null}. + */ + public MappingConfiguration(KeyspaceConfiguration keyspaceConfiguration) { + + this.keyspaceConfiguration = keyspaceConfiguration; + } + + /** + * @return never {@literal null}. + */ + public KeyspaceConfiguration getKeyspaceConfiguration() { + return keyspaceConfiguration; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/MappingRedisConverter.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/MappingRedisConverter.java new file mode 100644 index 000000000..8dd7a030d --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/MappingRedisConverter.java @@ -0,0 +1,1371 @@ +package com.redis.om.cache.common.convert; + +import java.lang.reflect.Array; +import java.util.*; +import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.springframework.beans.factory.InitializingBean; +import org.springframework.core.CollectionFactory; +import org.springframework.core.convert.ConversionService; +import org.springframework.core.convert.ConverterNotFoundException; +import org.springframework.core.convert.support.DefaultConversionService; +import org.springframework.core.convert.support.GenericConversionService; +import org.springframework.data.convert.CustomConversions; +import org.springframework.data.mapping.*; +import org.springframework.data.mapping.model.EntityInstantiator; +import org.springframework.data.mapping.model.EntityInstantiators; +import org.springframework.data.mapping.model.PersistentEntityParameterValueProvider; +import org.springframework.data.mapping.model.PropertyValueProvider; +import org.springframework.data.util.ProxyUtils; +import org.springframework.data.util.TypeInformation; +import org.springframework.lang.Nullable; +import org.springframework.stereotype.Indexed; +import org.springframework.util.*; +import org.springframework.util.comparator.NullSafeComparator; + +import com.redis.om.cache.common.convert.PartialUpdate.PropertyUpdate; +import com.redis.om.cache.common.convert.PartialUpdate.UpdateCommand; + +/** + * {@link RedisConverter} implementation creating flat binary map structure out + * of a given domain type. Considers {@link Indexed} annotation for enabling + * helper structures for finder operations.
+ *
+ * NOTE {@link MappingRedisConverter} is an + * {@link InitializingBean} and requires + * {@link MappingRedisConverter#afterPropertiesSet()} to be called. + * + *

+ * 
+ * @RedisHash("persons")
+ * class Person {
+ *
+ * @Id String id;
+ * String firstname;
+ *
+ * List<String> nicknames;
+ * List<Person> coworkers;
+ *
+ * Address address;
+ * @Reference Country nationality;
+ * }
+ * 
+ * 
+ * + * The above is represented as: + * + *
+ * 
+ * _class=org.example.Person
+ * id=1
+ * firstname=rand
+ * lastname=al'thor
+ * coworkers.[0].firstname=mat
+ * coworkers.[0].nicknames.[0]=prince of the ravens
+ * coworkers.[1].firstname=perrin
+ * coworkers.[1].address.city=two rivers
+ * nationality=nationality:andora
+ * 
+ * 
+ * + */ +@SuppressWarnings( + "deprecation" +) +public class MappingRedisConverter implements RedisConverter, InitializingBean { + + private static final String INVALID_TYPE_ASSIGNMENT = "Value of type %s cannot be assigned to property %s of type %s"; + + private final RedisMappingContext mappingContext; + private final GenericConversionService conversionService; + private final EntityInstantiators entityInstantiators; + private final RedisTypeMapper typeMapper; + private final Comparator listKeyComparator = new NullSafeComparator<>(NaturalOrderingKeyComparator.INSTANCE, + true); + + private @Nullable ReferenceResolver referenceResolver; + private CustomConversions customConversions; + + /** + * Creates new {@link MappingRedisConverter}. + * + * @param context can be {@literal null}. + * @since 2.4 + */ + public MappingRedisConverter(RedisMappingContext context) { + this(context, null, null); + } + + /** + * Creates new {@link MappingRedisConverter} and defaults + * {@link RedisMappingContext} when {@literal null}. + * + * @param mappingContext can be {@literal null}. + * @param referenceResolver can be not be {@literal null}. + */ + public MappingRedisConverter(@Nullable RedisMappingContext mappingContext, + @Nullable ReferenceResolver referenceResolver) { + this(mappingContext, referenceResolver, null); + } + + /** + * Creates new {@link MappingRedisConverter} and defaults + * {@link RedisMappingContext} when {@literal null}. + * + * @param mappingContext can be {@literal null}. + * @param referenceResolver can be {@literal null}. + * @param typeMapper can be {@literal null}. + */ + public MappingRedisConverter(@Nullable RedisMappingContext mappingContext, + @Nullable ReferenceResolver referenceResolver, @Nullable RedisTypeMapper typeMapper) { + + this.mappingContext = mappingContext != null ? mappingContext : new RedisMappingContext(); + + this.entityInstantiators = new EntityInstantiators(); + this.conversionService = new DefaultConversionService(); + this.customConversions = new RedisCustomConversions(); + this.typeMapper = typeMapper != null ? + typeMapper : + new DefaultRedisTypeMapper(DefaultRedisTypeMapper.DEFAULT_TYPE_KEY, this.mappingContext); + + this.referenceResolver = referenceResolver; + } + + @Override + @SuppressWarnings( + "unchecked" + ) + public R read(Class type, RedisData source) { + + TypeInformation readType = typeMapper.readType(source.getBucket().getPath(), TypeInformation.of(type)); + + return readType.isCollectionLike() ? + (R) readCollectionOrArray("", ArrayList.class, Object.class, source.getBucket()) : + doReadInternal("", type, source); + + } + + @Nullable + private R readInternal(String path, Class type, RedisData source) { + return source.getBucket().isEmpty() ? null : doReadInternal(path, type, source); + } + + @SuppressWarnings( + "unchecked" + ) + private R doReadInternal(String path, Class type, RedisData source) { + + TypeInformation readType = typeMapper.readType(source.getBucket().getPath(), TypeInformation.of(type)); + + if (customConversions.hasCustomReadTarget(Map.class, readType.getType())) { + + Map partial = new HashMap<>(); + + if (!path.isEmpty()) { + + for (Entry entry : source.getBucket().extract(path + ".").entrySet()) { + partial.put(entry.getKey().substring(path.length() + 1), entry.getValue()); + } + + } else { + partial.putAll(source.getBucket().asMap()); + } + R instance = (R) conversionService.convert(partial, readType.getType()); + + RedisPersistentEntity entity = mappingContext.getPersistentEntity(readType); + if (entity != null && entity.hasIdProperty()) { + + PersistentPropertyAccessor propertyAccessor = entity.getPropertyAccessor(instance); + + propertyAccessor.setProperty(entity.getRequiredIdProperty(), source.getId()); + instance = propertyAccessor.getBean(); + } + return instance; + } + + if (conversionService.canConvert(byte[].class, readType.getType())) { + return (R) conversionService.convert(source.getBucket().get(StringUtils.hasText(path) ? path : "_raw"), readType + .getType()); + } + + RedisPersistentEntity entity = mappingContext.getRequiredPersistentEntity(readType); + EntityInstantiator instantiator = entityInstantiators.getInstantiatorFor(entity); + + Object instance = instantiator.createInstance((RedisPersistentEntity) entity, + new PersistentEntityParameterValueProvider<>(entity, new ConverterAwareParameterValueProvider(path, source, + conversionService), this.conversionService)); + + PersistentPropertyAccessor accessor = entity.getPropertyAccessor(instance); + + entity.doWithProperties((PropertyHandler) persistentProperty -> { + + InstanceCreatorMetadata creator = entity.getInstanceCreatorMetadata(); + + if (creator != null && creator.isCreatorParameter(persistentProperty)) { + return; + } + + Object targetValue = readProperty(path, source, persistentProperty); + + if (targetValue != null) { + accessor.setProperty(persistentProperty, targetValue); + } + }); + + readAssociation(path, source, entity, accessor); + + return (R) accessor.getBean(); + } + + /** + * Reads a property value from the Redis data source. + * + * @param path the path to the property + * @param source the Redis data source + * @param persistentProperty the property to read + * @return the property value, or {@literal null} if not found + */ + @Nullable + protected Object readProperty(String path, RedisData source, RedisPersistentProperty persistentProperty) { + + String currentPath = !path.isEmpty() ? path + "." + persistentProperty.getName() : persistentProperty.getName(); + TypeInformation typeInformation = typeMapper.readType(source.getBucket().getPropertyPath(currentPath), + persistentProperty.getTypeInformation()); + + if (typeInformation.isMap()) { + + Class mapValueType = null; + + if (typeInformation.getMapValueType() != null) { + mapValueType = typeInformation.getMapValueType().getType(); + } + + if (mapValueType == null && persistentProperty.isMap()) { + mapValueType = persistentProperty.getMapValueType(); + } + + if (mapValueType == null) { + throw new IllegalArgumentException("Unable to retrieve MapValueType"); + } + + if (conversionService.canConvert(byte[].class, mapValueType)) { + return readMapOfSimpleTypes(currentPath, typeInformation.getType(), typeInformation.getRequiredComponentType() + .getType(), mapValueType, source); + } + + return readMapOfComplexTypes(currentPath, typeInformation.getType(), typeInformation.getRequiredComponentType() + .getType(), mapValueType, source); + } + + if (typeInformation.isCollectionLike()) { + + if (!isByteArray(typeInformation)) { + + return readCollectionOrArray(currentPath, typeInformation.getType(), typeInformation.getRequiredComponentType() + .getType(), source.getBucket()); + } + + if (!source.getBucket().hasValue(currentPath) && isByteArray(typeInformation)) { + + return readCollectionOrArray(currentPath, typeInformation.getType(), typeInformation.getRequiredComponentType() + .getType(), source.getBucket()); + } + } + + if (mappingContext.getPersistentEntity(typeInformation) != null && !conversionService.canConvert(byte[].class, + typeInformation.getRequiredActualType().getType())) { + + Bucket bucket = source.getBucket().extract(currentPath + "."); + + RedisData newBucket = new RedisData(bucket); + + return readInternal(currentPath, typeInformation.getType(), newBucket); + } + + byte[] sourceBytes = source.getBucket().get(currentPath); + + if (typeInformation.getType().isPrimitive() && sourceBytes == null) { + return null; + } + + if (persistentProperty.isIdProperty() && ObjectUtils.isEmpty(path)) { + return sourceBytes != null ? fromBytes(sourceBytes, typeInformation.getType()) : source.getId(); + } + + if (sourceBytes == null) { + return null; + } + + if (customConversions.hasCustomReadTarget(byte[].class, persistentProperty.getType())) { + return fromBytes(sourceBytes, persistentProperty.getType()); + } + + Class typeToUse = getTypeHint(currentPath, source.getBucket(), persistentProperty.getType()); + return fromBytes(sourceBytes, typeToUse); + } + + private void readAssociation(String path, RedisData source, RedisPersistentEntity entity, + PersistentPropertyAccessor accessor) { + + entity.doWithAssociations((AssociationHandler) association -> { + + String currentPath = !path.isEmpty() ? + path + "." + association.getInverse().getName() : + association.getInverse().getName(); + + if (association.getInverse().isCollectionLike()) { + + Bucket bucket = source.getBucket().extract(currentPath + ".["); + + Collection target = CollectionFactory.createCollection(association.getInverse().getType(), association + .getInverse().getComponentType(), bucket.size()); + + for (Entry entry : bucket.entrySet()) { + + String referenceKey = fromBytes(entry.getValue(), String.class); + + if (!KeyspaceIdentifier.isValid(referenceKey)) { + continue; + } + + KeyspaceIdentifier identifier = KeyspaceIdentifier.of(referenceKey); + Map rawHash = referenceResolver.resolveReference(identifier.getId(), identifier + .getKeyspace()); + + if (!CollectionUtils.isEmpty(rawHash)) { + target.add(read(association.getInverse().getActualType(), new RedisData(rawHash))); + } + } + + accessor.setProperty(association.getInverse(), target); + + } else { + + byte[] binKey = source.getBucket().get(currentPath); + if (binKey == null || binKey.length == 0) { + return; + } + + String referenceKey = fromBytes(binKey, String.class); + if (KeyspaceIdentifier.isValid(referenceKey)) { + + KeyspaceIdentifier identifier = KeyspaceIdentifier.of(referenceKey); + + Map rawHash = referenceResolver.resolveReference(identifier.getId(), identifier + .getKeyspace()); + + if (!CollectionUtils.isEmpty(rawHash)) { + accessor.setProperty(association.getInverse(), read(association.getInverse().getActualType(), new RedisData( + rawHash))); + } + } + } + }); + } + + @Override + @SuppressWarnings( + { "rawtypes" } + ) + public void write(Object source, RedisData sink) { + + if (source == null) { + return; + } + + RedisPersistentEntity entity = mappingContext.getPersistentEntity(source.getClass()); + + if (!customConversions.hasCustomWriteTarget(source.getClass())) { + typeMapper.writeType(ClassUtils.getUserClass(source), sink.getBucket().getPath()); + } + + if (entity == null) { + + typeMapper.writeType(ClassUtils.getUserClass(source), sink.getBucket().getPath()); + sink.getBucket().put("_raw", conversionService.convert(source, byte[].class)); + return; + } + + sink.setKeyspace(entity.getKeySpace()); + + if (entity.getTypeInformation().isCollectionLike()) { + writeCollection(entity.getKeySpace(), "", (List) source, entity.getTypeInformation().getRequiredComponentType(), + sink); + } else { + writeInternal(entity.getKeySpace(), "", source, entity.getTypeInformation(), sink); + } + + Object identifier = entity.getIdentifierAccessor(source).getIdentifier(); + + if (identifier != null) { + sink.setId(getConversionService().convert(identifier, String.class)); + } + + } + + /** + * Writes a partial update to the Redis data sink. + * + * @param update the partial update to write + * @param sink the Redis data sink to write to + */ + protected void writePartialUpdate(PartialUpdate update, RedisData sink) { + + RedisPersistentEntity entity = mappingContext.getRequiredPersistentEntity(update.getTarget()); + + write(update.getValue(), sink); + + for (String key : sink.getBucket().keySet()) { + if (typeMapper.isTypeKey(key)) { + sink.getBucket().remove(key); + break; + } + } + + for (PropertyUpdate pUpdate : update.getPropertyUpdates()) { + + String path = pUpdate.getPropertyPath(); + + if (UpdateCommand.SET.equals(pUpdate.getCmd())) { + writePartialPropertyUpdate(update, pUpdate, sink, entity, path); + } + } + } + + /** + * @param update + * @param pUpdate + * @param sink + * @param entity + * @param path + */ + private void writePartialPropertyUpdate(PartialUpdate update, PropertyUpdate pUpdate, RedisData sink, + RedisPersistentEntity entity, String path) { + + RedisPersistentProperty targetProperty = getTargetPropertyOrNullForPath(path, update.getTarget()); + + if (targetProperty == null) { + + targetProperty = getTargetPropertyOrNullForPath(path.replaceAll("\\.\\[.*\\]", ""), update.getTarget()); + + TypeInformation ti = targetProperty == null ? + TypeInformation.OBJECT : + (targetProperty.isMap() ? + (targetProperty.getTypeInformation().getMapValueType() != null ? + targetProperty.getTypeInformation().getRequiredMapValueType() : + TypeInformation.OBJECT) : + targetProperty.getTypeInformation().getActualType()); + + writeInternal(entity.getKeySpace(), pUpdate.getPropertyPath(), pUpdate.getValue(), ti, sink); + return; + } + + if (targetProperty.isAssociation()) { + + if (targetProperty.isCollectionLike()) { + + RedisPersistentEntity ref = mappingContext.getPersistentEntity(targetProperty.getRequiredAssociation() + .getInverse().getTypeInformation().getRequiredComponentType().getRequiredActualType()); + + int i = 0; + for (Object o : (Collection) pUpdate.getValue()) { + + Object refId = ref.getPropertyAccessor(o).getProperty(ref.getRequiredIdProperty()); + if (refId != null) { + sink.getBucket().put(pUpdate.getPropertyPath() + ".[" + i + "]", toBytes(ref.getKeySpace() + ":" + refId)); + i++; + } + } + } else { + + RedisPersistentEntity ref = mappingContext.getRequiredPersistentEntity(targetProperty + .getRequiredAssociation().getInverse().getTypeInformation()); + + Object refId = ref.getPropertyAccessor(pUpdate.getValue()).getProperty(ref.getRequiredIdProperty()); + if (refId != null) { + sink.getBucket().put(pUpdate.getPropertyPath(), toBytes(ref.getKeySpace() + ":" + refId)); + } + } + } else if (targetProperty.isCollectionLike() && !isByteArray(targetProperty)) { + + Collection collection = pUpdate.getValue() instanceof Collection ? + (Collection) pUpdate.getValue() : + Collections.singleton(pUpdate.getValue()); + writeCollection(entity.getKeySpace(), pUpdate.getPropertyPath(), collection, targetProperty.getTypeInformation() + .getRequiredActualType(), sink); + } else if (targetProperty.isMap()) { + + Map map = new HashMap<>(); + + if (pUpdate.getValue() instanceof Map) { + map.putAll((Map) pUpdate.getValue()); + } else if (pUpdate.getValue() instanceof Entry) { + map.put(((Entry) pUpdate.getValue()).getKey(), ((Entry) pUpdate.getValue()).getValue()); + } else { + throw new MappingException(String.format( + "Cannot set update value for map property '%s' to '%s'; Please use a Map or Map.Entry", pUpdate + .getPropertyPath(), pUpdate.getValue())); + } + + writeMap(entity.getKeySpace(), pUpdate.getPropertyPath(), targetProperty.getMapValueType(), map, sink); + } else { + + writeInternal(entity.getKeySpace(), pUpdate.getPropertyPath(), pUpdate.getValue(), targetProperty + .getTypeInformation(), sink); + + } + } + + @Nullable + RedisPersistentProperty getTargetPropertyOrNullForPath(String path, Class type) { + + try { + + PersistentPropertyPath persistentPropertyPath = mappingContext.getPersistentPropertyPath( + path, type); + return persistentPropertyPath.getLeafProperty(); + } catch (Exception ignore) { + } + + return null; + } + + /** + * @param keyspace + * @param path + * @param value + * @param typeHint + * @param sink + */ + private void writeInternal(@Nullable String keyspace, String path, @Nullable Object value, + TypeInformation typeHint, RedisData sink) { + + if (value == null) { + return; + } + + if (customConversions.hasCustomWriteTarget(value.getClass())) { + + Optional> targetType = customConversions.getCustomWriteTarget(value.getClass()); + + if (!StringUtils.hasText(path) && targetType.isPresent() && ClassUtils.isAssignable(byte[].class, targetType + .get())) { + sink.getBucket().put(StringUtils.hasText(path) ? path : "_raw", conversionService.convert(value, byte[].class)); + } else { + + if (!ClassUtils.isAssignable(typeHint.getType(), value.getClass())) { + throw new MappingException(String.format(INVALID_TYPE_ASSIGNMENT, value.getClass(), path, typeHint + .getType())); + } + writeToBucket(path, value, sink, typeHint.getType()); + } + return; + } + + if (value instanceof byte[] valueBytes) { + sink.getBucket().put(StringUtils.hasText(path) ? path : "_raw", valueBytes); + return; + } + + if (value.getClass() != typeHint.getType()) { + typeMapper.writeType(value.getClass(), sink.getBucket().getPropertyPath(path)); + } + + RedisPersistentEntity entity = mappingContext.getRequiredPersistentEntity(value.getClass()); + PersistentPropertyAccessor accessor = entity.getPropertyAccessor(value); + + entity.doWithProperties((PropertyHandler) persistentProperty -> { + + String propertyStringPath = (!path.isEmpty() ? path + "." : "") + persistentProperty.getName(); + + Object propertyValue = accessor.getProperty(persistentProperty); + if (persistentProperty.isIdProperty()) { + + if (propertyValue != null) { + sink.getBucket().put(propertyStringPath, toBytes(propertyValue)); + } + return; + } + + if (persistentProperty.isMap()) { + + if (propertyValue != null) { + writeMap(keyspace, propertyStringPath, persistentProperty.getMapValueType(), (Map) propertyValue, sink); + } + } else if (persistentProperty.isCollectionLike() && !isByteArray(persistentProperty)) { + + if (propertyValue == null) { + writeCollection(keyspace, propertyStringPath, null, persistentProperty.getTypeInformation() + .getRequiredComponentType(), sink); + } else { + + if (Iterable.class.isAssignableFrom(propertyValue.getClass())) { + + writeCollection(keyspace, propertyStringPath, (Iterable) propertyValue, persistentProperty + .getTypeInformation().getRequiredComponentType(), sink); + } else if (propertyValue.getClass().isArray()) { + + writeCollection(keyspace, propertyStringPath, CollectionUtils.arrayToList(propertyValue), persistentProperty + .getTypeInformation().getRequiredComponentType(), sink); + } else { + + throw new RuntimeException("Don't know how to handle " + propertyValue.getClass() + " type collection"); + } + } + + } else if (propertyValue != null) { + + if (customConversions.isSimpleType(ProxyUtils.getUserClass(propertyValue.getClass()))) { + + writeToBucket(propertyStringPath, propertyValue, sink, persistentProperty.getType()); + } else { + writeInternal(keyspace, propertyStringPath, propertyValue, persistentProperty.getTypeInformation() + .getRequiredActualType(), sink); + } + } + }); + + writeAssociation(path, entity, value, sink); + } + + private void writeAssociation(String path, RedisPersistentEntity entity, @Nullable Object value, RedisData sink) { + + if (value == null) { + return; + } + + PersistentPropertyAccessor accessor = entity.getPropertyAccessor(value); + + entity.doWithAssociations((AssociationHandler) association -> { + + Object refObject = accessor.getProperty(association.getInverse()); + if (refObject == null) { + return; + } + + if (association.getInverse().isCollectionLike()) { + + RedisPersistentEntity ref = mappingContext.getRequiredPersistentEntity(association.getInverse() + .getTypeInformation().getRequiredComponentType().getRequiredActualType()); + + String keyspace = ref.getKeySpace(); + String propertyStringPath = (!path.isEmpty() ? path + "." : "") + association.getInverse().getName(); + + int i = 0; + for (Object o : (Collection) refObject) { + + Object refId = ref.getPropertyAccessor(o).getProperty(ref.getRequiredIdProperty()); + if (refId != null) { + sink.getBucket().put(propertyStringPath + ".[" + i + "]", toBytes(keyspace + ":" + refId)); + i++; + } + } + + } else { + + RedisPersistentEntity ref = mappingContext.getRequiredPersistentEntity(association.getInverse() + .getTypeInformation()); + String keyspace = ref.getKeySpace(); + + if (keyspace != null) { + Object refId = ref.getPropertyAccessor(refObject).getProperty(ref.getRequiredIdProperty()); + + if (refId != null) { + String propertyStringPath = (!path.isEmpty() ? path + "." : "") + association.getInverse().getName(); + sink.getBucket().put(propertyStringPath, toBytes(keyspace + ":" + refId)); + } + } + } + }); + } + + /** + * @param keyspace + * @param path + * @param values + * @param typeHint + * @param sink + */ + private void writeCollection(@Nullable String keyspace, String path, @Nullable Iterable values, + TypeInformation typeHint, RedisData sink) { + + if (values == null) { + return; + } + + int i = 0; + for (Object value : values) { + + if (value == null) { + break; + } + + String currentPath = path + (path.equals("") ? "" : ".") + "[" + i + "]"; + + if (!ClassUtils.isAssignable(typeHint.getType(), value.getClass())) { + throw new MappingException(String.format(INVALID_TYPE_ASSIGNMENT, value.getClass(), currentPath, typeHint + .getType())); + } + + if (customConversions.hasCustomWriteTarget(value.getClass())) { + writeToBucket(currentPath, value, sink, typeHint.getType()); + } else { + writeInternal(keyspace, currentPath, value, typeHint, sink); + } + i++; + } + } + + private void writeToBucket(String path, @Nullable Object value, RedisData sink, Class propertyType) { + + if (value == null || (value instanceof Optional && !((Optional) value).isPresent())) { + return; + } + + if (value instanceof byte[]) { + sink.getBucket().put(path, toBytes(value)); + return; + } + + if (customConversions.hasCustomWriteTarget(value.getClass())) { + + Optional> targetType = customConversions.getCustomWriteTarget(value.getClass()); + + if (!propertyType.isPrimitive() && !targetType.filter(it -> ClassUtils.isAssignable(Map.class, it)) + .isPresent() && customConversions.isSimpleType(value.getClass()) && value.getClass() != propertyType) { + typeMapper.writeType(value.getClass(), sink.getBucket().getPropertyPath(path)); + } + + if (targetType.filter(it -> ClassUtils.isAssignable(Map.class, it)).isPresent()) { + + Map map = (Map) conversionService.convert(value, targetType.get()); + for (Entry entry : map.entrySet()) { + sink.getBucket().put(path + (StringUtils.hasText(path) ? "." : "") + entry.getKey(), toBytes(entry + .getValue())); + } + } else if (targetType.filter(it -> ClassUtils.isAssignable(byte[].class, it)).isPresent()) { + sink.getBucket().put(path, toBytes(value)); + } else { + throw new IllegalArgumentException(String.format("Cannot convert value '%s' of type %s to bytes", value, value + .getClass())); + } + } + } + + @Nullable + private Object readCollectionOrArray(String path, Class collectionType, Class valueType, Bucket bucket) { + + List keys = new ArrayList<>(bucket.extractAllKeysFor(path)); + keys.sort(listKeyComparator); + + boolean isArray = collectionType.isArray(); + Class collectionTypeToUse = isArray ? ArrayList.class : collectionType; + Collection target = CollectionFactory.createCollection(collectionTypeToUse, valueType, keys.size()); + + for (String key : keys) { + + if (typeMapper.isTypeKey(key)) { + continue; + } + + Bucket elementData = bucket.extract(key); + + TypeInformation typeInformation = typeMapper.readType(elementData.getPropertyPath(key), TypeInformation.of( + valueType)); + + Class typeToUse = typeInformation.getType(); + if (conversionService.canConvert(byte[].class, typeToUse)) { + target.add(fromBytes(elementData.get(key), typeToUse)); + } else { + target.add(readInternal(key, typeToUse, new RedisData(elementData))); + } + } + + return isArray ? toArray(target, collectionType, valueType) : (target.isEmpty() ? null : target); + } + + /** + * @param keyspace + * @param path + * @param mapValueType + * @param source + * @param sink + */ + private void writeMap(@Nullable String keyspace, String path, Class mapValueType, Map source, + RedisData sink) { + + if (CollectionUtils.isEmpty(source)) { + return; + } + + for (Entry entry : source.entrySet()) { + + if (entry.getValue() == null || entry.getKey() == null) { + continue; + } + + String currentPath = path + ".[" + mapMapKey(entry.getKey()) + "]"; + + if (!ClassUtils.isAssignable(mapValueType, entry.getValue().getClass())) { + throw new MappingException(String.format(INVALID_TYPE_ASSIGNMENT, entry.getValue().getClass(), currentPath, + mapValueType)); + } + + if (customConversions.hasCustomWriteTarget(entry.getValue().getClass())) { + writeToBucket(currentPath, entry.getValue(), sink, mapValueType); + } else { + writeInternal(keyspace, currentPath, entry.getValue(), TypeInformation.of(mapValueType), sink); + } + } + } + + private String mapMapKey(Object key) { + + if (conversionService.canConvert(key.getClass(), byte[].class)) { + return new String(conversionService.convert(key, byte[].class)); + } + + return conversionService.convert(key, String.class); + } + + /** + * @param path + * @param mapType + * @param keyType + * @param valueType + * @param source + * @return + */ + @Nullable + private Map readMapOfSimpleTypes(String path, Class mapType, Class keyType, Class valueType, + RedisData source) { + + Bucket partial = source.getBucket().extract(path + ".["); + + Map target = CollectionFactory.createMap(mapType, partial.size()); + + for (Entry entry : partial.entrySet()) { + + if (typeMapper.isTypeKey(entry.getKey())) { + continue; + } + + Object key = extractMapKeyForPath(path, entry.getKey(), keyType); + Class typeToUse = getTypeHint(path + ".[" + key + "]", source.getBucket(), valueType); + target.put(key, fromBytes(entry.getValue(), typeToUse)); + } + + return target.isEmpty() ? null : target; + } + + /** + * @param path + * @param mapType + * @param keyType + * @param valueType + * @param source + * @return + */ + @Nullable + private Map readMapOfComplexTypes(String path, Class mapType, Class keyType, Class valueType, + RedisData source) { + + Set keys = source.getBucket().extractAllKeysFor(path); + + Map target = CollectionFactory.createMap(mapType, keys.size()); + + for (String key : keys) { + + Bucket partial = source.getBucket().extract(key); + + Object mapKey = extractMapKeyForPath(path, key, keyType); + + TypeInformation typeInformation = typeMapper.readType(source.getBucket().getPropertyPath(key), TypeInformation + .of(valueType)); + + Object o = readInternal(key, typeInformation.getType(), new RedisData(partial)); + target.put(mapKey, o); + } + + return target.isEmpty() ? null : target; + } + + @Nullable + private Object extractMapKeyForPath(String path, String key, Class targetType) { + + String regex = "^(" + Pattern.quote(path) + "\\.\\[)(.*?)(\\])"; + Pattern pattern = Pattern.compile(regex); + + Matcher matcher = pattern.matcher(key); + if (!matcher.find()) { + throw new IllegalArgumentException(String.format("Cannot extract map value for key '%s' in path '%s'.", key, + path)); + } + + Object mapKey = matcher.group(2); + + if (ClassUtils.isAssignable(targetType, mapKey.getClass())) { + return mapKey; + } + + return conversionService.convert(toBytes(mapKey), targetType); + } + + private Class getTypeHint(String path, Bucket bucket, Class fallback) { + + TypeInformation typeInformation = typeMapper.readType(bucket.getPropertyPath(path), TypeInformation.of( + fallback)); + return typeInformation.getType(); + } + + /** + * Convert given source to binary representation using the underlying + * {@link ConversionService}. + * + * @param source the object to convert to binary representation + * @return the binary representation as byte array + * @throws ConverterNotFoundException if no suitable converter can be found + */ + public byte[] toBytes(Object source) { + + if (source instanceof byte[] bytes) { + return bytes; + } + + return conversionService.convert(source, byte[].class); + } + + /** + * Convert given binary representation to desired target type using the + * underlying {@link ConversionService}. + * + * @param the type of the returned object + * @param source the binary data to convert + * @param type the target type to convert to + * @return the converted object of the specified type + * @throws ConverterNotFoundException if no suitable converter can be found + */ + public T fromBytes(byte[] source, Class type) { + + if (type.isInstance(source)) { + return type.cast(source); + } + + return conversionService.convert(source, type); + } + + /** + * Converts a given {@link Collection} into an array considering primitive + * types. + * + * @param source {@link Collection} of values to be added to the array. + * @param arrayType {@link Class} of array. + * @param valueType to be used for conversion before setting the actual value. + * @return + */ + @Nullable + private Object toArray(Collection source, Class arrayType, Class valueType) { + + if (source.isEmpty()) { + return null; + } + + if (!ClassUtils.isPrimitiveArray(arrayType)) { + return source.toArray((Object[]) Array.newInstance(valueType, source.size())); + } + + Object targetArray = Array.newInstance(valueType, source.size()); + Iterator iterator = source.iterator(); + int i = 0; + while (iterator.hasNext()) { + Array.set(targetArray, i, conversionService.convert(iterator.next(), valueType)); + i++; + } + return i > 0 ? targetArray : null; + } + + /** + * Sets the reference resolver to be used for resolving references. + * + * @param referenceResolver the reference resolver to use + */ + public void setReferenceResolver(ReferenceResolver referenceResolver) { + this.referenceResolver = referenceResolver; + } + + /** + * Set {@link CustomConversions} to be applied. + * + * @param customConversions the custom conversions to be used for type conversion + */ + public void setCustomConversions(@Nullable CustomConversions customConversions) { + this.customConversions = customConversions != null ? customConversions : new RedisCustomConversions(); + } + + @Override + public RedisMappingContext getMappingContext() { + return this.mappingContext; + } + + @Override + public EntityInstantiators getEntityInstantiators() { + return entityInstantiators; + } + + @Override + public ConversionService getConversionService() { + return this.conversionService; + } + + @Override + public void afterPropertiesSet() { + this.initializeConverters(); + } + + private void initializeConverters() { + customConversions.registerConvertersIn(conversionService); + } + + private static boolean isByteArray(RedisPersistentProperty property) { + return property.getType().equals(byte[].class); + } + + private static boolean isByteArray(TypeInformation type) { + return type.getType().equals(byte[].class); + } + + private class ConverterAwareParameterValueProvider implements PropertyValueProvider { + + private final String path; + private final RedisData source; + private final ConversionService conversionService; + + ConverterAwareParameterValueProvider(String path, RedisData source, ConversionService conversionService) { + + this.path = path; + this.source = source; + this.conversionService = conversionService; + } + + @Override + @SuppressWarnings( + "unchecked" + ) + public T getPropertyValue(RedisPersistentProperty property) { + + Object value = readProperty(path, source, property); + + if (value == null || ClassUtils.isAssignableValue(property.getType(), value)) { + return (T) value; + } + + return (T) conversionService.convert(value, property.getType()); + } + } + + private enum NaturalOrderingKeyComparator implements Comparator { + + INSTANCE; + + public int compare(String s1, String s2) { + + int s1offset = 0; + int s2offset = 0; + + while (s1offset < s1.length() && s2offset < s2.length()) { + + Part thisPart = extractPart(s1, s1offset); + Part thatPart = extractPart(s2, s2offset); + + int result = thisPart.compareTo(thatPart); + + if (result != 0) { + return result; + } + + s1offset += thisPart.length(); + s2offset += thatPart.length(); + } + + return 0; + } + + private Part extractPart(String source, int offset) { + + StringBuilder builder = new StringBuilder(); + + char c = source.charAt(offset); + builder.append(c); + + boolean isDigit = Character.isDigit(c); + for (int i = offset + 1; i < source.length(); i++) { + + c = source.charAt(i); + if ((isDigit && !Character.isDigit(c)) || (!isDigit && Character.isDigit(c))) { + break; + } + builder.append(c); + } + + return new Part(builder.toString(), isDigit); + } + + private static class Part implements Comparable { + + private final String rawValue; + private final @Nullable Long longValue; + + Part(String value, boolean isDigit) { + + this.rawValue = value; + this.longValue = isDigit ? Long.valueOf(value) : null; + } + + boolean isNumeric() { + return longValue != null; + } + + int length() { + return rawValue.length(); + } + + @Override + public int compareTo(Part that) { + + if (this.isNumeric() && that.isNumeric()) { + return this.longValue.compareTo(that.longValue); + } + + return this.rawValue.compareTo(that.rawValue); + } + } + } + + /** + * Value object representing a Redis Hash/Object identifier composed from + * keyspace and object id in the form of {@literal keyspace:id}. + * + */ + public static class KeyspaceIdentifier { + + /** + * Constant representing a phantom key identifier. + */ + public static final String PHANTOM = "phantom"; + + /** + * Delimiter used to separate keyspace and ID in a key. + */ + public static final String DELIMITER = ":"; + + /** + * Suffix appended to phantom keys, consisting of the delimiter followed by the phantom identifier. + */ + public static final String PHANTOM_SUFFIX = DELIMITER + PHANTOM; + + private final String keyspace; + private final String id; + private final boolean phantomKey; + + private KeyspaceIdentifier(String keyspace, String id, boolean phantomKey) { + + this.keyspace = keyspace; + this.id = id; + this.phantomKey = phantomKey; + } + + /** + * Parse a {@code key} into {@link KeyspaceIdentifier}. + * + * @param key the key representation. + * @return {@link BinaryKeyspaceIdentifier} for binary key. + */ + public static KeyspaceIdentifier of(String key) { + + Assert.isTrue(isValid(key), String.format("Invalid key %s", key)); + + boolean phantomKey = key.endsWith(PHANTOM_SUFFIX); + int keyspaceEndIndex = key.indexOf(DELIMITER); + String keyspace = key.substring(0, keyspaceEndIndex); + String id; + + if (phantomKey) { + id = key.substring(keyspaceEndIndex + 1, key.length() - PHANTOM_SUFFIX.length()); + } else { + id = key.substring(keyspaceEndIndex + 1); + } + + return new KeyspaceIdentifier(keyspace, id, phantomKey); + } + + /** + * Check whether the {@code key} is valid, in particular whether the key + * contains a keyspace and an id part in the form of {@literal keyspace:id}. + * + * @param key the key. + * @return {@literal true} if the key is valid. + */ + public static boolean isValid(@Nullable String key) { + + if (key == null) { + return false; + } + + int keyspaceEndIndex = key.indexOf(DELIMITER); + + return keyspaceEndIndex > 0 && key.length() > keyspaceEndIndex; + } + + /** + * Returns the keyspace part of the identifier. + * + * @return the keyspace string + */ + public String getKeyspace() { + return this.keyspace; + } + + /** + * Returns the ID part of the identifier. + * + * @return the ID string + */ + public String getId() { + return this.id; + } + + /** + * Indicates whether this is a phantom key. + * + * @return true if this is a phantom key, false otherwise + */ + public boolean isPhantomKey() { + return this.phantomKey; + } + } + + /** + * Value object representing a binary Redis Hash/Object identifier composed from + * keyspace and object id in the form of {@literal keyspace:id}. + * + */ + public static class BinaryKeyspaceIdentifier { + + /** + * Binary representation of the phantom key identifier. + */ + public static final byte[] PHANTOM = KeyspaceIdentifier.PHANTOM.getBytes(); + + /** + * Delimiter byte used to separate keyspace and ID in a binary key. + */ + public static final byte DELIMITER = ':'; + + /** + * Binary suffix appended to phantom keys, consisting of the delimiter followed by the phantom identifier. + */ + public static final byte[] PHANTOM_SUFFIX = ByteUtils.concat(new byte[] { DELIMITER }, PHANTOM); + + private final byte[] keyspace; + private final byte[] id; + private final boolean phantomKey; + + private BinaryKeyspaceIdentifier(byte[] keyspace, byte[] id, boolean phantomKey) { + + this.keyspace = keyspace; + this.id = id; + this.phantomKey = phantomKey; + } + + /** + * Parse a binary {@code key} into {@link BinaryKeyspaceIdentifier}. + * + * @param key the binary key representation. + * @return {@link BinaryKeyspaceIdentifier} for binary key. + */ + public static BinaryKeyspaceIdentifier of(byte[] key) { + + Assert.isTrue(isValid(key), String.format("Invalid key %s", new String(key))); + + boolean phantomKey = ByteUtils.startsWith(key, PHANTOM_SUFFIX, key.length - PHANTOM_SUFFIX.length); + + int keyspaceEndIndex = ByteUtils.indexOf(key, DELIMITER); + byte[] keyspace = extractKeyspace(key, keyspaceEndIndex); + byte[] id = extractId(key, phantomKey, keyspaceEndIndex); + + return new BinaryKeyspaceIdentifier(keyspace, id, phantomKey); + } + + /** + * Check whether the {@code key} is valid, in particular whether the key + * contains a keyspace and an id part in the form of {@literal keyspace:id}. + * + * @param key the key. + * @return {@literal true} if the key is valid. + */ + public static boolean isValid(byte[] key) { + + if (key.length == 0) { + return false; + } + + int keyspaceEndIndex = ByteUtils.indexOf(key, DELIMITER); + + return keyspaceEndIndex > 0 && key.length > keyspaceEndIndex; + } + + private static byte[] extractId(byte[] key, boolean phantomKey, int keyspaceEndIndex) { + + int idSize; + + if (phantomKey) { + idSize = (key.length - PHANTOM_SUFFIX.length) - (keyspaceEndIndex + 1); + } else { + + idSize = key.length - (keyspaceEndIndex + 1); + } + + byte[] id = new byte[idSize]; + System.arraycopy(key, keyspaceEndIndex + 1, id, 0, idSize); + + return id; + } + + private static byte[] extractKeyspace(byte[] key, int keyspaceEndIndex) { + + byte[] keyspace = new byte[keyspaceEndIndex]; + System.arraycopy(key, 0, keyspace, 0, keyspaceEndIndex); + + return keyspace; + } + + /** + * Returns the binary keyspace part of the identifier. + * + * @return the keyspace as a byte array + */ + public byte[] getKeyspace() { + return this.keyspace; + } + + /** + * Returns the binary ID part of the identifier. + * + * @return the ID as a byte array + */ + public byte[] getId() { + return this.id; + } + + /** + * Indicates whether this is a phantom key. + * + * @return true if this is a phantom key, false otherwise + */ + public boolean isPhantomKey() { + return this.phantomKey; + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/PartialUpdate.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/PartialUpdate.java new file mode 100644 index 000000000..5e96b5fbb --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/PartialUpdate.java @@ -0,0 +1,236 @@ +package com.redis.om.cache.common.convert; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import org.springframework.util.ClassUtils; + +/** + * {@link PartialUpdate} allows to issue individual property updates without the need of rewriting the whole entity. It + * allows to define {@literal set}, {@literal delete} actions on existing objects while taking care of updating + * potential expiration times of the entity itself as well as index structures. + * + */ +public class PartialUpdate { + + private final Object id; + private final Class target; + private final @Nullable T value; + private boolean refreshTtl = false; + + private final List propertyUpdates = new ArrayList<>(); + + private PartialUpdate(Object id, Class target, @Nullable T value, boolean refreshTtl, + List propertyUpdates) { + + this.id = id; + this.target = target; + this.value = value; + this.refreshTtl = refreshTtl; + this.propertyUpdates.addAll(propertyUpdates); + } + + /** + * Create new {@link PartialUpdate} for given id and type. + * + * @param id must not be {@literal null}. + * @param targetType must not be {@literal null}. + */ + @SuppressWarnings( + "unchecked" + ) + public PartialUpdate(Object id, Class targetType) { + + Assert.notNull(id, "Id must not be null"); + Assert.notNull(targetType, "TargetType must not be null"); + + this.id = id; + this.target = (Class) ClassUtils.getUserClass(targetType); + this.value = null; + } + + /** + * Create new {@link PartialUpdate} for given id and object. + * + * @param id must not be {@literal null}. + * @param value must not be {@literal null}. + */ + @SuppressWarnings( + "unchecked" + ) + public PartialUpdate(Object id, T value) { + + Assert.notNull(id, "Id must not be null"); + Assert.notNull(value, "Value must not be null"); + + this.id = id; + this.target = (Class) ClassUtils.getUserClass(value.getClass()); + this.value = value; + } + + /** + * Create new {@link PartialUpdate} for given id and type. + * + * @param the type of the entity to be updated + * @param id must not be {@literal null}. + * @param targetType must not be {@literal null}. + * @return a new {@link PartialUpdate} instance for the given id and target type + */ + public static PartialUpdate newPartialUpdate(Object id, Class targetType) { + return new PartialUpdate<>(id, targetType); + } + + /** + * @return can be {@literal null}. + */ + @Nullable + public T getValue() { + return value; + } + + /** + * Set the value of a simple or complex {@literal value} reachable via given {@literal path}. + * + * @param path must not be {@literal null}. + * @param value must not be {@literal null}. If you want to remove a value use {@link #del(String)}. + * @return a new {@link PartialUpdate}. + */ + public PartialUpdate set(String path, Object value) { + + Assert.hasText(path, "Path to set must not be null or empty"); + + PartialUpdate update = new PartialUpdate<>(this.id, this.target, this.value, this.refreshTtl, + this.propertyUpdates); + update.propertyUpdates.add(new PropertyUpdate(UpdateCommand.SET, path, value)); + + return update; + } + + /** + * Remove the value reachable via given {@literal path}. + * + * @param path path must not be {@literal null}. + * @return a new {@link PartialUpdate}. + */ + public PartialUpdate del(String path) { + + Assert.hasText(path, "Path to remove must not be null or empty"); + + PartialUpdate update = new PartialUpdate<>(this.id, this.target, this.value, this.refreshTtl, + this.propertyUpdates); + update.propertyUpdates.add(new PropertyUpdate(UpdateCommand.DEL, path)); + + return update; + } + + /** + * Get the target type. + * + * @return never {@literal null}. + */ + public Class getTarget() { + return target; + } + + /** + * Get the id of the element to update. + * + * @return never {@literal null}. + */ + public Object getId() { + return id; + } + + /** + * Get the list of individual property updates. + * + * @return never {@literal null}. + */ + public List getPropertyUpdates() { + return Collections.unmodifiableList(propertyUpdates); + } + + /** + * @return true if expiration time of target should be updated. + */ + public boolean isRefreshTtl() { + return refreshTtl; + } + + /** + * Set indicator for updating expiration time of target. + * + * @param refreshTtl whether to refresh the TTL (Time To Live) of the target entity + * @return a new {@link PartialUpdate}. + */ + public PartialUpdate refreshTtl(boolean refreshTtl) { + return new PartialUpdate<>(this.id, this.target, this.value, refreshTtl, this.propertyUpdates); + } + + /** + * Inner class representing a property update operation with a command type, property path, and optional value. + * Used to track individual property changes within a {@link PartialUpdate}. + */ + public static class PropertyUpdate { + + private final UpdateCommand cmd; + private final String propertyPath; + private final @Nullable Object value; + + private PropertyUpdate(UpdateCommand cmd, String propertyPath) { + this(cmd, propertyPath, null); + } + + private PropertyUpdate(UpdateCommand cmd, String propertyPath, @Nullable Object value) { + + this.cmd = cmd; + this.propertyPath = propertyPath; + this.value = value; + } + + /** + * Get the associated {@link UpdateCommand}. + * + * @return never {@literal null}. + */ + public UpdateCommand getCmd() { + return cmd; + } + + /** + * Get the target path. + * + * @return never {@literal null}. + */ + public String getPropertyPath() { + return propertyPath; + } + + /** + * Get the value to set. + * + * @return can be {@literal null}. + */ + @Nullable + public Object getValue() { + return value; + } + } + + /** + * Enum representing the types of update commands that can be performed on properties. + */ + public enum UpdateCommand { + /** + * Command to set a property value + */ + SET, + /** + * Command to delete a property + */ + DEL + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisConverter.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisConverter.java new file mode 100644 index 000000000..23a9fc513 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisConverter.java @@ -0,0 +1,29 @@ +package com.redis.om.cache.common.convert; + +import org.springframework.data.convert.EntityConverter; +import org.springframework.data.mapping.model.EntityInstantiators; + +/** + * Redis specific {@link EntityConverter} that handles conversion between domain objects and + * Redis data structures. This interface extends the Spring Data {@link EntityConverter} with + * Redis-specific functionality. + */ +public interface RedisConverter extends + EntityConverter, RedisPersistentProperty, Object, RedisData> { + + /** + * Returns the mapping context used by this converter. + * + * @return the {@link RedisMappingContext} used by this converter + */ + @Override + RedisMappingContext getMappingContext(); + + /** + * Returns the entity instantiators used by this converter. + * + * @return the configured {@link EntityInstantiators} + * @since 3.2.4 + */ + EntityInstantiators getEntityInstantiators(); +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisCustomConversions.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisCustomConversions.java new file mode 100644 index 000000000..15c2cf104 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisCustomConversions.java @@ -0,0 +1,45 @@ +package com.redis.om.cache.common.convert; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.springframework.data.mapping.model.SimpleTypeHolder; + +/** + * Value object to capture custom conversion. That is essentially a {@link List} of converters and some additional logic + * around them. + * + */ +public class RedisCustomConversions extends org.springframework.data.convert.CustomConversions { + + private static final StoreConversions STORE_CONVERSIONS; + private static final List STORE_CONVERTERS; + + static { + + List converters = new ArrayList<>(35); + + converters.addAll(BinaryConverters.getConvertersToRegister()); + converters.addAll(Jsr310Converters.getConvertersToRegister()); + + STORE_CONVERTERS = Collections.unmodifiableList(converters); + STORE_CONVERSIONS = StoreConversions.of(SimpleTypeHolder.DEFAULT, STORE_CONVERTERS); + } + + /** + * Creates an empty {@link RedisCustomConversions} object. + */ + public RedisCustomConversions() { + this(Collections.emptyList()); + } + + /** + * Creates a new {@link RedisCustomConversions} instance registering the given converters. + * + * @param converters list of custom converters to register + */ + public RedisCustomConversions(List converters) { + super(STORE_CONVERSIONS, converters); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisData.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisData.java new file mode 100644 index 000000000..ca49e1dbb --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisData.java @@ -0,0 +1,168 @@ +package com.redis.om.cache.common.convert; + +import java.util.*; +import java.util.concurrent.TimeUnit; + +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +/** + * Data object holding {@link Bucket} representing the domain object to be stored in a Redis hash. Index information + * points to additional structures holding the objects is for searching. + * + */ +public class RedisData { + + private final Bucket bucket; + private final Set indexedData; + + private @Nullable String keyspace; + private @Nullable String id; + private @Nullable Long timeToLive; + + /** + * Creates new {@link RedisData} with empty {@link Bucket}. + */ + public RedisData() { + this(Collections.emptyMap()); + } + + /** + * Creates new {@link RedisData} with {@link Bucket} holding provided values. + * + * @param raw should not be {@literal null}. + */ + public RedisData(Map raw) { + this(Bucket.newBucketFromRawMap(raw)); + } + + /** + * Creates new {@link RedisData} with {@link Bucket} + * + * @param bucket must not be {@literal null}. + */ + public RedisData(Bucket bucket) { + + Assert.notNull(bucket, "Bucket must not be null"); + + this.bucket = bucket; + this.indexedData = new HashSet<>(); + } + + /** + * Set the id to be used as part of the key. + * + * @param id the ID to set, can be {@literal null} + */ + public void setId(@Nullable String id) { + this.id = id; + } + + /** + * Get the ID used as part of the key. + * + * @return the ID or {@literal null} if not set + */ + @Nullable + public String getId() { + return this.id; + } + + /** + * Get the time before expiration in seconds. + * + * @return {@literal null} if not set. + */ + @Nullable + public Long getTimeToLive() { + return timeToLive; + } + + /** + * Add indexed data for additional search structures. + * + * @param index must not be {@literal null}. + */ + public void addIndexedData(IndexedData index) { + + Assert.notNull(index, "IndexedData to add must not be null"); + this.indexedData.add(index); + } + + /** + * Add multiple indexed data entries for additional search structures. + * + * @param indexes must not be {@literal null}. + */ + public void addIndexedData(Collection indexes) { + + Assert.notNull(indexes, "IndexedData to add must not be null"); + this.indexedData.addAll(indexes); + } + + /** + * Get all indexed data entries for additional search structures. + * + * @return an unmodifiable set of indexed data, never {@literal null}. + */ + public Set getIndexedData() { + return Collections.unmodifiableSet(this.indexedData); + } + + /** + * Get the keyspace used for storing this data in Redis. + * + * @return the keyspace or {@literal null} if not set + */ + @Nullable + public String getKeyspace() { + return keyspace; + } + + /** + * Set the keyspace to be used for storing this data in Redis. + * + * @param keyspace the keyspace to set, can be {@literal null} + */ + public void setKeyspace(@Nullable String keyspace) { + this.keyspace = keyspace; + } + + /** + * Get the bucket containing the data to be stored in Redis. + * + * @return the bucket, never {@literal null} + */ + public Bucket getBucket() { + return bucket; + } + + /** + * Set the time before expiration in {@link TimeUnit#SECONDS}. + * + * @param timeToLive can be {@literal null}. + */ + public void setTimeToLive(Long timeToLive) { + this.timeToLive = timeToLive; + } + + /** + * Set the time before expiration converting the given arguments to {@link TimeUnit#SECONDS}. + * + * @param timeToLive must not be {@literal null} + * @param timeUnit must not be {@literal null} + */ + public void setTimeToLive(Long timeToLive, TimeUnit timeUnit) { + + Assert.notNull(timeToLive, "TimeToLive must not be null when used with TimeUnit"); + Assert.notNull(timeUnit, "TimeUnit must not be null"); + + setTimeToLive(TimeUnit.SECONDS.convert(timeToLive, timeUnit)); + } + + @Override + public String toString() { + return "RedisDataObject [key=" + keyspace + ":" + id + ", hash=" + bucket + "]"; + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisMappingContext.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisMappingContext.java new file mode 100644 index 000000000..3cdd27886 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisMappingContext.java @@ -0,0 +1,112 @@ +package com.redis.om.cache.common.convert; + +import org.springframework.data.keyvalue.annotation.KeySpace; +import org.springframework.data.keyvalue.core.mapping.KeySpaceResolver; +import org.springframework.data.keyvalue.core.mapping.context.KeyValueMappingContext; +import org.springframework.data.mapping.context.MappingContext; +import org.springframework.data.mapping.model.Property; +import org.springframework.data.mapping.model.SimpleTypeHolder; +import org.springframework.data.util.TypeInformation; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import org.springframework.util.ClassUtils; +import org.springframework.util.StringUtils; + +/** + * Redis specific {@link MappingContext}. + * + */ +public class RedisMappingContext extends KeyValueMappingContext, RedisPersistentProperty> { + + private static final SimpleTypeHolder SIMPLE_TYPE_HOLDER = new RedisCustomConversions().getSimpleTypeHolder(); + + private final MappingConfiguration mappingConfiguration; + + /** + * Creates new {@link RedisMappingContext} with empty {@link MappingConfiguration}. + */ + public RedisMappingContext() { + this(new MappingConfiguration(new KeyspaceConfiguration())); + } + + /** + * Creates new {@link RedisMappingContext}. + * + * @param mappingConfiguration can be {@literal null}. + */ + public RedisMappingContext(@Nullable MappingConfiguration mappingConfiguration) { + + this.mappingConfiguration = mappingConfiguration != null ? + mappingConfiguration : + new MappingConfiguration(new KeyspaceConfiguration()); + + setKeySpaceResolver(new ConfigAwareKeySpaceResolver(this.mappingConfiguration.getKeyspaceConfiguration())); + this.setSimpleTypeHolder(SIMPLE_TYPE_HOLDER); + } + + @Override + protected RedisPersistentEntity createPersistentEntity(TypeInformation typeInformation) { + return new BasicRedisPersistentEntity<>(typeInformation, getKeySpaceResolver()); + } + + @Override + protected RedisPersistentProperty createPersistentProperty(Property property, RedisPersistentEntity owner, + SimpleTypeHolder simpleTypeHolder) { + return new RedisPersistentProperty(property, owner, simpleTypeHolder); + } + + /** + * Get the {@link MappingConfiguration} used. + * + * @return never {@literal null}. + */ + public MappingConfiguration getMappingConfiguration() { + return mappingConfiguration; + } + + /** + * {@link KeySpaceResolver} implementation considering {@link KeySpace} and {@link KeyspaceConfiguration}. + * + */ + static class ConfigAwareKeySpaceResolver implements KeySpaceResolver { + + private final KeyspaceConfiguration keyspaceConfig; + + public ConfigAwareKeySpaceResolver(KeyspaceConfiguration keyspaceConfig) { + + this.keyspaceConfig = keyspaceConfig; + } + + @Override + public String resolveKeySpace(Class type) { + + Assert.notNull(type, "Type must not be null"); + if (keyspaceConfig.hasSettingsFor(type)) { + + String value = keyspaceConfig.getKeyspaceSettings(type).getKeyspace(); + if (StringUtils.hasText(value)) { + return value; + } + } + + return null; + } + } + + /** + * {@link KeySpaceResolver} implementation considering {@link KeySpace}. + * + */ + enum ClassNameKeySpaceResolver implements KeySpaceResolver { + + INSTANCE; + + @Override + public String resolveKeySpace(Class type) { + + Assert.notNull(type, "Type must not be null"); + return ClassUtils.getUserClass(type).getName(); + } + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisPersistentEntity.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisPersistentEntity.java new file mode 100644 index 000000000..19ed6a6ed --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisPersistentEntity.java @@ -0,0 +1,15 @@ +package com.redis.om.cache.common.convert; + +import org.springframework.data.keyvalue.core.mapping.KeyValuePersistentEntity; +import org.springframework.data.mapping.PersistentEntity; + +/** + * Redis specific {@link PersistentEntity} that represents a persistent entity stored in Redis. + * This interface extends the Spring Data {@link KeyValuePersistentEntity} with Redis-specific + * functionality for managing entity metadata and mapping between domain objects and Redis data structures. + * + * @param the type of the persistent entity + */ +public interface RedisPersistentEntity extends KeyValuePersistentEntity { + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisPersistentProperty.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisPersistentProperty.java new file mode 100644 index 000000000..d9ba0e03d --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisPersistentProperty.java @@ -0,0 +1,45 @@ +package com.redis.om.cache.common.convert; + +import java.util.HashSet; +import java.util.Set; + +import org.springframework.data.keyvalue.core.mapping.KeyValuePersistentProperty; +import org.springframework.data.mapping.PersistentEntity; +import org.springframework.data.mapping.PersistentProperty; +import org.springframework.data.mapping.model.Property; +import org.springframework.data.mapping.model.SimpleTypeHolder; + +/** + * Redis specific {@link PersistentProperty} implementation. + * + */ +public class RedisPersistentProperty extends KeyValuePersistentProperty { + + private static final Set SUPPORTED_ID_PROPERTY_NAMES = new HashSet<>(); + + static { + SUPPORTED_ID_PROPERTY_NAMES.add("id"); + } + + /** + * Creates new {@link RedisPersistentProperty}. + * + * @param property the property to be persisted + * @param owner the entity owning the property + * @param simpleTypeHolder holder of simple type information + */ + public RedisPersistentProperty(Property property, PersistentEntity owner, + SimpleTypeHolder simpleTypeHolder) { + super(property, owner, simpleTypeHolder); + } + + @Override + public boolean isIdProperty() { + + if (super.isIdProperty()) { + return true; + } + + return SUPPORTED_ID_PROPERTY_NAMES.contains(getName()); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisTypeMapper.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisTypeMapper.java new file mode 100644 index 000000000..914857c1d --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/RedisTypeMapper.java @@ -0,0 +1,20 @@ +package com.redis.om.cache.common.convert; + +import org.springframework.data.convert.TypeMapper; + +import com.redis.om.cache.common.convert.Bucket.BucketPropertyPath; + +/** + * Redis-specific {@link TypeMapper} exposing that {@link BucketPropertyPath}s might contain a type key. + * + */ +public interface RedisTypeMapper extends TypeMapper { + + /** + * Returns whether the given {@code key} is the type key. + * + * @param key the key to check + * @return {@literal true} if the given {@code key} is the type key. + */ + boolean isTypeKey(String key); +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/ReferenceResolver.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/ReferenceResolver.java new file mode 100644 index 000000000..b62256da7 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/convert/ReferenceResolver.java @@ -0,0 +1,21 @@ +package com.redis.om.cache.common.convert; + +import java.util.Map; + +import org.springframework.data.annotation.Reference; +import org.springframework.lang.Nullable; + +/** + * {@link ReferenceResolver} retrieves Objects marked with {@link Reference} from Redis. + * + */ +public interface ReferenceResolver { + + /** + * @param id must not be {@literal null}. + * @param keyspace must not be {@literal null}. + * @return {@literal null} if referenced object does not exist. + */ + @Nullable + Map resolveReference(Object id, String keyspace); +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/ByteArrayMapper.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/ByteArrayMapper.java new file mode 100644 index 000000000..e0539dbea --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/ByteArrayMapper.java @@ -0,0 +1,30 @@ +package com.redis.om.cache.common.mapping; + +import org.springframework.lang.Nullable; + +import com.redis.om.cache.common.RedisStringMapper; + +/** + * Implementation of {@link RedisStringMapper} that handles byte arrays. + * This mapper acts as a pass-through for byte array data, returning the byte arrays directly + * without any transformation in both directions. + */ +public class ByteArrayMapper implements RedisStringMapper { + + /** + * Singleton instance of ByteArrayMapper for convenient access. + */ + public static final ByteArrayMapper INSTANCE = new ByteArrayMapper(); + + @Nullable + @Override + public byte[] toString(@Nullable Object value) { + return (byte[]) value; + } + + @Nullable + @Override + public byte[] fromString(@Nullable byte[] bytes) { + return bytes; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/GenericJackson2JsonMapper.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/GenericJackson2JsonMapper.java new file mode 100644 index 000000000..39cd7f7b7 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/GenericJackson2JsonMapper.java @@ -0,0 +1,665 @@ +package com.redis.om.cache.common.mapping; + +import java.io.IOException; +import java.io.Serial; +import java.util.Collections; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import org.springframework.cache.support.NullValue; +import org.springframework.core.KotlinDetector; +import org.springframework.data.util.Lazy; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; +import org.springframework.util.ClassUtils; +import org.springframework.util.StringUtils; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.TreeNode; +import com.fasterxml.jackson.databind.*; +import com.fasterxml.jackson.databind.ObjectMapper.DefaultTyping; +import com.fasterxml.jackson.databind.jsontype.PolymorphicTypeValidator; +import com.fasterxml.jackson.databind.jsontype.TypeDeserializer; +import com.fasterxml.jackson.databind.jsontype.TypeSerializer; +import com.fasterxml.jackson.databind.jsontype.impl.StdTypeResolverBuilder; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.databind.ser.SerializerFactory; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import com.fasterxml.jackson.databind.type.TypeFactory; +import com.redis.om.cache.common.RedisStringMapper; +import com.redis.om.cache.common.SerializationException; + +/** + * Implementation of {@link RedisStringMapper} that uses Jackson's {@link ObjectMapper} to convert objects to and from + * JSON. + * This class provides various constructors to configure the Jackson mapper with different type handling options. + * It supports default typing, custom type hint property names, and custom object readers and writers. + */ +public class GenericJackson2JsonMapper implements RedisStringMapper { + + private final JacksonObjectReader reader; + + private final JacksonObjectWriter writer; + + private final Lazy defaultTypingEnabled; + + private final ObjectMapper mapper; + + private final TypeResolver typeResolver; + + /** + * Creates {@link GenericJackson2JsonMapper} initialized with an + * {@link ObjectMapper} configured for default typing. + */ + public GenericJackson2JsonMapper() { + this((String) null); + } + + /** + * Creates {@link GenericJackson2JsonMapper} initialized with an + * {@link ObjectMapper} configured for default typing using the given + * {@link String name}. + *

+ * In case {@link String name} is {@literal empty} or {@literal null}, then + * {@link JsonTypeInfo.Id#CLASS} will be used. + * + * @param typeHintPropertyName {@link String name} of the JSON property holding + * type information; can be {@literal null}. + * @see ObjectMapper#activateDefaultTypingAsProperty(PolymorphicTypeValidator, + * DefaultTyping, String) + * @see ObjectMapper#activateDefaultTyping(PolymorphicTypeValidator, + * DefaultTyping, As) + */ + public GenericJackson2JsonMapper(@Nullable String typeHintPropertyName) { + this(typeHintPropertyName, JacksonObjectReader.create(), JacksonObjectWriter.create()); + } + + /** + * Creates {@link GenericJackson2JsonMapper} initialized with an + * {@link ObjectMapper} configured for default typing using the given + * {@link String name} along with the given, required + * {@link JacksonObjectReader} and {@link JacksonObjectWriter} used to + * read/write {@link Object Objects} de/serialized as JSON. + *

+ * In case {@link String name} is {@literal empty} or {@literal null}, then + * {@link JsonTypeInfo.Id#CLASS} will be used. + * + * @param typeHintPropertyName {@link String name} of the JSON property holding + * type information; can be {@literal null}. + * @param reader {@link JacksonObjectReader} function to read + * objects using {@link ObjectMapper}. + * @param writer {@link JacksonObjectWriter} function to write + * objects using {@link ObjectMapper}. + * @see ObjectMapper#activateDefaultTypingAsProperty(PolymorphicTypeValidator, + * DefaultTyping, String) + * @see ObjectMapper#activateDefaultTyping(PolymorphicTypeValidator, + * DefaultTyping, As) + * @since 3.0 + */ + public GenericJackson2JsonMapper(@Nullable String typeHintPropertyName, JacksonObjectReader reader, + JacksonObjectWriter writer) { + + this(new ObjectMapper(), reader, writer, typeHintPropertyName); + + registerNullValueSerializer(this.mapper, typeHintPropertyName); + + this.mapper.setDefaultTyping(createDefaultTypeResolverBuilder(getObjectMapper(), typeHintPropertyName)); + } + + /** + * Setting a custom-configured {@link ObjectMapper} is one way to take further + * control of the JSON serialization process. For example, an extended + * {@link SerializerFactory} can be configured that provides custom serializers + * for specific types. + * + * @param mapper must not be {@literal null}. + */ + public GenericJackson2JsonMapper(ObjectMapper mapper) { + this(mapper, JacksonObjectReader.create(), JacksonObjectWriter.create()); + } + + /** + * Setting a custom-configured {@link ObjectMapper} is one way to take further + * control of the JSON serialization process. For example, an extended + * {@link SerializerFactory} can be configured that provides custom serializers + * for specific types. + * + * @param mapper must not be {@literal null}. + * @param reader the {@link JacksonObjectReader} function to read objects using + * {@link ObjectMapper}. + * @param writer the {@link JacksonObjectWriter} function to write objects using + * {@link ObjectMapper}. + * @since 3.0 + */ + public GenericJackson2JsonMapper(ObjectMapper mapper, JacksonObjectReader reader, JacksonObjectWriter writer) { + + this(mapper, reader, writer, null); + } + + private GenericJackson2JsonMapper(ObjectMapper mapper, JacksonObjectReader reader, JacksonObjectWriter writer, + @Nullable String typeHintPropertyName) { + + Assert.notNull(mapper, "ObjectMapper must not be null"); + Assert.notNull(reader, "Reader must not be null"); + Assert.notNull(writer, "Writer must not be null"); + + this.mapper = mapper; + this.reader = reader; + this.writer = writer; + + this.defaultTypingEnabled = Lazy.of(() -> mapper.getSerializationConfig().getDefaultTyper(null) != null); + + this.typeResolver = newTypeResolver(mapper, typeHintPropertyName, this.defaultTypingEnabled); + } + + private static TypeResolver newTypeResolver(ObjectMapper mapper, @Nullable String typeHintPropertyName, + Lazy defaultTypingEnabled) { + + Lazy lazyTypeFactory = Lazy.of(mapper::getTypeFactory); + + Lazy lazyTypeHintPropertyName = typeHintPropertyName != null ? + Lazy.of(typeHintPropertyName) : + newLazyTypeHintPropertyName(mapper, defaultTypingEnabled); + + return new TypeResolver(lazyTypeFactory, lazyTypeHintPropertyName); + } + + private static Lazy newLazyTypeHintPropertyName(ObjectMapper mapper, Lazy defaultTypingEnabled) { + + Lazy configuredTypeDeserializationPropertyName = getConfiguredTypeDeserializationPropertyName(mapper); + + Lazy resolvedLazyTypeHintPropertyName = Lazy.of(() -> defaultTypingEnabled.get() ? + null : + configuredTypeDeserializationPropertyName.get()); + + return resolvedLazyTypeHintPropertyName.or("@class"); + } + + private static Lazy getConfiguredTypeDeserializationPropertyName(ObjectMapper mapper) { + + return Lazy.of(() -> { + + DeserializationConfig deserializationConfig = mapper.getDeserializationConfig(); + + JavaType objectType = mapper.getTypeFactory().constructType(Object.class); + + TypeDeserializer typeDeserializer = deserializationConfig.getDefaultTyper(null).buildTypeDeserializer( + deserializationConfig, objectType, Collections.emptyList()); + + return typeDeserializer.getPropertyName(); + }); + } + + private static StdTypeResolverBuilder createDefaultTypeResolverBuilder(ObjectMapper objectMapper, + @Nullable String typeHintPropertyName) { + + StdTypeResolverBuilder typer = TypeResolverBuilder.forEverything(objectMapper).init(JsonTypeInfo.Id.CLASS, null) + .inclusion(As.PROPERTY); + + if (StringUtils.hasText(typeHintPropertyName)) { + typer = typer.typeProperty(typeHintPropertyName); + } + return typer; + } + + /** + * Factory method returning a {@literal Builder} used to construct and configure + * a {@link GenericJackson2JsonMapper}. + * + * @return new + * {@link GenericJackson2JsonRedisSerializerBuilder}. + * @since 3.3.1 + */ + public static GenericJackson2JsonRedisSerializerBuilder builder() { + return new GenericJackson2JsonRedisSerializerBuilder(); + } + + /** + * Register {@link NullValueSerializer} in the given {@link ObjectMapper} with + * an optional {@code typeHintPropertyName}. This method should be called by + * code that customizes {@link GenericJackson2JsonMapper} by providing an + * external {@link ObjectMapper}. + * + * @param objectMapper the object mapper to customize. + * @param typeHintPropertyName name of the type property. Defaults to + * {@code @class} if {@literal null}/empty. + * @since 2.2 + */ + public static void registerNullValueSerializer(ObjectMapper objectMapper, @Nullable String typeHintPropertyName) { + + // Simply setting {@code + // mapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS)} does not help here + // since we need the type hint embedded for deserialization using the default + // typing feature. + objectMapper.registerModule(new SimpleModule().addSerializer(new NullValueSerializer(typeHintPropertyName))); + } + + /** + * Gets the configured {@link ObjectMapper} used internally by this + * {@link GenericJackson2JsonMapper} to de/serialize {@link Object objects} as + * {@literal JSON}. + * + * @return the configured {@link ObjectMapper}. + */ + protected ObjectMapper getObjectMapper() { + return this.mapper; + } + + @Override + public byte[] toString(@Nullable Object value) throws SerializationException { + + if (value == null) { + return SerializationUtils.EMPTY_ARRAY; + } + + try { + return writer.write(mapper, value); + } catch (IOException ex) { + String message = String.format("Could not write JSON: %s", ex.getMessage()); + throw new SerializationException(message, ex); + } + } + + @Override + public Object fromString(@Nullable byte[] source) throws SerializationException { + return deserialize(source, Object.class); + } + + /** + * Deserialized the array of bytes containing {@literal JSON} as an + * {@link Object} of the given, required {@link Class type}. + * + * @param the type of the object to deserialize to + * @param source array of bytes containing the {@literal JSON} to deserialize; + * can be {@literal null}. + * @param type {@link Class type} of {@link Object} from which the + * {@literal JSON} will be deserialized; must not be + * {@literal null}. + * @return {@literal null} for an empty source, or an {@link Object} of the + * given {@link Class type} deserialized from the array of bytes + * containing {@literal JSON}. + * @throws IllegalArgumentException if the given {@link Class type} is + * {@literal null}. + * @throws SerializationException if the array of bytes cannot be deserialized + * as an instance of the given {@link Class + * type} + */ + @Nullable + @SuppressWarnings( + "unchecked" + ) + public T deserialize(@Nullable byte[] source, Class type) throws SerializationException { + + Assert.notNull(type, + "Deserialization type must not be null;" + " Please provide Object.class to make use of Jackson2 default typing."); + + if (SerializationUtils.isEmpty(source)) { + return null; + } + + try { + return (T) reader.read(mapper, source, resolveType(source, type)); + } catch (Exception ex) { + String message = String.format("Could not read JSON:%s ", ex.getMessage()); + throw new SerializationException(message, ex); + } + } + + /** + * Builder method used to configure and customize the internal Jackson + * {@link ObjectMapper} created by this {@link GenericJackson2JsonMapper} and + * used to de/serialize {@link Object objects} as {@literal JSON}. + * + * @param objectMapperConfigurer {@link Consumer} used to configure and + * customize the internal {@link ObjectMapper}; + * must not be {@literal null}. + * @return this {@link GenericJackson2JsonMapper}. + * @throws IllegalArgumentException if the {@link Consumer} used to configure + * and customize the internal + * {@link ObjectMapper} is {@literal null}. + * @since 3.1.5 + */ + public GenericJackson2JsonMapper configure(Consumer objectMapperConfigurer) { + + Assert.notNull(objectMapperConfigurer, "Consumer used to configure and customize ObjectMapper must not be null"); + + objectMapperConfigurer.accept(getObjectMapper()); + + return this; + } + + /** + * Resolves the JavaType for deserialization based on the source bytes and target type. + * If the target type is Object.class and default typing is enabled, attempts to resolve + * the type from the JSON content using the type hint. + * + * @param source the JSON source bytes + * @param type the target class type + * @return the resolved JavaType + * @throws IOException if an error occurs during type resolution + */ + protected JavaType resolveType(byte[] source, Class type) throws IOException { + + if (!type.equals(Object.class) || !defaultTypingEnabled.get()) { + return typeResolver.constructType(type); + } + + return typeResolver.resolveType(source, type); + } + + /** + * @since 3.0 + */ + static class TypeResolver { + + // need a separate instance to bypass class hint checks + private final ObjectMapper mapper = new ObjectMapper(); + + private final Supplier typeFactory; + private final Supplier hintName; + + TypeResolver(Supplier typeFactory, Supplier hintName) { + + this.typeFactory = typeFactory; + this.hintName = hintName; + } + + protected JavaType constructType(Class type) { + return typeFactory.get().constructType(type); + } + + /** + * Resolves the JavaType from the JSON source bytes by extracting the type hint. + * If a type hint is found in the JSON, constructs the JavaType from the canonical name. + * Otherwise, falls back to constructing the type from the provided class. + * + * @param source the JSON source bytes + * @param type the fallback class type + * @return the resolved JavaType + * @throws IOException if an error occurs during JSON parsing + */ + protected JavaType resolveType(byte[] source, Class type) throws IOException { + + JsonNode root = mapper.readTree(source); + JsonNode jsonNode = root.get(hintName.get()); + + if (jsonNode instanceof TextNode && jsonNode.asText() != null) { + return typeFactory.get().constructFromCanonical(jsonNode.asText()); + } + + return constructType(type); + } + } + + private static class NullValueSerializer extends StdSerializer { + + @Serial + private static final long serialVersionUID = 1999052150548658808L; + + private final String classIdentifier; + + /** + * @param classIdentifier can be {@literal null} and will be defaulted to + * {@code @class}. + */ + NullValueSerializer(@Nullable String classIdentifier) { + + super(NullValue.class); + this.classIdentifier = StringUtils.hasText(classIdentifier) ? classIdentifier : "@class"; + } + + @Override + public void serialize(NullValue value, JsonGenerator jsonGenerator, SerializerProvider provider) + throws IOException { + + jsonGenerator.writeStartObject(); + jsonGenerator.writeStringField(classIdentifier, NullValue.class.getName()); + jsonGenerator.writeEndObject(); + } + + @Override + public void serializeWithType(NullValue value, JsonGenerator jsonGenerator, SerializerProvider serializers, + TypeSerializer typeSerializer) throws IOException { + + serialize(value, jsonGenerator, serializers); + } + } + + /** + * Builder class for creating {@link GenericJackson2JsonMapper} instances with various configuration options. + * Provides methods for configuring default typing, type hint property name, ObjectMapper, reader, writer, + * and null value serializer. + */ + public static class GenericJackson2JsonRedisSerializerBuilder { + + private @Nullable String typeHintPropertyName; + + private JacksonObjectReader reader = JacksonObjectReader.create(); + + private JacksonObjectWriter writer = JacksonObjectWriter.create(); + + private @Nullable ObjectMapper objectMapper; + + private @Nullable Boolean defaultTyping; + + private boolean registerNullValueSerializer = true; + + private @Nullable StdSerializer nullValueSerializer; + + private GenericJackson2JsonRedisSerializerBuilder() { + } + + /** + * Enable or disable default typing. Enabling default typing will override + * {@link ObjectMapper#setDefaultTyping(com.fasterxml.jackson.databind.jsontype.TypeResolverBuilder)} + * for a given {@link ObjectMapper}. Default typing is enabled by default if no + * {@link ObjectMapper} is provided. + * + * @param defaultTyping whether to enable/disable default typing. Enabled by + * default if the {@link ObjectMapper} is not provided. + * @return this + * {@link GenericJackson2JsonRedisSerializerBuilder}. + */ + public GenericJackson2JsonRedisSerializerBuilder defaultTyping(boolean defaultTyping) { + this.defaultTyping = defaultTyping; + return this; + } + + /** + * Configure a property name to that represents the type hint. + * + * @param typeHintPropertyName {@link String name} of the JSON property holding + * type information. + * @return this + * {@link GenericJackson2JsonRedisSerializerBuilder}. + */ + public GenericJackson2JsonRedisSerializerBuilder typeHintPropertyName(String typeHintPropertyName) { + + Assert.hasText(typeHintPropertyName, "Type hint property name must bot be null or empty"); + + this.typeHintPropertyName = typeHintPropertyName; + return this; + } + + /** + * Configure a provided {@link ObjectMapper}. Note that the provided + * {@link ObjectMapper} can be reconfigured with a {@link #nullValueSerializer} + * or default typing depending on the builder configuration. + * + * @param objectMapper must not be {@literal null}. + * @return this + * {@link GenericJackson2JsonRedisSerializerBuilder}. + */ + public GenericJackson2JsonRedisSerializerBuilder objectMapper(ObjectMapper objectMapper) { + + Assert.notNull(objectMapper, "ObjectMapper must not be null"); + + this.objectMapper = objectMapper; + return this; + } + + /** + * Configure {@link JacksonObjectReader}. + * + * @param reader must not be {@literal null}. + * @return this + * {@link GenericJackson2JsonRedisSerializerBuilder}. + */ + public GenericJackson2JsonRedisSerializerBuilder reader(JacksonObjectReader reader) { + + Assert.notNull(reader, "JacksonObjectReader must not be null"); + + this.reader = reader; + return this; + } + + /** + * Configure {@link JacksonObjectWriter}. + * + * @param writer must not be {@literal null}. + * @return this + * {@link GenericJackson2JsonRedisSerializerBuilder}. + */ + public GenericJackson2JsonRedisSerializerBuilder writer(JacksonObjectWriter writer) { + + Assert.notNull(writer, "JacksonObjectWriter must not be null"); + + this.writer = writer; + return this; + } + + /** + * Register a {@link StdSerializer serializer} for {@link NullValue}. + * + * @param nullValueSerializer the {@link StdSerializer} to use for + * {@link NullValue} serialization, must not be + * {@literal null}. + * @return this + * {@link GenericJackson2JsonRedisSerializerBuilder}. + */ + public GenericJackson2JsonRedisSerializerBuilder nullValueSerializer(StdSerializer nullValueSerializer) { + + Assert.notNull(nullValueSerializer, "Null value serializer must not be null"); + + this.nullValueSerializer = nullValueSerializer; + return this; + } + + /** + * Configure whether to register a {@link StdSerializer serializer} for + * {@link NullValue} serialization. The default serializer considers + * {@link #typeHintPropertyName(String)}. + * + * @param registerNullValueSerializer {@code true} to register the default + * serializer; {@code false} otherwise. + * @return this + * {@link GenericJackson2JsonRedisSerializerBuilder}. + */ + public GenericJackson2JsonRedisSerializerBuilder registerNullValueSerializer(boolean registerNullValueSerializer) { + this.registerNullValueSerializer = registerNullValueSerializer; + return this; + } + + /** + * Creates a new instance of {@link GenericJackson2JsonMapper} with + * configuration options applied. Creates also a new {@link ObjectMapper} if + * none was provided. + * + * @return a new instance of {@link GenericJackson2JsonMapper}. + */ + public GenericJackson2JsonMapper build() { + + ObjectMapper objectMapper = this.objectMapper; + boolean providedObjectMapper = objectMapper != null; + + if (objectMapper == null) { + objectMapper = new ObjectMapper(); + } + + if (registerNullValueSerializer) { + objectMapper.registerModule(new SimpleModule("GenericJackson2JsonRedisSerializerBuilder").addSerializer( + this.nullValueSerializer != null ? + this.nullValueSerializer : + new NullValueSerializer(this.typeHintPropertyName))); + } + + if ((!providedObjectMapper && (defaultTyping == null || defaultTyping)) || (defaultTyping != null && defaultTyping)) { + objectMapper.setDefaultTyping(createDefaultTypeResolverBuilder(objectMapper, typeHintPropertyName)); + } + + return new GenericJackson2JsonMapper(objectMapper, this.reader, this.writer, this.typeHintPropertyName); + } + } + + @SuppressWarnings( + "serial" + ) + private static class TypeResolverBuilder extends ObjectMapper.DefaultTypeResolverBuilder { + + @SuppressWarnings( + "deprecation" + ) + static TypeResolverBuilder forEverything(ObjectMapper mapper) { + return new TypeResolverBuilder(DefaultTyping.EVERYTHING, mapper.getPolymorphicTypeValidator()); + } + + public TypeResolverBuilder(DefaultTyping typing, PolymorphicTypeValidator polymorphicTypeValidator) { + super(typing, polymorphicTypeValidator); + } + + @Override + public ObjectMapper.DefaultTypeResolverBuilder withDefaultImpl(Class defaultImpl) { + return this; + } + + /** + * Method called to check if the default type handler should be used for given + * type. Note: "natural types" (String, Boolean, Integer, Double) will never use + * typing; that is both due to them being concrete and final, and since actual + * serializers and deserializers will also ignore any attempts to enforce + * typing. + */ + public boolean useForType(JavaType javaType) { + + if (javaType.isJavaLangObject()) { + return true; + } + + javaType = resolveArrayOrWrapper(javaType); + + if (javaType.isEnumType() || ClassUtils.isPrimitiveOrWrapper(javaType.getRawClass())) { + return false; + } + + if (javaType.isFinal() && !KotlinDetector.isKotlinType(javaType.getRawClass()) && javaType.getRawClass() + .getPackageName().startsWith("java")) { + return false; + } + + // [databind#88] Should not apply to JSON tree models: + return !TreeNode.class.isAssignableFrom(javaType.getRawClass()); + } + + private JavaType resolveArrayOrWrapper(JavaType type) { + + while (type.isArrayType()) { + type = type.getContentType(); + if (type.isReferenceType()) { + type = resolveArrayOrWrapper(type); + } + } + + while (type.isReferenceType()) { + type = type.getReferencedType(); + if (type.isArrayType()) { + type = resolveArrayOrWrapper(type); + } + } + + return type; + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/JacksonObjectReader.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/JacksonObjectReader.java new file mode 100644 index 000000000..465e3b04d --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/JacksonObjectReader.java @@ -0,0 +1,38 @@ +package com.redis.om.cache.common.mapping; + +import java.io.IOException; +import java.io.InputStream; + +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Functional interface for reading JSON data into Java objects using Jackson. + * Provides a consistent way to deserialize JSON data with different Jackson configurations. + */ +@FunctionalInterface +public interface JacksonObjectReader { + + /** + * Read an object graph from the given root JSON into a Java object considering + * the {@link JavaType}. + * + * @param mapper the object mapper to use. + * @param source the JSON to deserialize. + * @param type the Java target type + * @return the deserialized Java object. + * @throws IOException if an I/O error or JSON deserialization error occurs. + */ + Object read(ObjectMapper mapper, byte[] source, JavaType type) throws IOException; + + /** + * Create a default {@link JacksonObjectReader} delegating to + * {@link ObjectMapper#readValue(InputStream, JavaType)}. + * + * @return the default {@link JacksonObjectReader}. + */ + static JacksonObjectReader create() { + return (mapper, source, type) -> mapper.readValue(source, 0, source.length, type); + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/JacksonObjectWriter.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/JacksonObjectWriter.java new file mode 100644 index 000000000..eda72318d --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/JacksonObjectWriter.java @@ -0,0 +1,34 @@ +package com.redis.om.cache.common.mapping; + +import java.io.IOException; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Functional interface for writing Java objects to JSON data using Jackson. + * Provides a consistent way to serialize Java objects with different Jackson configurations. + */ +@FunctionalInterface +public interface JacksonObjectWriter { + + /** + * Write the object graph with the given root {@code source} as byte array. + * + * @param mapper the object mapper to use. + * @param source the root of the object graph to marshal. + * @return a byte array containing the serialized object graph. + * @throws IOException if an I/O error or JSON serialization error occurs. + */ + byte[] write(ObjectMapper mapper, Object source) throws IOException; + + /** + * Create a default {@link JacksonObjectWriter} delegating to + * {@link ObjectMapper#writeValueAsBytes(Object)}. + * + * @return the default {@link JacksonObjectWriter}. + */ + static JacksonObjectWriter create() { + return ObjectMapper::writeValueAsBytes; + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/JdkSerializationStringMapper.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/JdkSerializationStringMapper.java new file mode 100644 index 000000000..eb19e69c8 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/JdkSerializationStringMapper.java @@ -0,0 +1,96 @@ +package com.redis.om.cache.common.mapping; + +import org.springframework.core.convert.converter.Converter; +import org.springframework.core.serializer.support.DeserializingConverter; +import org.springframework.core.serializer.support.SerializingConverter; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +import com.redis.om.cache.common.RedisStringMapper; +import com.redis.om.cache.common.SerializationException; + +/** + * Implementation of {@link RedisStringMapper} that uses JDK serialization to convert + * objects to and from byte arrays for storage in Redis. + */ +public class JdkSerializationStringMapper implements RedisStringMapper { + + /** + * Converter used to serialize objects to byte arrays. + */ + private final Converter serializer; + + /** + * Converter used to deserialize byte arrays back to objects. + */ + private final Converter deserializer; + + /** + * Creates a new {@link JdkSerializationStringMapper} using the default + * {@link ClassLoader}. + */ + public JdkSerializationStringMapper() { + this(new SerializingConverter(), new DeserializingConverter()); + } + + /** + * Creates a new {@link JdkSerializationStringMapper} with the given + * {@link ClassLoader} used to resolve {@link Class types} during + * deserialization. + * + * @param classLoader {@link ClassLoader} used to resolve {@link Class types} + * for deserialization; can be {@literal null}. + * @since 1.7 + */ + public JdkSerializationStringMapper(@Nullable ClassLoader classLoader) { + this(new SerializingConverter(), new DeserializingConverter(classLoader)); + } + + /** + * Creates a new {@link JdkSerializationStringMapper} using {@link Converter + * converters} to serialize and deserialize {@link Object objects}. + * + * @param serializer {@link Converter} used to serialize an {@link Object} to + * a byte array; must not be {@literal null}. + * @param deserializer {@link Converter} used to deserialize and convert a byte + * arra into an {@link Object}; must not be {@literal null} + * @throws IllegalArgumentException if either the given {@code serializer} or + * {@code deserializer} are {@literal null}. + * @since 1.7 + */ + public JdkSerializationStringMapper(Converter serializer, Converter deserializer) { + + Assert.notNull(serializer, "Serializer must not be null"); + Assert.notNull(deserializer, "Deserializer must not be null"); + this.serializer = serializer; + this.deserializer = deserializer; + } + + @Override + public byte[] toString(@Nullable Object value) { + + if (value == null) { + return SerializationUtils.EMPTY_ARRAY; + } + + try { + return serializer.convert(value); + } catch (Exception ex) { + throw new SerializationException("Cannot serialize", ex); + } + } + + @Override + public Object fromString(@Nullable byte[] bytes) { + + if (SerializationUtils.isEmpty(bytes)) { + return null; + } + + try { + return deserializer.convert(bytes); + } catch (Exception ex) { + throw new SerializationException("Cannot deserialize", ex); + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/ObjectHashMapper.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/ObjectHashMapper.java new file mode 100644 index 000000000..920a7f64e --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/ObjectHashMapper.java @@ -0,0 +1,165 @@ +package com.redis.om.cache.common.mapping; + +import java.util.Collections; +import java.util.Map; + +import org.springframework.data.convert.CustomConversions; +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +import com.redis.om.cache.common.RedisHashMapper; +import com.redis.om.cache.common.convert.*; + +/** + * {@link RedisHashMapper} based on {@link MappingRedisConverter}. Supports + * nested properties and simple types like {@link String}. + * + *

+ * 
+ * class Person {
+ *
+ * String firstname;
+ * String lastname;
+ *
+ * List<String> nicknames;
+ * List<Person> coworkers;
+ *
+ * Address address;
+ * }
+ * 
+ * 
+ * + * The above is represented as: + * + *
+ * 
+ * _class=org.example.Person
+ * firstname=rand
+ * lastname=al'thor
+ * coworkers.[0].firstname=mat
+ * coworkers.[0].nicknames.[0]=prince of the ravens
+ * coworkers.[1].firstname=perrin
+ * coworkers.[1].address.city=two rivers
+ * 
+ * 
+ * + */ +public class ObjectHashMapper implements RedisHashMapper { + + @Nullable + private volatile static ObjectHashMapper sharedInstance; + + private final RedisConverter converter; + + /** + * Creates new {@link ObjectHashMapper}. + */ + public ObjectHashMapper() { + this(new RedisCustomConversions()); + } + + /** + * Creates a new {@link ObjectHashMapper} using the given {@link RedisConverter} + * for conversion. + * + * @param converter must not be {@literal null}. + * @throws IllegalArgumentException if the given {@literal converter} is + * {@literal null}. + * @since 2.4 + */ + public ObjectHashMapper(RedisConverter converter) { + + Assert.notNull(converter, "Converter must not be null"); + this.converter = converter; + } + + /** + * Creates new {@link ObjectHashMapper}. + * + * @param customConversions can be {@literal null}. + * @since 2.0 + */ + public ObjectHashMapper(@Nullable CustomConversions customConversions) { + + MappingRedisConverter mappingConverter = new MappingRedisConverter(new RedisMappingContext(), + new NoOpReferenceResolver()); + mappingConverter.setCustomConversions(customConversions == null ? new RedisCustomConversions() : customConversions); + mappingConverter.afterPropertiesSet(); + + converter = mappingConverter; + } + + /** + * Return a shared default {@link ObjectHashMapper} instance, lazily building it + * once needed. + *

+ * NOTE: We highly recommend constructing individual + * {@link ObjectHashMapper} instances for customization purposes. This accessor + * is only meant as a fallback for code paths which need simple type coercion + * but cannot access a longer-lived {@link ObjectHashMapper} instance any other + * way. + * + * @return the shared {@link ObjectHashMapper} instance (never {@literal null}). + * @since 2.4 + */ + public static ObjectHashMapper getSharedInstance() { + + ObjectHashMapper cs = sharedInstance; + if (cs == null) { + synchronized (ObjectHashMapper.class) { + cs = sharedInstance; + if (cs == null) { + cs = new ObjectHashMapper(); + sharedInstance = cs; + } + } + } + return cs; + } + + @Override + public Map toHash(Object source) { + if (source == null) { + return Collections.emptyMap(); + } + RedisData sink = new RedisData(); + converter.write(source, sink); + return sink.getBucket().rawMap(); + } + + @Override + public Object fromHash(Map hash) { + if (hash == null || hash.isEmpty()) { + return null; + } + return converter.read(Object.class, new RedisData(hash)); + } + + /** + * Convert a {@code hash} (map) to an object and return the casted result. + * + * @param hash the hash map containing the object data to convert + * @param type the target class type to convert the hash to + * @param the generic type of the returned object + * @return the converted object of type T + */ + public T fromHash(Map hash, Class type) { + return type.cast(fromHash(hash)); + } + + /** + * {@link ReferenceResolver} implementation always returning an empty + * {@link Map}. + * + */ + private static class NoOpReferenceResolver implements ReferenceResolver { + + private static final Map NO_REFERENCE = Collections.emptyMap(); + + @Override + public Map resolveReference(Object id, String keyspace) { + return NO_REFERENCE; + } + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/SerializationUtils.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/SerializationUtils.java new file mode 100644 index 000000000..6a4f020e4 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/SerializationUtils.java @@ -0,0 +1,24 @@ +package com.redis.om.cache.common.mapping; + +import org.springframework.lang.Nullable; + +/** + * Utility class providing helper methods for serialization operations. + */ +public abstract class SerializationUtils { + + /** + * Constant representing an empty byte array. + */ + public static final byte[] EMPTY_ARRAY = new byte[0]; + + /** + * Checks if the given byte array is null or empty. + * + * @param data the byte array to check + * @return true if the array is null or empty, false otherwise + */ + public static boolean isEmpty(@Nullable byte[] data) { + return (data == null || data.length == 0); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/StringMapper.java b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/StringMapper.java new file mode 100644 index 000000000..e45da38bf --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/cache/common/mapping/StringMapper.java @@ -0,0 +1,74 @@ +package com.redis.om.cache.common.mapping; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +import org.springframework.lang.Nullable; +import org.springframework.util.Assert; + +import com.redis.om.cache.common.RedisStringMapper; + +/** + * Implementation of {@link RedisStringMapper} that converts between String objects and byte arrays + * using a specified character set encoding. + */ +public class StringMapper implements RedisStringMapper { + + private final Charset charset; + + /** + * {@link StringMapper} to use 7 bit ASCII, a.k.a. ISO646-US, a.k.a. the Basic + * Latin block of the Unicode character set. + * + * @see StandardCharsets#US_ASCII + * @since 2.1 + */ + public static final StringMapper US_ASCII = new StringMapper(StandardCharsets.US_ASCII); + + /** + * {@link StringMapper} to use ISO Latin Alphabet No. 1, a.k.a. ISO-LATIN-1. + * + * @see StandardCharsets#ISO_8859_1 + * @since 2.1 + */ + public static final StringMapper ISO_8859_1 = new StringMapper(StandardCharsets.ISO_8859_1); + + /** + * {@link StringMapper} to use 8 bit UCS Transformation Format. + * + * @see StandardCharsets#UTF_8 + * @since 2.1 + */ + public static final StringMapper UTF_8 = new StringMapper(StandardCharsets.UTF_8); + + /** + * Creates a new {@link StringMapper} using {@link StandardCharsets#UTF_8 + * UTF-8}. + */ + public StringMapper() { + this(StandardCharsets.UTF_8); + } + + /** + * Creates a new {@link StringMapper} using the given {@link Charset} to encode + * and decode strings. + * + * @param charset must not be {@literal null}. + */ + public StringMapper(Charset charset) { + + Assert.notNull(charset, "Charset must not be null"); + this.charset = charset; + } + + @Override + public byte[] toString(@Nullable Object value) { + return (value == null ? null : ((String) value).getBytes(charset)); + } + + @Override + public String fromString(@Nullable byte[] bytes) { + return (bytes == null ? null : new String(bytes, charset)); + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/CacheEntry.java b/redis-om-spring/src/main/java/com/redis/om/sessions/CacheEntry.java new file mode 100644 index 000000000..78a88776b --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/CacheEntry.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.util.Objects; + +import lombok.Getter; +import lombok.Setter; + +@Setter +@Getter +public class CacheEntry implements Comparable> { + long score; + T session; + + public CacheEntry(T session, long score) { + this.score = score; + this.session = session; + } + + public long getSize() { + return this.session.getSize(); + } + + /** + * {@inheritDoc} + */ + @Override + public int compareTo(CacheEntry o) { + int scoreComparison = Long.compare(this.score, o.score); + if (scoreComparison == 0) { + return this.session.getId().compareTo(o.session.getId()); + } + return scoreComparison; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null || getClass() != obj.getClass()) + return false; + CacheEntry that = (CacheEntry) obj; + return session.getId().equals(that.session.getId()); + } + + /** + * {@inheritDoc} + */ + @Override + public int hashCode() { + return Objects.hash(session.getId()); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/Constants.java b/redis-om-spring/src/main/java/com/redis/om/sessions/Constants.java new file mode 100644 index 000000000..4ca4b0ae6 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/Constants.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +public class Constants { + public static final String CREATED_AT_KEY = "createdAt"; + public static final String MAX_INACTIVE_INTERVAL_KEY = "maxInactiveInterval"; + public static final String LAST_ACCESSED_TIME_KEY = "lastAccessedTime"; + public static final String LAST_MODIFIED_TIME_KEY = "lastModifiedTime"; + public static final String SESSION_METRICS_KEY = "sessionMetrics"; + public static final String SIZE_FIELD_NAME = "sessionSize"; + public static final String INVALIDATION_CHANNEL_FORMAT = "redis-session-invalidate:%s"; + public static final Set reservedFields = new HashSet<>(Arrays.asList(Constants.CREATED_AT_KEY, + Constants.SIZE_FIELD_NAME, Constants.MAX_INACTIVE_INTERVAL_KEY, Constants.LAST_ACCESSED_TIME_KEY, + Constants.LAST_MODIFIED_TIME_KEY)); +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/Function.java b/redis-om-spring/src/main/java/com/redis/om/sessions/Function.java new file mode 100644 index 000000000..39f2227b3 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/Function.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; + +public enum Function { + touch_key, + read_key, + read_locally_cached_entry, + reserve_structs; + + private static final String LIBRARY_FILE = "redisSessions.lua"; + + private static String getLibraryFileLocation() { + return Paths.get("functions", LIBRARY_FILE).toString(); + } + + public static String getFunctionFile() { + try (InputStream stream = Function.class.getClassLoader().getResourceAsStream(getLibraryFileLocation())) { + if (stream == null) { + throw new IllegalArgumentException(String.format("Could not load %s from disk", getLibraryFileLocation())); + } + + return new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new IllegalArgumentException(String.format("Error while reading the function file %s", + getLibraryFileLocation())); + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/GeoLoc.java b/redis-om-spring/src/main/java/com/redis/om/sessions/GeoLoc.java new file mode 100644 index 000000000..5a6a4e137 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/GeoLoc.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import lombok.Getter; + +@Getter +public class GeoLoc { + private final double latitude; + private final double longitude; + + public GeoLoc(double longitude, double latitude) { + this.latitude = latitude; + this.longitude = longitude; + } + + @Override + public String toString() { + return String.format("%f,%f", this.longitude, this.latitude); + } + + public static GeoLoc parse(String s) { + String[] parts = s.split(","); + if (parts.length != 2) { + throw new IllegalArgumentException(String.format("unparseable point %s", s)); + } + + return new GeoLoc(Double.parseDouble(parts[0]), Double.parseDouble(parts[1])); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/GeoUnit.java b/redis-om-spring/src/main/java/com/redis/om/sessions/GeoUnit.java new file mode 100644 index 000000000..5d86873a4 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/GeoUnit.java @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +public enum GeoUnit { + mi, + km, + m, + ft; + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/LocalCache.java b/redis-om-spring/src/main/java/com/redis/om/sessions/LocalCache.java new file mode 100644 index 000000000..b5655bcc9 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/LocalCache.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.util.*; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import lombok.Getter; + +public class LocalCache { + private static final Logger logger = LoggerFactory.getLogger(LocalCache.class); + @Getter + private long cacheSize; + @Getter + private final long capacity; + private final LocalCacheType cacheType; + private final Map> sessions; + private final SortedSet> sessionRanking; + private final long minSessionSize; + + void removeEntry(String id, boolean unsubscribe) { + CacheEntry entry = sessions.remove(id); + if (entry != null) { + sessionRanking.remove(entry); + cacheSize -= entry.getSize(); + } + } + + boolean addEntry(T session) { + if (this.capacity == 0) { + return false; + } + + if (session.getSize() < this.minSessionSize) { + return false; + } + + if (!this.sessions.containsKey(session.getId()) && session.getSize() > this.capacity / 10) { + logger.warn("Session size {} exceeded 10% of local cache allocation {}, so it will not be cached locally", session + .getSize(), capacity); + return false; + } + + while (this.capacity < (this.cacheSize - session.getSize())) { + this.trimEntry(); + } + + long score = 0; + if (this.cacheType == LocalCacheType.LRU) { + score = System.currentTimeMillis(); + } + + CacheEntry entry = new CacheEntry<>(session, score); + + if (this.sessions.containsKey(session.getId())) { + this.removeEntry(session.getId(), false); + } + + this.sessions.put(session.getId(), entry); + this.sessionRanking.add(entry); + this.cacheSize += entry.getSize(); + return true; + } + + Optional readEntry(String id) { + CacheEntry session = sessions.get(id); + if (session == null) { + return Optional.empty(); + } + sessionRanking.remove(session); + if (session.getSession().isExpired()) { + return Optional.empty(); + } + + if (this.cacheType == LocalCacheType.LRU) { + session.setScore(System.currentTimeMillis()); + } + + sessionRanking.add(session); + + return Optional.of(session.getSession()); + } + + public LocalCacheStatistics getStats() { + double averageEntrySize = sessions.values().stream().mapToDouble(s -> (double) s.getSize()).average().orElse(0); + return new LocalCacheStatistics(capacity, cacheSize, sessions.size(), averageEntrySize); + } + + void trimEntry() { + CacheEntry entry = sessionRanking.first(); + if (entry != null) { + this.cacheSize -= entry.getSize(); + sessions.remove(entry.getSession().getId()); + } + } + + public LocalCache(LocalCacheType cacheType, long capacity, long minSize) { + this.cacheType = cacheType; + this.sessions = new HashMap<>(); + this.sessionRanking = Collections.synchronizedSortedSet(new TreeSet<>()); + this.capacity = capacity; + this.minSessionSize = minSize; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/LocalCacheInvalidator.java b/redis-om-spring/src/main/java/com/redis/om/sessions/LocalCacheInvalidator.java new file mode 100644 index 000000000..34cbe5338 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/LocalCacheInvalidator.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.lettuce.core.pubsub.RedisPubSubListener; + +public class LocalCacheInvalidator implements RedisPubSubListener { + private final static Logger logger = LoggerFactory.getLogger(LocalCacheInvalidator.class); + private final LocalCache localCache; + + public LocalCacheInvalidator(LocalCache localCache) { + this.localCache = localCache; + } + + @Override + public void message(String s, String s2) { + this.localCache.removeEntry(s2, true); + } + + @Override + public void message(String s, String k1, String s2) { + } + + @Override + public void subscribed(String s, long l) { + + } + + @Override + public void psubscribed(String s, long l) { + + } + + @Override + public void unsubscribed(String s, long l) { + + } + + @Override + public void punsubscribed(String s, long l) { + + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/LocalCacheStatistics.java b/redis-om-spring/src/main/java/com/redis/om/sessions/LocalCacheStatistics.java new file mode 100644 index 000000000..17537b8de --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/LocalCacheStatistics.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import lombok.Getter; + +@Getter +public class LocalCacheStatistics { + private final long cacheCapacity; + private final long cacheSize; + private final long numEntries; + private final double averageEntrySize; + + public LocalCacheStatistics(long cacheCapacity, long cacheSize, long numEntries, double averageEntrySize) { + this.cacheCapacity = cacheCapacity; + this.cacheSize = cacheSize; + this.numEntries = numEntries; + this.averageEntrySize = averageEntrySize; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/LocalCacheType.java b/redis-om-spring/src/main/java/com/redis/om/sessions/LocalCacheType.java new file mode 100644 index 000000000..a2841fbf2 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/LocalCacheType.java @@ -0,0 +1,9 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +public enum LocalCacheType { + LRU +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/MetricsMonitor.java b/redis-om-spring/src/main/java/com/redis/om/sessions/MetricsMonitor.java new file mode 100644 index 000000000..2e2c81a6e --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/MetricsMonitor.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.lettuce.core.KeyValue; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; + +public class MetricsMonitor { + private final static Logger logger = LoggerFactory.getLogger(MetricsMonitor.class); + private final MeterRegistry meterRegistry; + private final Optional appPrefix; + private final SessionProvider sessionProvider; + private final int numSessions; + private final ScheduledExecutorService scheduler; + private final Long[] largestSessionSizes; + private final Long[] mostAccessedSessions; + private final double[] sizeQuantiles; + private final double[] quantiles; + private Long numUniqueSessions = 0L; + private long localCacheCapacity; + private long localCacheSize; + private long numLocalCacheEntries; + private double averageCacheSize; + + public MetricsMonitor(MeterRegistry meterRegistry, SessionProvider provider, Optional appPrefix, + int numSessions, double[] quantiles) { + this.meterRegistry = meterRegistry; + this.sessionProvider = provider; + this.appPrefix = appPrefix; + this.numSessions = numSessions; + + int period = 5; + int initialDelay = 5; + this.scheduler = Executors.newScheduledThreadPool(1); + scheduler.scheduleAtFixedRate(this::monitorTopSessions, initialDelay, period, TimeUnit.SECONDS); + scheduler.scheduleAtFixedRate(this::monitorSessionAccess, initialDelay, period, TimeUnit.SECONDS); + scheduler.scheduleAtFixedRate(this::monitorSessionSizeStatistics, initialDelay, period, TimeUnit.SECONDS); + scheduler.scheduleAtFixedRate(this::monitorUniqueSessionCount, initialDelay, period, TimeUnit.SECONDS); + scheduler.scheduleAtFixedRate(this::monitorLocalCache, initialDelay, period, TimeUnit.SECONDS); + + this.quantiles = quantiles; + largestSessionSizes = new Long[numSessions]; + mostAccessedSessions = new Long[numSessions]; + sizeQuantiles = new double[this.quantiles.length]; + Gauge.builder("redis.sessions.unique.sessions", () -> this.numUniqueSessions).register(this.meterRegistry); + + for (int i = 0; i < quantiles.length; i++) { + int index = i; + Gauge.builder("redis.session.size.quantiles", () -> sizeQuantiles[index]).tag("quantile", String.valueOf( + quantiles[i])).register(this.meterRegistry); + } + + for (int i = 1; i <= numSessions; i++) { + int index = i - 1; + largestSessionSizes[index] = 0L; + mostAccessedSessions[index] = 0L; + Gauge.builder("redis.session.largest", () -> largestSessionSizes[index]).tag("sessionRank", String.valueOf(i)) + .register(this.meterRegistry); + + Gauge.builder("redis.session.most.accessed", () -> mostAccessedSessions[index]).tag("sessionRank", String.valueOf( + i)).register(this.meterRegistry); + } + + setupLocalCacheStats(); + } + + private void setupLocalCacheStats() { + Gauge.builder("redis.local.cache.capacity", () -> localCacheCapacity).register(this.meterRegistry); + Gauge.builder("redis.local.cache.size", () -> localCacheSize).register(this.meterRegistry); + Gauge.builder("redis.local.cache.num.entries", () -> numLocalCacheEntries).register(this.meterRegistry); + Gauge.builder("redis.local.cache.average.entry.size", () -> averageCacheSize).register(this.meterRegistry); + } + + public void monitorLocalCache() { + LocalCacheStatistics statistics = sessionProvider.getLocalCacheStatistics(); + localCacheCapacity = statistics.getCacheCapacity(); + localCacheSize = statistics.getCacheSize(); + numLocalCacheEntries = statistics.getNumEntries(); + averageCacheSize = statistics.getAverageEntrySize(); + } + + public void monitorTopSessions() { + try { + Map topSessions = this.sessionProvider.largestSessions(this.numSessions); + + int i = 0; + for (Map.Entry entry : topSessions.entrySet()) { + + this.largestSessionSizes[i] = entry.getValue(); + i++; + if (i > this.largestSessionSizes.length) { + break; + } + } + } catch (Exception e) { + logger.error("error checking largest sessions", e); + } + } + + public void monitorSessionSizeStatistics() { + try { + List quantileResults = this.sessionProvider.sessionSizeQuantiles(quantiles); + for (int i = 0; i < quantileResults.size(); i++) { + this.sizeQuantiles[i] = quantileResults.get(i); + } + + } catch (Exception e) { + logger.error("Encountered error while checking session size statistics", e); + } + } + + public void monitorUniqueSessionCount() { + try { + this.numUniqueSessions = sessionProvider.uniqueSessions(); + } catch (Exception e) { + logger.error("Encountered error while checking num unique sessions", e); + } + } + + public void monitorSessionAccess() { + try { + List> mostAccessedSessions = this.sessionProvider.mostAccessedSessions(); + int i = 0; + + for (KeyValue session : mostAccessedSessions) { + + this.mostAccessedSessions[i] = session.getValue(); + i++; + if (i >= this.mostAccessedSessions.length) { + break; + } + } + } catch (Exception e) { + logger.error("error checking session access", e); + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/RedisHashSerializer.java b/redis-om-spring/src/main/java/com/redis/om/sessions/RedisHashSerializer.java new file mode 100644 index 000000000..29a653fd5 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/RedisHashSerializer.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.io.*; + +public class RedisHashSerializer { + private static byte[] serialize(Object object) throws Exception { + if (!(object instanceof Serializable)) { + throw new IllegalArgumentException("Object must be serializable to serialize"); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(1024); + ObjectOutputStream objectOutputStream = new ObjectOutputStream(out); + objectOutputStream.writeObject(object); + objectOutputStream.flush(); + byte[] bytes = out.toByteArray(); + return bytes; + } + + private static Object Deserialize(byte[] obj) throws Exception { + ByteArrayInputStream in = new ByteArrayInputStream(obj); + ObjectInputStream objectInputStream = new ObjectInputStream(in); + return objectInputStream.readObject(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/RedisSession.java b/redis-om-spring/src/main/java/com/redis/om/sessions/RedisSession.java new file mode 100644 index 000000000..2d5fb5b48 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/RedisSession.java @@ -0,0 +1,344 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.concurrent.ExecutionException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.redis.lettucemod.api.StatefulRedisModulesConnection; +import com.redis.om.sessions.indexing.IndexedField; +import com.redis.om.sessions.indexing.RedisIndexConfiguration; + +import io.lettuce.core.RedisFuture; +import io.lettuce.core.ScriptOutputType; +import io.lettuce.core.json.JsonPath; + +public class RedisSession implements Session { + private final static Logger logger = LoggerFactory.getLogger(RedisSession.class); + private final Optional maxInactiveInterval; + private final Map sessionData; + private String sessionId; + private final Map updateData = new HashMap<>(); + private boolean isNew; + private final StatefulRedisModulesConnection connection; + private final StatefulRedisModulesConnection rawConnection; + private final Optional appPrefix; + private final RedisIndexConfiguration redisIndexConfiguration; + private final Serializer serializer; + + RedisSession(Map sessionData, String sessionId, boolean isNew, + StatefulRedisModulesConnection connection, + StatefulRedisModulesConnection rawConnection, Optional appPrefix, + RedisIndexConfiguration redisIndexConfiguration, Serializer serializer, + RedisSessionProviderConfiguration config) { + this.serializer = serializer; + this.sessionData = sessionData; + this.sessionId = sessionId; + this.isNew = isNew; + this.connection = connection; + this.appPrefix = appPrefix; + this.rawConnection = rawConnection; + this.redisIndexConfiguration = redisIndexConfiguration; + this.maxInactiveInterval = config.getTtl(); + + if (this.isNew) { + long currentUnixTimestamp = System.currentTimeMillis(); + this.sessionData.put(Constants.CREATED_AT_KEY, currentUnixTimestamp); + this.sessionData.put(Constants.LAST_ACCESSED_TIME_KEY, currentUnixTimestamp); + this.sessionData.put(Constants.LAST_MODIFIED_TIME_KEY, currentUnixTimestamp); + this.sessionData.put(Constants.SIZE_FIELD_NAME, 0); + maxInactiveInterval.ifPresent(d -> this.sessionData.put(Constants.MAX_INACTIVE_INTERVAL_KEY, maxInactiveInterval + .get().get(ChronoUnit.SECONDS))); + updateData.putAll(this.sessionData); + + } else if (this.getSize() == 0) { + throw new IllegalArgumentException("Created a session which is not new without a defined size"); + } + } + + public static RedisSession create(Map sessionData, String sessionId, boolean isNew, + StatefulRedisModulesConnection connection, + StatefulRedisModulesConnection rawConnection, Optional appPrefix, + RedisIndexConfiguration redisIndexConfiguration, Serializer serializer, + RedisSessionProviderConfiguration config) { + return new RedisSession(sessionData, sessionId, isNew, connection, rawConnection, appPrefix, + redisIndexConfiguration, serializer, config); + + } + + private SessionMetrics getSessionMetrics() { + if (this.sessionData.containsKey(Constants.SIZE_FIELD_NAME)) { + SessionMetrics sm = new SessionMetrics(); + sm.setSize(Long.parseLong(this.sessionData.get(Constants.SIZE_FIELD_NAME).toString())); + return sm; + } + + throw new IllegalStateException("Session did not contain session metrics"); + } + + /** + * {@inheritDoc} + */ + @Override + public String getId() { + return sessionId; + } + + /** + * {@inheritDoc} + */ + @Override + public String changeSessionId() { + String newSessionId = UUID.randomUUID().toString(); + return this.changeSessionId(newSessionId); + } + + /** + * {@inheritDoc} + */ + @Override + public String changeSessionId(String sessionId) { + String oldSessionKey = keyName(); + String oldSessionId = this.sessionId; + try { + this.sessionId = sessionId; + this.save(); + } catch (Exception e) { + this.sessionId = oldSessionId; + throw e; + } + + this.connection.sync().unlink(oldSessionKey); + return this.sessionId; + } + + /** + * {@inheritDoc} + */ + @Override + public Optional getAttribute(String attribute) { + if (!sessionData.containsKey(attribute)) { + return Optional.empty(); + } + + try { + Object obj = this.sessionData.get(attribute); + return Optional.of((T) obj); + } catch (Exception e) { + return Optional.empty(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public Set getAttributeNames() { + return Set.of(this.connection.sync().hkeys(keyName()).toArray(new String[0])); + } + + /** + * {@inheritDoc} + */ + @Override + public Long setAttribute(String attributeName, T attributeValue) { + this.sessionData.put(attributeName, attributeValue); + this.updateData.put(attributeName, attributeValue); + return save(); + } + + /** + * {@inheritDoc} + */ + @Override + public void removeAttribute(String attributeName) { + if (this.sessionData.containsKey(attributeName)) { + this.sessionData.remove(attributeName); + /** + * FIXME + * Needed to combine both Redis Sessions and Redis Cache together despite using different LettuceMod versions + * LettuceMod 4.3.0 works well for Redis Sessions + * LettuceMod 4.2.1 works well for Redis Cache + * Unfortunately they differ in LettuceMod dependency, and RedisJsonCommands API. + * The quick and dirty fix was to downgrade to version 4.2.1 (fixing Redis Cache was more complicated) + * and implement the fieldNameToJsonPath which provides a JsonPath object. + */ + // this.connection.sync().jsonDel(keyName(), fieldNameToPath(attributeName)); + this.connection.sync().jsonDel(keyName(), fieldNameToJsonPath(attributeName)); + } + } + + /** + * {@inheritDoc} + */ + @Override + public Instant getCreationTime() { + Optional creationTime = this.getAttribute(Constants.CREATED_AT_KEY); + if (creationTime.isEmpty()) { + throw new IllegalStateException("Creation Time not Found on Session"); + } + + return Instant.ofEpochMilli(creationTime.get()); + } + + /** + * {@inheritDoc} + */ + @Override + public void setLastAccessedTime(Instant lastAccessedTime) { + setAttribute(Constants.LAST_ACCESSED_TIME_KEY, lastAccessedTime.toEpochMilli()); + } + + /** + * {@inheritDoc} + */ + @Override + public Instant getLastAccessedTime() { + Optional lastAccessedTime = getAttribute(Constants.LAST_ACCESSED_TIME_KEY); + if (lastAccessedTime.isEmpty()) { + throw new IllegalStateException("Last Accessed Time not Found on Session"); + } + + return Instant.ofEpochMilli(lastAccessedTime.get()); + } + + /** + * {@inheritDoc} + */ + @Override + public void setMaxInactiveInterval(Duration interval) { + setAttribute(Constants.MAX_INACTIVE_INTERVAL_KEY, interval); + } + + /** + * {@inheritDoc} + */ + @Override + public Optional getMaxInactiveInterval() { + Optional seconds = getAttribute(Constants.MAX_INACTIVE_INTERVAL_KEY); + if (seconds.isEmpty()) { + return Optional.of(Duration.ofSeconds(600)); + } + return seconds.map(Duration::ofSeconds); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isExpired() { + Optional maxInactiveInterval = getMaxInactiveInterval(); + return maxInactiveInterval.filter(duration -> Duration.between(getLastAccessedTime(), Instant.now()).compareTo( + duration) > 0).isPresent(); + } + + /** + * {@inheritDoc} + */ + @Override + public Long save() { + long startTime = System.nanoTime(); + if (this.updateData.isEmpty() || this.updateData.size() == 1 && this.updateData.containsKey( + Constants.LAST_ACCESSED_TIME_KEY)) { + return this.getSize(); + } + + String[] keys = { keyName() }; + List args = new ArrayList<>(); + Map fieldValues = new HashMap<>(); + maxInactiveInterval.ifPresent(d -> args.add(String.valueOf(d.get(ChronoUnit.SECONDS)))); + ; + for (Map.Entry entry : this.updateData.entrySet()) { + if (Constants.reservedFields.contains(entry.getKey())) { + fieldValues.put(entry.getKey().getBytes(StandardCharsets.UTF_8), entry.getValue().toString().getBytes( + StandardCharsets.UTF_8)); + continue; + } + try { + if (this.redisIndexConfiguration.getFields().containsKey(entry.getKey())) { + IndexedField indexedField = this.redisIndexConfiguration.getFields().get(entry.getKey()); + if (indexedField.isKnownOrDefaultClass(entry.getValue().getClass())) { + fieldValues.put(entry.getKey().getBytes(StandardCharsets.UTF_8), entry.getValue().toString().getBytes( + StandardCharsets.UTF_8)); + } else { + throw new IllegalArgumentException(String.format( + "Object provided for serialization did not match a known or default type: %s", entry.getValue() + .getClass().getName())); + } + } else { + byte[] raw = this.serializer.Serialize(entry.getValue()); + fieldValues.put(entry.getKey().getBytes(StandardCharsets.UTF_8), raw); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + byte[][] argsArr = args.stream().map(String::getBytes).toArray(byte[][]::new); + byte[][] keyBytesArray = Arrays.stream(keys).map(String::getBytes).toArray(byte[][]::new); + + this.rawConnection.async().hset(keys[0].getBytes(StandardCharsets.UTF_8), fieldValues); + RedisFuture sizeFuture = this.rawConnection.async().fcall(Function.touch_key.name(), ScriptOutputType.INTEGER, + keyBytesArray, argsArr); + + if (fieldValues.size() > 1 || !fieldValues.containsKey(Constants.LAST_ACCESSED_TIME_KEY)) { + this.connection.async().publish(String.format(Constants.INVALIDATION_CHANNEL_FORMAT, this.sessionId), + this.sessionId); + } + + this.connection.flushCommands(); + this.rawConnection.flushCommands(); + this.updateData.clear(); + try { + Long size = sizeFuture.get(); + return size; + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + + /** + * {@inheritDoc} + */ + @Override + public long getSize() { + return this.getSessionMetrics().getSize(); + } + + /** + * {@inheritDoc} + */ + @Override + public long getLastModifiedTime() { + return (Long) this.sessionData.get(Constants.LAST_MODIFIED_TIME_KEY); + } + + private static String fieldNameToPath(String fieldName) { + return String.format("$.%s", fieldName); + } + + private static JsonPath fieldNameToJsonPath(String fieldName) { + return new JsonPath(fieldNameToPath(fieldName)); + } + + private String keyName() { + return buildKeyName(this.appPrefix, this.sessionId); + } + + public static String buildKeyName(Optional appPrefix, String sessionId) { + if (appPrefix.isPresent()) { + return String.format("%s:session:%s", appPrefix.get(), sessionId); + } + + return String.format("session:%s", sessionId); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/RedisSessionProvider.java b/redis-om-spring/src/main/java/com/redis/om/sessions/RedisSessionProvider.java new file mode 100644 index 000000000..55afec828 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/RedisSessionProvider.java @@ -0,0 +1,517 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.redis.lettucemod.RedisModulesClient; +import com.redis.lettucemod.api.StatefulRedisModulesConnection; +import com.redis.lettucemod.api.async.RedisModulesAsyncCommands; +import com.redis.lettucemod.api.sync.RediSearchCommands; +import com.redis.lettucemod.cluster.RedisModulesClusterClient; +import com.redis.lettucemod.search.*; +import com.redis.om.sessions.filtering.Filter; +import com.redis.om.sessions.indexing.IndexedField; +import com.redis.om.sessions.indexing.RedisIndexConfiguration; + +import io.lettuce.core.AbstractRedisClient; +import io.lettuce.core.FlushMode; +import io.lettuce.core.KeyValue; +import io.lettuce.core.ScriptOutputType; +import io.lettuce.core.codec.ByteArrayCodec; +import io.lettuce.core.pubsub.StatefulRedisPubSubConnection; +import lombok.Getter; + +@Getter +public class RedisSessionProvider implements SessionProvider { + private static final Logger logger = LoggerFactory.getLogger(RedisSessionProvider.class); + private final StatefulRedisModulesConnection connection; + private final StatefulRedisModulesConnection rawConnection; + private final Optional appPrefix; + private final ScheduledExecutorService scheduler; + private final ConcurrentLinkedQueue sessionsAccessed = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue sessionSizes = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue createdSessions = new ConcurrentLinkedQueue<>(); + private final String topKKey; + private final int topK; + private final LocalCache localCache; + private final RedisIndexConfiguration redisIndexConfiguration; + private final Serializer serializer; + private final StatefulRedisPubSubConnection pubsub; + private final RedisSessionProviderConfiguration configuration; + + private RedisSessionProvider(RedisSessionProviderConfiguration configuration, + StatefulRedisModulesConnection connection, + StatefulRedisModulesConnection rawConnection, + StatefulRedisPubSubConnection pubsub) { + this.connection = connection; + this.rawConnection = rawConnection; + this.pubsub = pubsub; + this.serializer = configuration.getSerializer(); + this.appPrefix = configuration.getAppPrefix(); + this.configuration = configuration; + + topK = 1000; + topKKey = this.appPrefix.isPresent() ? + String.format("%s:redisSessions:topAccessedSessions", this.appPrefix.get()) : + "redisSessions:topAccessedSessions"; + scheduler = Executors.newScheduledThreadPool(5); + int initialDelay = 5; + int period = 5; + scheduler.scheduleAtFixedRate(this::writeSessionSizes, initialDelay, period, TimeUnit.SECONDS); + scheduler.scheduleAtFixedRate(this::writeSessionsAccessed, initialDelay, period, TimeUnit.SECONDS); + scheduler.scheduleAtFixedRate(this::writeCreatedSessions, initialDelay, period, TimeUnit.SECONDS); + this.localCache = new LocalCache<>(LocalCacheType.LRU, configuration.getLocalCacheMaxSize(), configuration + .getMinLocalRecordSize()); + this.redisIndexConfiguration = configuration.getIndexConfiguration(); + + this.pubsub.addListener(new LocalCacheInvalidator<>(localCache)); + } + + private void writeCreatedSessions() { + try { + String[] sessionsToAppendToHll = new String[this.createdSessions.size()]; + for (int i = 0; i < sessionsToAppendToHll.length; i++) { + String nextEntry = this.createdSessions.poll(); + if (nextEntry == null) { + break; + } + + sessionsToAppendToHll[i] = nextEntry; + } + + if (sessionsToAppendToHll.length > 0) { + this.connection.sync().pfadd(uniqueSessionsHll(), sessionsToAppendToHll); + } + + } catch (Exception e) { + logger.error("Encountered error while appending new sessions to hll", e); + } + + } + + private void writeSessionsAccessed() { + try { + String[] sessionsAccessed = new String[this.sessionsAccessed.size()]; + for (int i = 0; i < sessionsAccessed.length; i++) { + String nextEntry = this.sessionsAccessed.poll(); + if (nextEntry == null) { + break; + } + + sessionsAccessed[i] = nextEntry; + } + + if (sessionsAccessed.length > 0) { + logger.debug("Adding {} sessions", sessionsAccessed.length); + this.connection.sync().topKAdd(topKKey, sessionsAccessed); + } + } catch (Exception e) { + logger.error("Error encountered adding session accesses", e); + } + } + + private void writeSessionSizes() { + try { + double[] entries = new double[this.sessionSizes.size()]; + for (int i = 0; i < entries.length; i++) { + Double nextEntry = sessionSizes.poll(); + if (nextEntry == null) { + break; + } + entries[i] = nextEntry; + } + + if (entries.length > 0) { + String result = this.connection.sync().tDigestAdd(tDigestKey(), entries); + logger.debug("Result from tdigest.add was: {}", result); + } + } catch (Exception e) { + logger.error("encountered error while writing session sizes"); + } + } + + /** + * Create a new RedisSessionProvider + * + * @param client The client that the Provider will use to connect to Redis + * @param configuration The configuration for the provider + * @return A new RedisSessionProvider + */ + public static RedisSessionProvider create(AbstractRedisClient client, + RedisSessionProviderConfiguration configuration) { + StatefulRedisModulesConnection connection; + StatefulRedisPubSubConnection pubSubConnection; + StatefulRedisModulesConnection rawConnection; + if (client instanceof RedisModulesClusterClient) { + connection = ((RedisModulesClusterClient) client).connect(); + pubSubConnection = ((RedisModulesClusterClient) client).connectPubSub(); + rawConnection = ((RedisModulesClusterClient) client).connect(new ByteArrayCodec()); + } else { + connection = ((RedisModulesClient) client).connect(); + pubSubConnection = ((RedisModulesClient) client).connectPubSub(); + rawConnection = ((RedisModulesClient) client).connect(new ByteArrayCodec()); + } + + return new RedisSessionProvider(configuration, connection, rawConnection, pubSubConnection); + } + + /** + * {@inheritDoc} + */ + @Override + public RedisSession createSession(String sessionId, Map sessionData) { + RedisSession session = RedisSession.create(sessionData, sessionId, true, this.connection, this.rawConnection, + this.appPrefix, this.redisIndexConfiguration, this.serializer, this.configuration); + + Long size = session.save(); + this.createdSessions.add(sessionId); + boolean locallyCached = this.localCache.addEntry(session); + if (locallyCached) { + subscribeToSessionUpdates(sessionId); + } + this.sessionSizes.add(size.doubleValue()); + return session; + } + + private void logDuration(String operationName, long startTime) { + long endTime = System.nanoTime(); + long duration = (endTime - startTime) / 1000000; + logger.info("{} took {}ms", operationName, duration); + } + + /** + * {@inheritDoc} + */ + @Override + public RedisSession findSessionById(String id) { + String key = RedisSession.buildKeyName(this.getAppPrefix(), id); + Optional localSession = this.localCache.readEntry(id); + + if (localSession.isPresent()) { + this.sessionsAccessed.add(id); + this.localCache.addEntry(localSession.get()); + return localSession.get(); + } + + try { + Map readResult = this.rawConnection.async().hgetall(key.getBytes(StandardCharsets.UTF_8)).get(); + if (readResult.size() < 2) { + return null; + } + + Map sessionData = readResultToMap(readResult); + if (sessionData.isEmpty()) { + throw new IllegalArgumentException("Session Data not found."); + } + + this.sessionsAccessed.add(id); + RedisSession session = new RedisSession(sessionData, id, false, this.connection, this.rawConnection, + this.appPrefix, this.redisIndexConfiguration, serializer, this.configuration); + boolean cached = this.localCache.addEntry(session); + if (cached) { + subscribeToSessionUpdates(id); + } + return session; + } catch (Exception e) { + throw new IllegalArgumentException("Could not look up session.", e); + } + } + + private void subscribeToSessionUpdates(String sessionId) { + this.pubsub.async().subscribe(String.format(Constants.INVALIDATION_CHANNEL_FORMAT, sessionId)); + } + + Map readResultToMap(Map inputMap) throws Exception { + Map sessionData = new HashMap<>(); + for (Map.Entry entry : inputMap.entrySet()) { + byte[] fieldName = entry.getKey(); + byte[] fieldValueStr = entry.getValue(); + putResultFieldInMap(sessionData, fieldName, fieldValueStr); + } + + return sessionData; + } + + private void putResultFieldInMap(Map sessionData, byte[] fieldName, byte[] fieldValueStr) + throws Exception { + String fieldNameString = new String(fieldName, StandardCharsets.UTF_8); + if (Constants.reservedFields.contains(fieldNameString)) { + sessionData.put(new String(fieldName, StandardCharsets.UTF_8), Long.parseLong(new String(fieldValueStr, + StandardCharsets.UTF_8))); + } else if (this.redisIndexConfiguration.getFields().containsKey(fieldNameString)) { + IndexedField indexedField = this.redisIndexConfiguration.getFields().get(fieldNameString); + sessionData.put(new String(fieldName, StandardCharsets.UTF_8), indexedField.getConverter().parse(new String( + fieldValueStr, StandardCharsets.UTF_8))); + } else { + + sessionData.put(new String(fieldName, StandardCharsets.UTF_8), serializer.Deserialize(fieldValueStr)); + } + } + + /** + * {@inheritDoc} + */ + @Override + public void deleteSessionById(String id) { + this.connection.sync().unlink(RedisSession.buildKeyName(this.appPrefix, id)); + this.localCache.removeEntry(id, true); + this.publishInvalidationMessage(id); + } + + /** + * {@inheritDoc} + */ + @Override + public Map findSessionsByExactMatch(String fieldName, String fieldValue) throws Exception { + RediSearchCommands commands = this.rawConnection.sync(); + SearchResults results = commands.ftSearch(indexName().getBytes(StandardCharsets.UTF_8), String + .format("@%s:{%s}", fieldName, fieldValue).getBytes(StandardCharsets.UTF_8)); + Map sessions = new HashMap<>(); + for (Document doc : results) { + RedisSession session = searchDocToSession(doc); + boolean cached = this.localCache.addEntry(session); + subscribeToSessionUpdates(session.getId()); + sessions.put(session.getId(), session); + } + + return sessions; + } + + RedisSession searchDocToSession(Document doc) throws Exception { + Map sessionData = searchDocToSessionData(doc); + String sessionId = new String(doc.getId(), StandardCharsets.UTF_8).split(keyPrefix())[1]; + return new RedisSession(sessionData, sessionId, false, this.connection, this.rawConnection, this.appPrefix, + this.redisIndexConfiguration, serializer, this.configuration); + } + + private Map searchDocToSessionData(Document doc) throws Exception { + Map sessionData = new HashMap<>(); + + for (Map.Entry entry : doc.entrySet()) { + putResultFieldInMap(sessionData, entry.getKey(), entry.getValue()); + } + + return sessionData; + } + + /** + * {@inheritDoc} + */ + @Override + public void bootstrap() { + connection.sync().functionFlush(FlushMode.SYNC); + String lua = Function.getFunctionFile(); + connection.sync().functionLoad(lua); + + CreateOptions createOptions = CreateOptions.builder().prefix(keyPrefix()).build(); + + List> fields = redisIndexConfiguration.getFields().entrySet().stream().map(f -> f.getValue() + .toLettuceModField()).collect(Collectors.toList()); + + connection.sync().ftCreate(indexName(), createOptions, fields.toArray(Field[]::new)); + + if (connection.sync().exists(tDigestKey()) != 1) { + connection.sync().tDigestCreate(tDigestKey()); + } + + String[] evalKeys = { topKKey }; + connection.sync().fcall(Function.reserve_structs.name(), ScriptOutputType.BOOLEAN, evalKeys, String.valueOf(topK)); + } + + /** + * {@inheritDoc} + */ + @Override + public void dropIndex(boolean dropAssociatedRecords) { + try { + if (dropAssociatedRecords) { + connection.sync().ftDropindexDeleteDocs(indexName()); + } else { + connection.sync().ftDropindex(indexName()); + } + } catch (Exception e) { + if (!e.getMessage().toLowerCase().contains("no such index") && !e.getMessage().toLowerCase().contains( + "unknown index name")) { + throw e; + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public Map largestSessions(int topK) { + String returnPath = "sessionSize"; + SearchResults largestSessions = connection.sync().ftSearch(indexName(), "*", SearchOptions + .builder().sortBy(SearchOptions.SortBy.desc("sessionSize")).limit(0, topK).returnField( + returnPath).build()); + return largestSessions.stream().filter(d -> d.containsKey(returnPath)).collect(Collectors.toMap(d -> d.getId() + .split(keyPrefix())[1], d -> Long.parseLong(d.get(returnPath)), (x, y) -> y, LinkedHashMap::new)); + } + + /** + * {@inheritDoc} + */ + @Override + public List> mostAccessedSessions() { + return this.connection.sync().topKListWithScores(this.topKKey); + + } + + /** + * {@inheritDoc} + */ + @Override + public Map findSessions(Filter filter, int limit) { + SearchOptions searchOptions = SearchOptions.builder().limit(0, limit).build(); + return findSessions(searchOptions, filter, new HashMap<>()); + } + + /** + * {@inheritDoc} + */ + @Override + public Map findSessions(Filter filter, String sortBy, boolean ascending, int limit) { + SearchOptions.SortBy orderBy = ascending ? + SearchOptions.SortBy.asc(sortBy.getBytes(StandardCharsets.UTF_8)) : + SearchOptions.SortBy.desc(sortBy.getBytes(StandardCharsets.UTF_8)); + SearchOptions searchOptions = SearchOptions.builder().limit(0, limit).sortBy( + orderBy).build(); + return findSessions(searchOptions, filter, new LinkedHashMap<>()); + } + + private Map findSessions(SearchOptions searchOptions, Filter filter, + Map resultMap) { + SearchResults results = rawConnection.sync().ftSearch(indexName().getBytes(StandardCharsets.UTF_8), + filter.getQuery().getBytes(StandardCharsets.UTF_8), searchOptions); + + for (Document doc : results) { + try { + RedisSession session = searchDocToSession(doc); + boolean cached = this.localCache.addEntry(session); + if (cached) { + subscribeToSessionUpdates(session.getId()); + } + resultMap.put(session.getId(), session); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return resultMap; + } + + /** + * {@inheritDoc} + */ + @Override + public Set deleteSessions(Filter filter, int limit) { + Map sessions = findSessions(filter, limit); + RedisModulesAsyncCommands commands = connection.async(); + for (String session : sessions.keySet()) { + commands.unlink(session); + } + + connection.flushCommands(); + for (String session : sessions.keySet()) { + localCache.removeEntry(session, true); + } + return sessions.keySet(); + } + + /** + * {@inheritDoc} + */ + @Override + public Set updateSessions(Filter filter, int limit, String field, Object value) { + Map sessions = findSessions(filter, limit); + for (Map.Entry entry : sessions.entrySet()) { + entry.getValue().setAttribute(field, value); + boolean cached = localCache.addEntry(entry.getValue()); + if (cached) { + subscribeToSessionUpdates(entry.getValue().getId()); + } + + publishInvalidationMessage(entry.getValue().getId()); + } + + return sessions.keySet(); + } + + private void publishInvalidationMessage(String sessionId) { + this.pubsub.async().publish(String.format(Constants.INVALIDATION_CHANNEL_FORMAT, sessionId), sessionId); + } + + /** + * {@inheritDoc} + */ + @Override + public List sessionSizeQuantiles(double[] quantiles) { + return this.connection.sync().tDigestQuantile(tDigestKey(), quantiles); + } + + /** + * {@inheritDoc} + */ + @Override + public Long uniqueSessions() { + return this.connection.sync().pfcount(uniqueSessionsHll()); + } + + /** + * {@inheritDoc} + */ + @Override + public void addSessionSize(Long sessionSize) { + this.sessionSizes.add(sessionSize.doubleValue()); + } + + /** + * {@inheritDoc} + */ + @Override + public LocalCacheStatistics getLocalCacheStatistics() { + return localCache.getStats(); + } + + public String tDigestKey() { + return this.appPrefix.isPresent() ? + String.format("%s:redisSessions:sessionSizeTd", appPrefix.get()) : + "redisSessions:sessionSizeTd"; + } + + private String keyPrefix() { + return this.appPrefix.isPresent() ? String.format("%s:session:", this.appPrefix.get()) : "session:"; + } + + private String indexName() { + return this.appPrefix.isPresent() ? String.format("%s:sessions-idx", this.appPrefix.get()) : "session-idx"; + } + + public String uniqueSessionsHll() { + return this.appPrefix.isPresent() ? + String.format("%s:redisSessions:uniqueSessionsHll", appPrefix.get()) : + "redisSessions:uniqueSessionsHll"; + } + + /** + * {@inheritDoc} + */ + @Override + public void close() { + this.connection.close(); + this.pubsub.close(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/RedisSessionProviderConfiguration.java b/redis-om-spring/src/main/java/com/redis/om/sessions/RedisSessionProviderConfiguration.java new file mode 100644 index 000000000..25433e6e3 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/RedisSessionProviderConfiguration.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.time.Duration; +import java.util.Optional; + +import com.redis.om.sessions.indexing.RedisIndexConfiguration; +import com.redis.om.sessions.serializers.JdkSerializer; + +import lombok.Getter; + +@Getter +public class RedisSessionProviderConfiguration { + private final RedisIndexConfiguration indexConfiguration; + private final long localCacheMaxSize; + private final long minLocalRecordSize; + private final Serializer serializer; + private final Optional appPrefix; + private final Optional ttl; + + private RedisSessionProviderConfiguration(RedisIndexConfiguration indexConfiguration, long localCacheMaxSize, + long minLocalRecordSize, Serializer serializer, Optional appPrefix, Optional ttl) { + this.indexConfiguration = indexConfiguration; + this.localCacheMaxSize = localCacheMaxSize; + this.minLocalRecordSize = minLocalRecordSize; + this.serializer = serializer; + this.appPrefix = appPrefix; + this.ttl = ttl; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private RedisIndexConfiguration indexConfiguration = RedisIndexConfiguration.builder().build(); + private long localCacheMaxSize = 0; + private long minLocalRecordSize = 0; + private Serializer serializer = new JdkSerializer(); + private Optional appPrefix = Optional.empty(); + private Optional ttl = Optional.of(Duration.ofMinutes(30)); + + public Builder indexConfiguration(RedisIndexConfiguration indexConfiguration) { + this.indexConfiguration = indexConfiguration; + return this; + } + + public Builder localCacheMaxSize(long localCacheMaxSize) { + this.localCacheMaxSize = localCacheMaxSize; + return this; + } + + public Builder minLocalRecordSize(long minLocalRecordSize) { + this.minLocalRecordSize = minLocalRecordSize; + return this; + } + + public Builder serializer(Serializer serializer) { + this.serializer = serializer; + return this; + } + + public Builder appPrefix(String appPrefix) { + this.appPrefix = Optional.of(appPrefix); + return this; + } + + public Builder appPrefix(Optional appPrefix) { + this.appPrefix = appPrefix; + return this; + } + + public Builder ttl(Duration duration) { + this.ttl = Optional.of(duration); + return this; + } + + public Builder ttlSeconds(long secondsToLive) { + this.ttl = Optional.of(Duration.ofSeconds(secondsToLive)); + return this; + } + + public Builder ttlMinutes(long minutesToLive) { + this.ttl = Optional.of(Duration.ofMinutes(minutesToLive)); + return this; + } + + public Builder ttlHours(long hoursToLive) { + this.ttl = Optional.of(Duration.ofHours(hoursToLive)); + return this; + } + + public Builder ttlDays(long days) { + this.ttl = Optional.of(Duration.ofDays(days)); + return this; + } + + public RedisSessionProviderConfiguration build() { + return new RedisSessionProviderConfiguration(indexConfiguration, localCacheMaxSize, minLocalRecordSize, + serializer, appPrefix, ttl); + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/RedisSessionRepository.java b/redis-om-spring/src/main/java/com/redis/om/sessions/RedisSessionRepository.java new file mode 100644 index 000000000..6d42e618c --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/RedisSessionRepository.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import org.springframework.session.*; + +public class RedisSessionRepository implements SessionRepository, + FindByIndexNameSessionRepository { + + public static final String DEFAULT_KEY_NAMESPACE = "spring:"; + private final SessionIdGenerator sessionIdGenerator = UuidSessionIdGenerator.getInstance(); + + private final RedisSessionProvider sessionProvider; + + private final Duration maxInactiveInterval = Duration.ofSeconds(MapSession.DEFAULT_MAX_INACTIVE_INTERVAL_SECONDS); + + public RedisSessionRepository(RedisSessionProvider sessionProvider) { + this.sessionProvider = sessionProvider; + } + + @Override + public RedisSession createSession() { + MapSession mapSession = new MapSession(this.sessionIdGenerator); + String sessionId = this.sessionIdGenerator.generate(); + mapSession.setMaxInactiveInterval(this.maxInactiveInterval); + RedisSession session = new RedisSession(new HashMap<>(), sessionId, sessionProvider); + session.save(); + return session; + } + + @Override + public void save(RedisSession session) { + session.save(); + } + + @Override + public RedisSession findById(String s) { + com.redis.om.sessions.RedisSession session = this.sessionProvider.findSessionById(s); + if (session == null) { + return null; + } + + return new RedisSession(session, sessionProvider); + } + + @Override + public void deleteById(String s) { + this.sessionProvider.deleteSessionById(s); + } + + @Override + public Map findByIndexNameAndIndexValue(String indexName, String indexValue) { + Map sessions; + try { + sessions = this.sessionProvider.findSessionsByExactMatch(indexName, indexValue); + } catch (Exception e) { + throw new RuntimeException(e); + } + + return sessions.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> new RedisSession(e.getValue(), + sessionProvider))); + } + + @Override + public Map findByPrincipalName(String principalName) { + return FindByIndexNameSessionRepository.super.findByPrincipalName(principalName); + } + + public static final class RedisSession implements org.springframework.session.Session { + private final com.redis.om.sessions.RedisSession internalSession; + private final RedisSessionProvider provider; + + private RedisSession(Map sessionData, String sessionId, RedisSessionProvider provider) { + this.provider = provider; + this.internalSession = provider.createSession(sessionId, sessionData); + } + + private RedisSession(com.redis.om.sessions.RedisSession internalSession, RedisSessionProvider provider) { + this.provider = provider; + this.internalSession = internalSession; + } + + private void save() { + this.internalSession.save(); + } + + @Override + public String getId() { + return this.internalSession.getId(); + } + + @Override + public String changeSessionId() { + return this.internalSession.changeSessionId(); + } + + @Override + public T getAttribute(String s) { + Optional opt = this.internalSession.getAttribute(s); + return opt.orElse(null); + } + + @Override + public Set getAttributeNames() { + return this.internalSession.getAttributeNames(); + } + + @Override + public void setAttribute(String s, Object o) { + Long newSize = this.internalSession.setAttribute(s, o); + provider.addSessionSize(newSize); + + } + + @Override + public void removeAttribute(String s) { + this.internalSession.removeAttribute(s); + } + + @Override + public Instant getCreationTime() { + return this.internalSession.getCreationTime(); + } + + @Override + public void setLastAccessedTime(Instant instant) { + this.internalSession.setLastAccessedTime(instant); + } + + @Override + public Instant getLastAccessedTime() { + return this.internalSession.getLastAccessedTime(); + } + + @Override + public void setMaxInactiveInterval(Duration duration) { + this.internalSession.setMaxInactiveInterval(duration); + } + + @Override + public Duration getMaxInactiveInterval() { + return this.internalSession.getMaxInactiveInterval().orElse(null); + } + + @Override + public boolean isExpired() { + return this.internalSession.isExpired(); + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/Script.java b/redis-om-spring/src/main/java/com/redis/om/sessions/Script.java new file mode 100644 index 000000000..492bccd89 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/Script.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; + +import lombok.Getter; + +public enum Script { + readKey("readKey.lua"), + reserveStructs("reserveStructs.lua"); + + @Getter + public final String scriptFile; + @Getter + public final String code; + + private Script(String scriptFile) { + this.scriptFile = scriptFile; + this.code = getScriptFromFile(); + } + + private static String getScriptName(String script) { + return Paths.get("scripts", script).toString(); + } + + private String getScriptFromFile() { + + try (InputStream stream = getClass().getClassLoader().getResourceAsStream(getScriptName(scriptFile))) { + if (stream == null) { + throw new IllegalArgumentException(String.format("Could not load %s from disk", getScriptName(scriptFile))); + } + + return new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } catch (Exception e) { + throw new IllegalArgumentException(String.format("Error while reading the script file %s", getScriptName( + scriptFile)), e); + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/ScriptRunner.java b/redis-om-spring/src/main/java/com/redis/om/sessions/ScriptRunner.java new file mode 100644 index 000000000..f0db86752 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/ScriptRunner.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.util.HashMap; +import java.util.Map; + +import com.redis.lettucemod.api.StatefulRedisModulesConnection; + +import io.lettuce.core.ScriptOutputType; + +public class ScriptRunner { + private final static String MISSING_SCRIPT_ERROR = "NOSCRIPT No matching script"; + public static final ScriptRunner INSTANCE = new ScriptRunner(); + public Map hashMapping; + + private ScriptRunner() { + hashMapping = new HashMap<>(); + } + + T run(StatefulRedisModulesConnection connection, Script script, ScriptOutputType outputType, K[] keys, + V... values) { + if (!hashMapping.containsKey(script)) { + return loadAndRun(connection, script, outputType, keys, values); + } + + try { + return connection.sync().evalsha(hashMapping.get(script), outputType, keys, values); + } catch (Exception e) { + if (e.getMessage().contains(MISSING_SCRIPT_ERROR)) { + return loadAndRun(connection, script, outputType, keys, values); + } + + throw e; + } + } + + private T loadAndRun(StatefulRedisModulesConnection connection, Script script, + ScriptOutputType outputType, K[] keys, V... values) { + hashMapping.put(script, connection.sync().scriptLoad(script.code)); + return connection.sync().evalsha(hashMapping.get(script), outputType, keys, values); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/Serializer.java b/redis-om-spring/src/main/java/com/redis/om/sessions/Serializer.java new file mode 100644 index 000000000..aaacbd7be --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/Serializer.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +public interface Serializer { + /** + * Serialize an object to a byte array + * + * @param object the object to serialize + * @return the byte arrary representation of the object + * @param the type of the object + * @throws Exception thrown if the object cannot be serialized + */ + byte[] Serialize(T object) throws Exception; + + /** + * Deserialize a byte array to an object + * + * @param redisObj the byte array to deserialize + * @return the object + * @param the type of the object + * @throws Exception thrown if the object cannot be deserialized + */ + T Deserialize(byte[] redisObj) throws Exception; +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/Session.java b/redis-om-spring/src/main/java/com/redis/om/sessions/Session.java new file mode 100644 index 000000000..a20b0062e --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/Session.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.Set; + +public interface Session { + /** + * Get the id of the session + * + * @return the sessions id + */ + String getId(); + + /** + * Change the session id + * + * @return the new session id + */ + String changeSessionId(); + + /** + * Change the session id + * + * @param sessionId the new session id + * @return the new session id + */ + String changeSessionId(String sessionId); + + /** + * Get the value of an attribute + * + * @param attribute the name of the attribute + * @return the value of the attribute + */ + Optional getAttribute(String attribute); + + /** + * Get all the attribute names + * + * @return the set of attribute names + */ + Set getAttributeNames(); + + /** + * Set an attribute + * + * @param attributeName the name of the attribute + * @param attributeValue the value of the attribute + * @return the session + */ + Long setAttribute(String attributeName, T attributeValue); + + /** + * Remove an attribute + * + * @param attributeName the name of the attribute + */ + void removeAttribute(String attributeName); + + /** + * Get the creation time of the session + * + * @return the creation time + */ + Instant getCreationTime(); + + /** + * Set the last accessed time of the session + * + * @param lastAccessedTime the last accessed time + */ + void setLastAccessedTime(Instant lastAccessedTime); + + /** + * Get the last accessed time of the session + * + * @return the last accessed time + */ + Instant getLastAccessedTime(); + + /** + * Set the max inactive interval of the session + * + * @param interval the max inactive interval + */ + void setMaxInactiveInterval(Duration interval); + + /** + * Get the max inactive interval of the session + * + * @return the max inactive interval + */ + Optional getMaxInactiveInterval(); + + /** + * Check if the session is expired + * + * @return true if the session is expired, false otherwise + */ + boolean isExpired(); + + /** + * Save the session + * + * @return the size of the session + */ + Long save(); + + /** + * Get the size of the session + * + * @return the size of the session + */ + long getSize(); + + /** + * Get the last modified time of the session + * + * @return the last modified time of the session + */ + long getLastModifiedTime(); +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/SessionMetrics.java b/redis-om-spring/src/main/java/com/redis/om/sessions/SessionMetrics.java new file mode 100644 index 000000000..ef6d62fff --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/SessionMetrics.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +public class SessionMetrics { + private long size; + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/SessionProvider.java b/redis-om-spring/src/main/java/com/redis/om/sessions/SessionProvider.java new file mode 100644 index 000000000..40990ce01 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/SessionProvider.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.redis.om.sessions.filtering.Filter; + +import io.lettuce.core.KeyValue; + +public interface SessionProvider extends AutoCloseable { + /** + * Create a new Session + * + * @param sessionId the id of the session + * @param sessionData the data for the session + * @return the new session + */ + T createSession(String sessionId, Map sessionData); + + /** + * Retrieves a session by its id + * + * @param id the id of the session to retrive + * @return the session + */ + T findSessionById(String id); + + /** + * Deletes a session with a given id + * + * @param id the id of the session to delete + */ + void deleteSessionById(String id); + + /** + * Finds a session with an exact string match + * + * @param fieldName + * @param fieldValue + * @return sessions matching the exact string match. + * @throws Exception + */ + Map findSessionsByExactMatch(String fieldName, String fieldValue) throws Exception; + + /** + * Bootstraps the session provider, Creating the necessary indexes and metrics tracking data structures for the + * provider + */ + void bootstrap(); + + /** + * Deletes the index associated with the session provider + * + * @param dropAssociatedRecords whether or not to delete all the sessions currently mapped by the index. + */ + void dropIndex(boolean dropAssociatedRecords); + + /** + * Returns the top-k largest sessions + * + * @param topK the number of largest sessions to return + * @return the session Ids along with their sizes + */ + Map largestSessions(int topK); + + /** + * Returns the sessions that have been most heavily accessed. + * + * @return the session Ids along with the number of times they have been accessed. + */ + List> mostAccessedSessions(); + + /** + * Finds the sessions that match the provided filter + * + * @param filter the filter to use to search for sessions + * @param limit the number of sessions to return + * @return session Ids along with the sessions that match the filter + */ + Map findSessions(Filter filter, int limit); + + /** + * Finds sessions that match the provided filter, ordering the results by the provided field + * + * @param filter the filter to use to search for sessions + * @param sortBy the field to order the results by + * @param ascending whether to order the results in ascending or descending order + * @param limit the number of sessions to return + * @return session Ids along with the sessions that match the filter + */ + Map findSessions(Filter filter, String sortBy, boolean ascending, int limit); + + /** + * Deletes sessions that match the provided filter + * + * @param filter the filter to use to search for sessions + * @param limit the number of sessions to delete + * @return the session Ids of the sessions that were deleted + */ + Set deleteSessions(Filter filter, int limit); + + /** + * Updates the sessions that match the provided filter by setting the provided field to the provided value + * + * @param filter the filter to use to search for sessions + * @param limit the number of sessions to update + * @param field the name of the field to update. + * @param value the value to set the field to. + * @return the session Ids of the sessions that were updated. + */ + Set updateSessions(Filter filter, int limit, String field, Object value); + + /** + * Retrieves the size of the session at the provided quantiles + * + * @param quantiles a set of numbers between 0 and 1 representing the quantiles to retrieve + * @return the session sizes at the provided quantiles. + */ + List sessionSizeQuantiles(double[] quantiles); + + /** + * Retrieves the approximate number of unique sessions + * + * @return the approximate number of unique sessions + */ + Long uniqueSessions(); + + /** + * Adds a recording of the session size to the redis session provider, for internal use only, to help with metrics + * tracking + * + * @param sessionSize the size of the session to record + */ + void addSessionSize(Long sessionSize); + + /** + * Retrieves the session size distribution + * + * @return returns the Local cache statistics. + */ + LocalCacheStatistics getLocalCacheStatistics(); + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/Util.java b/redis-om-spring/src/main/java/com/redis/om/sessions/Util.java new file mode 100644 index 000000000..f0b33a9a0 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/Util.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +public class Util { + static final Charset charset = StandardCharsets.UTF_8; + + static ByteBuffer s2BB(String s) { + return ByteBuffer.wrap(s.getBytes(charset)); + } + + static String bB2S(ByteBuffer byteBuffer) { + byte[] arr = new byte[byteBuffer.remaining()]; + byteBuffer.get(arr); + return new String(arr, charset); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/codecs/SessionProviderCodec.java b/redis-om-spring/src/main/java/com/redis/om/sessions/codecs/SessionProviderCodec.java new file mode 100644 index 000000000..f21102b19 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/codecs/SessionProviderCodec.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.codecs; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +import io.lettuce.core.codec.RedisCodec; + +public class SessionProviderCodec implements RedisCodec { + private Charset charset = StandardCharsets.UTF_8; + + @Override + public String decodeKey(ByteBuffer byteBuffer) { + return charset.decode(byteBuffer).toString(); + } + + @Override + public ByteBuffer decodeValue(ByteBuffer byteBuffer) { + return clone(byteBuffer); + } + + @Override + public ByteBuffer encodeKey(String s) { + ByteBuffer buffer = charset.encode(s); + return buffer; + } + + @Override + public ByteBuffer encodeValue(ByteBuffer bytes) { + return clone(bytes); + } + + public static ByteBuffer clone(ByteBuffer original) { + ByteBuffer clone = ByteBuffer.allocate(original.capacity()); + clone.put(original); + original.rewind(); + clone.flip(); + return clone; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/EnableRedisSessionMetrics.java b/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/EnableRedisSessionMetrics.java new file mode 100644 index 000000000..8f6fb0ec9 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/EnableRedisSessionMetrics.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.config.annotation.web.http; + +import java.lang.annotation.*; + +import org.springframework.context.annotation.Import; + +@Retention( + RetentionPolicy.RUNTIME +) +@Target( + ElementType.TYPE +) +@Documented +@Import( + RedisSessionsMetrics.class +) +public @interface EnableRedisSessionMetrics { +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/EnableRedisSessions.java b/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/EnableRedisSessions.java new file mode 100644 index 000000000..2570c743c --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/EnableRedisSessions.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.config.annotation.web.http; + +import java.lang.annotation.*; + +import org.springframework.context.annotation.Import; + +@Retention( + RetentionPolicy.RUNTIME +) +@Target( + ElementType.TYPE +) +@Documented +@Import( + RedisSessionsConfiguration.class +) +public @interface EnableRedisSessions { +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/RedisSessionProperties.java b/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/RedisSessionProperties.java new file mode 100644 index 000000000..f3ac5040f --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/RedisSessionProperties.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.config.annotation.web.http; + +import java.util.Optional; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties( + RedisSessionProperties.CONFIG_PREFIX +) +public class RedisSessionProperties { + public static final String CONFIG_PREFIX = "redis"; + private String host = "localhost"; + private Optional prefix = Optional.empty(); + private int port = 6379; + private double[] sessionSizeQuantiles = new double[] { .5, .75, .9, .99, 1 }; + private Cache cache; + + public double[] getSessionSizeQuantiles() { + return sessionSizeQuantiles; + } + + public void setSessionSizeQuantiles(double[] sessionSizeQuantiles) { + this.sessionSizeQuantiles = sessionSizeQuantiles; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public Optional getPrefix() { + return prefix; + } + + public void setPrefix(Optional prefix) { + this.prefix = prefix; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public Cache getCache() { + return cache; + } + + public void setCache(Cache cache) { + this.cache = cache; + } + + public static class Cache { + private long cap; + private int min; + + public long getCap() { + return cap; + } + + public void setCap(long cap) { + this.cap = cap; + } + + public int getMin() { + return min; + } + + public void setMin(int min) { + this.min = min; + } + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/RedisSessionsConfiguration.java b/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/RedisSessionsConfiguration.java new file mode 100644 index 000000000..69b9e0858 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/RedisSessionsConfiguration.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.config.annotation.web.http; + +import java.time.Duration; +import java.util.Optional; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.session.FlushMode; +import org.springframework.session.MapSession; +import org.springframework.session.SaveMode; +import org.springframework.session.config.annotation.web.http.SpringHttpSessionConfiguration; + +import com.redis.lettucemod.RedisModulesClient; +import com.redis.om.sessions.RedisSessionProvider; +import com.redis.om.sessions.RedisSessionProviderConfiguration; +import com.redis.om.sessions.RedisSessionRepository; +import com.redis.om.sessions.indexing.RedisIndexConfiguration; + +@Configuration +@Import( + SpringHttpSessionConfiguration.class +) +@EnableConfigurationProperties( + RedisSessionProperties.class +) +public class RedisSessionsConfiguration { + private Duration maxInactiveInterval = MapSession.DEFAULT_MAX_INACTIVE_INTERVAL; + private String redisAppPrefix = RedisSessionRepository.DEFAULT_KEY_NAMESPACE; + private FlushMode flushMode = FlushMode.ON_SAVE; + private SaveMode saveMode = SaveMode.ON_SET_ATTRIBUTE; + + @Bean + public RedisModulesClient redisClient(RedisSessionProperties properties) { + String redisUri = String.format("redis://%s:%d", properties.getHost(), properties.getPort()); + return RedisModulesClient.create(redisUri); + } + + @Bean + public RedisSessionProvider redisSessionProvider(RedisModulesClient client, + Optional redisIndexConfigurationOpt, RedisSessionProperties properties) { + + RedisIndexConfiguration redisIndexConfiguration = redisIndexConfigurationOpt.orElse(RedisIndexConfiguration + .builder().build()); + RedisSessionProviderConfiguration config = RedisSessionProviderConfiguration.builder().appPrefix(properties + .getPrefix()).localCacheMaxSize(properties.getCache().getCap()).indexConfiguration(redisIndexConfiguration) + .minLocalRecordSize(properties.getCache().getMin()).build(); + return RedisSessionProvider.create(client, config); + } + + public void createIndex(RedisSessionProvider provider) { + provider.bootstrap(); + } + + @Bean + public RedisSessionRepository redisSessionRepository(RedisSessionProvider provider) { + return new RedisSessionRepository(provider); + } + + @Bean( + initMethod = "createIndex" + ) + public StartupBean startup() { + return new StartupBean(); + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/RedisSessionsMetrics.java b/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/RedisSessionsMetrics.java new file mode 100644 index 000000000..5678411dc --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/RedisSessionsMetrics.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.config.annotation.web.http; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.redis.om.sessions.MetricsMonitor; +import com.redis.om.sessions.RedisSession; +import com.redis.om.sessions.RedisSessionProvider; + +import io.micrometer.core.instrument.MeterRegistry; + +@Configuration +public class RedisSessionsMetrics { + @Bean + public MetricsMonitor metricsMonitor(RedisSessionProvider provider, MeterRegistry registry, + RedisSessionProperties properties) { + return new MetricsMonitor<>(registry, provider, provider.getAppPrefix(), 5, properties.getSessionSizeQuantiles()); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/StartupBean.java b/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/StartupBean.java new file mode 100644 index 000000000..3a961aa4d --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/config/annotation/web/http/StartupBean.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.config.annotation.web.http; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import com.redis.om.sessions.RedisSessionProvider; + +@Component +public class StartupBean { + @Autowired + private RedisSessionProvider provider; + + public void createIndex() { + try { + provider.dropIndex(false); + } catch (Exception ex) { + // ignored + } + + try { + provider.bootstrap(); + } catch (Exception ex) { + if (!ex.getMessage().equals("Index already exists")) { + throw ex; + } + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/converters/BooleanConverter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/BooleanConverter.java new file mode 100644 index 000000000..7657f6f3b --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/BooleanConverter.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.converters; + +public class BooleanConverter implements Converter { + @Override + public Boolean parse(String s) { + return Boolean.parseBoolean(s); + } + + @Override + public String toRedisString(Boolean o) { + return o.toString(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/converters/ByteConverter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/ByteConverter.java new file mode 100644 index 000000000..98a8ae46d --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/ByteConverter.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.converters; + +public class ByteConverter implements Converter { + @Override + public Byte parse(String s) { + return Byte.parseByte(s); + } + + @Override + public String toRedisString(Byte o) { + return o.toString(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/converters/Converter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/Converter.java new file mode 100644 index 000000000..f7438589d --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/Converter.java @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.converters; + +public interface Converter { + + T parse(String s); + + String toRedisString(T o); + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/converters/DoubleConverter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/DoubleConverter.java new file mode 100644 index 000000000..b4bce6525 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/DoubleConverter.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.converters; + +public class DoubleConverter implements Converter { + @Override + public Double parse(String s) { + return Double.parseDouble(s); + } + + @Override + public String toRedisString(Double o) { + return o.toString(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/converters/EnumConverter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/EnumConverter.java new file mode 100644 index 000000000..ea3defa69 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/EnumConverter.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.converters; + +public class EnumConverter implements Converter { + private final Class clazz; + + public EnumConverter(Class clazz) { + if (!Enum.class.isAssignableFrom(clazz)) { + throw new IllegalArgumentException(String.format("Attempted to initialize enum converter from non-enum type: %s", + clazz.getName())); + } + this.clazz = clazz; + } + + @Override + public Enum parse(String s) { + return Enum.valueOf(clazz, s); + } + + @Override + public String toRedisString(Enum o) { + return o.name(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/converters/FloatConverter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/FloatConverter.java new file mode 100644 index 000000000..ae00b727e --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/FloatConverter.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.converters; + +public class FloatConverter implements Converter { + @Override + public Float parse(String s) { + return Float.parseFloat(s); + } + + @Override + public String toRedisString(Float o) { + return o.toString(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/converters/GeoLocConverter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/GeoLocConverter.java new file mode 100644 index 000000000..8ae48ca28 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/GeoLocConverter.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.converters; + +import com.redis.om.sessions.GeoLoc; + +public class GeoLocConverter implements Converter { + @Override + public GeoLoc parse(String s) { + return GeoLoc.parse(s); + } + + @Override + public String toRedisString(GeoLoc o) { + return o.toString(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/converters/IntegerConverter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/IntegerConverter.java new file mode 100644 index 000000000..d8472bcce --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/IntegerConverter.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.converters; + +public class IntegerConverter implements Converter { + @Override + public Integer parse(String s) { + return Integer.parseInt(s); + } + + @Override + public String toRedisString(Integer o) { + return o.toString(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/converters/LongConverter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/LongConverter.java new file mode 100644 index 000000000..0f26d8199 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/LongConverter.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.converters; + +public class LongConverter implements Converter { + @Override + public Long parse(String s) { + return Long.parseLong(s); + } + + @Override + public String toRedisString(Long o) { + return o.toString(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/converters/ShortConverter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/ShortConverter.java new file mode 100644 index 000000000..eb6647110 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/ShortConverter.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.converters; + +public class ShortConverter implements Converter { + @Override + public Short parse(String s) { + return Short.parseShort(s); + } + + @Override + public String toRedisString(Short o) { + return o.toString(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/converters/StringConverter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/StringConverter.java new file mode 100644 index 000000000..85e3ae69e --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/StringConverter.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.converters; + +public class StringConverter implements Converter { + @Override + public String parse(String s) { + return s; + } + + @Override + public String toRedisString(String o) { + return o; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/converters/URIConverter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/URIConverter.java new file mode 100644 index 000000000..90e84165a --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/URIConverter.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.converters; + +import java.net.URI; + +public class URIConverter implements Converter { + @Override + public URI parse(String s) { + return URI.create(s); + } + + @Override + public String toRedisString(URI o) { + return o.toString(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/converters/UrlConverter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/UrlConverter.java new file mode 100644 index 000000000..b9f887942 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/UrlConverter.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.converters; + +import java.net.MalformedURLException; +import java.net.URL; + +public class UrlConverter implements Converter { + + @Override + public URL parse(String s) { + try { + return new URL(s); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + } + + @Override + public String toRedisString(URL o) { + return o.toString(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/converters/UuidConverter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/UuidConverter.java new file mode 100644 index 000000000..e74307055 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/converters/UuidConverter.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.converters; + +import java.util.UUID; + +public class UuidConverter implements Converter { + @Override + public UUID parse(String s) { + return UUID.fromString(s); + } + + @Override + public String toRedisString(UUID o) { + return o.toString(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/AndFilter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/AndFilter.java new file mode 100644 index 000000000..122a334cb --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/AndFilter.java @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +public class AndFilter extends LogicalFilter { + @Override + public String getQuery() { + return " "; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/AnyFilter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/AnyFilter.java new file mode 100644 index 000000000..8d892a02e --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/AnyFilter.java @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +public class AnyFilter extends Filter { + @Override + public String getQuery() { + return "*"; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/BetweenFilter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/BetweenFilter.java new file mode 100644 index 000000000..37d9fb0de --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/BetweenFilter.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +public class BetweenFilter extends Filter { + private final String fieldName; + private final L lowerBound; + private final U upperBound; + + public BetweenFilter(String fieldName, L lowerBound, U upperBound) { + this.fieldName = fieldName; + this.lowerBound = lowerBound; + this.upperBound = upperBound; + } + + @Override + public String getQuery() { + return String.format("@%s:[%s %s]", fieldName, lowerBound.toString(), upperBound.toString()); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/CompositeFilter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/CompositeFilter.java new file mode 100644 index 000000000..efde5b661 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/CompositeFilter.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import com.redis.om.sessions.GeoLoc; +import com.redis.om.sessions.GeoUnit; +import com.redis.om.sessions.converters.Converter; + +public class CompositeFilter extends Filter { + private final List filters = new ArrayList<>(); + + CompositeFilter(Filter firstFilter, LogicalFilter logicalFilter) { + filters.add(firstFilter); + filters.add(logicalFilter); + } + + public Filter equals(String fieldName, String fieldValue) { + this.filters.add(new ExactStringMatchFilter(fieldName, fieldValue)); + return this; + } + + public Filter equals(String fieldName, T value, Converter converter) { + this.filters.add(new ExactStringMatchFilter(fieldName, converter.toRedisString(value))); + return this; + } + + public Filter textMatch(String fieldName, String matchValue) { + this.filters.add(new TextMatchFilter(fieldName, matchValue)); + return this; + } + + public Filter equals(String fieldName, T fieldValue) { + this.filters.add(new ExactNumericMatchFilter<>(fieldName, fieldValue)); + return this; + } + + public Filter between(String fieldName, L lowerBound, U upperBound) { + this.filters.add(new BetweenFilter<>(fieldName, lowerBound, upperBound)); + return this; + } + + public Filter greaterThan(String fieldName, L lowerBound) { + this.filters.add(new GreaterThanFilter<>(fieldName, lowerBound)); + return this; + } + + public Filter lessThan(String fieldName, U upperBound) { + this.filters.add(new LessThanFilter<>(fieldName, upperBound)); + return this; + } + + public Filter geoRadius(String fieldName, GeoLoc point, double distance, GeoUnit geoUnit) { + this.filters.add(new GeoFilter(fieldName, point, distance, geoUnit)); + return this; + } + + @Override + public String getQuery() { + StringBuilder sb = new StringBuilder(); + List applicableFilters = filters.stream().filter(f -> !(f instanceof AnyFilter)).collect(Collectors + .toList()); + if (applicableFilters.isEmpty()) { // all filters are AnyFilters, so we can just pass a * back + return "*"; + } + + if (applicableFilters.size() == 2 && applicableFilters.get(0) instanceof LogicalFilter) { // handle case where first filter was any with a logical filter proceeding it. + return applicableFilters.get(1).getQuery(); + } + + for (Filter filter : applicableFilters) { + sb.append(filter.getQuery()); + } + + return String.format("(%s)", sb); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/ExactNumericMatchFilter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/ExactNumericMatchFilter.java new file mode 100644 index 000000000..8b031a629 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/ExactNumericMatchFilter.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +public class ExactNumericMatchFilter extends Filter { + T fieldValue; + String fieldName; + + public ExactNumericMatchFilter(String fieldName, T fieldValue) { + this.fieldName = fieldName; + this.fieldValue = fieldValue; + } + + @Override + public String getQuery() { + return String.format("@%s:[%s %s]", this.fieldName, this.fieldValue, this.fieldValue); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/ExactStringMatchFilter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/ExactStringMatchFilter.java new file mode 100644 index 000000000..3df0a1189 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/ExactStringMatchFilter.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +public class ExactStringMatchFilter extends Filter { + private final String fieldName; + private final String fieldValue; + + public ExactStringMatchFilter(String fieldName, String fieldValue) { + this.fieldName = fieldName; + this.fieldValue = fieldValue; + } + + @Override + public String getQuery() { + return String.format("@%s:{%s}", fieldName, fieldValue); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/Filter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/Filter.java new file mode 100644 index 000000000..4eb53d5db --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/Filter.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +public abstract class Filter { + public abstract String getQuery(); + + public CompositeFilter and() { + return new CompositeFilter(this, new AndFilter()); + } + + public CompositeFilter or() { + return new CompositeFilter(this, new OrFilter()); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/GeoFilter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/GeoFilter.java new file mode 100644 index 000000000..a9cc240db --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/GeoFilter.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +import com.redis.om.sessions.GeoLoc; +import com.redis.om.sessions.GeoUnit; + +public class GeoFilter extends Filter { + private final GeoLoc geoLoc; + private final String fieldName; + private final double radius; + private final GeoUnit geoUnit; + + public GeoFilter(String fieldName, GeoLoc geoLoc, double radius, GeoUnit geoUnit) { + this.geoLoc = geoLoc; + this.fieldName = fieldName; + this.radius = radius; + this.geoUnit = geoUnit; + } + + @Override + public String getQuery() { + return String.format("@%s:[%s %s %s %s]", fieldName, geoLoc.getLongitude(), geoLoc.getLatitude(), radius, geoUnit + .name()); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/GreaterThanFilter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/GreaterThanFilter.java new file mode 100644 index 000000000..fc61ece38 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/GreaterThanFilter.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +public class GreaterThanFilter extends Filter { + private final String fieldName; + private final L lowerBound; + + public GreaterThanFilter(String fieldName, L lowerBound) { + this.fieldName = fieldName; + this.lowerBound = lowerBound; + } + + @Override + public String getQuery() { + return String.format("@%s:[%s +inf]", fieldName, lowerBound.toString()); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/LessThanFilter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/LessThanFilter.java new file mode 100644 index 000000000..f0c503726 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/LessThanFilter.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +public class LessThanFilter extends Filter { + private final String fieldName; + private final U upperBound; + + public LessThanFilter(String fieldName, U upperBound) { + this.fieldName = fieldName; + this.upperBound = upperBound; + } + + @Override + public String getQuery() { + return String.format("@%s:[-inf %s]", this.fieldName, this.upperBound.toString()); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/LogicalFilter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/LogicalFilter.java new file mode 100644 index 000000000..81f408fa2 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/LogicalFilter.java @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +public abstract class LogicalFilter extends Filter { +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/OrFilter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/OrFilter.java new file mode 100644 index 000000000..e9a4a9d2f --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/OrFilter.java @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +public class OrFilter extends LogicalFilter { + @Override + public String getQuery() { + return " | "; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/QueryBuilder.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/QueryBuilder.java new file mode 100644 index 000000000..6c3056388 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/QueryBuilder.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +import com.redis.om.sessions.GeoLoc; +import com.redis.om.sessions.GeoUnit; +import com.redis.om.sessions.converters.Converter; + +public class QueryBuilder { + public static AnyFilter any() { + return new AnyFilter(); + } + + public static ExactStringMatchFilter equals(String fieldName, String fieldValue) { + return new ExactStringMatchFilter(fieldName, fieldValue); + } + + public static ExactStringMatchFilter equals(String fieldName, T value, Converter converter) { + return new ExactStringMatchFilter(fieldName, converter.toRedisString(value)); + } + + public static TextMatchFilter textMatch(String fieldName, String matchValue) { + return new TextMatchFilter(fieldName, matchValue); + } + + public static ExactNumericMatchFilter equals(String fieldName, T fieldValue) { + return new ExactNumericMatchFilter<>(fieldName, fieldValue); + } + + public static BetweenFilter between(String fieldName, L lowerBound, + U upperBound) { + return new BetweenFilter<>(fieldName, lowerBound, upperBound); + } + + public static GreaterThanFilter greaterThan(String fieldName, L lowerBound) { + return new GreaterThanFilter<>(fieldName, lowerBound); + } + + public static LessThanFilter lessThan(String fieldName, U upperBound) { + return new LessThanFilter<>(fieldName, upperBound); + } + + public static GeoFilter geoRadius(String fieldName, GeoLoc point, double distance, GeoUnit geoUnit) { + return new GeoFilter(fieldName, point, distance, geoUnit); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/RawFilter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/RawFilter.java new file mode 100644 index 000000000..3a3863ce5 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/RawFilter.java @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +public class RawFilter extends Filter { + private final String predicate; + + public RawFilter(String predicate) { + this.predicate = predicate; + } + + @Override + public String getQuery() { + return predicate; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/TextMatchFilter.java b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/TextMatchFilter.java new file mode 100644 index 000000000..dcc9b282f --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/filtering/TextMatchFilter.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.filtering; + +public class TextMatchFilter extends Filter { + private final String fieldName; + private final String fieldValue; + + public TextMatchFilter(String fieldName, String fieldValue) { + this.fieldName = fieldName; + this.fieldValue = fieldValue; + } + + @Override + public String getQuery() { + return String.format("@%s:%s", fieldName, fieldValue); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/FieldType.java b/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/FieldType.java new file mode 100644 index 000000000..a7d68c87f --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/FieldType.java @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.indexing; + +public enum FieldType { + tag, + numeric, + geo, + text, + vector, +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/GeoField.java b/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/GeoField.java new file mode 100644 index 000000000..2f4cc3039 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/GeoField.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.indexing; + +import java.util.Optional; + +import com.redis.lettucemod.search.Field; +import com.redis.om.sessions.converters.Converter; + +public class GeoField extends IndexedField { + protected GeoField(String name, Optional> javaType, Optional> converter, boolean sortable) { + super(FieldType.geo, name, javaType, converter, sortable); + } + + @Override + public Field toLettuceModField() { + return Field.geo(name).sortable(sortable).build(); + } + + public static class Builder extends IndexedField.Builder { + + public Builder(String name) { + super(FieldType.geo, name); + } + + @Override + public IndexedField build() { + return new GeoField(name, javaType, converter, sortable); + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/IndexedField.java b/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/IndexedField.java new file mode 100644 index 000000000..8841328fe --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/IndexedField.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.indexing; + +import java.util.Optional; + +import com.redis.lettucemod.search.Field; +import com.redis.om.sessions.GeoLoc; +import com.redis.om.sessions.converters.*; + +import lombok.Getter; + +@Getter +public abstract class IndexedField { + protected final FieldType fieldType; + protected final String name; + protected final Class javaType; + protected final Converter converter; + protected final boolean sortable; + + protected IndexedField(FieldType fieldType, String name, Optional> javaType, + Optional> converter, boolean sortable) { + this.fieldType = fieldType; + this.name = name; + this.sortable = sortable; + if (javaType.isPresent()) { + this.javaType = javaType.get(); + } else { + this.javaType = defaultClass(); + } + + if (converter.isPresent()) { + this.converter = converter.get(); + } else { + this.converter = defaultConverter(); + } + } + + private Converter defaultConverter() { + switch (this.fieldType) { + case geo: + return new GeoLocConverter(); + case tag: + if (Enum.class.isAssignableFrom(this.javaType)) { + return new EnumConverter(this.javaType); + } + return new StringConverter(); + case text: + return new StringConverter(); + case numeric: + if (this.javaType == Long.class) { + return new LongConverter(); + } else if (this.javaType == Short.class) { + return new ShortConverter(); + } else if (this.javaType == Double.class) { + return new DoubleConverter(); + } else if (this.javaType == Byte.class) { + return new ByteConverter(); + } else if (this.javaType == Integer.class) { + return new IntegerConverter(); + } else if (this.javaType == Number.class) { + return new DoubleConverter(); + } else { + throw new IllegalArgumentException(String.format("Passed Numeric index type without a valid java type %s", + this.javaType.getName())); + } + case vector: + default: // TODO build converter for vectors + throw new IllegalArgumentException("Unusable fieldType"); + } + } + + private Class defaultClass() { + switch (this.fieldType) { + case geo: + return GeoLoc.class; + case tag: + case text: + return String.class; + case numeric: + return Number.class; + case vector: // TODO how to get float[] clazz? + default: + throw new IllegalArgumentException("Unusable fieldType"); + } + } + + public boolean isKnownOrDefaultClass(Class clazz) { + if (clazz == this.javaType) { + return true; + } + + if (Number.class.isAssignableFrom(clazz)) { + return true; + } + + return clazz == defaultClass(); + } + + public static TagField.Builder tag(String name) { + return new TagField.Builder(name); + } + + public static TextField.Builder text(String name) { + return new TextField.Builder(name); + } + + public static GeoField.Builder geo(String name) { + return new GeoField.Builder(name); + } + + public static NumericField.Builder numeric(String name) { + return new NumericField.Builder(name); + } + + public abstract Field toLettuceModField(); + + public abstract static class Builder { + protected final FieldType fieldType; + protected final String name; + protected boolean sortable; + protected Optional> javaType = Optional.empty(); + protected Optional> converter = Optional.empty(); + + protected Builder(FieldType fieldType, String name) { + this.name = name; + this.fieldType = fieldType; + this.sortable = false; + } + + public Builder javaType(Class javaType) { + this.javaType = Optional.of(javaType); + return this; + } + + public Builder converter(Converter converter) { + this.converter = Optional.of(converter); + return this; + } + + public Builder sortable(boolean sortable) { + this.sortable = sortable; + return this; + } + + public abstract IndexedField build(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/NumericField.java b/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/NumericField.java new file mode 100644 index 000000000..2558027c0 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/NumericField.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.indexing; + +import java.util.Optional; + +import com.redis.lettucemod.search.Field; +import com.redis.om.sessions.converters.Converter; + +public class NumericField extends IndexedField { + protected NumericField(String name, Optional> javaType, Optional> converter, boolean sortable) { + super(FieldType.numeric, name, javaType, converter, sortable); + } + + @Override + public Field toLettuceModField() { + return Field.numeric(name).sortable(sortable).build(); + } + + public static class Builder extends IndexedField.Builder { + + protected Builder(String name) { + super(FieldType.numeric, name); + } + + @Override + public IndexedField build() { + return new NumericField(name, javaType, converter, sortable); + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/RedisIndexConfiguration.java b/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/RedisIndexConfiguration.java new file mode 100644 index 000000000..3d58663b0 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/RedisIndexConfiguration.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.indexing; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import com.redis.om.sessions.Constants; + +import lombok.Getter; + +@Getter +public class RedisIndexConfiguration { + private Map fields; + private Optional name; + + private RedisIndexConfiguration(Optional name, Map fields) { + this.fields = fields; + this.name = name; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private final Map fields = new HashMap<>(); + private Optional name; + + private Builder() { + fields.put(Constants.SIZE_FIELD_NAME, IndexedField.numeric(Constants.SIZE_FIELD_NAME).sortable(true).javaType( + Long.class).build()); + } + + public Builder fields(List indexedFields) { + indexedFields.forEach(f -> this.fields.put(f.getName(), f)); + return this; + } + + public Builder withField(IndexedField indexedField) { + this.fields.put(indexedField.getName(), indexedField); + return this; + } + + public Builder name(String name) { + this.name = Optional.of(name); + return this; + } + + public RedisIndexConfiguration build() { + return new RedisIndexConfiguration(this.name, this.fields); + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/TagField.java b/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/TagField.java new file mode 100644 index 000000000..b9ffab502 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/TagField.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.indexing; + +import java.util.Optional; + +import com.redis.lettucemod.search.Field; +import com.redis.om.sessions.converters.Converter; + +public class TagField extends IndexedField { + private final Optional separator; + + private TagField(String name, Optional> javaType, Optional> converter, boolean sortable, + Optional separator) { + super(FieldType.tag, name, javaType, converter, sortable); + this.separator = separator; + } + + @Override + public Field toLettuceModField() { + com.redis.lettucemod.search.TagField.Builder builder = Field.tag(name).sortable(sortable); + separator.ifPresent(builder::separator); + return builder.build(); + } + + public static class Builder extends IndexedField.Builder { + private Optional separator = Optional.empty(); + + public Builder(String fieldName) { + super(FieldType.tag, fieldName); + } + + public IndexedField.Builder separator(Character separator) { + this.separator = Optional.of(separator); + return this; + } + + @Override + public IndexedField build() { + return new TagField(name, javaType, converter, sortable, separator); + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/TextField.java b/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/TextField.java new file mode 100644 index 000000000..219ce09ac --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/indexing/TextField.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.indexing; + +import java.util.Optional; + +import com.redis.lettucemod.search.Field; +import com.redis.om.sessions.converters.Converter; + +public class TextField extends IndexedField { + private TextField(String name, Optional> javaType, Optional> converter, boolean sortable) { + super(FieldType.text, name, javaType, converter, sortable); + } + + @Override + public Field toLettuceModField() { + return Field.text(name).sortable(sortable).build(); + } + + public static class Builder extends IndexedField.Builder { + public Builder(String name) { + super(FieldType.text, name); + } + + @Override + public IndexedField build() { + return new TextField(name, javaType, converter, sortable); + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/serializers/JdkSerializer.java b/redis-om-spring/src/main/java/com/redis/om/sessions/serializers/JdkSerializer.java new file mode 100644 index 000000000..ae86ac57d --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/serializers/JdkSerializer.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.serializers; + +import java.io.*; + +import com.redis.om.sessions.Serializer; + +public class JdkSerializer implements Serializer { + + /** + * {@inheritDoc} + */ + @Override + public byte[] Serialize(T object) throws Exception { + if (!(object instanceof Serializable)) { + throw new IllegalArgumentException("Object must be serializable to serialize"); + } + ByteArrayOutputStream out = new ByteArrayOutputStream(1024); + ObjectOutputStream objectOutputStream = new ObjectOutputStream(out); + objectOutputStream.writeObject(object); + objectOutputStream.flush(); + return out.toByteArray(); + } + + /** + * {@inheritDoc} + */ + @Override + public T Deserialize(byte[] redisObj) throws Exception { + ByteArrayInputStream in = new ByteArrayInputStream(redisObj); + ObjectInputStream objectInputStream = new ObjectInputStream(in); + return (T) objectInputStream.readObject(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/sessions/serializers/JsonSerializer.java b/redis-om-spring/src/main/java/com/redis/om/sessions/serializers/JsonSerializer.java new file mode 100644 index 000000000..fd1317e31 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/sessions/serializers/JsonSerializer.java @@ -0,0 +1,45 @@ + +/* + * Copyright (c) 2024. Redis Ltd. + */ + +package com.redis.om.sessions.serializers; + +import java.nio.charset.StandardCharsets; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.redis.om.sessions.Serializer; + +public class JsonSerializer implements Serializer { + + private final ObjectMapper objectMapper; + + public JsonSerializer() { + this.objectMapper = JsonMapper.builder().addModule(new JavaTimeModule()).build(); + this.objectMapper.activateDefaultTyping(this.objectMapper.getPolymorphicTypeValidator(), + ObjectMapper.DefaultTyping.EVERYTHING, JsonTypeInfo.As.PROPERTY); + } + + public JsonSerializer(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + /** + * {@inheritDoc} + */ + @Override + public byte[] Serialize(T object) throws Exception { + return this.objectMapper.writeValueAsString(object).getBytes(StandardCharsets.UTF_8); + } + + /** + * {@inheritDoc} + */ + @Override + public T Deserialize(byte[] buffer) throws Exception { + return (T) this.objectMapper.readValue(new String(buffer, StandardCharsets.UTF_8), Object.class); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/AckMessage.java b/redis-om-spring/src/main/java/com/redis/om/streams/AckMessage.java new file mode 100644 index 000000000..46a93b861 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/AckMessage.java @@ -0,0 +1,90 @@ +package com.redis.om.streams; + +import com.redis.om.streams.utils.Util; + +import lombok.Data; +import redis.clients.jedis.StreamEntryID; + +/** + * Represents an acknowledgment message for Redis Streams. + *

+ * This class encapsulates the information needed to acknowledge a message in a Redis Stream, + * including the stream name, consumer group name, and the message ID. It can be constructed + * from different types of entries or directly with the required information. + *

+ */ +@Data +public class AckMessage { + + /** + * The name of the Redis Stream. + */ + private final String streamName; + + /** + * The name of the consumer group. + */ + private final String groupName; + + /** + * The ID of the message to acknowledge. + */ + private final TopicEntryId id; + + /** + * Constructs an AckMessage from a TopicEntry. + * + * @param entry The TopicEntry containing the stream name, group name, and ID information + */ + public AckMessage(TopicEntry entry) { + this.streamName = entry.getStreamName(); + this.groupName = entry.getGroupName(); + this.id = entry.getId(); + } + + /** + * Constructs an AckMessage from a PendingEntry. + * + * @param pendingEntry The PendingEntry containing the stream name, group name, and ID information + */ + public AckMessage(PendingEntry pendingEntry) { + this.streamName = pendingEntry.getStreamName(); + this.groupName = pendingEntry.getGroupName(); + this.id = new TopicEntryId(pendingEntry.getId(), pendingEntry.getStreamId()); + } + + /** + * Constructs an AckMessage with the specified stream name, group name, and StreamEntryID. + * + * @param streamName The name of the Redis Stream + * @param groupName The name of the consumer group + * @param id The StreamEntryID of the message to acknowledge + */ + public AckMessage(String streamName, String groupName, StreamEntryID id) { + this.streamName = streamName; + this.groupName = groupName; + this.id = new TopicEntryId(id, Util.streamIdFromStreamName(streamName)); + } + + /** + * Constructs an AckMessage with the specified stream name, group name, and ID as a string. + * + * @param streamName The name of the Redis Stream + * @param groupName The name of the consumer group + * @param id The ID of the message to acknowledge as a string + */ + public AckMessage(String streamName, String groupName, String id) { + this.streamName = streamName; + this.groupName = groupName; + this.id = new TopicEntryId(new StreamEntryID(id), Util.streamIdFromStreamName(streamName)); + } + + /** + * Returns the string representation of the stream entry ID. + * + * @return The string representation of the stream entry ID + */ + public String getStreamEntryId() { + return id.getStreamEntryId().toString(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/ConsumerGroupStatus.java b/redis-om-spring/src/main/java/com/redis/om/streams/ConsumerGroupStatus.java new file mode 100644 index 000000000..b6a881e0c --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/ConsumerGroupStatus.java @@ -0,0 +1,55 @@ +package com.redis.om.streams; + +import lombok.Value; + +/** + * Represents the status of a consumer group in Redis Streams. + * This class provides information about the consumer group's lag, pending entries, + * total entries in the topic, and identifiers for the topic and group. + */ +@Value +public class ConsumerGroupStatus { + + /** + * The number of entries that the consumer group is behind the latest entry in the topic. + */ + private final long consumerLag; + + /** + * The number of entries that have been delivered to the consumer group but not yet acknowledged. + */ + private final long pendingEntryCount; + + /** + * The total number of entries in the topic. + */ + private final long topicEntryCount; + + /** + * The name of the topic. + */ + private final String topicName; + + /** + * The name of the consumer group. + */ + private final String groupName; + + /** + * Constructs a ConsumerGroupStatus with the specified parameters. + * + * @param topicName The name of the topic + * @param groupName The name of the consumer group + * @param pendingEntryCount The number of entries that have been delivered but not yet acknowledged + * @param topicEntryCount The total number of entries in the topic + * @param consumerLag The number of entries that the consumer group is behind the latest entry + */ + public ConsumerGroupStatus(String topicName, String groupName, long pendingEntryCount, long topicEntryCount, + long consumerLag) { + this.topicName = topicName; + this.groupName = groupName; + this.pendingEntryCount = pendingEntryCount; + this.topicEntryCount = topicEntryCount; + this.consumerLag = consumerLag; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/PendingEntry.java b/redis-om-spring/src/main/java/com/redis/om/streams/PendingEntry.java new file mode 100644 index 000000000..4eedfa2e5 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/PendingEntry.java @@ -0,0 +1,87 @@ +package com.redis.om.streams; + +import com.redis.om.streams.utils.Util; + +import lombok.Data; +import redis.clients.jedis.StreamEntryID; + +/** + * Represents a pending entry in a Redis Stream. + * Pending entries are messages that have been delivered to consumers but not yet acknowledged. + * This class encapsulates the metadata associated with such entries, including their IDs, + * the stream and consumer group they belong to, and delivery statistics. + */ +@Data +public class PendingEntry { + + /** + * The Redis StreamEntryID of this pending entry. + */ + private final StreamEntryID id; + + /** + * The name of the topic this entry belongs to. + */ + private final String topicName; + + /** + * The name of the stream this entry belongs to. + */ + private final String streamName; + + /** + * The name of the consumer that this entry was delivered to. + */ + private final String consumerName; + + /** + * The name of the consumer group this entry belongs to. + */ + private final String groupName; + + /** + * The time in milliseconds that this entry has been idle (not acknowledged). + */ + private final Long idleTimeMs; + + /** + * The number of times this entry has been delivered. + */ + private final Long deliveryCount; + + /** + * The identifier of the stream this entry belongs to. + */ + private final Long streamId; + + /** + * The TopicEntryId representation of this pending entry's ID. + */ + private final TopicEntryId topicEntryId; + + /** + * Constructs a PendingEntry with the specified parameters. + * + * @param id The Redis StreamEntryID of this pending entry + * @param topicName The name of the topic this entry belongs to + * @param streamName The name of the stream this entry belongs to + * @param groupName The name of the consumer group this entry belongs to + * @param consumerName The name of the consumer that this entry was delivered to + * @param idleTimeMs The time in milliseconds that this entry has been idle + * @param deliveryCount The number of times this entry has been delivered + */ + public PendingEntry(StreamEntryID id, String topicName, String streamName, String groupName, String consumerName, + Long idleTimeMs, Long deliveryCount) { + this.id = id; + this.topicName = topicName; + this.streamName = streamName; + this.groupName = groupName; + this.consumerName = consumerName; + this.idleTimeMs = idleTimeMs; + this.deliveryCount = deliveryCount; + + this.streamId = Util.streamIdFromStreamName(streamName); + this.topicEntryId = new TopicEntryId(id, streamId); + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/Producer.java b/redis-om-spring/src/main/java/com/redis/om/streams/Producer.java new file mode 100644 index 000000000..e267a72f0 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/Producer.java @@ -0,0 +1,26 @@ +package com.redis.om.streams; + +import java.util.Map; + +import com.redis.om.streams.exception.InvalidMessageException; +import com.redis.om.streams.exception.ProducerTimeoutException; +import com.redis.om.streams.exception.TopicNotFoundException; + +/** + * Interface for producing messages to Redis Streams. + * Implementations of this interface handle the details of sending messages to Redis Streams topics. + */ +public interface Producer { + + /** + * Produces a message to a Redis Stream topic. + * + * @param message The message to produce, represented as key-value pairs + * @return The unique identifier assigned to the produced message + * @throws TopicNotFoundException If the specified topic does not exist + * @throws InvalidMessageException If the message format is invalid + * @throws ProducerTimeoutException If the operation times out + */ + public TopicEntryId produce(Map message) throws TopicNotFoundException, InvalidMessageException, + ProducerTimeoutException; +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/TopicEntry.java b/redis-om-spring/src/main/java/com/redis/om/streams/TopicEntry.java new file mode 100644 index 000000000..5512ea39d --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/TopicEntry.java @@ -0,0 +1,69 @@ +package com.redis.om.streams; + +import java.util.List; +import java.util.Map; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import redis.clients.jedis.resps.StreamEntry; + +/** + * Represents an entry in a Redis Stream topic. + * This class encapsulates the data and metadata associated with a stream entry, + * including the stream name, group name, entry ID, and the message content. + */ +@ToString +@EqualsAndHashCode +public class TopicEntry { + /** + * Creates a TopicEntry from a Redis Stream entry result. + * + * @param groupName The name of the consumer group + * @param result The entry result from Redis, containing the stream name and entry data + * @param streamId The identifier of the stream + * @return A new TopicEntry instance + */ + public static TopicEntry create(String groupName, Map.Entry> result, long streamId) { + return new TopicEntry(result.getKey(), groupName, result.getValue().get(0), streamId); + } + + /** + * The name of the stream. + */ + @Getter + private final String streamName; + + /** + * The name of the consumer group. + */ + @Getter + private final String groupName; + + /** + * The unique identifier of this topic entry. + */ + @Getter + private final TopicEntryId id; + + /** + * The message content of this topic entry, represented as key-value pairs. + */ + @Getter + private Map message; + + /** + * Constructs a TopicEntry with the specified parameters. + * + * @param streamName The name of the stream + * @param groupName The name of the consumer group + * @param entry The Redis StreamEntry containing the entry data + * @param streamId The identifier of the stream + */ + public TopicEntry(String streamName, String groupName, StreamEntry entry, long streamId) { + this.streamName = streamName; + this.groupName = groupName; + this.message = entry.getFields(); + this.id = new TopicEntryId(entry.getID(), streamId); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/TopicEntryId.java b/redis-om-spring/src/main/java/com/redis/om/streams/TopicEntryId.java new file mode 100644 index 000000000..ec63e1a77 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/TopicEntryId.java @@ -0,0 +1,152 @@ +package com.redis.om.streams; + +import java.nio.ByteBuffer; + +import com.redis.om.streams.utils.Util; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import redis.clients.jedis.StreamEntryID; + +/** + * Represents a unique identifier for a topic entry in Redis Streams. + * The ID consists of three components: time, sequence, and streamId. + * This class provides methods to create, manipulate, and convert these IDs. + */ +@EqualsAndHashCode +public class TopicEntryId { + /** + * The timestamp component of the ID. + */ + @Getter + private long time; + + /** + * The sequence number component of the ID. + */ + @Getter + private long sequence; + + /** + * The stream identifier component of the ID. + */ + @Getter + private long streamId; + + /** + * A string of zeros used for padding long values in string representation. + */ + final private String PADDED_LONG = "0000000000000000000"; + + /** + * Represents the minimum possible TopicEntryId with all components set to 0. + */ + static final public TopicEntryId MIN_ID = new TopicEntryId(0, 0, 0); + + /** + * Represents the maximum possible TopicEntryId with all components set to a large value. + */ + static final public TopicEntryId MAX_ID = new TopicEntryId(5999999999999L, 5999999999999L, 5999999999999L); + + /** + * Constructs a TopicEntryId with the specified time, sequence, and streamId. + * + * @param time The timestamp component of the ID + * @param sequence The sequence number component of the ID + * @param streamId The stream identifier component of the ID + */ + public TopicEntryId(long time, long sequence, long streamId) { + this.time = time; + this.sequence = sequence; + this.streamId = streamId; + } + + /** + * Constructs a TopicEntryId from a StreamEntryID and a stream identifier. + * + * @param streamEntryId The Redis StreamEntryID containing time and sequence components + * @param streamId The stream identifier component of the ID + */ + public TopicEntryId(StreamEntryID streamEntryId, long streamId) { + this.time = streamEntryId.getTime(); + this.sequence = streamEntryId.getSequence(); + this.streamId = streamId; + } + + /** + * Constructs a TopicEntryId from a StreamEntryID and a stream name. + * The stream identifier is derived from the stream name. + * + * @param streamEntryId The Redis StreamEntryID containing time and sequence components + * @param streamName The name of the stream, used to derive the stream identifier + */ + public TopicEntryId(StreamEntryID streamEntryId, String streamName) { + this.time = streamEntryId.getTime(); + this.sequence = streamEntryId.getSequence(); + this.streamId = Util.streamIdFromStreamName(streamName); + } + + /** + * Constructs a TopicEntryId from its string representation. + * The string format is: TIME-SEQUENCE-STREAMID + * + * @param id The string representation of the TopicEntryId + */ + public TopicEntryId(String id) { + String[] split = id.split("-"); + this.time = Long.parseLong(split[0]); + this.sequence = Long.parseLong(split[1]); + this.streamId = Long.parseLong(split[2]); + } + + /** + * Constructs a TopicEntryId from a byte array. + * + * @param bytes The byte array containing the TopicEntryId data + */ + public TopicEntryId(byte[] bytes) { + ByteBuffer buffer = ByteBuffer.wrap(bytes); + } + + /** + * Returns a string representation of this TopicEntryId. + * The format is: TIME-SEQUENCE-STREAMID + * + * @return The string representation of this TopicEntryId + */ + public String toString() { + return String.valueOf(time) + "-" + String.valueOf(sequence) + "-" + String.valueOf(streamId); + } + + /** + * Returns a padded string representation of this TopicEntryId. + * Each component is padded with leading zeros to ensure consistent string length. + * + * @return The padded string representation of this TopicEntryId + */ + public String toPaddedString() { + return getPaddedLong(time) + "-" + getPaddedLong(sequence) + "-" + getPaddedLong(streamId); + } + + /** + * Pads a long value with leading zeros to ensure consistent string length. + * + * @param value The long value to pad + * @return The padded string representation of the long value + */ + private String getPaddedLong(long value) { + String longStringValue = String.valueOf(value); + int zeros = PADDED_LONG.length() - longStringValue.length(); + return PADDED_LONG.substring(0, zeros) + longStringValue; + } + + /** + * Converts this TopicEntryId to a Redis StreamEntryID. + * Only the time and sequence components are used; the streamId is not included. + * + * @return A new StreamEntryID with the time and sequence from this TopicEntryId + */ + public StreamEntryID getStreamEntryId() { + return new StreamEntryID(time, sequence); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/annotation/EnableRedisStreams.java b/redis-om-spring/src/main/java/com/redis/om/streams/annotation/EnableRedisStreams.java new file mode 100644 index 000000000..eb359d8ca --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/annotation/EnableRedisStreams.java @@ -0,0 +1,61 @@ +package com.redis.om.streams.annotation; + +import java.lang.annotation.*; + +import org.springframework.context.annotation.Import; +import org.springframework.core.annotation.AliasFor; +import org.springframework.scheduling.annotation.EnableScheduling; + +import com.redis.om.streams.config.RedisStreamConsumerRegistrar; + +/** + * Enables Redis Streams support in a Spring application. + * This annotation should be applied to a Spring @Configuration class to scan for + * and register Redis Stream consumers annotated with {@link RedisStreamConsumer}. + *

+ * Example usage: + *

+ * @Configuration
+ * @EnableRedisStreams(basePackages = "com.example.streams")
+ * public class AppConfig {
+ * // configuration
+ * }
+ * 
+ */ +@Target( + ElementType.TYPE +) +@Retention( + RetentionPolicy.RUNTIME +) +@Documented +@Import( + RedisStreamConsumerRegistrar.class +) +@EnableScheduling +public @interface EnableRedisStreams { + + /** + * Alias for the {@link #basePackages()} attribute. Allows for more concise annotation declarations e.g.: + * {@code @EnableRedisRepositories("org.my.pkg")} instead of + * {@code @EnableRedisRepositories(basePackages="org.my.pkg")}. + * + * @return basePackages + */ + @AliasFor( + "basePackages" + ) + String[] value() default {}; + + /** + * Base packages to scan for annotated components. {@link #value()} is an alias for (and mutually exclusive with) this + * attribute. + * + * @return basePackages as a String + */ + @AliasFor( + "value" + ) + String[] basePackages() default {}; + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/annotation/RedisStreamConsumer.java b/redis-om-spring/src/main/java/com/redis/om/streams/annotation/RedisStreamConsumer.java new file mode 100644 index 000000000..aa3157fa6 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/annotation/RedisStreamConsumer.java @@ -0,0 +1,57 @@ +package com.redis.om.streams.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation to mark a class as a Redis Stream consumer. + * Classes annotated with this will be registered as consumers for the specified Redis Stream. + */ +@Target( + ElementType.TYPE +) +@Retention( + RetentionPolicy.RUNTIME +) +public @interface RedisStreamConsumer { + /** + * The name of the Redis Stream topic to consume messages from. + * + * @return the topic name + */ + String topicName(); + + /** + * The name of the consumer group this consumer belongs to. + * Consumer groups allow multiple consumers to cooperatively consume messages from a stream. + * + * @return the consumer group name + */ + String groupName(); + + /** + * The name of the consumer within the consumer group. + * If not specified, a default name will be generated. + * + * @return the consumer name + */ + String consumerName() default ""; + + /** + * Whether to automatically acknowledge messages after processing. + * If set to false, messages must be explicitly acknowledged. + * + * @return true if messages should be auto-acknowledged, false otherwise + */ + boolean autoAck() default false; + + /** + * Whether the Redis deployment is a cluster. + * This affects how the consumer interacts with Redis. + * + * @return true if Redis is deployed as a cluster, false otherwise + */ + boolean cluster() default false; +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/command/ConsumerGroupBase.java b/redis-om-spring/src/main/java/com/redis/om/streams/command/ConsumerGroupBase.java new file mode 100644 index 000000000..553d4c6ab --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/command/ConsumerGroupBase.java @@ -0,0 +1,407 @@ +package com.redis.om.streams.command; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import com.redis.om.streams.TopicEntry; +import com.redis.om.streams.command.serial.SerialTopicConfig; +import com.redis.om.streams.command.serial.TopicManager; +import com.redis.om.streams.exception.TopicNotFoundException; +import com.redis.om.streams.utils.Util; + +import lombok.Getter; +import redis.clients.jedis.JedisPooled; +import redis.clients.jedis.StreamEntryID; +import redis.clients.jedis.exceptions.JedisDataException; +import redis.clients.jedis.resps.ScanResult; +import redis.clients.jedis.resps.StreamEntry; +import redis.clients.jedis.resps.Tuple; + +/** + * Abstract base class for consumer groups in Redis Streams. + *

+ * This class provides common functionality for different types of consumer groups, + * such as managing stream connections, tracking consumer group state, and handling + * message consumption. It serves as the foundation for specific consumer group + * implementations like {@code ConsumerGroup}, {@code NoAckConsumerGroup}, and + * {@code SingleClusterPelConsumerGroup}. + *

+ * Consumer groups in Redis Streams allow multiple consumers to cooperatively + * consume messages from the same stream, with each message being delivered to + * only one consumer within the group. + */ +public abstract class ConsumerGroupBase { + + /** The Jedis connection used to execute Redis commands. */ + protected final JedisPooled connection; + + /** The LuaCommandRunner used to execute Lua scripts. */ + protected final LuaCommandRunner luaCommandRunner; + + /** The configuration for the topic this consumer group is consuming from. */ + @Getter + protected SerialTopicConfig config; + + /** The name of this consumer group. */ + @Getter + protected final String groupName; + + /** The name of the topic this consumer group is consuming from. */ + @Getter + protected final String topicName; + + /** The current stream this consumer group is consuming from. */ + @Getter + protected String currentStream; + + /** The ID of the current stream this consumer group is consuming from. */ + @Getter + protected long currentStreamId; + + /** Whether this consumer group has been initialized. */ + protected boolean initialized; + + /** The time at which the configuration should be refreshed. */ + protected Instant nextConfigurationRefresh; + + /** + * The last reported lag (number of unprocessed messages) for this consumer group. + * Did not utilize the lombok.getter tag because I want to be explicit with the logic. + */ + protected long lastReportedLagForTopicAndGroup = 0; + + /** + * Consumes a message from the topic for the specified consumer. + * + * @param consumerName the name of the consumer + * @return the consumed message, or null if no message is available + * @throws TopicNotFoundException if the topic does not exist + */ + public abstract TopicEntry consume(String consumerName) throws TopicNotFoundException; + + /** + * Gets the next message from the topic for the specified consumer. + * + * @param consumerName the name of the consumer + * @return a list of stream entries, or null if no message is available + */ + public abstract List>> getNextMessage(String consumerName); + + /** + * Constructs a new ConsumerGroupBase with the specified connection, topic name, and group name. + * + * @param connection the Jedis connection to use + * @param topicName the name of the topic to consume from + * @param groupName the name of the consumer group + */ + public ConsumerGroupBase(JedisPooled connection, String topicName, String groupName) { + this.connection = connection; + this.topicName = topicName; + this.luaCommandRunner = new LuaCommandRunner(connection); + this.groupName = groupName; + this.initialized = false; + } + + /** + * Returns the last reported lag (number of unprocessed messages) for this consumer group. + * Note that this is a cached value, derived from a prior call to + * {@link #getEstimatedTopicEntriesReadForGroup(JedisPooled, String, String)}. + * + * @return a locally cached value for the lag for this group + */ + public long getLastReportedLagForThisInstanceTopicAndGroup() { + return this.lastReportedLagForTopicAndGroup; + } + + /** + * Calculates the current lag (number of unprocessed messages) for this consumer group. + * This method forces the calculation of the count of read objects + * and from that, it calculates the best estimated known lag for this group. + * + * @return the best possible estimated known lag for this group + * @throws TopicNotFoundException if the topic does not exist + */ + public long getCurrentLagForThisInstanceTopicAndGroup() throws TopicNotFoundException { + this.getEstimatedTopicEntriesReadForGroup(this.connection, this.topicName, this.groupName); + return this.getLastReportedLagForThisInstanceTopicAndGroup(); + } + + /** + * Calculates the estimated number of entries read from the topic by this consumer group. + * It also calculates the best estimated known lag and sets that value locally on this class instance. + * + * @param connection the Jedis connection to use + * @param topicName the name of the topic + * @param groupName the name of the consumer group + * @return the estimated number of entries read + * @throws TopicNotFoundException if the topic does not exist + */ + public long getEstimatedTopicEntriesReadForGroup(JedisPooled connection, String topicName, String groupName) + throws TopicNotFoundException { + initialize(); + long result = 0; + // below, we use the entries read keys: + //__entries_read_{__rsj:topic:stream:Topic-79342252526761:0}_consumer:noack:1 + // and the __rsj:topic:index... key and its members: + // __rsj:topic:stream:Topic-79342252526761:0 + // __rsj:topic:stream:Topic-79342252526761:1 + // to determine total read for group & topic + + String z_topic_streams_index = "__rsj:topic:index:{" + topicName + "}"; + ScanResult scanResult = connection.zscan(z_topic_streams_index, "0"); + List tl = scanResult.getResult(); + int totalNumStreamsInGroup = 0; + String streamName = ""; + //"GET" "__entries_read_{__rsj:topic:stream:Topic-81870797064923:0}_consumer:noack:1" + for (Tuple t : tl) { + streamName = t.getElement(); + String readEntriesKey = "__entries_read_{" + streamName + "}_" + groupName; + try { + result += Long.parseLong(connection.get(readEntriesKey)); + } catch (NumberFormatException nfe) { + //TODO: unclear if this kind of notification is necessary or desired: + //System.out.println("Zero reads reported for group: "+groupName+" on stream"+t.getElement()); + } + totalNumStreamsInGroup++; + } + refreshEstimatedLag(connection, topicName, totalNumStreamsInGroup, streamName, result); + return result; + } + + /** + * Refreshes the estimated lag (number of unprocessed messages) for this consumer group. + * This method is only useful to a caller that has intimate knowledge of the state of the streams + * that belong to a topic - as well as the currently known count of read entries for this group. + * Therefore, it only makes sense to allow it to be invoked from the method that + * retrieves the currently known count of read entries for the group. + * + * @param connection the Jedis connection to use + * @param topicNameVal the name of the topic + * @param totalNumStreamsInGroup the total number of streams in the group + * @param lastStreamInGroup the name of the last stream in the group + * @param estimatedTotalEntriesRead the estimated total number of entries read + * @throws TopicNotFoundException if the topic does not exist + */ + protected void refreshEstimatedLag(JedisPooled connection, String topicNameVal, int totalNumStreamsInGroup, + String lastStreamInGroup, long estimatedTotalEntriesRead) throws TopicNotFoundException { + //related data gathering for lag in case it is wanted: + //__rsj:topic:config:{Topic-82180446799153} <--sample topic keyname construction + //System.out.println("ot:0502:debug ConsumerGroupBase:refreshEstimatedLag -->\n\t This.topicName == "+topicName+" passed-in topicName is: "+topicNameVal); + String topicConfigKeyName = "__rsj:topic:config:{" + topicNameVal + "}"; + long totalEntriesForTopic = 0; + long lastStreamLength = 0; + try { + totalEntriesForTopic = Long.parseLong(connection.hget(topicConfigKeyName, "maxStreamLength")); + totalEntriesForTopic = totalEntriesForTopic * (totalNumStreamsInGroup - 1);//don't count the most recent stream + lastStreamLength = connection.xlen(lastStreamInGroup); + } catch (Throwable t) { + System.out.println( + "ot:0502 ConsumerGroupBase:refreshEstimatedLag --> maybe weird issue reading topicConfig maxStreamLength " + topicConfigKeyName); + t.printStackTrace(); + } + this.lastReportedLagForTopicAndGroup = (totalEntriesForTopic + lastStreamLength) - estimatedTotalEntriesRead; + } + + /** + * Peeks at the next messages in the topic without consuming them. + * This method allows you to see what messages are available without removing them from the stream. + * + * @param count the maximum number of messages to peek at (limited to 100) + * @return a list of topic entries, or an empty list if no messages are available + * @throws TopicNotFoundException if the topic does not exist + */ + public List peek(int count) throws TopicNotFoundException { + initialize(); + + if (count <= 0 || !isInitialized()) { + return Collections.emptyList(); + } + + // TODO: Address this limitation. We don't want to hard code this. + if (count > 100) { + count = 100; + } + + SerialTopicConfig config; + try { + config = TopicManager.loadConfig(connection, topicName); + } catch (TopicNotFoundException e) { + return Collections.emptyList(); + } + TopicManager manager = new TopicManager(connection, config); + + if (manager.getTopicSize() == 0) { + return Collections.emptyList(); + } + + String currentStreamForGroup = manager.getCurrentStreamForGroup(groupName); + long currentStreamIdForGroup = Util.streamIdFromStreamName(currentStreamForGroup); + long latestStreamId = manager.latestStreamId(); + StreamEntryID lastDeliveredId = getLastDeliveredId(currentStreamForGroup, groupName); + + // TODO: Get the lag for the group as an optimization. If lag is >= count, + // TODO: then we might not need to look at any subsequent streams + + List resultStreamEntries = new ArrayList<>(); + + // Because the lastDeliveredId will be included in the result set, we need to + // add one to the number of responses to return. This works around the + // fact that there's no way to add the exclusive operator "(" to the lastDeliveredId + // in Jedis. + List entries = connection.xrange(currentStreamForGroup, lastDeliveredId, StreamEntryID.MAXIMUM_ID, + count + 1); + + final String streamName = currentStreamForGroup; + final long streamId = currentStreamIdForGroup; + + // If the first entry's ID matches the last delivered id, then remove that entry + if (!entries.isEmpty()) { + if (entries.get(0).getID().equals(lastDeliveredId)) { + entries.remove(0); + } + } + + resultStreamEntries.addAll(entries.stream().map(e -> new TopicEntry(streamName, groupName, e, streamId)).toList()); + + while (resultStreamEntries.size() < count && (currentStreamIdForGroup < latestStreamId)) { + currentStreamIdForGroup += 1; + currentStreamForGroup = manager.getStreamForId(currentStreamIdForGroup); + final long sid = currentStreamIdForGroup; + final String sname = currentStreamForGroup; + int entriesNeeded = count - resultStreamEntries.size(); + entries = connection.xrange(currentStreamForGroup, StreamEntryID.MINIMUM_ID, StreamEntryID.MAXIMUM_ID, + entriesNeeded); + resultStreamEntries.addAll(entries.stream().map(e -> new TopicEntry(sname, groupName, e, sid)).toList()); + } + + return resultStreamEntries; + } + + /** + * Active-Active databases do not appear to replicate the last-delivered-id field of a consumer + * group. For this reason, the Lua scripts that we use store the last delivered ID for each group + * always write the last delivered ID to a stream whose length is capped at 1. This means + * that getting the last delivered ID for a consumer group means getting the ID of single + * message stored in this stream. If the stream is empty or does not exist, we return + * the minimum ID (0-0). + * + * @param streamName + * @param groupName + * @return + */ + protected StreamEntryID getLastDeliveredId(String streamName, String groupName) { + String lastDeliveredIdStreamName = config.getLastDeliveredIdKey(streamName, groupName); + List results = connection.xrange(lastDeliveredIdStreamName, StreamEntryID.MINIMUM_ID, + StreamEntryID.MAXIMUM_ID, 1); + if (results.isEmpty()) { + return SerialTopicConfig.MIN_STREAM_ENTRY_ID; + } else { + StreamEntry entry = results.get(0); + return entry.getID(); + } + } + + /** + * Gets the next stream for the consumer group to read from. + * + * @param streamToAdvance the current stream + * @return the name of the next stream + */ + protected String getNextStream(String streamToAdvance) { + return luaCommandRunner.advanceConsumerGroupStream(config, groupName, streamToAdvance); + } + + /** + * Initializes the consumer group. + * This method loads the topic configuration, creates the consumer group if it doesn't exist, + * and sets the current stream. If the configuration has expired, it refreshes the configuration + * and updates the current stream. + * + * @throws TopicNotFoundException if the topic does not exist + */ + protected synchronized void initialize() throws TopicNotFoundException { + if (!initialized) { + this.config = TopicManager.loadConfig(connection, topicName); + TopicManager manager = new TopicManager(connection, config); + String stream = manager.createConsumerGroup(groupName); + ensureConsumerGroupExists(stream); + setCurrentStream(stream); + this.initialized = true; + } else if (configurationExpired()) { + this.config = TopicManager.loadConfig(connection, topicName); + TopicManager manager = new TopicManager(connection, config); + String firstUnexpiredStream = manager.getFirstUnexpiredStream(); + + String latestStreamForGroup = connection.hget(config.getConsumerGroupConfigKey(groupName), config + .getGroupCurrentStreamField()); + long firstStreamId = Util.streamIdFromStreamName(firstUnexpiredStream); + long storedStreamId = Util.streamIdFromStreamName(latestStreamForGroup); + if (storedStreamId > firstStreamId) { + ensureConsumerGroupExists(latestStreamForGroup); + setCurrentStream(latestStreamForGroup); + } else { + ensureConsumerGroupExists(firstUnexpiredStream); + setCurrentStream(firstUnexpiredStream); + } + } + } + + /** + * Sets the current stream for this consumer group. + * + * @param streamName the name of the stream to set as current + */ + protected void setCurrentStream(String streamName) { + this.currentStream = streamName; + this.currentStreamId = Util.streamIdFromStreamName(currentStream); + setNextConfigurationRefresh(); + } + + /** + * Sets the time at which the configuration should be refreshed. + * This is based on the TTL of the current stream or the retention time of the topic. + */ + protected void setNextConfigurationRefresh() { + long currentStreamTTL = connection.ttl(currentStream); + if (currentStreamTTL > 0) { + this.nextConfigurationRefresh = Instant.now().plusSeconds(currentStreamTTL - 1); + } else { + this.nextConfigurationRefresh = Instant.now().plusSeconds(config.getRetentionTimeSeconds()); + } + } + + /** + * Checks if the configuration has expired and needs to be refreshed. + * + * @return true if the configuration has expired, false otherwise + */ + protected boolean configurationExpired() { + return Instant.now().isAfter(nextConfigurationRefresh); + } + + /** + * Checks if this consumer group has been initialized. + * + * @return true if initialized, false otherwise + */ + public synchronized boolean isInitialized() { + return initialized; + } + + /** + * Ensures that the consumer group exists for the specified stream. + * If the consumer group already exists, this method does nothing. + * + * @param stream the name of the stream + */ + public void ensureConsumerGroupExists(String stream) { + try { + connection.xgroupCreate(stream, groupName, new StreamEntryID(0), true); + } catch (JedisDataException e) { + // TODO: Should we confirm that the text includes "BUSYGROUP"? + } + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/command/LuaCommand.java b/redis-om-spring/src/main/java/com/redis/om/streams/command/LuaCommand.java new file mode 100644 index 000000000..fb84ff159 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/command/LuaCommand.java @@ -0,0 +1,94 @@ +package com.redis.om.streams.command; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; + +import lombok.Getter; + +/** + * Enum representing Lua script commands used for Redis Streams operations. + * Each enum constant corresponds to a Lua script file that is loaded and executed + * by the {@link LuaCommandRunner} to perform various operations on Redis Streams. + */ +public enum LuaCommand { + + /** Script to calculate the lag (number of unprocessed messages) for a consumer group. */ + LAG("lag.script"), + + /** Script to acknowledge a message has been processed by a consumer group. */ + ACK("xack.script"), + + /** Script to set the last delivered ID for a consumer group. */ + GROUP_SET_ID("xgroup_setid.script"), + + /** + * Script to retrieve pending entries (messages that were delivered but not yet acknowledged) for a consumer group. + */ + PENDING("xpending.script"), + + /** Script to read messages from a stream as part of a consumer group. */ + READGROUP("xreadgroup.script"), + + /** Script to read messages from a stream without requiring acknowledgment. */ + NOACK_READGROUP("xreadgroup_noack.script"), + + /** Script to read messages from a stream with single cluster pending entry list. */ + SINGLE_DB_PEL_READGROUP("xreadgroup_singleClusterPel.script"), + + /** Script to get the next stream in an active-active configuration. */ + NEXT_SERIAL_ACTIVE_ACTIVE_STREAM("serial_get_next_stream.script"), + + /** Script to advance a consumer group to the next stream. */ + SERIAL_ADVANCE_CONSUMER_STREAM("serial_advance_consumer.script"), + + /** Script to publish a message to a serial stream. */ + SERIAL_PUBLISH_MESSAGE("serial_publish_message.script"),; + + /** The content of the Lua script loaded from the script file. */ + @Getter + public final String script; + + /** The filename of the Lua script. */ + @Getter + public final String scriptFile; + + /** + * Constructs a LuaCommand with the specified script file. + * + * @param scriptFile the name of the script file to load + */ + LuaCommand(String scriptFile) { + this.scriptFile = scriptFile; + this.script = getScriptFromFile(scriptFile); + } + + /** + * Constructs the full path to the script file. + * + * @param script the name of the script file + * @return the full path to the script file + */ + private String getScriptFilename(String script) { + return Paths.get("scripts", script).toString(); + } + + /** + * Loads the content of the script file. + * + * @param scriptFile the name of the script file to load + * @return the content of the script file as a string + * @throws IllegalArgumentException if the script file cannot be read + */ + private String getScriptFromFile(String scriptFile) { + String code = null; + try (InputStream stream = getClass().getClassLoader().getResourceAsStream(getScriptFilename(scriptFile))) { + if (stream != null) { + code = new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } + } catch (Exception e) { + throw new IllegalArgumentException("Error while reading the script file " + scriptFile, e); + } + return code; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/command/LuaCommandRunner.java b/redis-om-spring/src/main/java/com/redis/om/streams/command/LuaCommandRunner.java new file mode 100644 index 000000000..8aea0a7be --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/command/LuaCommandRunner.java @@ -0,0 +1,413 @@ +package com.redis.om.streams.command; + +import static redis.clients.jedis.Protocol.Command.EVALSHA; + +import java.nio.charset.StandardCharsets; +import java.util.*; + +import com.redis.om.streams.command.serial.SerialTopicConfig; +import com.redis.om.streams.exception.InvalidMessageException; +import com.redis.om.streams.exception.TopicOrGroupNotFoundException; +import com.redis.om.streams.utils.Util; + +import redis.clients.jedis.*; +import redis.clients.jedis.commands.ProtocolCommand; +import redis.clients.jedis.exceptions.JedisDataException; +import redis.clients.jedis.resps.StreamEntry; +import redis.clients.jedis.util.SafeEncoder; + +/** + * A utility class for executing Lua scripts on Redis. + *

+ * This class loads and executes Lua scripts defined in the {@link LuaCommand} enum. + * It provides methods for various Redis Streams operations such as publishing messages, + * reading messages, acknowledging messages, and managing consumer groups. + *

+ * The class uses the Jedis client to interact with Redis and execute the Lua scripts. + */ +public class LuaCommandRunner { + + /** Maximum number of key-value pairs allowed in a published message. */ + private static final int MAX_PUBLISH_MESSAGE_KEYS = 2000; + + /** + * Factory method to create a new LuaCommandRunner instance. + * + * @param jedis the JedisPooled connection to use + * @return a new LuaCommandRunner instance + */ + public static LuaCommandRunner getInstance(JedisPooled jedis) { + return new LuaCommandRunner(jedis); + } + + /** The Jedis connection used to execute Redis commands. */ + private final JedisPooled connection; + + /** Map of LuaCommand to SHA1 hash of the loaded script. */ + private final Map commandMap; + + /** + * Constructs a new LuaCommandRunner with the specified Jedis connection. + * Loads all Lua scripts defined in the LuaCommand enum. + * + * @param connection the JedisPooled connection to use + */ + public LuaCommandRunner(JedisPooled connection) { + this.connection = connection; + this.commandMap = loadScripts(this.connection); + } + + /** + * Advances a consumer group to the next stream if necessary. + * + * @param config the topic configuration + * @param groupName the name of the consumer group + * @param currentStream the stream to advance from. If this is not the same as the current stream, stream will not + * be advanced. + * @return the name of the next stream that the consumer group should use. The Lua function will + * advance the stream only if the current stream is no longer in use and a new stream exists. + */ + public String advanceConsumerGroupStream(SerialTopicConfig config, String groupName, String currentStream) { + String sha1 = this.commandMap.get(LuaCommand.SERIAL_ADVANCE_CONSUMER_STREAM); + if (currentStream == null) { + currentStream = ""; + } + long currentStreamId = Util.streamIdFromStreamName(currentStream); + Object response = executeLuaCommand(sha1, List.of(config.getTopicConfigKey(), config.getConsumerGroupConfigKey( + groupName), config.getStreamIndexKey(), config.getFullStreamsKey()), + + List.of(groupName, currentStream, String.valueOf(currentStreamId), String.valueOf(config + .getRetentionTimeSeconds()), config.getGroupCurrentStreamField(), config.getTopicStreamIdField(), config + .getCurrentStreamBaseName())); + return response.toString(); + } + + /** + * Gets the next stream in an active-active configuration. + * + * @param config the topic configuration + * @param currentStreamId the ID of the current stream + * @return the name of the next stream + */ + public String getNextSerialActiveActiveStream(SerialTopicConfig config, long currentStreamId) { + String sha1 = this.commandMap.get(LuaCommand.NEXT_SERIAL_ACTIVE_ACTIVE_STREAM); + Object response = executeLuaCommand(sha1, List.of(config.getTopicConfigKey(), config.getStreamIndexKey(), config + .getFullStreamsKey()), List.of(config.getTopicStreamIdField(), config.getCurrentStreamBaseName(), String + .valueOf(config.getRetentionTimeSeconds()), String.valueOf(currentStreamId), String.valueOf(Util + .getExpiryAtSeconds(connection, config.getRetentionTimeSeconds())))); + return response.toString(); + } + + /** + * Gets the next stream in an active-active configuration, starting from the beginning. + * + * @param config the topic configuration + * @return the name of the next stream + */ + public String getNextSerialActiveActiveStream(SerialTopicConfig config) { + return getNextSerialActiveActiveStream(config, -1); + } + + /** + * Publishes a message to a serial stream. + * + * @param streamName the name of the stream to publish to + * @param retentionTimeSeconds the number of seconds to retain the message + * @param maxStreamLength the maximum number of entries in the stream + * @param streamSwitchTTL the time-to-live for stream switching + * @param message the message to publish as a map of key-value pairs + * @return the ID of the published message + * @throws InvalidMessageException if the message is null, empty, or contains null keys or values, + * or if the message contains more than MAX_PUBLISH_MESSAGE_KEYS key-value pairs + */ + public StreamEntryID publishSerialStreamMessage(String streamName, long retentionTimeSeconds, long maxStreamLength, + long streamSwitchTTL, Map message) throws InvalidMessageException { + List items = new ArrayList<>(); + items.add(streamName); + items.add(String.valueOf(retentionTimeSeconds)); + items.add(String.valueOf(maxStreamLength)); + items.add(String.valueOf(streamSwitchTTL)); + + if (message == null) { + throw new InvalidMessageException("Message may not be null."); + } + + if (message.isEmpty()) { + throw new InvalidMessageException("Message may not be empty."); + } + + Set keys = message.keySet(); + + if (keys.size() > MAX_PUBLISH_MESSAGE_KEYS) { + throw new InvalidMessageException("Message contains " + keys + .size() + " keys. May not exceed " + MAX_PUBLISH_MESSAGE_KEYS); + } + + for (String key : keys) { + if (key == null) { + throw new InvalidMessageException("Message may not contain any null keys."); + } + items.add(key); + String value = message.get(key); + if (value == null) { + throw new InvalidMessageException("Message may not contain any null values."); + } + items.add(message.get(key)); + } + + return connection.executeCommand(readStreamEntry(this.commandMap.get(LuaCommand.SERIAL_PUBLISH_MESSAGE), 1, items + .toArray(String[]::new))); + } + + /** + * Gets a message from a stream for a consumer in a consumer group. + * + * @param streamName the name of the stream to read from + * @param groupName the name of the consumer group + * @param consumerName the name of the consumer + * @return a list of stream entries, or null if no messages are available + */ + public List>> getStreamMessage(String streamName, String groupName, + String consumerName) { + return connection.executeCommand(readStreamMessageObject(this.commandMap.get(LuaCommand.READGROUP), 1, streamName, + groupName, consumerName)); + } + + /** + * Gets pending entries (messages that were delivered but not yet acknowledged) for a consumer group. + * Uses default range and count parameters. + * + * @param streamName the name of the stream + * @param groupName the name of the consumer group + * @return a list of pending stream entries + */ + public List getPending(String streamName, String groupName) { + return connection.executeCommand(readListObject(this.commandMap.get(LuaCommand.PENDING), 1, streamName, groupName, + "-", "+", "1000")); + } + + /** + * Gets pending entries (messages that were delivered but not yet acknowledged) for a consumer group + * within the specified ID range and count. + * + * @param streamName the name of the stream + * @param groupName the name of the consumer group + * @param startId the minimum ID to include in the results + * @param endId the maximum ID to include in the results + * @param count the maximum number of entries to return + * @return a list of pending stream entries + */ + public List getPending(String streamName, String groupName, String startId, String endId, int count) { + try { + return connection.executeCommand(readListObject(this.commandMap.get(LuaCommand.PENDING), 1, streamName, groupName, + startId, endId, String.valueOf(count))); + } catch (JedisDataException e) { + if (e.getMessage().contains("NOGROUP")) { + return Collections.emptyList(); + } else { + throw e; + } + } + } + + /** + * Gets the lag (number of unprocessed messages) for a consumer group. + * + * @param streamName the name of the stream + * @param groupName the name of the consumer group + * @return the number of unprocessed messages, or null if the information is not available + * @throws TopicOrGroupNotFoundException if the topic or group does not exist + */ + public Long getConsumerGroupLag(String streamName, String groupName) throws TopicOrGroupNotFoundException { + String sha1 = this.commandMap.get(LuaCommand.LAG); + try { + Object response = executeLuaCommand(sha1, List.of(streamName), List.of(groupName)); + if (response != null) { + return Long.valueOf(response.toString()); + } + } catch (JedisDataException e) { + throw new TopicOrGroupNotFoundException("Cannot find topic or group: " + e.getMessage()); + } + return null; + } + + /** + * Acknowledges that a message has been processed by a consumer group. + * + * @param streamName the name of the stream + * @param groupName the name of the consumer group + * @param messageID the ID of the message to acknowledge + * @return true if the message was acknowledged, false otherwise + */ + public boolean ackMessage(String streamName, String groupName, String messageID) { + String sha1 = this.commandMap.get(LuaCommand.ACK); + Object response = executeLuaCommand(sha1, List.of(streamName), List.of(groupName, messageID)); + return response.toString().equals("1"); + } + + /** + * Sets the last delivered ID for a consumer group. + * + * @param streamName the name of the stream + * @param groupName the name of the consumer group + * @param messageID the ID to set as the last delivered ID + * @return the result of the operation + */ + public String setID(String streamName, String groupName, String messageID) { + String sha1 = this.commandMap.get(LuaCommand.GROUP_SET_ID); + Object response = executeLuaCommand(sha1, List.of(streamName), List.of(groupName, messageID)); + return response.toString(); + } + + /** + * Executes a Lua command on Redis. + * + * @param commandSha1 the SHA1 hash of the Lua script to execute + * @param keys the Redis keys to pass to the script + * @param args the arguments to pass to the script + * @return the result of the script execution + */ + private Object executeLuaCommand(String commandSha1, List keys, List args) { + return connection.evalsha(commandSha1, keys, args); + } + + /** + * Gets a message from a stream for a consumer in a consumer group without requiring acknowledgment. + * The message will be delivered to the consumer but will not be added to the pending entries list. + * + * @param streamName the name of the stream to read from + * @param groupName the name of the consumer group + * @param consumerName the name of the consumer + * @return a list of stream entries, or null if no messages are available + */ + public List>> noAckAndGetStreamMessage(String streamName, String groupName, + String consumerName) { + return connection.executeCommand(readStreamMessageObject(this.commandMap.get(LuaCommand.NOACK_READGROUP), 1, + streamName, groupName, consumerName)); + } + + /** + * Gets a message from a stream for a consumer in a consumer group with single cluster pending entry list. + * This is used in a single database cluster configuration where the pending entries list is maintained + * only on the local database and not replicated to other databases in the cluster. + * + * @param streamName the name of the stream to read from + * @param groupName the name of the consumer group + * @param consumerName the name of the consumer + * @return a list of stream entries, or null if no messages are available + */ + public List>> singleDBPELGetStreamMessage(String streamName, String groupName, + String consumerName) { + return connection.executeCommand(readStreamMessageObject(this.commandMap.get(LuaCommand.SINGLE_DB_PEL_READGROUP), 1, + streamName, groupName, consumerName)); + } + + /* + private void scriptChecksum() { + MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-256"); + + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + Arrays.stream(LuaCommand.values()).sorted().forEach(command -> { + digest.update(command.getScript().getBytes(StandardCharsets.UTF_8)); + }); + digest.toString(); + + Arrays.stream(LuaCommand.values()).forEach(command -> { + command.getScript() + if (command.script == null || command.script.isEmpty()) { + throw new RuntimeException("Missing script content for script: " + command); + } + String sha1 = jedis.scriptLoad(command.script); + }); + } + */ + + /** + * Loads all Lua scripts defined in the LuaCommand enum into Redis. + * + * @param connection the JedisPooled connection to use + * @return a map of LuaCommand to SHA1 hash of the loaded script + * @throws RuntimeException if a script is null or empty + */ + private Map loadScripts(JedisPooled connection) { + Map sha1Map = new HashMap<>(); + Arrays.stream(LuaCommand.values()).forEach(command -> { + if (command.script == null || command.script.isEmpty()) { + throw new RuntimeException("Missing script content for script: " + command); + } + String sha1 = connection.scriptLoad(command.script); + sha1Map.put(command, new String(SafeEncoder.encode(sha1), StandardCharsets.UTF_8)); + }); + return sha1Map; + } + + /** + * Creates command arguments for Jedis internal functionality. + * Necessary for the Jedis internal functionality provided below. + * + * @param command the protocol command to create arguments for + * @return the command arguments + */ + protected CommandArguments commandArguments(ProtocolCommand command) { + return new CommandArguments(command); + } + + /** + * Creates a command object to read a stream-like response from a Lua script. + * Method using Jedis internals to read a stream-like response from a Lua script. + * + * @param sha1 the SHA1 hash of the Lua script to execute + * @param keyCount the number of keys to pass to the script + * @param params the parameters to pass to the script + * @return a command object to read a stream-like response + */ + private CommandObject>>> readStreamMessageObject(String sha1, int keyCount, + String... params) { + return new CommandObject<>(commandArguments(EVALSHA).add(sha1).add(keyCount).addObjects((Object[]) params), + BuilderFactory.STREAM_READ_RESPONSE); + } + + /** + * Creates a command object to read a list of stream entries from a Lua script. + * Method using Jedis internals to read a stream entry from a Lua script. + * + * @param sha1 the SHA1 hash of the Lua script to execute + * @param keyCount the number of keys to pass to the script + * @param params the parameters to pass to the script + * @return a command object to read a list of stream entries + */ + private CommandObject> readListObject(String sha1, int keyCount, String... params) { + return new CommandObject<>(commandArguments(EVALSHA).add(sha1).add(keyCount).addObjects((Object[]) params), + BuilderFactory.STREAM_ENTRY_LIST); + } + + /** + * Creates a command object to read a stream entry ID from a Lua script. + * + * @param sha1 the SHA1 hash of the Lua script to execute + * @param keyCount the number of keys to pass to the script + * @param params the parameters to pass to the script + * @return a command object to read a stream entry ID + */ + private CommandObject readStreamEntry(String sha1, int keyCount, String... params) { + return new CommandObject<>(commandArguments(EVALSHA).add(sha1).add(keyCount).addObjects((Object[]) params), + BuilderFactory.STREAM_ENTRY_ID); + } + + /** + * Creates a command object to read a list of strings from a Lua script. + * + * @param sha1 the SHA1 hash of the Lua script to execute + * @param keyCount the number of keys to pass to the script + * @param params the parameters to pass to the script + * @return a command object to read a list of strings + */ + private CommandObject> readMapEntry(String sha1, int keyCount, String... params) { + return new CommandObject<>(commandArguments(EVALSHA).add(sha1).add(keyCount).addObjects((Object[]) params), + BuilderFactory.STRING_LIST); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/command/noack/NoAckConsumerGroup.java b/redis-om-spring/src/main/java/com/redis/om/streams/command/noack/NoAckConsumerGroup.java new file mode 100644 index 000000000..691d838a2 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/command/noack/NoAckConsumerGroup.java @@ -0,0 +1,101 @@ +package com.redis.om.streams.command.noack; + +import java.util.List; +import java.util.Map; + +import com.redis.om.streams.TopicEntry; +import com.redis.om.streams.command.ConsumerGroupBase; +import com.redis.om.streams.exception.TopicNotFoundException; + +import redis.clients.jedis.JedisPooled; +import redis.clients.jedis.resps.StreamEntry; + +/** + * A consumer group implementation that consumes messages from Redis Streams without acknowledging them. + * This is useful for scenarios where message processing is idempotent or when acknowledgment is not required. + */ +public class NoAckConsumerGroup extends ConsumerGroupBase { + + /** + * Constructs a new NoAckConsumerGroup. + * + * @param connection the Jedis connection to use + * @param topicName the name of the topic (stream) to consume from + * @param groupName the name of the consumer group + */ + public NoAckConsumerGroup(JedisPooled connection, String topicName, String groupName) { + super(connection, topicName, groupName); + } + + /** + * Consumes a message from the topic without acknowledging it. + * + * @param consumerName the name of the consumer + * @return a TopicEntry containing the consumed message, or null if no message is available + * @throws TopicNotFoundException if the topic does not exist + */ + @Override + public TopicEntry consume(String consumerName) throws TopicNotFoundException { + return this.consumeWithNoAck(consumerName); + } + + /** + * Consumes a message from the topic without acknowledging it. + * This method initializes the consumer group if necessary and retrieves the next message. + * + * @param consumerName the name of the consumer + * @return a TopicEntry containing the consumed message, or null if no message is available + * @throws TopicNotFoundException if the topic does not exist + */ + protected TopicEntry consumeWithNoAck(String consumerName) throws TopicNotFoundException { + initialize(); + List>> response = noAckAndGetNextMessage(consumerName); + if (response == null) { + return null; + } else { + return TopicEntry.create(groupName, response.get(0), currentStreamId); + } + } + + /** + * Gets the next message from the topic without acknowledging it. + * + * @param consumerName the name of the consumer + * @return a list of stream entries, or null if no message is available + */ + @Override + public List>> getNextMessage(String consumerName) { + return this.noAckAndGetNextMessage(consumerName); + } + + /** + * Gets the next message from the topic without acknowledging it. + * This method first tries to get a message from the current stream. If no message is available, + * it tries to get a message from the next stream in the topic. + * + * @param consumerName the name of the consumer + * @return a list of stream entries, or null if no message is available in any stream of the topic + */ + protected List>> noAckAndGetNextMessage(String consumerName) { + List>> response; + + // Make a defensive copy of the current stream name so that we + // guarantee that we're working with the same stream name throughout this method invocation. + String streamToUse = currentStream; + response = luaCommandRunner.noAckAndGetStreamMessage(streamToUse, groupName, consumerName); + if (response != null) { + return response; + } else { + // See if there's a next stream that we're not using. + // If there is, then advance to the next stream and try to get a message from it. + String nextStream = getNextStream(streamToUse); + if (!nextStream.equals(streamToUse)) { + ensureConsumerGroupExists(nextStream); + setCurrentStream(nextStream); + response = luaCommandRunner.noAckAndGetStreamMessage(nextStream, groupName, consumerName); + return response; + } + } + return null; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/ConsumerGroup.java b/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/ConsumerGroup.java new file mode 100644 index 000000000..9a8ca5512 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/ConsumerGroup.java @@ -0,0 +1,196 @@ +package com.redis.om.streams.command.serial; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import com.redis.om.streams.AckMessage; +import com.redis.om.streams.TopicEntry; +import com.redis.om.streams.command.ConsumerGroupBase; +import com.redis.om.streams.exception.TopicNotFoundException; +import com.redis.om.streams.utils.Util; + +import redis.clients.jedis.JedisPooled; +import redis.clients.jedis.StreamEntryID; +import redis.clients.jedis.exceptions.JedisDataException; +import redis.clients.jedis.resps.StreamEntry; + +/** + * Implementation of a Redis Stream consumer group that processes messages serially. + * This class extends ConsumerGroupBase and provides functionality for consuming, + * acknowledging, and peeking at messages in a Redis Stream. + */ +public class ConsumerGroup extends ConsumerGroupBase { + + /** + * Constructs a new ConsumerGroup instance. + * + * @param connection The JedisPooled connection to Redis + * @param topicName The name of the topic to consume from + * @param groupName The name of the consumer group + */ + public ConsumerGroup(JedisPooled connection, String topicName, String groupName) { + super(connection, topicName, groupName); + } + + /** + * Consumes a message from the Redis Stream as the specified consumer. + * This method initializes the consumer group if necessary, retrieves the next message, + * and converts it to a TopicEntry. + * + * @param consumerName The name of the consumer within the group + * @return A TopicEntry containing the consumed message, or null if no message is available + * @throws TopicNotFoundException If the topic does not exist + */ + @Override + public TopicEntry consume(String consumerName) throws TopicNotFoundException { + initialize(); + List>> response = getNextMessage(consumerName); + if (response == null) { + return null; + } else { + return TopicEntry.create(groupName, response.get(0), currentStreamId); + } + } + + /** + * Acknowledges a message that has been processed by the consumer group. + * This removes the message from the pending entries list (PEL). + * + * @param ack The AckMessage containing information about the message to acknowledge + * @return true if the message was successfully acknowledged, false otherwise + */ + public boolean acknowledge(AckMessage ack) { + return luaCommandRunner.ackMessage(ack.getStreamName(), ack.getGroupName(), ack.getStreamEntryId()); + } + + /** + * Peeks at messages in the stream without consuming them. + * This method allows viewing a specified number of messages that would be delivered + * to this consumer group, starting from the last delivered ID. + * + * @param count The maximum number of messages to peek (limited to 100) + * @return A list of TopicEntry objects representing the messages + * @throws TopicNotFoundException If the topic does not exist + */ + public List peek(int count) throws TopicNotFoundException { + initialize(); + + if (count <= 0 || !isInitialized()) { + return Collections.emptyList(); + } + + // TODO: Address this limitation. We don't want to hard code this. + if (count > 100) { + count = 100; + } + + SerialTopicConfig config; + try { + config = TopicManager.loadConfig(connection, topicName); + } catch (TopicNotFoundException e) { + return Collections.emptyList(); + } + TopicManager manager = new TopicManager(connection, config); + + if (manager.getTopicSize() == 0) { + return Collections.emptyList(); + } + + String currentStreamForGroup = manager.getCurrentStreamForGroup(groupName); + long currentStreamIdForGroup = Util.streamIdFromStreamName(currentStreamForGroup); + long latestStreamId = manager.latestStreamId(); + StreamEntryID lastDeliveredId = getLastDeliveredId(currentStreamForGroup, groupName); + + // TODO: Get the lag for the group as an optimization. If lag is >= count, + // TODO: then we might not need to look at any subsequent streams + + List resultStreamEntries = new ArrayList<>(); + + // Because the lastDeliveredId will be included in the result set, we need to + // add one to the number of responses to return. This works around the + // fact that there's no way to add the exclusive operator "(" to the lastDeliveredId + // in Jedis. + List entries = connection.xrange(currentStreamForGroup, lastDeliveredId, StreamEntryID.MAXIMUM_ID, + count + 1); + + final String streamName = new String(currentStreamForGroup); + final long streamId = currentStreamIdForGroup; + + // If the first entry's ID matches the last delivered id, then remove that entry + if (entries.size() >= 1) { + if (entries.get(0).getID().equals(lastDeliveredId)) { + entries.remove(0); + } + } + + resultStreamEntries.addAll(entries.stream().map(e -> new TopicEntry(streamName, groupName, e, streamId)).toList()); + + while (resultStreamEntries.size() < count && (currentStreamIdForGroup < latestStreamId)) { + currentStreamIdForGroup += 1; + currentStreamForGroup = manager.getStreamForId(currentStreamIdForGroup); + final long sid = currentStreamIdForGroup; + final String sname = currentStreamForGroup; + int entriesNeeded = count - resultStreamEntries.size(); + entries = connection.xrange(currentStreamForGroup, StreamEntryID.MINIMUM_ID, StreamEntryID.MAXIMUM_ID, + entriesNeeded); + resultStreamEntries.addAll(entries.stream().map(e -> new TopicEntry(sname, groupName, e, sid)).toList()); + } + + return resultStreamEntries; + } + + /** + * Gets the next message from the stream for the specified consumer. + * If no message is available in the current stream, it attempts to get a message + * from the next stream in the topic. + * + * @param consumerName The name of the consumer within the group + * @return A list containing the next message, or null if no message is available + */ + @Override + public List>> getNextMessage(String consumerName) { + List>> response; + + // Make a defensive copy of the current stream name so that we + // guarantee that we're working with the same stream name throughout this method invocation. + String streamToUse = new String(currentStream); + try { + response = luaCommandRunner.getStreamMessage(streamToUse, groupName, consumerName); + } catch (JedisDataException e) { + if (e.getMessage().contains("NOGROUP")) { + response = null; + } else { + throw e; + } + } + + if (response != null) { + return response; + } else { + // See if there's a next stream that we're not using. + // If there is, then advance to the next stream and try to get a message from it. + String nextStream = getNextStream(streamToUse); + if (!nextStream.equals(streamToUse)) { + ensureConsumerGroupExists(nextStream); + setCurrentStream(nextStream); + response = luaCommandRunner.getStreamMessage(nextStream, groupName, consumerName); + return response; + } + } + return null; + } + + /** + * Sets the current stream for this consumer group. + * Updates the current stream name and ID, and schedules the next configuration refresh. + * + * @param streamName The name of the stream to set as current + */ + public void setCurrentStream(String streamName) { + this.currentStream = streamName; + this.currentStreamId = Util.streamIdFromStreamName(currentStream); + setNextConfigurationRefresh(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/PendingEntryQuery.java b/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/PendingEntryQuery.java new file mode 100644 index 000000000..d102a32d8 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/PendingEntryQuery.java @@ -0,0 +1,49 @@ +package com.redis.om.streams.command.serial; + +import com.redis.om.streams.TopicEntryId; + +import lombok.Data; +import lombok.NonNull; + +/** + * A query object for retrieving pending entries from a Redis Stream consumer group. + * This class defines the parameters for filtering pending entries based on ID range, + * idle time, and count. + */ +@Data +public class PendingEntryQuery { + + /** + * The starting ID (inclusive) for the range of pending entries to retrieve. + */ + @NonNull + private TopicEntryId startId; + + /** + * The ending ID (exclusive) for the range of pending entries to retrieve. + */ + @NonNull + private TopicEntryId endId; + + /** + * The minimum idle time in milliseconds for pending entries to be included in the result. + */ + private int minIdleTimeMilliSeconds; + + /** + * The maximum number of pending entries to retrieve. + */ + private int count; + + /** + * Constructs a new PendingEntryQuery with default values. + * By default, it retrieves up to 1000 entries across the entire ID range + * with no minimum idle time. + */ + public PendingEntryQuery() { + this.startId = TopicEntryId.MIN_ID; + this.endId = TopicEntryId.MAX_ID; + this.minIdleTimeMilliSeconds = 0; + this.count = 1000; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/SerialTopicConfig.java b/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/SerialTopicConfig.java new file mode 100644 index 000000000..2b718b41b --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/SerialTopicConfig.java @@ -0,0 +1,314 @@ +package com.redis.om.streams.command.serial; + +import java.time.Duration; +import java.util.Map; +import java.util.Random; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import redis.clients.jedis.StreamEntryID; + +/** + * Configuration for a serial topic in Redis Streams. + * A serial topic is a logical topic that consists of one or more Redis Streams. + * This class defines the properties and behavior of the topic, including retention time, + * stream length limits, and key naming conventions. + */ +@ToString +@EqualsAndHashCode +public class SerialTopicConfig { + + /** + * Creates a SerialTopicConfig instance from a map of configuration values. + * + * @param storedConfig A map containing the configuration values + * @return A new SerialTopicConfig instance + */ + public static SerialTopicConfig fromMap(Map storedConfig) { + return new SerialTopicConfig(storedConfig.get("topicName"), Long.valueOf(storedConfig.get("retentionTimeSeconds")), + Long.valueOf(storedConfig.get("maxStreamLength")), Long.valueOf(storedConfig.get("streamCycleSeconds")), + TTLFuzzMode.valueOf(storedConfig.get("ttlFuzzMode"))); + } + + /** Key for the index of all topics */ + public static final String TOPIC_INDEX_KEY = "__rsj:index:topics__"; + + /** Key for the index of all consumer groups */ + public static final String CONSUMER_GROUP_INDEX_KEY = "__rsj:index:groups__"; + + /** Special group name used during creation of new groups */ + public static final String BLANK_GROUP_FOR_CREATE = "__rsj:blank:group:c61a6c36-8850-4f16-adb9-ba3ee6d7c859__"; + + /** Minimum stream entry ID */ + public static final StreamEntryID MIN_STREAM_ENTRY_ID = new StreamEntryID(0); + + /** + * Enumeration of TTL fuzz modes. + * RANDOM: Adds a random amount of time to the retention time + * NONE: Uses the exact retention time + */ + public enum TTLFuzzMode { + /** Adds a random amount of time to the retention time */ + RANDOM, + /** Uses the exact retention time */ + NONE + } + + /** Field name for topic name in hash */ + public static final String TOPIC_FIELD_NAME = "topicName"; + + /** Field name for group name in hash */ + public static final String GROUP_FIELD_NAME = "groupName"; + + /** Default maximum number of entries in a stream before creating a new one */ + private static final int DEFAULT_MAX_STREAM_LENGTH = 250000; + + /** Default retention time for streams (8 days) */ + private static final long DEFAULT_RETENTION_TIME_SECONDS = Duration.ofDays(8).toSeconds(); + + /** Default minimum time between stream cycles (1 day) */ + private static final long DEFAULT_MIN_STREAM_CYCLE_SECONDS = Duration.ofDays(1).toSeconds(); + + /** Maximum value for stream index */ + public static final double MAX_STREAM_INDEX_VALUE = 9999999999999.00; + + /** The TTL fuzz mode for this topic */ + @Getter + private final TTLFuzzMode ttlFuzzMode; + + /** Maximum number of entries in a stream before creating a new one */ + @Getter + private final long maxStreamLength; + + /** Time in seconds between stream cycles */ + @Getter + private final long streamCycleSeconds; + + /** Minimum time to live for streams in seconds */ + @Getter + private final long minStreamTTL; + + /** Retention time for streams in seconds */ + @Getter + private final long retentionTimeSeconds; + + /** Name of this topic */ + @Getter + private final String topicName; + + /** Random number generator for TTL fuzzing */ + private Random random = new Random(); + + // TODO: Validate topic config and raise exception if invalid + /** + * Constructs a new SerialTopicConfig with the specified topic name and default settings. + * + * @param topicName The name of the topic + */ + public SerialTopicConfig(String topicName) { + this(topicName, DEFAULT_RETENTION_TIME_SECONDS); + } + + /** + * Constructs a new SerialTopicConfig with the specified topic name and retention time. + * + * @param topicName The name of the topic + * @param retentionTimeSeconds The retention time in seconds + */ + public SerialTopicConfig(String topicName, long retentionTimeSeconds) { + this.topicName = topicName; + this.retentionTimeSeconds = retentionTimeSeconds; + this.maxStreamLength = DEFAULT_MAX_STREAM_LENGTH; + this.streamCycleSeconds = DEFAULT_MIN_STREAM_CYCLE_SECONDS; + this.minStreamTTL = retentionTimeSeconds - streamCycleSeconds; + this.ttlFuzzMode = TTLFuzzMode.RANDOM; + } + + /** + * Constructs a new SerialTopicConfig with fully customized settings. + * + * @param topicName The name of the topic + * @param retentionTimeSeconds The retention time in seconds + * @param maxStreamLength The maximum number of entries in a stream before creating a new one + * @param streamCycleSeconds The time in seconds between stream cycles + * @param fuzzMode The TTL fuzz mode + */ + public SerialTopicConfig(String topicName, long retentionTimeSeconds, long maxStreamLength, long streamCycleSeconds, + TTLFuzzMode fuzzMode) { + this.topicName = topicName; + this.retentionTimeSeconds = retentionTimeSeconds; + this.maxStreamLength = maxStreamLength; + this.streamCycleSeconds = streamCycleSeconds; + this.minStreamTTL = retentionTimeSeconds - streamCycleSeconds; + this.ttlFuzzMode = fuzzMode; + } + + /** + * Generates a TTL (Time To Live) value for a stream based on the configured TTL fuzz mode. + * If the fuzz mode is RANDOM, adds a random amount of time to the retention time. + * If the fuzz mode is NONE, returns the exact retention time. + * + * @return The TTL value in seconds + */ + public long generateStreamTTL() { + if (ttlFuzzMode.equals(TTLFuzzMode.RANDOM)) { + return retentionTimeSeconds + 10 + random.nextInt(60); + } else { + return retentionTimeSeconds; + } + } + + /** + * Converts this configuration object to a map of string key-value pairs. + * This is useful for storing the configuration in Redis. + * + * @return A map representation of this configuration + */ + public Map asMap() { + ObjectMapper objectMapper = new ObjectMapper(); + Map map = objectMapper.convertValue(this, new TypeReference>() { + }); + return map; + } + + // TODO: Compute these on construction. No need to build new strings every time + // these values are needed. + + /** + * Gets the Redis key for storing this topic's configuration. + * + * @return The topic configuration key + */ + public String getTopicConfigKey() { + return getTopicConfigKeyPrefix() + "{" + topicName + "}"; + } + + /** + * Gets the prefix for topic configuration keys. + * + * @return The topic configuration key prefix + */ + public String getTopicConfigKeyPrefix() { + return "__rsj:topic:config:"; + } + + /** + * Gets the prefix for consumer group keys. + * + * @return The consumer group key prefix + */ + public String getConsumerGroupKeyPrefix() { + return "__rsj:group:"; + } + + /** + * Gets the Redis key for storing a consumer group's configuration. + * + * @param groupName The name of the consumer group + * @return The consumer group configuration key + */ + public String getConsumerGroupConfigKey(String groupName) { + return getConsumerGroupKeyPrefix() + groupName + ":config:{" + topicName + "}"; + } + + /** + * Gets the Redis key for the set that contains all streams marked as full. + * Full streams will not receive additional messages from a producer. + * + * @return The full streams key + */ + public String getFullStreamsKey() { + return "__rsj:topic:full_streams:{" + topicName + "}"; + } + + /** + * Gets the name of the initial stream for this topic. + * + * @return The initial stream name + */ + public String getInitialStreamName() { + return getCurrentStreamBaseName() + getFirstStreamId(); + } + + /** + * Gets the Redis key for the index of streams in this topic. + * + * @return The stream index key + */ + public String getStreamIndexKey() { + return "__rsj:topic:index:{" + topicName + "}"; + } + + /** + * Gets the base name for streams in this topic. + * + * @return The stream base name + */ + public String getCurrentStreamBaseName() { + return "__rsj:topic:stream:" + topicName + ":"; + } + + /** + * Gets the Redis key for the Pending Entries List (PEL) for a consumer group. + * + * @param streamName The name of the stream + * @param groupName The name of the consumer group + * @return The PEL key + */ + public String getPELKey(String streamName, String groupName) { + return "__PEL_{" + streamName + "}_" + groupName; + } + + /** + * Gets the Redis key for storing the last delivered ID for a consumer group. + * + * @param streamName The name of the stream + * @param groupName The name of the consumer group + * @return The last delivered ID key + */ + public String getLastDeliveredIdKey(String streamName, String groupName) { + return "__last_dlvr_id_{" + streamName + "}_" + groupName; + } + + /** + * Gets the Redis key for storing the number of entries read by a consumer group. + * + * @param streamName The name of the stream + * @param groupName The name of the consumer group + * @return The entries read key + */ + public String getEntriesReadKey(String streamName, String groupName) { + return "__entries_read_{" + streamName + "}_" + groupName; + } + + /** + * Gets the field name for the latest stream ID in the topic configuration. + * + * @return The topic stream ID field name + */ + public String getTopicStreamIdField() { + return "latestStreamId"; + } + + /** + * Gets the field name for the current stream in the consumer group configuration. + * + * @return The group current stream field name + */ + public String getGroupCurrentStreamField() { + return "currentStream"; + } + + /** + * Gets the ID for the first stream in a topic. + * + * @return The first stream ID + */ + public String getFirstStreamId() { + return "0"; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/TopicManager.java b/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/TopicManager.java new file mode 100644 index 000000000..7a68512c5 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/TopicManager.java @@ -0,0 +1,726 @@ +package com.redis.om.streams.command.serial; + +import java.time.Duration; +import java.util.*; + +import com.redis.om.streams.ConsumerGroupStatus; +import com.redis.om.streams.PendingEntry; +import com.redis.om.streams.TopicEntryId; +import com.redis.om.streams.command.LuaCommandRunner; +import com.redis.om.streams.exception.InvalidTopicException; +import com.redis.om.streams.exception.TopicNotFoundException; +import com.redis.om.streams.exception.TopicOrGroupNotFoundException; +import com.redis.om.streams.utils.Util; + +import lombok.Getter; +import redis.clients.jedis.Connection; +import redis.clients.jedis.JedisPooled; +import redis.clients.jedis.ReliableTransaction; +import redis.clients.jedis.StreamEntryID; +import redis.clients.jedis.exceptions.JedisDataException; +import redis.clients.jedis.resps.StreamEntry; +import redis.clients.jedis.search.*; +import redis.clients.jedis.search.schemafields.TagField; +import redis.clients.jedis.util.Pool; + +/** + * Manages Redis Stream topics and their associated consumer groups. + * This class provides functionality for creating, loading, and managing topics, + * as well as working with consumer groups, pending entries, and stream operations. + * A topic in this context is a logical entity that consists of one or more Redis Streams. + */ +public class TopicManager { + + /** Separator character for tag indexes */ + protected static final char TAG_INDEX_SEPARATOR = '¦'; + + /** Configuration for this topic */ + @Getter + protected final SerialTopicConfig config; + + /** Connection to Redis */ + protected final JedisPooled connection; + + /** Runner for Lua commands */ + protected LuaCommandRunner luaCommandRunner; + + /** Minimum entry ID for streams */ + public static final StreamEntryID MIN_ENTRY_ID = new StreamEntryID(0, 0); + + /** Maximum entry ID for streams */ + public static final StreamEntryID MAX_ENTRY_ID = new StreamEntryID(9999999999999L, 9999999L); + + /** Random number generator for various operations */ + protected static Random random = new Random(); + + /** + * Returns the configuration object for an existing named topic. + * + * @param connection The JedisPooled connection to Redis + * @param topicName The name of the topic to load + * @return The SerialTopicConfig for the specified topic + * @throws TopicNotFoundException If the topic does not exist + */ + public static SerialTopicConfig loadConfig(JedisPooled connection, String topicName) throws TopicNotFoundException { + SerialTopicConfig config = new SerialTopicConfig(topicName); + Map storedConfig = connection.hgetAll(config.getTopicConfigKey()); + if (storedConfig == null || storedConfig.isEmpty()) { + throw new TopicNotFoundException( + "Cannot find topic: " + topicName + "\n" + "" + "Did you remember to execute: TopicManager.createTopic(connection, topicConfig) or similar?"); + } + return SerialTopicConfig.fromMap(storedConfig); + } + + /** + * Gets a topic manager instance from an existing named topic. + * + * @param connection The JedisPooled connection to Redis + * @param topicName The name of the topic to load + * @return A TopicManager instance for the specified topic + * @throws TopicNotFoundException If the topic does not exist + */ + public static TopicManager load(JedisPooled connection, String topicName) throws TopicNotFoundException { + return new TopicManager(connection, loadConfig(connection, topicName)); + } + + /** + * Creates a new topic with the given name using topic defaults. + * + * @param connection The JedisPooled connection to Redis + * @param topicName The name of the topic to create + * @return A TopicManager instance for the newly created topic + * @throws InvalidTopicException If the topic name is invalid + */ + public static TopicManager createTopic(JedisPooled connection, String topicName) throws InvalidTopicException { + SerialTopicConfig config = new SerialTopicConfig(topicName); + return createTopic(connection, config); + } + + /** + * Creates a new topic with the given name and retention time using topic defaults. + * + * @param connection The JedisPooled connection to Redis + * @param topicName The name of the topic to create + * @param retentionTimeSeconds The retention time in seconds for messages in the topic + * @return A TopicManager instance for the newly created topic + * @throws InvalidTopicException If the topic name is invalid + */ + public static TopicManager createTopic(JedisPooled connection, String topicName, long retentionTimeSeconds) + throws InvalidTopicException { + SerialTopicConfig config = new SerialTopicConfig(topicName, retentionTimeSeconds); + return createTopic(connection, config); + } + + /** + * Creates a new topic with the provided topic configuration. + * If the topic already exists, returns a TopicManager for the existing topic. + * TODO: Validate that topic name does not contain '¦' + * + * @param connection The JedisPooled connection to Redis + * @param config The configuration for the topic + * @return A TopicManager instance for the newly created or existing topic + * @throws InvalidTopicException If the topic configuration is invalid + */ + public static TopicManager createTopic(JedisPooled connection, SerialTopicConfig config) + throws InvalidTopicException { + Map localConfig = connection.hgetAll(config.getTopicConfigKey()); + if (localConfig == null || localConfig.isEmpty()) { + ensureTopicValid(config); + ensureIndexesExist(connection, config); + Pool pool = connection.getPool(); + try (Connection con = pool.getResource()) { + ReliableTransaction transaction = new ReliableTransaction(con); + Map initialConfig = config.asMap(); + transaction.hset(config.getTopicConfigKey(), initialConfig); + transaction.exec(); + } + + } + return new TopicManager(connection, config); + } + + /** + * Validates that a topic configuration is valid. + * Checks that the topic name is not null, has the correct length, + * and contains only valid characters. + * + * @param config The configuration to validate + * @throws InvalidTopicException If the topic configuration is invalid + */ + protected static void ensureTopicValid(SerialTopicConfig config) throws InvalidTopicException { + String topicName = config.getTopicName(); + if (topicName == null) { + throw new InvalidTopicException("Topic name cannot be null"); + } else { + if (!Util.nameCorrectLength(topicName)) { + throw new InvalidTopicException("Invalid topic name length: " + topicName + .length() + ". Topic name must be between 0 and 300 characters."); + } + + if (!Util.nameValid(topicName)) { + throw new InvalidTopicException( + "Invalid topic name: '" + topicName + "'. All topic characters must match the following regular expression: [a-zA-Z0-9._-]+"); + } + } + } + + /** + * Ensures that indexes exist for the topics and consumer groups. + * Creates the necessary RediSearch indexes if they don't already exist. + * + * @param connection The JedisPooled connection to Redis + * @param config The configuration for the topic + */ + protected static void ensureIndexesExist(JedisPooled connection, SerialTopicConfig config) { + FTCreateParams params = new FTCreateParams(); + params.on(IndexDataType.HASH); + params.prefix(config.getTopicConfigKeyPrefix()); + TagField topicField = new TagField(SerialTopicConfig.TOPIC_FIELD_NAME); + //topicField.separator(TAG_INDEX_SEPARATOR); + topicField.caseSensitive(); + + try { + connection.ftCreate(SerialTopicConfig.TOPIC_INDEX_KEY, params, List.of(topicField)); + } catch (JedisDataException e) { + // If an exception is thrown here, it means that the index already exists. + } + + params = new FTCreateParams(); + params.on(IndexDataType.HASH); + params.prefix(config.getConsumerGroupKeyPrefix()); + TagField groupField = new TagField(SerialTopicConfig.GROUP_FIELD_NAME); + //groupField.separator(TAG_INDEX_SEPARATOR); + groupField.caseSensitive(); + + try { + connection.ftCreate(SerialTopicConfig.CONSUMER_GROUP_INDEX_KEY, params, List.of(topicField)); + } catch (JedisDataException e) { + // If an exception is thrown here, it means that the index already exists. + } + } + + /** + * Returns the names of all non-expired topics on this Redis cluster. + * This method searches the topic index for all topics. + * + * @param connection The JedisPooled connection to Redis + * @return A list of topic names + */ + public static List getTopicNames(JedisPooled connection) { + int limit = 10000; + List names = new ArrayList<>(); + FTSearchParams params = new FTSearchParams(); + params.limit(0, limit); + SearchResult responses = connection.ftSearch(SerialTopicConfig.TOPIC_INDEX_KEY, "*", params); + for (Document document : responses.getDocuments()) { + Object response = document.get(SerialTopicConfig.TOPIC_FIELD_NAME); + if (response != null) { + names.add(String.valueOf(response)); + } + } + + return names; + } + + /** + * Returns the names of all consumer groups for the provided topic. + * This method searches the consumer group index for groups associated with the specified topic. + * + * @param connection The JedisPooled connection to Redis + * @param topicName The name of the topic to get groups for + * @return A list of consumer group names + */ + public static List getGroupsForTopic(JedisPooled connection, String topicName) { + int limit = 10000; + List names = new ArrayList<>(); + FTSearchParams params = new FTSearchParams(); + params.limit(0, limit); + String query = "@" + SerialTopicConfig.TOPIC_FIELD_NAME + ":{" + RediSearchUtil.escape(topicName) + "}"; + SearchResult responses = connection.ftSearch(SerialTopicConfig.CONSUMER_GROUP_INDEX_KEY, query, params); + for (Document document : responses.getDocuments()) { + Object response = document.get(SerialTopicConfig.GROUP_FIELD_NAME); + if (response != null) { + names.add(String.valueOf(response)); + } + } + + return names; + } + + /** + * Returns the status of all consumer groups for the provided topic. + * This method gets the status of each consumer group associated with the specified topic. + * + * @param connection The JedisPooled connection to Redis + * @param topicName The name of the topic to get group statuses for + * @return A list of ConsumerGroupStatus objects + */ + public static List getGroupStatusForTopic(JedisPooled connection, String topicName) { + List groupNames = getGroupsForTopic(connection, topicName); + List results = new ArrayList<>(); + for (String groupName : groupNames) { + TopicManager manager = new TopicManager(connection, new SerialTopicConfig(topicName)); + results.add(manager.getConsumerGroupStatus(groupName)); + } + + return results; + } + + /** + * Removes a topic, its underlying streams, and all relevant configuration keys. + * + * To avoid simultaneous cleanup of streams, all streams will have an expiry set + * within the following 24 hours, and these streams will be renamed to prevent them + * from being associated with any future topics. + * + * @param connection The JedisPooled connection to Redis + * @param topicName The name of the topic to destroy + * @return true if the topic was successfully destroyed, false if the topic did not exist + */ + public static boolean destroy(JedisPooled connection, String topicName) { + SerialTopicConfig config = new SerialTopicConfig(topicName); + Map localConfig = connection.hgetAll(config.getTopicConfigKey()); + if (localConfig != null) { + config = SerialTopicConfig.fromMap(localConfig); + TopicManager manager = new TopicManager(connection, config); + UUID deleteId = UUID.randomUUID(); + long expireTime = 90 + random.nextLong(Duration.ofMinutes(60).toSeconds()); + List groupNames = TopicManager.getGroupsForTopic(connection, topicName); + for (String streamName : manager.getStreamNames()) { + String deletedStreamName = "{" + streamName + "}_DELETED_" + deleteId; + connection.expire(streamName, expireTime); + connection.rename(streamName, deletedStreamName); + expireTime += 90; + + for (String groupName : groupNames) { + String pelKeyName = config.getPELKey(streamName, groupName); + String deletedPelKeyName = "__DELETED__" + deleteId + "_" + pelKeyName; + try { + connection.rename(pelKeyName, deletedPelKeyName); + } catch (JedisDataException e) { + // If there's no such key, that's entirely okay. + } + + // The Lua scripts create a last_delivered_id key and entries_read key for each group and stream + // Here we delete these keys. + connection.del(config.getEntriesReadKey(streamName, groupName)); + connection.del(config.getLastDeliveredIdKey(streamName, groupName)); + } + } + + for (String groupName : groupNames) { + connection.del(config.getConsumerGroupConfigKey(groupName)); + } + + connection.del(config.getStreamIndexKey()); + connection.del(config.getFullStreamsKey()); + connection.del(config.getTopicConfigKey()); + + return true; + } + return false; + } + + /** + * Constructs a new TopicManager with the specified connection and configuration. + * + * @param connection The JedisPooled connection to Redis + * @param config The configuration for the topic + */ + public TopicManager(JedisPooled connection, SerialTopicConfig config) { + this.config = config; + this.connection = connection; + this.luaCommandRunner = new LuaCommandRunner(connection); + } + + /** + * Gets the total number of messages in the streams that make up this topic. + * + * @return The total number of messages in the topic + */ + public long getTopicSize() { + long size = 0L; + for (String streamName : getStreamNames()) { + size += connection.xlen(streamName); + } + return size; + } + + /** + * Returns the status of the specified consumer group. + * The status includes information about the group's lag, pending entries, + * and the total size of the topic. + * + * @param groupName The name of the consumer group + * @return A ConsumerGroupStatus object containing the group's status + */ + public ConsumerGroupStatus getConsumerGroupStatus(String groupName) { + long topicSize = getTopicSize(); + long lag = 0; + try { + lag = getConsumerGroupLag(groupName); + } catch (TopicOrGroupNotFoundException e) { + lag = topicSize; + } + + ConsumerGroupStatus status = new ConsumerGroupStatus(config.getTopicName(), groupName, getPendingEntryCount( + groupName), topicSize, lag); + + return status; + } + + /** + * Gets the lag for the specified consumer group. + * The lag is the number of messages in the topic that have not yet been processed by the group. + * + * @param groupName The name of the consumer group + * @return The lag for the consumer group + * @throws TopicOrGroupNotFoundException If the consumer group does not exist + */ + public Long getConsumerGroupLag(String groupName) throws TopicOrGroupNotFoundException { + String currentStream = getCurrentStreamForGroup(groupName); + if (currentStream == null) { + throw new TopicOrGroupNotFoundException("Cannot find consumer group: " + groupName); + } else { + long currentStreamLag = luaCommandRunner.getConsumerGroupLag(currentStream, groupName); + List nextStreams = getNextStreams(currentStream); + long subsequentStreamLag = 0; + for (String stream : nextStreams) { + subsequentStreamLag += connection.xlen(stream); + } + return currentStreamLag + subsequentStreamLag; + } + } + + /** + * Returns the total number of pending entries for the given consumer group. + * Pending entries are messages that have been delivered to the group but not yet acknowledged. + * + * @param groupName The name of the consumer group + * @return The number of pending entries + */ + public long getPendingEntryCount(String groupName) { + long count = 0; + for (String streamName : getStreamNames()) { + String pelKey = config.getPELKey(streamName, groupName); + count += connection.xlen(pelKey); + } + + return count; + } + + /** + * Get pending entries for this consumer group. This method returns up to 1000 pending entries, even if a number + * greater + * than 1000 is specified in the count. This limit prevents the user from returning large buffers from Redis. + * + * @param groupName + * @return + */ + public List getPendingEntries(String groupName) { + PendingEntryQuery query = new PendingEntryQuery(); + return getPendingEntries(groupName, query.getStartId(), query.getEndId(), query.getMinIdleTimeMilliSeconds(), query + .getCount()); + } + + /** + * Get pending entries for this consumer group. This method returns up to 1000 pending entries, even if a number + * greater + * than 1000 is specified in the count. This limit prevents the user from returning large buffers from Redis. + * + * All returned pending entries will have IDs start from startId, inclusive, up to endId, exclusive. + * + * @param groupName + * @param query + * @return + */ + public List getPendingEntries(String groupName, PendingEntryQuery query) { + return getPendingEntries(groupName, query.getStartId(), query.getEndId(), query.getMinIdleTimeMilliSeconds(), query + .getCount()); + } + + /** + * Get pending entries for this consumer group. This method returns up to 1000 pending entries, even if a number + * greater + * than 1000 is specified in the count. This limit prevents the user from returning large buffers from Redis. + * + * All returned pending entries will have IDs start from startId, inclusive, up to endId, exclusive. + * + * @param groupName + * @param startId + * @param endId + * @param count + * @return + */ + public List getPendingEntries(String groupName, TopicEntryId startId, TopicEntryId endId, int count) { + int minIdleTimeSeconds = 0; + return getPendingEntries(groupName, startId, endId, minIdleTimeSeconds, count); + } + + /** + * Get pending entries for this consumer group. This method returns up to 1000 pending entries, even if a number + * greater + * than 1000 is specified in the count. This limit prevents the user from returning large buffers from Redis. + * + * All returned pending entries will have IDs start from startId, inclusive, up to endId, exclusive. + * + * @param groupName + * @param startId + * @param endId + * @param minIdleTimeMilliSeconds + * @param count + * @return + */ + public List getPendingEntries(String groupName, TopicEntryId startId, TopicEntryId endId, + int minIdleTimeMilliSeconds, int count) { + // Don't return more than 1000 messages. + if (count > 1000) { + count = 1000; + } else if (count <= 0) { + return Collections.emptyList(); + } + + if (getTopicSize() == 0) { + return Collections.emptyList(); + } + + // Return the first stream that may contain pending entries. + String streamName = getStreamForID(startId); + + // We need the boundary IDs as Doubles so that we can use them for comparisons later. + Double startIdAsDouble = idAsDouble(startId.getStreamEntryId()); + Double endIdAsDouble = idAsDouble(endId.getStreamEntryId()); + + // We need a flag to mark the iteration as complete. + boolean complete = false; + + // Get the server time so that we can calculate pending entry idle time. + long serverTimeMs = Util.getServerTimeMs(connection); + + // Now get up to _count_ entries from the first stream. + List pendingEntries = new ArrayList<>(); + List streamEntries = luaCommandRunner.getPending(streamName, groupName, startId.getStreamEntryId() + .toString(), endId.getStreamEntryId().toString(), count); + for (StreamEntry entry : streamEntries) { + // TODO: Change comparison logic here to compare long values instead of synthetic doubles + Double entryAsDouble = idAsDouble(entry.getID()); + if (entryAsDouble > endIdAsDouble) { + complete = true; + break; + } + + if (entryAsDouble <= startIdAsDouble) { + continue; + } + + if (entryIdleTimeMilliSeconds(entry, serverTimeMs) >= minIdleTimeMilliSeconds) { + pendingEntries.add(streamEntryToPendingEntry(entry, serverTimeMs, streamName, groupName)); + } + } + + // If we have enough entries, return. + if (pendingEntries.size() >= count || complete) { + return pendingEntries; + } + + streamName = getNextStream(streamName, endId); + while (streamName != null && pendingEntries.size() < count && !complete) { + streamEntries = luaCommandRunner.getPending(streamName, groupName, startId.getStreamEntryId().toString(), endId + .getStreamEntryId().toString(), count - pendingEntries.size()); + + // Add the pending entries one at a time until we reach the endID. + for (StreamEntry entry : streamEntries) { + if (idAsDouble(entry.getID()) > endIdAsDouble) { + complete = true; + break; + } + + // Filter by idle time if a min idle time is provided + if (entryIdleTimeMilliSeconds(entry, serverTimeMs) >= minIdleTimeMilliSeconds) { + pendingEntries.add(streamEntryToPendingEntry(entry, serverTimeMs, streamName, groupName)); + } + + if (pendingEntries.size() >= count) { + break; + } + } + streamName = getNextStream(streamName, endId); + } + return pendingEntries; + } + + protected long entryIdleTimeMilliSeconds(StreamEntry entry, long serverTimeMs) { + String deliveryTime = entry.getFields().get("delivery_time"); + Long idleTimeMs = serverTimeMs - Long.valueOf(deliveryTime); + return idleTimeMs; + } + + protected PendingEntry streamEntryToPendingEntry(StreamEntry entry, long serverTimeMs, String streamName, + String groupName) { + StreamEntryID id = entry.getID(); + Map fields = entry.getFields(); + String consumer = fields.get("consumer"); + String deliveryTime = fields.get("delivery_time"); + Long idleTimeMs = serverTimeMs - Long.valueOf(deliveryTime); + return new PendingEntry(id, config.getTopicName(), streamName, groupName, consumer, idleTimeMs, 1L); + } + + /** + * Return true if the provided stream is the latest stream in the topic + * + * @param streamName + * @return True or false + */ + public boolean isLatestStream(String streamName) { + long streamId = Util.streamIdFromStreamName(streamName); + return (streamId == latestStreamId()); + } + + /** + * Return the latest generated stream ID for this topic. + * + * @return + */ + public long latestStreamId() { + String latestId = connection.hget(config.getTopicConfigKey(), config.getTopicStreamIdField()); + return Long.valueOf(latestId); + } + + /** + * Find the stream that must contain the provided ID, if such an ID exists in the topic. + * + * @param id + * @return + */ + public String getStreamForID(TopicEntryId id) { + return getStreamForId(id.getStreamId()); + } + + public String getStreamForId(long streamId) { + return config.getCurrentStreamBaseName() + streamId; + } + + /** + * Return the next stream for this topic created after the provided stream. + * + * @param startStream + * @return + */ + public String getNextStream(String startStream) { + return getNextStream(startStream, TopicEntryId.MAX_ID); + } + + public String getNextStream(String startStream, TopicEntryId maxId) { + String latestId = connection.hget(config.getTopicConfigKey(), config.getTopicStreamIdField()); + long startStreamId = Util.streamIdFromStreamName(startStream); + long latestIdLong = Long.valueOf(latestId); + if (latestIdLong > startStreamId && latestIdLong <= maxId.getStreamId()) { + long nextStreamId = startStreamId + 1; + return config.getCurrentStreamBaseName() + nextStreamId; + } else { + return null; + } + } + + protected Double idAsDouble(StreamEntryID maxId) { + return Double.valueOf(maxId.toString().replace("-", ".")); + } + + /** + * Return all streams for this topic created after the provided stream. + * + * @param startStream + * @return + * + * TODO: This method is far from optimized. This should be optimized in the future. + */ + public List getNextStreams(String startStream) { + List streams = connection.zrangeByScore(config.getStreamIndexKey(), "-inf", "+inf"); + List results = new ArrayList<>(); + long startStreamId = Util.streamIdFromStreamName(startStream); + for (String stream : streams) { + long streamId = Util.streamIdFromStreamName(stream); + if (streamId > startStreamId) { + results.add(stream); + } + } + + return results; + } + + /** + * Return the stream currently assigned to the provided consumer group. + * + * @param groupName + * @return + */ + public String getCurrentStreamForGroup(String groupName) { + return connection.hget(config.getConsumerGroupConfigKey(groupName), "currentStream"); + } + + /** + * Return a list of all streams in the topic, ordered from oldest to newest. + * + * @return + */ + public List getStreamNames() { + return connection.zrangeByScore(config.getStreamIndexKey(), "-inf", "+inf"); + } + + public List getOrderedStreamNames() { + List names = connection.zrangeByScore(config.getStreamIndexKey(), "-inf", "+inf"); + Collections.sort(names, new StreamNameComparator()); + return names; + } + + protected static class StreamNameComparator implements Comparator { + @Override + public int compare(String a, String b) { + long aId = Util.streamIdFromStreamName(a); + long bId = Util.streamIdFromStreamName(b); + if (aId < bId) + return -1; + if (aId > bId) + return 1; + return 0; + } + } + + /** + * Return the set of stream marked as full. Full streams will not receive additional messages from a producer. + * + * @return + */ + public List getFullStreamNames() { + return connection.zrangeByLex(config.getFullStreamsKey(), "-", "+"); + } + + public String createConsumerGroup(String groupName) { + String consumerGroupConfigKey = config.getConsumerGroupConfigKey(groupName); + Map currentConfig = connection.hgetAll(consumerGroupConfigKey); + if (currentConfig != null && !currentConfig.isEmpty()) { + return currentConfig.get(config.getGroupCurrentStreamField()); + } else { + Map consumerGroupFields = new HashMap<>(); + consumerGroupFields.put("topicName", config.getTopicName()); + consumerGroupFields.put("groupName", groupName); + String firstStream = getFirstUnexpiredStream(); + consumerGroupFields.put(config.getGroupCurrentStreamField(), firstStream); + connection.hset(config.getConsumerGroupConfigKey(groupName), consumerGroupFields); + return firstStream; + } + } + + public String getFirstUnexpiredStream() { + for (String streamName : getOrderedStreamNames()) { + if (connection.exists(streamName)) { + return streamName; + } else { + connection.zrem(config.getStreamIndexKey(), streamName); + connection.zrem(config.getFullStreamsKey(), streamName); + } + } + + return config.getInitialStreamName(); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/TopicProducer.java b/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/TopicProducer.java new file mode 100644 index 000000000..c1df1db06 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/command/serial/TopicProducer.java @@ -0,0 +1,166 @@ +package com.redis.om.streams.command.serial; + +import java.time.Instant; +import java.util.Map; + +import com.redis.om.streams.TopicEntryId; +import com.redis.om.streams.command.LuaCommandRunner; +import com.redis.om.streams.exception.InvalidMessageException; +import com.redis.om.streams.exception.ProducerTimeoutException; +import com.redis.om.streams.exception.TopicNotFoundException; +import com.redis.om.streams.utils.Util; + +import lombok.Getter; +import redis.clients.jedis.JedisPooled; +import redis.clients.jedis.StreamEntryID; + +/** + * A producer for Redis Streams that implements a serial topic policy. + * + * This topic policy creates a logical topic consisting of one or more streams. + * Once a maximum stream size is reached, a new stream is created. + * For each topic, the following keys are created: + * - Topic config key - a hash. + * - Streams key - a list of created streams, in order. + * - Full streams key - a set containing the names of all streams that have reached max capacity. + * - Topic index - a sorted set. Members are stream names and scores are the last IDs written to each stream. + * - Topic producer lock. Check this lock to determine whether producers are still writing to a given stream. + * - Streams - the individual streams comprising this topic. + * + * The keys listed above are hash-keyed on topic name. Therefore, these keys will always appear together on a + * single shard. The streams themselves are not hash-keyed and thus will be distributed across the cluster. + */ +public class TopicProducer implements com.redis.om.streams.Producer { + + /** Connection to Redis */ + private final JedisPooled connection; + + /** Runner for Lua commands */ + private final LuaCommandRunner luaCommandRunner; + + /** Flag indicating whether the producer has been initialized */ + private boolean initialized; + + /** Configuration for this topic */ + @Getter + private SerialTopicConfig config; + + /** Name of this topic */ + @Getter + private final String topicName; + + /** Current stream being written to */ + @Getter + private String currentStream; + + /** ID of the current stream */ + @Getter + private long currentStreamId; + + /** + * Constructs a new TopicProducer for the specified topic. + * + * @param connection The JedisPooled connection to Redis + * @param topicName The name of the topic to produce to + */ + public TopicProducer(JedisPooled connection, String topicName) { + this.connection = connection; + this.topicName = topicName; + this.luaCommandRunner = new LuaCommandRunner(connection); + this.initialized = false; + } + + /** + * Produces a message to the topic. + * This method initializes the producer if necessary, publishes the message to the current stream, + * and handles stream switching if the current stream is full. + * + * @param message A map containing the message fields and values + * @return A TopicEntryId containing the ID and stream name of the produced message + * @throws TopicNotFoundException If the topic does not exist + * @throws InvalidMessageException If the message is invalid + * @throws ProducerTimeoutException If the producer times out after multiple retries + */ + @Override + public TopicEntryId produce(Map message) throws TopicNotFoundException, InvalidMessageException, + ProducerTimeoutException { + Instant end = Instant.now().plusSeconds(5); + initializeStream(); + String streamToPublish = this.currentStream; + StreamEntryID entryId = luaCommandRunner.publishSerialStreamMessage(streamToPublish, config.generateStreamTTL(), + config.getMaxStreamLength(), config.getMinStreamTTL(), message); + + // If the entry ID is null, then we need to switch to the next stream + // and then try to write again. + int retries = 0; + while (entryId == null) { + markComplete(streamToPublish); + setNextStream(Util.streamIdFromStreamName(streamToPublish)); + String nextStream = this.currentStream; + streamToPublish = nextStream; + entryId = luaCommandRunner.publishSerialStreamMessage(nextStream, config.generateStreamTTL(), config + .getMaxStreamLength(), config.getMinStreamTTL(), message); + retries += 1; + if (Instant.now().isAfter(end)) { + throw new ProducerTimeoutException( + "Unable to produce to topic " + "within 5 seconds and " + retries + " retries. " + "To address this, increase the maximum number of" + "messages per stream and/or reduce the number of " + "parallel threads using the same producer instance."); + } + } + + return new TopicEntryId(entryId, streamToPublish); + } + + /** + * Initializes the producer by loading the topic configuration and setting the initial stream. + * This method is synchronized to ensure thread safety during initialization. + * + * @throws TopicNotFoundException If the topic does not exist + */ + private synchronized void initializeStream() throws TopicNotFoundException { + if (!this.initialized) { + this.config = TopicManager.loadConfig(connection, topicName); + setNextStream(); + this.initialized = true; + } + } + + /** + * Sets the next stream to use for producing messages. + * This method is synchronized to ensure thread safety when changing streams. + * + * @param streamId The ID of the current stream, or -1 to get the first available stream + */ + private synchronized void setNextStream(long streamId) { + String response = luaCommandRunner.getNextSerialActiveActiveStream(config, streamId); + this.currentStream = response; + this.currentStreamId = Util.streamIdFromStreamName(response); + Util.ensureStreamWithTTLExists(connection, currentStream, config.generateStreamTTL()); + } + + /** + * Sets the next stream to use for producing messages, starting from the first available stream. + * This is a convenience method that calls setNextStream(-1). + */ + private void setNextStream() { + setNextStream(-1); + } + + /** + * Marks a stream as complete (full) so that no more messages will be produced to it. + * This method also cleans up old entries in the full streams set and stream index. + * + * @param streamToPublish The name of the stream to mark as complete + */ + private void markComplete(String streamToPublish) { + long serverTimeSeconds = Util.getServerTimeMs(connection) / 1000; + long expiryAtSeconds = serverTimeSeconds + config.getRetentionTimeSeconds(); + connection.zadd(config.getFullStreamsKey(), expiryAtSeconds, streamToPublish); + + // Remove all old full streams entries + String expiredTimeSeconds = String.valueOf(serverTimeSeconds - config.getRetentionTimeSeconds()); + connection.zremrangeByScore(config.getFullStreamsKey(), "-inf", expiredTimeSeconds); + + // Remove all old stream list entries + connection.zremrangeByScore(config.getStreamIndexKey(), "-inf", expiredTimeSeconds); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/command/singleclusterpel/SingleClusterPelConsumerGroup.java b/redis-om-spring/src/main/java/com/redis/om/streams/command/singleclusterpel/SingleClusterPelConsumerGroup.java new file mode 100644 index 000000000..a556d86b0 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/command/singleclusterpel/SingleClusterPelConsumerGroup.java @@ -0,0 +1,118 @@ +package com.redis.om.streams.command.singleclusterpel; + +import java.util.List; +import java.util.Map; + +import com.redis.om.streams.AckMessage; +import com.redis.om.streams.TopicEntry; +import com.redis.om.streams.command.ConsumerGroupBase; +import com.redis.om.streams.exception.TopicNotFoundException; + +import redis.clients.jedis.JedisPooled; +import redis.clients.jedis.StreamEntryID; +import redis.clients.jedis.resps.StreamEntry; + +/** + * A specialized implementation of ConsumerGroupBase that manages consumer groups in a single cluster + * with Pending Entry List (PEL) functionality. This class provides methods for consuming messages, + * acknowledging messages, and managing stream operations in a single Redis cluster environment. + */ +public class SingleClusterPelConsumerGroup extends ConsumerGroupBase { + /** + * Constructs a new SingleClusterPelConsumerGroup with the specified connection, topic name, and group name. + * + * @param connection The JedisPooled connection to the Redis server + * @param topicName The name of the topic + * @param groupName The name of the consumer group + */ + public SingleClusterPelConsumerGroup(JedisPooled connection, String topicName, String groupName) { + super(connection, topicName, groupName); + } + + /** + * Consumes a message from the topic using default ACK behavior. + * This means client code must ACK at some point to indicate complete processing. + * Any ACK will only be for the local DB and will not be propagated to Active-Active peers. + * + * @param consumerName The name of the consumer + * @return A TopicEntry containing the consumed message, or null if no message is available + * @throws TopicNotFoundException If the specified topic does not exist + */ + @Override + public TopicEntry consume(String consumerName) throws TopicNotFoundException { + return consumeSingleCluster(consumerName); + } + + /** + * Acknowledges a message that has been processed. + * For this class, no Active-Active replicated PEL occurs. + * + * @param ack The AckMessage containing information about the message to acknowledge + * @return true if the message was successfully acknowledged, false otherwise + */ + public boolean acknowledge(AckMessage ack) { + //List streamEntryIDS = List.of(new StreamEntryID(ack.getStreamEntryId()); + long ackValue = connection.xack(ack.getStreamName(), groupName, new StreamEntryID(ack.getStreamEntryId())); + return ackValue == 1; + } + + /** + * Gets the next message for the specified consumer. + * + * @param consumerName The name of the consumer + * @return A list of map entries containing stream entries, or null if no message is available + */ + @Override + public List>> getNextMessage(String consumerName) { + return this.singleClusterGetNextMessage(consumerName); + } + + /** + * Consumes a message from the topic in a single cluster environment. + * This is a private helper method used by the public consume method. + * + * @param consumerName The name of the consumer + * @return A TopicEntry containing the consumed message, or null if no message is available + * @throws TopicNotFoundException If the specified topic does not exist + */ + private TopicEntry consumeSingleCluster(String consumerName) throws TopicNotFoundException { + initialize(); + List>> response = singleClusterGetNextMessage(consumerName); + if (response == null) { + return null; + } else { + return TopicEntry.create(groupName, response.get(0), currentStreamId); + } + } + + /** + * Gets the next message for the specified consumer from a single cluster environment. + * This method first tries to get a message from the current stream. If no message is available, + * it checks if there's a next stream and tries to get a message from it. + * + * @param consumerName The name of the consumer + * @return A list of map entries containing stream entries, or null if no message is available + */ + protected List>> singleClusterGetNextMessage(String consumerName) { + List>> response; + + // Make a defensive copy of the current stream name so that we + // guarantee that we're working with the same stream name throughout this method invocation. + String streamToUse = currentStream; + response = luaCommandRunner.singleDBPELGetStreamMessage(streamToUse, groupName, consumerName); + if (response != null) { + return response; + } else { + // See if there's a next stream that we're not using. + // If there is, then advance to the next stream and try to get a message from it. + String nextStream = getNextStream(streamToUse); + if (!nextStream.equals(streamToUse)) { + ensureConsumerGroupExists(nextStream); + setCurrentStream(nextStream); + response = luaCommandRunner.singleDBPELGetStreamMessage(nextStream, groupName, consumerName); + return response; + } + } + return null; + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/command/singleclusterpel/SingleClusterPelTopicManager.java b/redis-om-spring/src/main/java/com/redis/om/streams/command/singleclusterpel/SingleClusterPelTopicManager.java new file mode 100644 index 000000000..0f22fd4a6 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/command/singleclusterpel/SingleClusterPelTopicManager.java @@ -0,0 +1,292 @@ +package com.redis.om.streams.command.singleclusterpel; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import com.redis.om.streams.ConsumerGroupStatus; +import com.redis.om.streams.PendingEntry; +import com.redis.om.streams.TopicEntryId; +import com.redis.om.streams.command.serial.PendingEntryQuery; +import com.redis.om.streams.command.serial.SerialTopicConfig; +import com.redis.om.streams.command.serial.TopicManager; +import com.redis.om.streams.exception.InvalidTopicException; +import com.redis.om.streams.exception.TopicOrGroupNotFoundException; +import com.redis.om.streams.utils.Util; + +import redis.clients.jedis.JedisPooled; +import redis.clients.jedis.StreamEntryID; +import redis.clients.jedis.params.XPendingParams; +import redis.clients.jedis.resps.StreamGroupInfo; +import redis.clients.jedis.resps.StreamPendingEntry; + +/** + * A specialized implementation of TopicManager that manages topics in a single cluster with + * Pending Entry List (PEL) functionality. This class provides methods for managing consumer groups, + * retrieving pending entries, and handling stream operations in a single Redis cluster environment. + */ +public class SingleClusterPelTopicManager extends TopicManager { + /** + * Constructs a new SingleClusterPelTopicManager with the specified connection and configuration. + * + * @param connection The JedisPooled connection to the Redis server + * @param config The configuration for the serial topic + */ + public SingleClusterPelTopicManager(JedisPooled connection, SerialTopicConfig config) { + super(connection, config); + } + + /** + * Creates a new topic with the specified configuration and returns a manager for it. + * + * @param connection The JedisPooled connection to the Redis server + * @param config The configuration for the serial topic + * @return A new SingleClusterPelTopicManager instance for the created topic + * @throws InvalidTopicException If the topic configuration is invalid + */ + public static SingleClusterPelTopicManager createTopic(JedisPooled connection, SerialTopicConfig config) + throws InvalidTopicException { + TopicManager.createTopic(connection, config); + return new SingleClusterPelTopicManager(connection, config); + } + + /** + * Retrieves the status of a consumer group including topic size, lag, and pending entry count. + * + * @param groupName The name of the consumer group + * @return A ConsumerGroupStatus object containing status information + */ + public ConsumerGroupStatus getConsumerGroupStatus(String groupName) { + long topicSize = getTopicSize(); + long lag = 0; + try { + lag = getConsumerGroupLag(groupName); + } catch (TopicOrGroupNotFoundException e) { + lag = topicSize; + } + + ConsumerGroupStatus status = new ConsumerGroupStatus(config.getTopicName(), groupName, getPendingEntryCount( + groupName), topicSize, lag); + + return status; + } + + /** + * Gets the count of pending entries for a consumer group. + * + * @param groupName The name of the consumer group + * @return The number of pending entries for the specified consumer group + */ + public long getPendingEntryCount(String groupName) { + return getSingleDBPendingEntryCount(groupName); + } + + /** + * Retrieves pending entries for a consumer group using default query parameters. + * + * @param groupName The name of the consumer group + * @return A list of pending entries for the specified consumer group + */ + public List getPendingEntries(String groupName) { + PendingEntryQuery query = new PendingEntryQuery(); + return getSingleDBPendingEntries(groupName, query.getStartId(), query.getEndId(), query + .getMinIdleTimeMilliSeconds(), query.getCount()); + } + + /** + * Retrieves pending entries for a consumer group within the specified ID range and count limit. + * Uses a default idle time of 0 milliseconds. + * + * @param groupName The name of the consumer group + * @param startId The starting ID for the range of entries to retrieve + * @param endId The ending ID for the range of entries to retrieve + * @param count The maximum number of entries to retrieve + * @return A list of pending entries for the specified consumer group within the given parameters + */ + public List getPendingEntries(String groupName, TopicEntryId startId, TopicEntryId endId, int count) { + int minIdleTimeMilliSeconds = 0; + return getSingleDBPendingEntries(groupName, startId, endId, minIdleTimeMilliSeconds, count); + } + + /** + * Retrieves pending entries for a consumer group within the specified ID range, idle time, and count limit. + * + * @param groupName The name of the consumer group + * @param startId The starting ID for the range of entries to retrieve + * @param endId The ending ID for the range of entries to retrieve + * @param minIdleTimeMilliSeconds The minimum idle time in milliseconds for entries to be included + * @param count The maximum number of entries to retrieve + * @return A list of pending entries for the specified consumer group within the given parameters + */ + public List getPendingEntries(String groupName, TopicEntryId startId, TopicEntryId endId, + int minIdleTimeMilliSeconds, int count) { + return getSingleDBPendingEntries(groupName, startId, endId, minIdleTimeMilliSeconds, count); + } + + /** + * Retrieves pending entries for a consumer group using the parameters specified in the query object. + * + * @param groupName The name of the consumer group + * @param query The query object containing parameters for retrieving pending entries + * @return A list of pending entries for the specified consumer group based on the query parameters + */ + public List getPendingEntries(String groupName, PendingEntryQuery query) { + return getSingleDBPendingEntries(groupName, query.getStartId(), query.getEndId(), query + .getMinIdleTimeMilliSeconds(), query.getCount()); + } + + /** + * Returns the total number of pending entries for the given consumer group in a single database PEL scenario. + * This method calls out to every stream serving the specified group to calculate the total count. + * + * @param groupName The name of the consumer group + * @return The total number of pending entries for the specified consumer group across all streams + */ + public long getSingleDBPendingEntryCount(String groupName) { + long count = 0; + long streamCheckCountForGroup = 0; + long streamNameLastDigits = 0; + for (String streamName : getStreamNames()) { //only streams for this topic will be returned? + try { + streamNameLastDigits = Long.parseLong(streamName.split(":")[4]); + Optional info = connection.xinfoGroups(streamName).stream().filter(group -> group.getName() + .equals(groupName)).findFirst(); + if (info.isPresent()) { + count += info.get().getPending(); + } + + streamCheckCountForGroup++; + } catch (redis.clients.jedis.exceptions.JedisDataException jde) { + //This happens whenever a group has not caught up to total # of possible stream keys + // let's check to see if we are at or beyond the currentStream: + String groupConfigKeyName = config.getConsumerGroupConfigKey(groupName); + String currentStreamFieldName = config.getGroupCurrentStreamField(); + String currentStreamForGroup = connection.hget(groupConfigKeyName, currentStreamFieldName); + // sample stream name: __rsj:topic:stream:Topic-SINGLEDBPEL-1618:0 + // there are 4 segments to the name divided by : tokens + if ((currentStreamForGroup.equals(streamName)) || (streamNameLastDigits < streamCheckCountForGroup)) { + System.out.println( + "streamNameLastDigits: " + streamNameLastDigits + " streamCheckCountForGroup: " + streamCheckCountForGroup + " groupConfigKeyName: " + groupConfigKeyName + " currentStreamForGroup: " + currentStreamForGroup); + throw jde; + } else { + //we can assume we are beyond the Streams currently being consumed by our group + //do nothing but break out of the loop + break; + } + } + } + return count; + } + + /** + * Retrieves pending entries for a consumer group in a single database PEL scenario. + * In this scenario, there is no __PEL__ key created, and pending entries are managed locally. + * + * @param groupName The name of the consumer group + * @param startId The starting ID for the range of entries to retrieve + * @param endId The ending ID for the range of entries to retrieve + * @param minIdleTimeMilliSeconds The minimum idle time in milliseconds for entries to be included + * @param count The maximum number of entries to retrieve + * @return A list of pending entries for the specified consumer group within the given parameters + */ + public List getSingleDBPendingEntries(String groupName, TopicEntryId startId, TopicEntryId endId, + int minIdleTimeMilliSeconds, int count) { + // Don't return more than 1000 messages. + if (count > 1000) { + count = 1000; + } else if (count <= 0) { + return Collections.emptyList(); + } + + if (getTopicSize() == 0) { + return Collections.emptyList(); + } + + // We need the boundary IDs as Doubles so that we can use them for comparisons later. + Double startIdAsDouble = idAsDouble(startId.getStreamEntryId()); + Double endIdAsDouble = idAsDouble(endId.getStreamEntryId()); + + // We need a flag to mark the iteration as complete. + boolean complete = false; + + // Get the server time so that we can calculate pending entry idle time. + long serverTimeMs = Util.getServerTimeMs(connection); + + // Now get up to _count_ entries from the streams: + List pendingEntries = new ArrayList<>(); + + //ot working here to use standard stream API: + // Return the first stream that may contain pending entries. + String streamName = ""; + streamName = getStreamForID(startId); + // ot : this implementation relies on count to optimize the check for pending entries within a single stream: + // it also checks to ensure this group is consuming from the nextStream + // the XPendingParams use MAXIMUM and MINIMUM IDs to check for Pending entries + // however: + // the initial stream in the loop will be as young as the startId demands + // the last stream in the loop will be as young as the count and/or endId demand + String groupConfigKeyName = config.getConsumerGroupConfigKey(groupName); + String currentStreamFieldName = config.getGroupCurrentStreamField(); + String currentStreamForGroup = connection.hget(groupConfigKeyName, currentStreamFieldName); + long streamCheckCountForGroup = 0; + long streamNameLastDigits = 0; + while (streamName != null && pendingEntries.size() < count && !complete) { + //System.out.println("DEBUG:\nminIdleTimeMilliSeconds arg == "+minIdleTimeMilliSeconds+ + // "\n also: if(streamName != null && pendingEntries.size() < count && !complete) IS TRUE"); + List streamEntries = null; + try { + XPendingParams params = new XPendingParams(StreamEntryID.MINIMUM_ID, StreamEntryID.MAXIMUM_ID, count); + streamEntries = connection.xpending(streamName, groupName, params); + } catch (Throwable t) { + if ((currentStreamForGroup.equals(streamName)) || (streamNameLastDigits < streamCheckCountForGroup)) { + throw t; + } else { + break; + } + } + for (StreamPendingEntry entry : streamEntries) { + if (idAsDouble(entry.getID()) > endIdAsDouble) { + complete = true; + //System.out.println("DEBUG:\nentry.getIdleTime(): "+entry.getIdleTime()+"\n"+ + // "minIdleTimeMilliSeconds arg == "+minIdleTimeMilliSeconds+"\n also: if(idAsDouble(entry.getID()) > endIdAsDouble) IS TRUE"); + break; + } + // Add the pending entries one at a time until we reach the endID. + // Filter by idle time if a min idle time is provided + if (entry.getIdleTime() >= minIdleTimeMilliSeconds) { + //if (entryIdleTimeMilliSeconds(entry, serverTimeMs) >= minIdleTimeMilliSeconds) { + //Thought : do we need to / can we construct a PendingEntry from a StreamPendingEntry?? + String consumer = entry.getConsumerName(); + String deliveryTime = String.valueOf(entry.getDeliveredTimes()); + Long idleTimeMs = entry.getIdleTime();//serverTimeMs - Long.valueOf(deliveryTime); + PendingEntry nextPendingEntry = new PendingEntry(entry.getID(), config.getTopicName(), streamName, groupName, + consumer, idleTimeMs, 1L); + pendingEntries.add(nextPendingEntry); + //System.out.println("OTMay3rd: PENDING ENTRIES INSIDE LOOP\n"+pendingEntries); + } + } + streamName = getNextStream(streamName, endId); + //end of if((currentStreamForGroup.equals(streamName))||(streamNameLastDigits < streamCheckCountForGroup)){ + if ((pendingEntries.size() >= count) || (null == streamName)) { + break; // stop the loop - we don't want to count anymore PELs + } + streamNameLastDigits = Long.parseLong(streamName.split(":")[4]); + streamCheckCountForGroup++; + }//loop continues... + return pendingEntries; + } + + /** + * Retrieves pending entries for a consumer group using the parameters specified in the query object. + * This is a convenience method that delegates to the more detailed getSingleDBPendingEntries method. + * + * @param consumerGroupName The name of the consumer group + * @param query The query object containing parameters for retrieving pending entries + * @return A list of pending entries for the specified consumer group based on the query parameters + */ + public List getSingleDBPendingEntries(String consumerGroupName, PendingEntryQuery query) { + return getSingleDBPendingEntries(consumerGroupName, query.getStartId(), query.getEndId(), query + .getMinIdleTimeMilliSeconds(), query.getCount()); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/config/ConfigManager.java b/redis-om-spring/src/main/java/com/redis/om/streams/config/ConfigManager.java new file mode 100644 index 000000000..a11891dda --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/config/ConfigManager.java @@ -0,0 +1,47 @@ +package com.redis.om.streams.config; + +import java.io.File; +import java.nio.file.Paths; +import java.util.Properties; + +import com.redis.om.streams.exception.RedisStreamsException; +import com.redis.om.streams.utils.Util; + +import lombok.Getter; + +/** + * Singleton enum for managing Redis Streams configuration properties. + * This class loads configuration from a properties file specified by the STREAMS_CONFIG_PATH + * system property, or from a default config.properties file in the current directory. + */ +@Getter +public enum ConfigManager { + + /** + * The singleton instance of the ConfigManager. + */ + INSTANCE; + + /** + * The loaded configuration properties for Redis Streams. + */ + private final Properties streamsConfig; + + /** + * Constructor that loads the configuration properties from the specified file. + * The file path is determined by the STREAMS_CONFIG_PATH system property, + * or defaults to config.properties in the current directory. + * + * @throws RedisStreamsException if the configuration file does not exist or is not a valid file + */ + ConfigManager() { + String path = System.getProperty("STREAMS_CONFIG_PATH", String.valueOf(Paths.get("config.properties"))); + + File file = new File(path); + if (!file.exists() || !file.isFile()) + throw new RedisStreamsException( + "STREAMS_CONFIG_PATH needs to be passed as a SYSTEM PROPERTY and must be a valid file"); + + this.streamsConfig = Util.loadPropertiesFile(path); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/config/CustomConfigConstants.java b/redis-om-spring/src/main/java/com/redis/om/streams/config/CustomConfigConstants.java new file mode 100644 index 000000000..8d35f04c3 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/config/CustomConfigConstants.java @@ -0,0 +1,52 @@ +package com.redis.om.streams.config; + +/** + * Constants used for Redis Streams configuration. + * This class provides property names used for configuring Redis Streams behavior. + */ +public class CustomConfigConstants { + /** + * Private constructor to prevent instantiation of this utility class. + */ + private CustomConfigConstants() { + } + + /** + * Property name for the maximum length of Redis streams. + * Defines the maximum number of entries a stream can hold. + */ + public static final String REDIS_STREAMS_MAX_LENGTH = "redis.streams.max.length"; + + /** + * Property name for the maximum allowed streams. + * Defines the maximum number of streams allowed in the system. + */ + public static final String REDIS_STREAMS_MAX_ALLOWED = "redis.streams.max.allowed"; + + /** + * Property name for the pending message count. + * Defines the number of pending messages to retrieve when checking pending entries. + */ + public static final String REDIS_STREAMS_PENDING_MSG_COUNT = "redis.streams.pending.msg.count"; + + // Consumer Properties + + /** + * Property name for the consumer read count. + * Defines how many entries to read at once when consuming from a stream. + */ + public static final String REDIS_STREAMS_CONSUMER_READ_COUNT = "redis.streams.consumer.read.count"; + + /** + * Property name for the consumer block count. + * Defines how long (in milliseconds) to block when reading from a stream if no entries are available. + */ + public static final String REDIS_STREAMS_CONSUMER_BLOCK_COUNT = "redis.streams.consumer.block.count"; + + /** + * Property name for the consumer read acknowledgment setting. + * Defines whether entries should be automatically acknowledged when read by a consumer. + */ + public static final String REDIS_STREAMS_CONSUMER_READ_ACK = "redis.streams.consumer.read.ack"; + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/config/RedisStreamConsumerBeanDefinitionRegistrarSupport.java b/redis-om-spring/src/main/java/com/redis/om/streams/config/RedisStreamConsumerBeanDefinitionRegistrarSupport.java new file mode 100644 index 000000000..5ae0a382b --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/config/RedisStreamConsumerBeanDefinitionRegistrarSupport.java @@ -0,0 +1,97 @@ +package com.redis.om.streams.config; + +import java.lang.annotation.Annotation; + +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.beans.factory.support.BeanNameGenerator; +import org.springframework.context.EnvironmentAware; +import org.springframework.context.ResourceLoaderAware; +import org.springframework.context.annotation.ImportBeanDefinitionRegistrar; +import org.springframework.core.env.Environment; +import org.springframework.core.io.ResourceLoader; +import org.springframework.core.type.AnnotationMetadata; +import org.springframework.lang.NonNull; +import org.springframework.util.Assert; + +/** + * Abstract support class for Redis Stream consumer bean definition registrars. + *

+ * This class provides the basic infrastructure for registering Redis Stream consumer beans + * based on annotations. It implements the Spring {@link ImportBeanDefinitionRegistrar} interface + * to allow for programmatic registration of bean definitions, as well as {@link ResourceLoaderAware} + * and {@link EnvironmentAware} to provide access to the Spring environment and resource loader. + *

+ * Subclasses must implement the {@link #getAnnotation()} method to specify which annotation + * triggers the registration process, and the + * {@link #registerBeanDefinitions(AnnotationMetadata, BeanDefinitionRegistry)} + * method to define the actual registration logic. + * + * @see ImportBeanDefinitionRegistrar + * @see ResourceLoaderAware + * @see EnvironmentAware + */ +public abstract class RedisStreamConsumerBeanDefinitionRegistrarSupport implements ImportBeanDefinitionRegistrar, + ResourceLoaderAware, EnvironmentAware { + /** + * The Spring ResourceLoader, injected by Spring. + */ + @NonNull + private ResourceLoader resourceLoader; + + /** + * The Spring Environment, injected by Spring. + */ + @NonNull + private Environment environment; + + /** + * Sets the ResourceLoader. + * This method is called by Spring to inject the ResourceLoader. + * + * @param resourceLoader the Spring ResourceLoader + */ + public void setResourceLoader(ResourceLoader resourceLoader) { + this.resourceLoader = resourceLoader; + } + + /** + * Sets the Environment. + * This method is called by Spring to inject the Environment. + * + * @param environment the Spring Environment + */ + public void setEnvironment(Environment environment) { + this.environment = environment; + } + + /** + * Registers bean definitions based on the given metadata. + * This method is called by Spring when processing the import annotation. + * It validates the input parameters and delegates to the child class implementation + * if the annotation specified by {@link #getAnnotation()} is present. + * + * @param metadata the annotation metadata of the importing class + * @param registry the bean definition registry + * @param generator the bean name generator + */ + public void registerBeanDefinitions(AnnotationMetadata metadata, BeanDefinitionRegistry registry, + BeanNameGenerator generator) { + Assert.notNull(metadata, "AnnotationMetadata must not be null"); + Assert.notNull(registry, "BeanDefinitionRegistry must not be null"); + Assert.notNull(this.resourceLoader, "ResourceLoader must not be null"); + if (metadata.getAnnotationAttributes(this.getAnnotation().getName()) != null) { + // Delegate to the child class implementation + registerBeanDefinitions(metadata, registry); + } + } + + /** + * Returns the annotation class that triggers the registration process. + * Subclasses must implement this method to specify which annotation + * should be detected to activate the bean registration. + * + * @return the annotation class + */ + protected abstract Class getAnnotation(); + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/config/RedisStreamConsumerRegistrar.java b/redis-om-spring/src/main/java/com/redis/om/streams/config/RedisStreamConsumerRegistrar.java new file mode 100644 index 000000000..ab4e08ff8 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/config/RedisStreamConsumerRegistrar.java @@ -0,0 +1,275 @@ +package com.redis.om.streams.config; + +import java.lang.annotation.Annotation; +import java.util.Map; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider; +import org.springframework.core.type.AnnotationMetadata; +import org.springframework.core.type.filter.AnnotationTypeFilter; +import org.springframework.util.StringUtils; + +import com.redis.om.streams.annotation.EnableRedisStreams; +import com.redis.om.streams.annotation.RedisStreamConsumer; +import com.redis.om.streams.command.noack.NoAckConsumerGroup; +import com.redis.om.streams.command.serial.ConsumerGroup; +import com.redis.om.streams.command.serial.SerialTopicConfig; +import com.redis.om.streams.command.serial.TopicManager; +import com.redis.om.streams.command.singleclusterpel.SingleClusterPelConsumerGroup; + +import jakarta.annotation.PostConstruct; + +/** + * Registrar for Redis Stream consumers that scans for classes annotated with {@link RedisStreamConsumer} + * and registers them as Spring beans. This class is responsible for setting up the necessary infrastructure + * for Redis Streams integration, including topic configurations, consumer groups, and the actual consumer beans. + *

+ * This registrar is activated by the {@link EnableRedisStreams} annotation and will scan the specified base + * packages for consumer classes. + * + * @see EnableRedisStreams + * @see RedisStreamConsumer + * @see RedisStreamConsumerBeanDefinitionRegistrarSupport + */ +public class RedisStreamConsumerRegistrar extends RedisStreamConsumerBeanDefinitionRegistrarSupport { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + /** + * Default constructor for the RedisStreamConsumerRegistrar. + */ + public RedisStreamConsumerRegistrar() { + } + + /** + * Initialization method called after the bean has been constructed. + * Logs an informational message indicating that the registrar has been initialized. + */ + @PostConstruct + private void init() { + logger.info("RedisStreamConsumerRegistrar init"); + } + + /** + * Registers bean definitions for Redis Stream consumers. + * This method scans for classes annotated with {@link RedisStreamConsumer} in the specified base packages + * and registers them as Spring beans along with the necessary infrastructure components. + * + * @param importingClassMetadata metadata about the importing class, which contains the {@link EnableRedisStreams} + * annotation + * @param registry the bean definition registry to register beans with + */ + @Override + public void registerBeanDefinitions(AnnotationMetadata importingClassMetadata, BeanDefinitionRegistry registry) { + Map attributes = importingClassMetadata.getAnnotationAttributes(EnableRedisStreams.class.getName()); + String[] basePackages = (String[]) attributes.get("basePackages"); + + if (basePackages == null || basePackages.length == 0) { + logger.warn("No base packages specified for @EnableRedisStreams, using default package"); + basePackages = new String[] { "com.redis.om.streams" }; + } + + ClassPathScanningCandidateComponentProvider scanner = new ClassPathScanningCandidateComponentProvider(false); + scanner.addIncludeFilter(new AnnotationTypeFilter(RedisStreamConsumer.class)); + + for (String basePackage : basePackages) { + if (StringUtils.hasText(basePackage)) { + Set candidateComponents = scanner.findCandidateComponents(basePackage); + for (BeanDefinition beanDefinition : candidateComponents) { + try { + Class clazz = Class.forName(beanDefinition.getBeanClassName()); + RedisStreamConsumer annotation = clazz.getAnnotation(RedisStreamConsumer.class); + if (annotation != null) { + registerConsumerBeans(clazz, annotation, registry); + } + } catch (ClassNotFoundException e) { + logger.error("Could not load class: {}", beanDefinition.getBeanClassName(), e); + } + } + } + } + } + + /** + * Registers all the necessary beans for a Redis Stream consumer. + * This includes the topic configuration, topic manager, consumer group, and the consumer class itself. + * + * @param consumerClass the class annotated with {@link RedisStreamConsumer} + * @param annotation the {@link RedisStreamConsumer} annotation instance + * @param registry the bean definition registry to register beans with + */ + private void registerConsumerBeans(Class consumerClass, RedisStreamConsumer annotation, + BeanDefinitionRegistry registry) { + String topicName = annotation.topicName(); + String groupName = annotation.groupName(); + String consumerName = annotation.consumerName(); + boolean autoAck = annotation.autoAck(); + boolean cluster = annotation.cluster(); + + logger.info("Registering beans for consumer: {} with topic: {}, group: {}, autoAck: {}, cluster: {}", consumerClass + .getSimpleName(), topicName, groupName, autoAck, cluster); + + // Register SerialTopicConfig + registerSerialTopicConfig(topicName, registry); + + // Register TopicManager + registerTopicManager(topicName, registry); + + // Register ConsumerGroup based on configuration + if (cluster) { + registerSingleClusterPelConsumerGroup(topicName, groupName, consumerClass, registry); + } else if (autoAck) { + registerConsumerGroup(topicName, groupName, consumerClass, registry); + } else { + registerNoAckConsumerGroup(topicName, groupName, consumerClass, registry); + } + + // Register the consumer class itself as a bean + registerConsumerClass(consumerClass, registry); + } + + /** + * Registers a SerialTopicConfig bean for the specified topic. + * This bean contains the configuration for a Redis Stream topic. + * + * @param topicName the name of the Redis Stream topic + * @param registry the bean definition registry to register the bean with + */ + private void registerSerialTopicConfig(String topicName, BeanDefinitionRegistry registry) { + String beanName = topicName + "SerialTopicConfig"; + if (!registry.containsBeanDefinition(beanName)) { + BeanDefinitionBuilder builder = BeanDefinitionBuilder.genericBeanDefinition(SerialTopicConfig.class); + builder.addConstructorArgValue(topicName); + registry.registerBeanDefinition(beanName, builder.getBeanDefinition()); + logger.info("Registered SerialTopicConfig bean: {}", beanName); + } + } + + /** + * Registers a TopicManager bean for the specified topic. + * The TopicManager is responsible for creating and managing the Redis Stream topic. + * + * @param topicName the name of the Redis Stream topic + * @param registry the bean definition registry to register the bean with + */ + private void registerTopicManager(String topicName, BeanDefinitionRegistry registry) { + String beanName = topicName + "TopicManager"; + if (!registry.containsBeanDefinition(beanName)) { + BeanDefinitionBuilder builder = BeanDefinitionBuilder.genericBeanDefinition(TopicManager.class); + builder.setFactoryMethod("createTopic"); + builder.addConstructorArgReference("jedisPooled"); + builder.addConstructorArgReference(topicName + "SerialTopicConfig"); + registry.registerBeanDefinition(beanName, builder.getBeanDefinition()); + logger.info("Registered TopicManager bean: {}", beanName); + } else { + logger.info("TopicManager bean already exists for topic: {}", topicName); + } + } + + /** + * Registers a ConsumerGroup bean for the specified topic and group. + * This consumer group automatically acknowledges messages after processing. + * + * @param topicName the name of the Redis Stream topic + * @param groupName the name of the consumer group + * @param consumerClass the class that will consume messages from this group + * @param registry the bean definition registry to register the bean with + */ + private void registerConsumerGroup(String topicName, String groupName, Class consumerClass, + BeanDefinitionRegistry registry) { + String beanName = groupName + "ConsumerGroup"; + if (!registry.containsBeanDefinition(beanName)) { + BeanDefinitionBuilder builder = BeanDefinitionBuilder.genericBeanDefinition(ConsumerGroup.class); + builder.addConstructorArgReference("jedisPooled"); + builder.addConstructorArgValue(topicName); + builder.addConstructorArgValue(groupName); + registry.registerBeanDefinition(beanName, builder.getBeanDefinition()); + logger.info("Registered ConsumerGroup bean: {}", beanName); + } else { + logger.info("ConsumerGroup bean already exists for group: {}", groupName); + } + } + + /** + * Registers a NoAckConsumerGroup bean for the specified topic and group. + * This consumer group does not automatically acknowledge messages after processing, + * requiring explicit acknowledgment by the consumer. + * + * @param topicName the name of the Redis Stream topic + * @param groupName the name of the consumer group + * @param consumerClass the class that will consume messages from this group + * @param registry the bean definition registry to register the bean with + */ + private void registerNoAckConsumerGroup(String topicName, String groupName, Class consumerClass, + BeanDefinitionRegistry registry) { + String beanName = groupName + "NoAckConsumerGroup"; + if (!registry.containsBeanDefinition(beanName)) { + BeanDefinitionBuilder builder = BeanDefinitionBuilder.genericBeanDefinition(NoAckConsumerGroup.class); + builder.addConstructorArgReference("jedisPooled"); + builder.addConstructorArgValue(topicName); + builder.addConstructorArgValue(groupName); + registry.registerBeanDefinition(beanName, builder.getBeanDefinition()); + logger.info("Registered NoAckConsumerGroup bean: {}", beanName); + } else { + logger.info("NoAckConsumerGroup bean already exists for group: {}", groupName); + } + } + + /** + * Registers a SingleClusterPelConsumerGroup bean for the specified topic and group. + * This consumer group is designed for use in a clustered environment, where multiple + * instances of the application can consume messages from the same stream. + * + * @param topicName the name of the Redis Stream topic + * @param groupName the name of the consumer group + * @param consumerClass the class that will consume messages from this group + * @param registry the bean definition registry to register the bean with + */ + private void registerSingleClusterPelConsumerGroup(String topicName, String groupName, Class consumerClass, + BeanDefinitionRegistry registry) { + String beanName = groupName + "SingleClusterPelConsumerGroup"; + if (!registry.containsBeanDefinition(beanName)) { + BeanDefinitionBuilder builder = BeanDefinitionBuilder.genericBeanDefinition(SingleClusterPelConsumerGroup.class); + builder.addConstructorArgReference("jedisPooled"); + builder.addConstructorArgValue(topicName); + builder.addConstructorArgValue(groupName); + registry.registerBeanDefinition(beanName, builder.getBeanDefinition()); + logger.info("Registered SingleClusterPelConsumerGroup bean: {}", beanName); + } else { + logger.info("SingleClusterPelConsumerGroup bean already exists for group: {}", groupName); + } + } + + /** + * Registers the consumer class itself as a Spring bean. + * This allows the consumer to be autowired into other components. + * + * @param consumerClass the class to register as a bean + * @param registry the bean definition registry to register the bean with + */ + private void registerConsumerClass(Class consumerClass, BeanDefinitionRegistry registry) { + String beanName = StringUtils.uncapitalize(consumerClass.getSimpleName()); + if (!registry.containsBeanDefinition(beanName)) { + BeanDefinitionBuilder builder = BeanDefinitionBuilder.genericBeanDefinition(consumerClass); + registry.registerBeanDefinition(beanName, builder.getBeanDefinition()); + logger.info("Registered consumer class bean: {}", beanName); + } + } + + /** + * Returns the annotation class that this registrar is looking for. + * In this case, it's the {@link EnableRedisStreams} annotation. + * + * @return the annotation class + */ + @Override + protected Class getAnnotation() { + return EnableRedisStreams.class; + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/consumer/RedisStreamsConsumer.java b/redis-om-spring/src/main/java/com/redis/om/streams/consumer/RedisStreamsConsumer.java new file mode 100644 index 000000000..8919e5575 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/consumer/RedisStreamsConsumer.java @@ -0,0 +1,172 @@ +package com.redis.om.streams.consumer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.util.ObjectUtils; + +import com.redis.om.streams.AckMessage; +import com.redis.om.streams.TopicEntry; +import com.redis.om.streams.annotation.RedisStreamConsumer; +import com.redis.om.streams.command.noack.NoAckConsumerGroup; +import com.redis.om.streams.command.serial.ConsumerGroup; +import com.redis.om.streams.command.singleclusterpel.SingleClusterPelConsumerGroup; +import com.redis.om.streams.exception.TopicNotFoundException; + +import jakarta.annotation.PostConstruct; + +/** + * Abstract base class for Redis Streams consumers. + *

+ * This class provides the core functionality for consuming messages from Redis Streams + * and acknowledging them. It works in conjunction with the {@link RedisStreamConsumer} + * annotation to configure how messages are consumed. + *

+ * Concrete implementations should extend this class and implement the necessary + * business logic for processing messages. + */ +public abstract class RedisStreamsConsumer implements ApplicationContextAware { + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + private ApplicationContext applicationContext; + + /** + * Sets the Spring application context. + *

+ * This method is called by the Spring container to inject the application context. + * It's used internally to access beans and other Spring resources. + * + * @param applicationContext the Spring application context + * @throws BeansException if an error occurs during bean instantiation + */ + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } + + /** + * Initializes the consumer after construction. + *

+ * This method is called automatically after the bean is constructed and + * dependencies are injected. + */ + @PostConstruct + private void init() { + logger.info("{} init", getClass().getSimpleName()); + } + + /** + * Gets the appropriate consumer group implementation based on the annotation configuration. + *

+ * This method determines which type of consumer group to use based on the + * autoAck and cluster settings in the {@link RedisStreamConsumer} annotation. + * + * @return the consumer group implementation + */ + private Object getConsumerGroup() { + RedisStreamConsumer annotation = getClass().getAnnotation(RedisStreamConsumer.class); + assert annotation != null; + String beanName; + if (annotation.autoAck()) { + if (annotation.cluster()) { + beanName = annotation.groupName() + "SingleClusterPelConsumerGroup"; + return applicationContext.getBean(beanName, SingleClusterPelConsumerGroup.class); + } else { + beanName = annotation.groupName() + "ConsumerGroup"; + return applicationContext.getBean(beanName, ConsumerGroup.class); + } + } else { + beanName = annotation.groupName() + "NoAckConsumerGroup"; + return applicationContext.getBean(beanName, NoAckConsumerGroup.class); + } + } + + /** + * Consumes a message from the Redis Stream. + *

+ * This method reads a message from the Redis Stream based on the configuration + * specified in the {@link RedisStreamConsumer} annotation. It handles different + * consumer group implementations based on whether auto-acknowledgment is enabled + * and whether Redis is deployed as a cluster. + * + * @return the consumed message as a {@link TopicEntry}, or null if no message could be consumed + */ + protected TopicEntry consume() { + RedisStreamConsumer annotation = getClass().getAnnotation(RedisStreamConsumer.class); + assert annotation != null; + if (annotation.autoAck()) { + if (annotation.cluster()) { + try { + SingleClusterPelConsumerGroup consumerGroup = (SingleClusterPelConsumerGroup) getConsumerGroup(); + return consumerGroup.consume(getConsumerName(annotation.consumerName())); + } catch (TopicNotFoundException e) { + logger.error(e.getMessage(), e); + } + } else { + ConsumerGroup consumerGroup = (ConsumerGroup) getConsumerGroup(); + try { + return consumerGroup.consume(getConsumerName(annotation.consumerName())); + } catch (TopicNotFoundException e) { + logger.error(e.getMessage(), e); + } + } + } else { + NoAckConsumerGroup consumerGroup = (NoAckConsumerGroup) getConsumerGroup(); + try { + return consumerGroup.consume(getConsumerName(annotation.consumerName())); + } catch (TopicNotFoundException e) { + logger.error(e.getMessage(), e); + } + } + return null; + } + + /** + * Acknowledges a message that has been processed. + *

+ * This method acknowledges that a message has been successfully processed, + * which removes it from the pending entries list (PEL) in Redis Streams. + * The behavior depends on the configuration in the {@link RedisStreamConsumer} annotation. + * If auto-acknowledgment is disabled, this method will log a debug message and return false. + * + * @param topicEntry the message to acknowledge + * @return true if the message was successfully acknowledged, false otherwise + */ + protected boolean acknowledge(TopicEntry topicEntry) { + if (topicEntry == null) { + logger.debug("Skipping acknowledge because TopicEntry is null."); + return false; + } + RedisStreamConsumer annotation = getClass().getAnnotation(RedisStreamConsumer.class); + assert annotation != null; + if (annotation.autoAck()) { + if (annotation.cluster()) { + SingleClusterPelConsumerGroup consumerGroup = (SingleClusterPelConsumerGroup) getConsumerGroup(); + return consumerGroup.acknowledge(new AckMessage(topicEntry)); + } else { + ConsumerGroup consumerGroup = (ConsumerGroup) getConsumerGroup(); + return consumerGroup.acknowledge(new AckMessage(topicEntry)); + } + } else { + logger.debug("Ignoring acknowledge of topic {}", topicEntry); + } + return false; + } + + /** + * Gets the consumer name to use for Redis Streams operations. + *

+ * If a consumer name is specified in the annotation, that name is used. + * Otherwise, the simple name of the implementing class is used as the consumer name. + * + * @param annotationConsumerName the consumer name from the annotation + * @return the consumer name to use + */ + private String getConsumerName(String annotationConsumerName) { + return ObjectUtils.isEmpty(annotationConsumerName) ? getClass().getSimpleName() : annotationConsumerName; + } + +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/exception/InvalidMessageException.java b/redis-om-spring/src/main/java/com/redis/om/streams/exception/InvalidMessageException.java new file mode 100644 index 000000000..63c14360e --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/exception/InvalidMessageException.java @@ -0,0 +1,16 @@ +package com.redis.om.streams.exception; + +/** + * Exception thrown when an invalid message is encountered in Redis Streams operations. + * This typically occurs when a message format is incorrect or cannot be processed. + */ +public class InvalidMessageException extends TopicException { + /** + * Constructs a new InvalidMessageException with the specified detail message. + * + * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method) + */ + public InvalidMessageException(String message) { + super(message); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/exception/InvalidTopicException.java b/redis-om-spring/src/main/java/com/redis/om/streams/exception/InvalidTopicException.java new file mode 100644 index 000000000..b57580e30 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/exception/InvalidTopicException.java @@ -0,0 +1,16 @@ +package com.redis.om.streams.exception; + +/** + * Exception thrown when an invalid topic is specified in Redis Streams operations. + * This typically occurs when a topic name is malformed or does not meet the required criteria. + */ +public class InvalidTopicException extends TopicException { + /** + * Constructs a new InvalidTopicException with the specified detail message. + * + * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method) + */ + public InvalidTopicException(String message) { + super(message); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/exception/ProducerTimeoutException.java b/redis-om-spring/src/main/java/com/redis/om/streams/exception/ProducerTimeoutException.java new file mode 100644 index 000000000..52e639342 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/exception/ProducerTimeoutException.java @@ -0,0 +1,16 @@ +package com.redis.om.streams.exception; + +/** + * Exception thrown when a producer operation times out in Redis Streams. + * This typically occurs when a message production operation takes longer than the configured timeout period. + */ +public class ProducerTimeoutException extends TopicException { + /** + * Constructs a new ProducerTimeoutException with the specified detail message. + * + * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method) + */ + public ProducerTimeoutException(String message) { + super(message); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/exception/RedisStreamsException.java b/redis-om-spring/src/main/java/com/redis/om/streams/exception/RedisStreamsException.java new file mode 100644 index 000000000..f5e36224a --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/exception/RedisStreamsException.java @@ -0,0 +1,18 @@ +package com.redis.om.streams.exception; + +/** + * Base exception class for Redis Streams related exceptions. + * This exception is thrown for general errors that occur during Redis Streams operations. + * + * TODO: RedisStreamsException should inherit from Exception. + */ +public class RedisStreamsException extends RuntimeException { + /** + * Constructs a new RedisStreamsException with the specified detail message. + * + * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method) + */ + public RedisStreamsException(String message) { + super(message); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/exception/TopicException.java b/redis-om-spring/src/main/java/com/redis/om/streams/exception/TopicException.java new file mode 100644 index 000000000..968ca0e54 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/exception/TopicException.java @@ -0,0 +1,18 @@ +package com.redis.om.streams.exception; + +/** + * Base exception class for Redis Streams topic-related exceptions. + * This class serves as the parent for more specific topic-related exceptions + * and is thrown when an error occurs related to Redis Stream topics. + */ +public class TopicException extends Exception { + + /** + * Constructs a new TopicException with the specified detail message. + * + * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method) + */ + public TopicException(String message) { + super(message); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/exception/TopicNotFoundException.java b/redis-om-spring/src/main/java/com/redis/om/streams/exception/TopicNotFoundException.java new file mode 100644 index 000000000..698e4dee2 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/exception/TopicNotFoundException.java @@ -0,0 +1,16 @@ +package com.redis.om.streams.exception; + +/** + * Exception thrown when a specified Redis Stream topic cannot be found. + * This typically occurs when attempting to perform operations on a non-existent topic. + */ +public class TopicNotFoundException extends TopicException { + /** + * Constructs a new TopicNotFoundException with the specified error message. + * + * @param errorMessage the detail message (which is saved for later retrieval by the {@link #getMessage()} method) + */ + public TopicNotFoundException(String errorMessage) { + super(errorMessage); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/exception/TopicOrGroupNotFoundException.java b/redis-om-spring/src/main/java/com/redis/om/streams/exception/TopicOrGroupNotFoundException.java new file mode 100644 index 000000000..6667423bf --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/exception/TopicOrGroupNotFoundException.java @@ -0,0 +1,18 @@ +package com.redis.om.streams.exception; + +/** + * Exception thrown when either a Redis Stream topic or a consumer group cannot be found. + * This typically occurs when attempting to perform operations on a non-existent topic + * or when trying to use a consumer group that doesn't exist. + */ +public class TopicOrGroupNotFoundException extends TopicException { + + /** + * Constructs a new TopicOrGroupNotFoundException with the specified detail message. + * + * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method) + */ + public TopicOrGroupNotFoundException(String message) { + super(message); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/utils/StreamsCommand.java b/redis-om-spring/src/main/java/com/redis/om/streams/utils/StreamsCommand.java new file mode 100644 index 000000000..073b369c1 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/utils/StreamsCommand.java @@ -0,0 +1,179 @@ +package com.redis.om.streams.utils; + +import static com.redis.om.streams.config.CustomConfigConstants.*; +import static java.util.Collections.singletonMap; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import com.redis.om.streams.config.ConfigManager; + +import redis.clients.jedis.JedisPooled; +import redis.clients.jedis.StreamEntryID; +import redis.clients.jedis.params.XPendingParams; +import redis.clients.jedis.params.XReadGroupParams; +import redis.clients.jedis.params.XReadParams; +import redis.clients.jedis.resps.StreamEntry; +import redis.clients.jedis.resps.StreamPendingEntry; +import redis.clients.jedis.resps.StreamPendingSummary; + +/** + * Utility class for executing Redis Streams commands. + * Provides methods for subscribing to streams, retrieving pending entries, + * and building parameters for Redis Streams operations. + */ +public class StreamsCommand { + + /** + * Factory method to create a new StreamsCommand instance. + * + * @param jedis the JedisPooled connection to use + * @return a new StreamsCommand instance + */ + public static StreamsCommand getInstance(JedisPooled jedis) { + return new StreamsCommand(jedis); + } + + /** + * The JedisPooled connection used for Redis operations. + */ + private final JedisPooled jedis; + + /** + * Parameters for XPENDING commands. + */ + private final XPendingParams pendingParams; + + /** + * Parameters for XREADGROUP commands. + */ + private final XReadGroupParams groupParams; + + /** + * Parameters for XREAD commands. + */ + private final XReadParams readParams; + + /** + * Constructs a new StreamsCommand with the specified JedisPooled connection. + * Initializes command parameters from the configuration. + * + * @param jedis the JedisPooled connection to use + */ + public StreamsCommand(JedisPooled jedis) { + Properties config = ConfigManager.INSTANCE.getStreamsConfig(); + + this.jedis = jedis; + this.pendingParams = buildXPendingParams(Integer.parseInt(String.valueOf(config.getOrDefault( + REDIS_STREAMS_PENDING_MSG_COUNT, String.valueOf(0))))); + this.groupParams = buildXReadGroupParams(Integer.parseInt(String.valueOf(config.getOrDefault( + REDIS_STREAMS_CONSUMER_BLOCK_COUNT, String.valueOf(0)))), Integer.parseInt(String.valueOf(config.getOrDefault( + REDIS_STREAMS_CONSUMER_READ_COUNT, String.valueOf(0)))), Boolean.parseBoolean(String.valueOf(config + .getOrDefault(REDIS_STREAMS_CONSUMER_READ_ACK, Boolean.valueOf("false"))))); + this.readParams = buildXReadParams(Integer.parseInt(String.valueOf(config.getOrDefault( + REDIS_STREAMS_CONSUMER_BLOCK_COUNT, String.valueOf(0)))), Integer.parseInt(String.valueOf(config.getOrDefault( + REDIS_STREAMS_CONSUMER_READ_COUNT, String.valueOf(0))))); + } + + /** + * Subscribes to messages from the specified stream. + * Uses the XREAD command to read entries from the stream. + * + * @param streamName the name of the stream to subscribe to + * @return a list of stream entries + */ + public List>> subscribeMessage(String streamName) { + Map streamEntryIDMap = singletonMap(streamName, StreamEntryID.UNRECEIVED_ENTRY); + + return this.jedis.xread(readParams, streamEntryIDMap); + } + + /** + * Subscribes to messages from the specified stream as part of a consumer group. + * Uses the XREADGROUP command to read entries from the stream. + * + * @param streamName the name of the stream to subscribe to + * @param groupName the name of the consumer group + * @param consumerNames the names of the consumers + * @return a list of stream entries + */ + public List>> subscribeMessage(String streamName, String groupName, + String... consumerNames) { + Map streamEntryIDMap = singletonMap(streamName, StreamEntryID.UNRECEIVED_ENTRY); + + return this.jedis.xreadGroup(groupName, Arrays.toString(consumerNames), groupParams, streamEntryIDMap); + } + + /** + * Gets a summary of pending messages for a consumer group. + * Executes the XPENDING command: XPENDING key group + * + * @param streamName the name of the stream + * @param groupName the name of the consumer group + * @return a summary of pending messages + */ + public StreamPendingSummary getPendingSummary(String streamName, String groupName) { + return this.jedis.xpending(streamName, groupName); + } + + /** + * Gets detailed information about pending messages for a consumer group. + * Executes the XPENDING command: XPENDING key group [[IDLE min-idle-time] start end count [consumer]] + * + * @param streamName the name of the stream + * @param groupName the name of the consumer group + * @return a list of pending entries + */ + public List getPendingEntry(String streamName, String groupName) { + return this.jedis.xpending(streamName, groupName, pendingParams); + } + + /** + * Builds parameters for the XPENDING command. + * + * @param count the maximum number of entries to return + * @return the XPendingParams object, or null if count is not positive + */ + private XPendingParams buildXPendingParams(int count) { + XPendingParams params = null; + + if (count > 0) { + params = new XPendingParams().start(StreamEntryID.MINIMUM_ID).end(StreamEntryID.MAXIMUM_ID).count(count); + } + + return params; + } + + /** + * Builds parameters for the XREADGROUP command. + * + * @param block the number of milliseconds to block waiting for new entries + * @param count the maximum number of entries to return + * @param ack whether to automatically acknowledge the entries + * @return the XReadGroupParams object + */ + private XReadGroupParams buildXReadGroupParams(int block, int count, boolean ack) { + XReadGroupParams params; + + if (ack) { + params = new XReadGroupParams().block(block).count(count); + } else { + params = new XReadGroupParams().block(block).count(count).noAck(); + } + + return params; + } + + /** + * Builds parameters for the XREAD command. + * + * @param block the number of milliseconds to block waiting for new entries + * @param count the maximum number of entries to return + * @return the XReadParams object + */ + private XReadParams buildXReadParams(int block, int count) { + return new XReadParams().block(block).count(count); + } +} diff --git a/redis-om-spring/src/main/java/com/redis/om/streams/utils/Util.java b/redis-om-spring/src/main/java/com/redis/om/streams/utils/Util.java new file mode 100644 index 000000000..bb7e9be55 --- /dev/null +++ b/redis-om-spring/src/main/java/com/redis/om/streams/utils/Util.java @@ -0,0 +1,172 @@ +package com.redis.om.streams.utils; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.redis.om.streams.command.serial.SerialTopicConfig; +import com.redis.om.streams.exception.RedisStreamsException; + +import redis.clients.jedis.Connection; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPooled; +import redis.clients.jedis.StreamEntryID; +import redis.clients.jedis.exceptions.JedisDataException; +import redis.clients.jedis.util.Pool; + +/** + * Utility class providing helper methods for Redis Streams operations. + * Contains methods for configuration loading, time calculations, stream management, + * and name validation. + */ +public class Util { + + /** + * Maximum allowed length for stream names. + */ + private static final short MAX_NAME_LENGTH = 300; + + /** + * Pattern for validating stream names. Valid names contain only alphanumeric characters, + * underscores, dots, and hyphens. + */ + static Pattern validNamePattern = Pattern.compile("[\\w._-]+"); + + /** + * Loads configuration properties from the specified file. + * + * @param path the path to the file containing the configuration properties; may not be null + * @return the loaded properties; never null + * @throws RedisStreamsException if the file cannot be read or is not a valid properties file + */ + public static Properties loadPropertiesFile(String path) { + Properties properties = new Properties(); + + try (InputStream inputStream = Files.newInputStream(Path.of(path))) { + properties.load(inputStream); + } catch (IOException e) { + throw new RedisStreamsException(path + " is not a valid properties file. " + e); + } + return properties; + } + + /** + * Gets the current server time in milliseconds. + * This is a workaround for the absence of the time() command in JedisPooled. + * + * @param conn the JedisPooled connection to use + * @return the current server time in milliseconds + */ + public static long getServerTimeMs(JedisPooled conn) { + Pool pool = conn.getPool(); + try (Connection connection = pool.getResource()) { + Jedis jedis = new Jedis(connection); + List times = jedis.time(); + if (times.size() >= 1) { + String micros = times.get(1); + if (micros.length() >= 3) { + return Long.valueOf(times.get(0) + micros.substring(0, 3)); + } else { + return Long.valueOf(times.get(0)) + 1000; + } + } else { + return 0; + } + } + } + + /** + * Calculates the expiry time in seconds based on the current server time + * plus the specified retention time. + * + * @param connection the JedisPooled connection to use + * @param retentionTimeSeconds the number of seconds to retain the data + * @return the calculated expiry time in seconds (unix timestamp) + */ + public static long getExpiryAtSeconds(JedisPooled connection, long retentionTimeSeconds) { + return (getServerTimeMs(connection) / 1000) + retentionTimeSeconds; + } + + /** + * Extracts the stream ID from a stream name. + * The stream name is expected to be in a format with 5 colon-separated elements, + * with the last element being the stream ID. + * + * @param streamName the name of the stream + * @return the extracted stream ID + * @throws RedisStreamsException if the stream name format is invalid or the ID cannot be parsed + */ + public static long streamIdFromStreamName(String streamName) { + String[] elements = streamName.split(":"); + if (elements.length != 5) { + throw new RedisStreamsException("Invalid stream name: " + streamName); + } + long result; + try { + result = Long.valueOf(elements[4]); + } catch (Exception e) { + throw new RedisStreamsException("Cannot parse stream id from stream name: " + streamName + ". Source: " + e + .getMessage()); + } + return result; + } + + /** + * Ensures that a stream with the specified name exists and has the specified TTL. + * If the stream doesn't exist, it creates it. If it already exists, it sets the TTL. + * + * @param connection the JedisPooled connection to use + * @param streamName the name of the stream to ensure exists + * @param ttl the time-to-live in seconds for the stream + */ + public static void ensureStreamWithTTLExists(JedisPooled connection, String streamName, long ttl) { + try { + connection.xgroupCreate(streamName, SerialTopicConfig.BLANK_GROUP_FOR_CREATE, StreamEntryID.LAST_ENTRY, true); + } catch (JedisDataException e) { + // If a JedisDataException is thrown, then the stream already exists. + return; + } + + connection.expire(streamName, ttl); + } + + /** + * Validates that a name matches the allowed pattern for stream names. + * + * @param name the name to validate + * @return true if the name is valid, false otherwise + */ + public static boolean nameValid(String name) { + Matcher matcher = validNamePattern.matcher(name); + if (matcher.find()) { + String match = matcher.group(0); + return match.equals(name); + } + return false; + } + + /** + * Validates that a name has the correct length (greater than 0 and less than or equal to MAX_NAME_LENGTH). + * + * @param name the name to validate + * @return true if the name has the correct length, false otherwise + */ + public static boolean nameCorrectLength(String name) { + return ((name.length() > 0) && (name.length() <= MAX_NAME_LENGTH)); + } + + /** + * Validates a name for use in Redis Streams. + * This method is a placeholder for future implementation. + * + * @param name the name to validate + */ + public static void validateName(String name) { + // This method is currently empty and appears to be a placeholder for future implementation + } +} diff --git a/redis-om-spring/src/main/resources/functions/redisSessions.lua b/redis-om-spring/src/main/resources/functions/redisSessions.lua new file mode 100644 index 000000000..f0f9e87b6 --- /dev/null +++ b/redis-om-spring/src/main/resources/functions/redisSessions.lua @@ -0,0 +1,58 @@ +#!lua name=redisSessions + +local function touch_key(keys, args) + -- check if an expiration was passed into the function. + if args[1] then + redis.call('EXPIRE', keys[1], args[1]) + end + + local size = redis.call('MEMORY', 'USAGE', keys[1], 'SAMPLES', 0) + redis.call('HSET', keys[1], 'sessionSize', size) + return size +end + +local function read_key(keys, args) + local str = redis.call('HGET', keys[1], 'maxInactiveInterval') + + if str == false then + return nil + end + + local expiry = tonumber(str) + redis.call('EXPIRE', keys[1], expiry) + return redis.call('HGETALL', keys[1]) + +end + +local function read_locally_cached_entry(keys, args) + local lastModifiedTimeStr = redis.call('HGET', keys[1], 'lastModifiedTime') + + if lastModifiedTimeStr == false then + return {false, nil} + end + + local lastModifiedTime = tonumber(lastModifiedTimeStr) + + redis.call('HSET', keys[1], "lastAccessedTime", args[2]) + + if lastModifiedTime > tonumber(args[1]) then + local body = redis.call("HGETALL", keys[1]) + return {false, body} + end + + return {true, nil} +end + +local function reserve_structs(keys, args) + local keyExists = redis.call('EXISTS', keys[1]) + if keyExists == 0 then + redis.call('TOPK.RESERVE', keys[1], args[1]) + return true + end + return false +end + +redis.register_function('touch_key', touch_key) +redis.register_function('read_key', read_key) +redis.register_function('read_locally_cached_entry', read_locally_cached_entry) +redis.register_function('reserve_structs', reserve_structs) \ No newline at end of file diff --git a/redis-om-spring/src/main/resources/scripts/lag.script b/redis-om-spring/src/main/resources/scripts/lag.script new file mode 100644 index 000000000..51d2745e2 --- /dev/null +++ b/redis-om-spring/src/main/resources/scripts/lag.script @@ -0,0 +1,56 @@ + local function group_last_dlvr_id(key, group) + local xinfo = redis.call("XINFO", "GROUPS", key) + for i, entry in ipairs(xinfo) do + if (entry[2] == group) then + return entry[8] + end + end + end + + -- The first ID location depends on Redis version (6 vs. 7) + -- In 7, it's stored at "recorded-first-entry-id". + -- In 6, it's stored in a literal "first-entry" struct. + local function first_id(key) + local xinfo = redis.call("XINFO", "STREAM", KEYS[1]) + if (xinfo[13] == "recorded-first-entry-id") then + return xinfo[14] + else + return xinfo[12][1] + end + end + + local function sid2d(sid) + local tmp = string.gsub(sid, "-", ".") + return tonumber(tmp) + end + + -- Just to raise an error if key or group don't exist + redis.call("XPENDING", KEYS[1], ARGV[1]) + + local entries_read_key = string.format("__entries_read_{%s}_%s", KEYS[1], ARGV[1]) + local entries_read = tonumber(redis.call("GET", entries_read_key)) + if (entries_read == nil) then + -- compare last-dlvr-id with stream first and last IDs + -- 1. if smaller than min, entries read is 0 + -- 2. if equal to last, entries read is xlen + -- 3. otherwise, we can't tell so return nil + local xinfo = redis.call("XINFO", "STREAM", KEYS[1]) + local stream_first_id = sid2d(first_id(KEYS[1])) + local stream_last_id = sid2d(xinfo[8]) + local last_dlvr_id = sid2d(group_last_dlvr_id(KEYS[1], ARGV[1])) + if (last_dlvr_id == stream_last_id) then + local xlen = redis.call("XLEN", KEYS[1]) + entries_read = xlen + elseif (last_dlvr_id < stream_first_id) then + entries_read = 0 + else + entries_read = -1 + end + end + + if (entries_read == -1) then + return nil + end + + local xlen = redis.call("XLEN", KEYS[1]) + return xlen - entries_read diff --git a/redis-om-spring/src/main/resources/scripts/readKey.lua b/redis-om-spring/src/main/resources/scripts/readKey.lua new file mode 100644 index 000000000..42b1ccbaf --- /dev/null +++ b/redis-om-spring/src/main/resources/scripts/readKey.lua @@ -0,0 +1,9 @@ +local str = redis.call('HGET', KEYS[1], 'maxInactiveInterval') + +if str == false then + return nil +end + +local expiry = tonumber(str) +redis.call('EXPIRE', KEYS[1], expiry) +return redis.call('HGETALL', KEYS[1]) diff --git a/redis-om-spring/src/main/resources/scripts/readLocallyCachedEntry.lua b/redis-om-spring/src/main/resources/scripts/readLocallyCachedEntry.lua new file mode 100644 index 000000000..61a69ac1a --- /dev/null +++ b/redis-om-spring/src/main/resources/scripts/readLocallyCachedEntry.lua @@ -0,0 +1,24 @@ +-- Reads a cache entry that is resident within the memory of the calling application +-- params KEYS[1] - key being interrogated +-- params ARGV[1] - the lastModifiedTime according to the client +-- params ARGV[2] - the new lastAccessedTime to write +-- return [false, nil] if cacheEntry is not present +-- return [false, array] if cacheEntry is present, but has been modified since the provided last modified lastModifiedTime +-- return [true, nil] if cacheEntry is present and has not been modified since the provided lastModifiedTime + +local lastModifiedTimeStr = redis.call('HGET', KEYS[1], 'lastModifiedTime') + +if lastModifiedTimeStr == false then + return {false, nil} +end + +local lastModifiedTime = tonumber(lastModifiedTimeStr) + +redis.call('HSET', KEYS[1], "lastAccessedTime", ARGV[2]) + +if lastModifiedTime > tonumber(ARGV[1]) then + local body = redis.call("HGETALL", KEYS[1], '$') + return {false, body} +end + +return {true, nil} \ No newline at end of file diff --git a/redis-om-spring/src/main/resources/scripts/reserveStructs.lua b/redis-om-spring/src/main/resources/scripts/reserveStructs.lua new file mode 100644 index 000000000..0316483b8 --- /dev/null +++ b/redis-om-spring/src/main/resources/scripts/reserveStructs.lua @@ -0,0 +1,6 @@ +local keyExists = redis.call('EXISTS', KEYS[1]) +if keyExists == 0 then + redis.call('TOPK.RESERVE', KEYS[1], ARGV[1]) + return true +end +return false \ No newline at end of file diff --git a/redis-om-spring/src/main/resources/scripts/serial_advance_consumer.script b/redis-om-spring/src/main/resources/scripts/serial_advance_consumer.script new file mode 100644 index 000000000..d1bc08396 --- /dev/null +++ b/redis-om-spring/src/main/resources/scripts/serial_advance_consumer.script @@ -0,0 +1,45 @@ +local topic_config_key = KEYS[1] +local group_config_key = KEYS[2] +local stream_index_key = KEYS[3] +local full_streams_key = KEYS[4] + +local group_name = ARGV[1] +local empty_stream_name = ARGV[2] +local empty_stream_id = ARGV[3] +local retention_time_seconds = ARGV[4] +local current_stream_field_name = ARGV[5] +local topic_config_stream_id_field = ARGV[6] +local stream_base_name = ARGV[7] + +-- Get the name of the current stream for the consumer group +local current_latest_stream_name = redis.call("HGET", group_config_key, current_stream_field_name) + +if (current_latest_stream_name ~= empty_stream_name) then + return current_latest_stream_name + +-- Otherwise, it's time to advance the stream. Advance only if the current stream +-- has been marked full and there's a new stream to advance to. +else + local full = redis.call("ZSCORE", full_streams_key, empty_stream_name) + + -- If the current stream has not been marked full, return the current stream. + if (full == nil) then + return current_latest_stream_name + -- The current stream is full, so try to get the next one. + else + -- If the current stream id is less than the latest stream id, then there's a new stream + local latest_stream_id = redis.call("HGET", topic_config_key, topic_config_stream_id_field) + if (latest_stream_id == nil or latest_stream_id == false) then + return current_latest_stream_name + else + if (tonumber(latest_stream_id) > tonumber(empty_stream_id)) then + local subsequent_stream_id = tostring(tonumber(empty_stream_id) + 1) + local next_current_stream = stream_base_name .. subsequent_stream_id + redis.call("HSET", group_config_key, current_stream_field_name, next_current_stream) + return next_current_stream + else + return current_latest_stream_name + end + end + end +end diff --git a/redis-om-spring/src/main/resources/scripts/serial_get_next_stream.script b/redis-om-spring/src/main/resources/scripts/serial_get_next_stream.script new file mode 100644 index 000000000..2ad01dadf --- /dev/null +++ b/redis-om-spring/src/main/resources/scripts/serial_get_next_stream.script @@ -0,0 +1,34 @@ + local topic_config_key = KEYS[1] + local stream_index_key = KEYS[2] + local full_streams_key = KEYS[3] + local topic_config_stream_id_field = ARGV[1] + local stream_base_name = ARGV[2] + local retention_time_seconds = ARGV[3] + local producer_stream_id = tonumber(ARGV[4]) + local stream_expiry_at = ARGV[5] + + -- Determine whether a current stream id exists in the config + local current_stream_id = redis.call("HGET", topic_config_key, topic_config_stream_id_field) + + -- If it's null, no stream name has ever been created. + -- So, set the current stream id to 0. Then add the stream name to the list of stream names. + if ((current_stream_id == false) or (current_stream_id == nil)) then + local stream_id = "0" + redis.call("HSET", topic_config_key, topic_config_stream_id_field, stream_id) + local next_stream_name = stream_base_name .. stream_id + redis.call("ZADD", stream_index_key, stream_expiry_at, next_stream_name) + return next_stream_name + else + -- We need to perform this check to confirm that we should create a new stream name. + local latest_stream_name = stream_base_name .. current_stream_id + local full = redis.call("ZSCORE", full_streams_key, latest_stream_name) + + if ((full ~= nil) and (producer_stream_id == tonumber(current_stream_id))) then + local next_stream_id = redis.call("HINCRBY", topic_config_key, topic_config_stream_id_field, 1) + local next_stream_name = stream_base_name .. next_stream_id + redis.call("ZADD", stream_index_key, stream_expiry_at, next_stream_name) + return next_stream_name + else + return latest_stream_name + end + end \ No newline at end of file diff --git a/redis-om-spring/src/main/resources/scripts/serial_publish_message.script b/redis-om-spring/src/main/resources/scripts/serial_publish_message.script new file mode 100644 index 000000000..e546a24f2 --- /dev/null +++ b/redis-om-spring/src/main/resources/scripts/serial_publish_message.script @@ -0,0 +1,40 @@ +local stream_key = KEYS[1] +local retention_time_seconds = tonumber(ARGV[1]) +local max_stream_length = tonumber(ARGV[2]) +local stream_switch_ttl = tonumber(ARGV[3]) + +local function slice(data, start) + local table_copy = {} + for i, element in ipairs(data) do + if (i >= start) then + table.insert(table_copy, element) + end + end + + return table_copy +end + +local length = redis.call("XLEN", stream_key) +if (tonumber(length) > (tonumber(max_stream_length) - 1)) then + return nil +end + +-- Get the current TTL +local ttl = tonumber(redis.call("TTL", stream_key)) + +-- If the TTL is greater than 0, then the key exists and the TTL was set at some point +if (ttl >= 0) then + -- If the TTL is less than the stream switch value, then we return nil + -- so that the caller will select the next stream + if (ttl <= stream_switch_ttl) then + return nil + end +else + -- In this case, the stream was created at some point and is now expired. Therefore, return nil + return nil +end + +local message = slice(ARGV, 4) +local id = redis.call("XADD", stream_key, "*", unpack(message)) + +return id \ No newline at end of file diff --git a/redis-om-spring/src/main/resources/scripts/xack.script b/redis-om-spring/src/main/resources/scripts/xack.script new file mode 100644 index 000000000..42d670ebe --- /dev/null +++ b/redis-om-spring/src/main/resources/scripts/xack.script @@ -0,0 +1,4 @@ + local pel_key = string.format("__PEL_{%s}_%s", KEYS[1], ARGV[1]) + local response = redis.call("XDEL", pel_key, ARGV[2]) + redis.call("XACK", KEYS[1], ARGV[1], ARGV[2]) + return response diff --git a/redis-om-spring/src/main/resources/scripts/xgroup_setid.script b/redis-om-spring/src/main/resources/scripts/xgroup_setid.script new file mode 100644 index 000000000..91a3ab86a --- /dev/null +++ b/redis-om-spring/src/main/resources/scripts/xgroup_setid.script @@ -0,0 +1,31 @@ + -- Just to raise an error if key or group doesn't exist + redis.call("XPENDING", KEYS[1], ARGV[1]) + + -- First call the actual command because we want an early-exit if it fails (try to set the ID to be smaller than + -- last consecutive ack) + local res = redis.call("XGROUP", "SETID", KEYS[1], ARGV[1], ARGV[2]) + + local entries_read_key = string.format("__entries_read_{%s}_%s", KEYS[1], ARGV[1]) + local entries_read_key_ttl = redis.call("TTL", entries_read_key) + + local ttl = redis.call("TTL", KEYS[1]) + local xinfo = redis.call("XINFO", "STREAM", KEYS[1]) + + if (xinfo[8] == ARGV[2]) then + local xlen = redis.call("XLEN", KEYS[1]) + redis.call("SET", entries_read_key, xlen, "KEEPTTL") + else + redis.call("SET", entries_read_key, "-1", "KEEPTTL") + end + if (ttl > 0 and entries_read_key_ttl < 0) then + redis.call("EXPIRE", entries_read_key, ttl) + end + + local last_dlvr_id_key = string.format("__last_dlvr_id_{%s}_%s", KEYS[1], ARGV[1]) + local last_dlvr_id_key_ttl = redis.call("TTL", last_dlvr_id_key) + redis.call("SET", last_dlvr_id_key, ARGV[2], "KEEPTTL") + if (ttl > 0 and last_dlvr_id_key_ttl < 0) then + redis.call("EXPIRE", last_dlvr_id_key, ttl) + end + + return res diff --git a/redis-om-spring/src/main/resources/scripts/xpending.script b/redis-om-spring/src/main/resources/scripts/xpending.script new file mode 100644 index 000000000..2278c5a4f --- /dev/null +++ b/redis-om-spring/src/main/resources/scripts/xpending.script @@ -0,0 +1,15 @@ + local stream_key = KEYS[1] + local group_name = ARGV[1] + local min_id = ARGV[2] + local max_id = ARGV[3] + local max_entries = ARGV[4] + + if ((max_entries == nil) or (tonumber(max_entries)) > 1000) then + max_entries = "1000" + end + + -- Just to raise an error if key or group do not exist + -- redis.call("XPENDING", stream_key, group_name) + + local pel_key = string.format("__PEL_{%s}_%s", stream_key, group_name) + return redis.call("XRANGE", pel_key, min_id, max_id, "COUNT", max_entries) \ No newline at end of file diff --git a/redis-om-spring/src/main/resources/scripts/xreadgroup.script b/redis-om-spring/src/main/resources/scripts/xreadgroup.script new file mode 100644 index 000000000..34a25912f --- /dev/null +++ b/redis-om-spring/src/main/resources/scripts/xreadgroup.script @@ -0,0 +1,178 @@ + local function mstime() + local res = redis.call("TIME") + local s = tonumber(res[1]) + local us = tonumber(res[2]) + return math.floor(s * 1000 + us / 1000) + end + + local function group_last_dlvr_id(key, group) + local xinfo = redis.call("XINFO", "GROUPS", key) + for i, entry in ipairs(xinfo) do + if (entry[2] == group) then + return entry[8] + end + end + end + + local function mysplit(inputstr, sep) + if sep == nil then + sep = "%s" + end + local t={} + for str in string.gmatch(inputstr, "([^"..sep.."]+)") do + table.insert(t, str) + end + return t + end + + local function sid_cmp(a, b) + local a_pairs = mysplit(a, "-") + local a_ms = tonumber(a_pairs[1]) + local a_seq = tonumber(a_pairs[2]) + local b_pairs = mysplit(b, "-") + local b_ms = tonumber(b_pairs[1]) + local b_seq = tonumber(b_pairs[2]) + if (a_ms > b_ms) then + return 1 + elseif (b_ms > a_ms) then + return -1 + elseif (a_seq > b_seq) then + return 1 + elseif (b_seq > a_seq) then + return -1 + end + return 0 + end + + -- The first ID location depends on Redis version (6 vs. 7) + local function first_id(xinfo) + if (tonumber(xinfo[2]) == 0) then + return "0-0" + elseif (xinfo[13] == "recorded-first-entry-id") then + return xinfo[14] + else + return xinfo[12][1] + end + end + + -- See comment above first_id + local function last_id(xinfo) + if (tonumber(xinfo[2]) == 0) then + return "0-0" + elseif (xinfo[7] == "last-generated-id") then + return xinfo[7] + else + return xinfo[14][1] + end + end + + -- Just to raise an error if key or group don't exist + redis.call("XPENDING", KEYS[1], ARGV[1]) + + local ttl = redis.call("TTL", KEYS[1]) + local xlen = redis.call("XLEN", KEYS[1]) + local xinfo = redis.call("XINFO", "STREAM", KEYS[1]) + + local entries_read_key = string.format("__entries_read_{%s}_%s", KEYS[1], ARGV[1]) + local entries_read_key_ttl = redis.call("TTL", entries_read_key) + -- Can be nil in case entries_read_key doesn't exist + local entries_read = tonumber(redis.call("GET", entries_read_key)) + if (entries_read == nil) then + -- edge case, stream is empty, the group lag is 0 because there's + -- nothing more to read + if (xlen == 0) then + entries_read = xlen + else + -- compare last-dlvr-id with stream first and last IDs + -- 1. if smaller than min, entries read is 0 + -- 2. if equal to last, entries read is xlen + -- 3. otherwise, we can't tell so return nil + local stream_first_id = first_id(xinfo) + local stream_last_id = last_id(xinfo) + local actual_last_dlvr_id = group_last_dlvr_id(KEYS[1], ARGV[1]) + if (actual_last_dlvr_id == stream_last_id) then + entries_read = xlen + elseif (sid_cmp(actual_last_dlvr_id, stream_first_id) < 0) then + entries_read = 0 + else + entries_read = -1 + end + end + + redis.call("SET", entries_read_key, entries_read) + if (ttl > 0 and entries_read_key_ttl < 0) then + redis.call("EXPIRE", entries_read_key, ttl) + end + end + + local pel_key = string.format("__PEL_{%s}_%s", KEYS[1], ARGV[1]) + local pel_key_ttl = redis.call("TTL", pel_key) + + -- We need to make sure that XREADGROUP moves the CG's last_dlvr_id (to minimize over-processing of messages + -- in case of a failover) + -- We do that by using a helper key (last_dlvr_id_key) which is updated with the maximal message ID returned + -- by XREADGROUP. + -- Before executing any XREADGROUP we need to set the last_dlvr_id of the group to whatever is in last_dlvr_id_key + -- (the block of code below) + -- last_dlvr_id_key is a stream, and we use the most recent entry as the CG's last_dlvr_id. + -- It is a stream because we want to keep this value monotonically increasing, as the stream's head is. + -- Here's an exmaple of how it could break monotonicity if it were a string: + -- R2 XREADGROUP 1-0 (last_dlvr_id_key is 1-0 in R2) + -- R2 XREADGROUP 2-0 (last_dlvr_id_key is 2-0 in R2) + -- R2 XACK 1-0 2-0 (last consecutive ack is 2-0 in R2) + -- R1 XREADGROUP 1-0 (last_dlvr_id_key is 1-0 in R1) + -- R1->R2 + -- Now, R2's last_dlvr_id_key is 1-0 but last consecutive ack os 2-0. If the XREADGROUP script runs on R2 again, + -- XGROUP SETID will raise an error ("ERR This ID is in the initial set of consecutive messages ...") + local last_dlvr_id_key = string.format("__last_dlvr_id_{%s}_%s", KEYS[1], ARGV[1]) + local last_dlvr_id_key_ttl = redis.call("TTL", last_dlvr_id_key) + -- Using redis.pcall because XINFO returns an error if key doesn't exist + local last_dlvr_id_key_xinfo = redis.pcall("XINFO", "STREAM", last_dlvr_id_key) + if last_dlvr_id_key_xinfo['err'] == nil then + -- XINFO[8] is the last-generated-id + redis.call("XGROUP", "SETID", KEYS[1], ARGV[1], last_dlvr_id_key_xinfo[8]) + end + + local res = redis.call("XREADGROUP", "GROUP", ARGV[1], ARGV[2], "COUNT", 1, "STREAMS", KEYS[1], ">") + + if (res == false) then + return res + end + + -- We always use COUNT 1 but having a loop is mre bulletproof + local now = mstime() + local last_dlvr_id = "" + for i, entry in ipairs(res[1][2]) do + last_dlvr_id = entry[1] + redis.call("XADD", pel_key, last_dlvr_id, "consumer", ARGV[2], "delivery_time", now) + + if (ttl > 0 and pel_key_ttl < 0) then + pel_key_ttl = ttl + redis.call("EXPIRE", pel_key, ttl) + end + + if (entries_read ~= -1) then + entries_read = entries_read + 1 + end + end + + if (entries_read == -1) then + if (xinfo[8] == last_dlvr_id) then + -- CG has caught up with the stream's tip + redis.call("SET", entries_read_key, xlen, "KEEPTTL") + if (ttl > 0 and entries_read_key_ttl < 0) then + redis.call("EXPIRE", entries_read_key, ttl) + end + end + else + redis.call("SET", entries_read_key, entries_read, "KEEPTTL") + if (ttl > 0 and entries_read_key_ttl < 0) then + redis.call("EXPIRE", entries_read_key, ttl) + end + end + + redis.call("XADD", last_dlvr_id_key, "MAXLEN", "1", last_dlvr_id, "f", "v") + if (ttl > 0 and last_dlvr_id_key_ttl < 0) then + redis.call("EXPIRE", last_dlvr_id_key, ttl) + end + return res \ No newline at end of file diff --git a/redis-om-spring/src/main/resources/scripts/xreadgroup_noack.script b/redis-om-spring/src/main/resources/scripts/xreadgroup_noack.script new file mode 100644 index 000000000..47603da5c --- /dev/null +++ b/redis-om-spring/src/main/resources/scripts/xreadgroup_noack.script @@ -0,0 +1,184 @@ +-- invoke this script thusly: eval xreadgroup_noack.script 1 streamname groupname consumerID +-- KEYS[1] is the name of the stream that holds the next interesting entry for the calling thread +-- ARGV[1] == groupname for current thread ARGV[2] == consumername in group for current thread + + local function mstime() + local res = redis.call("TIME") + local s = tonumber(res[1]) + local us = tonumber(res[2]) + return math.floor(s * 1000 + us / 1000) + end + + local function group_last_dlvr_id(key, group) + local xinfo = redis.call("XINFO", "GROUPS", key) + for i, entry in ipairs(xinfo) do + if (entry[2] == group) then + return entry[8] + end + end + end + + local function mysplit(inputstr, sep) + if sep == nil then + sep = "%s" + end + local t={} + for str in string.gmatch(inputstr, "([^"..sep.."]+)") do + table.insert(t, str) + end + return t + end + + local function sid_cmp(a, b) + local a_pairs = mysplit(a, "-") + local a_ms = tonumber(a_pairs[1]) + local a_seq = tonumber(a_pairs[2]) + local b_pairs = mysplit(b, "-") + local b_ms = tonumber(b_pairs[1]) + local b_seq = tonumber(b_pairs[2]) + if (a_ms > b_ms) then + return 1 + elseif (b_ms > a_ms) then + return -1 + elseif (a_seq > b_seq) then + return 1 + elseif (b_seq > a_seq) then + return -1 + end + return 0 + end + + -- The first ID location depends on Redis version (6 vs. 7) + local function first_id(xinfo) + if (tonumber(xinfo[2]) == 0) then + return "0-0" + elseif (xinfo[13] == "recorded-first-entry-id") then + return xinfo[14] + else + return xinfo[12][1] + end + end + + -- See comment above first_id + local function last_id(xinfo) + if (tonumber(xinfo[2]) == 0) then + return "0-0" + elseif (xinfo[7] == "last-generated-id") then + return xinfo[7] + else + return xinfo[14][1] + end + end + + -- Just to raise an error if key or group don't exist + -- KEYS[1] is the name of the stream that holds the next interesting entry for the calling thread + redis.call("XPENDING", KEYS[1], ARGV[1]) + + local ttl = redis.call("TTL", KEYS[1]) + local xlen = redis.call("XLEN", KEYS[1]) + local xinfo = redis.call("XINFO", "STREAM", KEYS[1]) + + local entries_read_key = string.format("__entries_read_{%s}_%s", KEYS[1], ARGV[1]) + local entries_read_key_ttl = redis.call("TTL", entries_read_key) + -- Can be nil in case entries_read_key doesn't exist + local entries_read = tonumber(redis.call("GET", entries_read_key)) + if (entries_read == nil) then + -- edge case, stream is empty, the group lag is 0 because there's + -- nothing more to read + if (xlen == 0) then + entries_read = xlen + else + -- compare last-dlvr-id with stream first and last IDs + -- 1. if smaller than min, entries read is 0 + -- 2. if equal to last, entries read is xlen + -- 3. otherwise, we can't tell so return nil + local stream_first_id = first_id(xinfo) + local stream_last_id = last_id(xinfo) + local actual_last_dlvr_id = group_last_dlvr_id(KEYS[1], ARGV[1]) + if (actual_last_dlvr_id == stream_last_id) then + entries_read = xlen + elseif (sid_cmp(actual_last_dlvr_id, stream_first_id) < 0) then + entries_read = 0 + else + entries_read = -1 + end + end + + redis.call("SET", entries_read_key, entries_read) + if (ttl > 0 and entries_read_key_ttl < 0) then + redis.call("EXPIRE", entries_read_key, ttl) + end + end + +-- not needed with noack: local pel_key = string.format("__PEL_{%s}_%s", KEYS[1], ARGV[1]) +-- not needed with noack: local pel_key_ttl = redis.call("TTL", pel_key) + + -- We need to make sure that XREADGROUP moves the CG's last_dlvr_id (to minimize over-processing of messages + -- in case of a failover) + -- We do that by using a helper key (last_dlvr_id_key) which is updated with the maximal message ID returned + -- by XREADGROUP. + -- Before executing any XREADGROUP we need to set the last_dlvr_id of the group to whatever is in last_dlvr_id_key + -- (the block of code below) + -- last_dlvr_id_key is a stream, and we use the most recent entry as the CG's last_dlvr_id. + -- It is a stream because we want to keep this value monotonically increasing, as the stream's head is. + -- Here's an exmaple of how it could break monotonicity if it were a string: + -- R2 XREADGROUP 1-0 (last_dlvr_id_key is 1-0 in R2) + -- R2 XREADGROUP 2-0 (last_dlvr_id_key is 2-0 in R2) + -- R2 XACK 1-0 2-0 (last consecutive ack is 2-0 in R2) + -- R1 XREADGROUP 1-0 (last_dlvr_id_key is 1-0 in R1) + -- R1->R2 + -- Now, R2's last_dlvr_id_key is 1-0 but last consecutive ack os 2-0. If the XREADGROUP script runs on R2 again, + -- XGROUP SETID will raise an error ("ERR This ID is in the initial set of consecutive messages ...") + local last_dlvr_id_key = string.format("__last_dlvr_id_{%s}_%s", KEYS[1], ARGV[1]) + local last_dlvr_id_key_ttl = redis.call("TTL", last_dlvr_id_key) + -- Using redis.pcall because XINFO returns an error if key doesn't exist + local last_dlvr_id_key_xinfo = redis.pcall("XINFO", "STREAM", last_dlvr_id_key) + if last_dlvr_id_key_xinfo['err'] == nil then + -- XINFO[8] is the last-generated-id + redis.call("XGROUP", "SETID", KEYS[1], ARGV[1], last_dlvr_id_key_xinfo[8]) + end + -- ARGV[1] == groupname for current thread ARGV[2] == consumername in group for current thread + local res = redis.call("XREADGROUP", "GROUP", ARGV[1], ARGV[2], "COUNT", 1, "NOACK", "STREAMS", KEYS[1], ">") + + -- if res == false there is no processing to do - no entry is available now + if (res == false) then + return res + end + + -- We always use COUNT 1 but having a loop is mre bulletproof + local now = mstime() + local last_dlvr_id = "" + for i, entry in ipairs(res[1][2]) do + last_dlvr_id = entry[1] +-- not needed with noack: redis.call("XADD", pel_key, last_dlvr_id, "consumer", ARGV[2], "delivery_time", now) + +-- not needed with noack: if (ttl > 0 and pel_key_ttl < 0) then +-- not needed with noack: pel_key_ttl = ttl +-- not needed with noack: redis.call("EXPIRE", pel_key, ttl) +-- not needed with noack: end + + if (entries_read ~= -1) then + entries_read = entries_read + 1 + end + end + + if (entries_read == -1) then + if (xinfo[8] == last_dlvr_id) then + -- CG has caught up with the stream's tip + redis.call("SET", entries_read_key, xlen, "KEEPTTL") + if (ttl > 0 and entries_read_key_ttl < 0) then + redis.call("EXPIRE", entries_read_key, ttl) + end + end + else + redis.call("SET", entries_read_key, entries_read, "KEEPTTL") + if (ttl > 0 and entries_read_key_ttl < 0) then + redis.call("EXPIRE", entries_read_key, ttl) + end + end + + redis.call("XADD", last_dlvr_id_key, "MAXLEN", "1", last_dlvr_id, "f", "v") + if (ttl > 0 and last_dlvr_id_key_ttl < 0) then + redis.call("EXPIRE", last_dlvr_id_key, ttl) + end + return res \ No newline at end of file diff --git a/redis-om-spring/src/main/resources/scripts/xreadgroup_singleClusterPel.script b/redis-om-spring/src/main/resources/scripts/xreadgroup_singleClusterPel.script new file mode 100644 index 000000000..74a33a12e --- /dev/null +++ b/redis-om-spring/src/main/resources/scripts/xreadgroup_singleClusterPel.script @@ -0,0 +1,184 @@ +-- invoke this script thusly: eval xreadgroup_singleRegionPEL.script 1 streamname groupname consumerID +-- KEYS[1] is the name of the stream that holds the next interesting entry for the calling thread +-- ARGV[1] == groupname for current thread ARGV[2] == consumername in group for current thread + + local function mstime() + local res = redis.call("TIME") + local s = tonumber(res[1]) + local us = tonumber(res[2]) + return math.floor(s * 1000 + us / 1000) + end + + local function group_last_dlvr_id(key, group) + local xinfo = redis.call("XINFO", "GROUPS", key) + for i, entry in ipairs(xinfo) do + if (entry[2] == group) then + return entry[8] + end + end + end + + local function mysplit(inputstr, sep) + if sep == nil then + sep = "%s" + end + local t={} + for str in string.gmatch(inputstr, "([^"..sep.."]+)") do + table.insert(t, str) + end + return t + end + + local function sid_cmp(a, b) + local a_pairs = mysplit(a, "-") + local a_ms = tonumber(a_pairs[1]) + local a_seq = tonumber(a_pairs[2]) + local b_pairs = mysplit(b, "-") + local b_ms = tonumber(b_pairs[1]) + local b_seq = tonumber(b_pairs[2]) + if (a_ms > b_ms) then + return 1 + elseif (b_ms > a_ms) then + return -1 + elseif (a_seq > b_seq) then + return 1 + elseif (b_seq > a_seq) then + return -1 + end + return 0 + end + + -- The first ID location depends on Redis version (6 vs. 7) + local function first_id(xinfo) + if (tonumber(xinfo[2]) == 0) then + return "0-0" + elseif (xinfo[13] == "recorded-first-entry-id") then + return xinfo[14] + else + return xinfo[12][1] + end + end + + -- See comment above first_id + local function last_id(xinfo) + if (tonumber(xinfo[2]) == 0) then + return "0-0" + elseif (xinfo[7] == "last-generated-id") then + return xinfo[7] + else + return xinfo[14][1] + end + end + + -- Just to raise an error if key or group don't exist + -- KEYS[1] is the name of the stream that holds the next interesting entry for the calling thread + redis.call("XPENDING", KEYS[1], ARGV[1]) + + local ttl = redis.call("TTL", KEYS[1]) + local xlen = redis.call("XLEN", KEYS[1]) + local xinfo = redis.call("XINFO", "STREAM", KEYS[1]) + + local entries_read_key = string.format("__entries_read_{%s}_%s", KEYS[1], ARGV[1]) + local entries_read_key_ttl = redis.call("TTL", entries_read_key) + -- Can be nil in case entries_read_key doesn't exist + local entries_read = tonumber(redis.call("GET", entries_read_key)) + if (entries_read == nil) then + -- edge case, stream is empty, the group lag is 0 because there's + -- nothing more to read + if (xlen == 0) then + entries_read = xlen + else + -- compare last-dlvr-id with stream first and last IDs + -- 1. if smaller than min, entries read is 0 + -- 2. if equal to last, entries read is xlen + -- 3. otherwise, we can't tell so return nil + local stream_first_id = first_id(xinfo) + local stream_last_id = last_id(xinfo) + local actual_last_dlvr_id = group_last_dlvr_id(KEYS[1], ARGV[1]) + if (actual_last_dlvr_id == stream_last_id) then + entries_read = xlen + elseif (sid_cmp(actual_last_dlvr_id, stream_first_id) < 0) then + entries_read = 0 + else + entries_read = -1 + end + end + + redis.call("SET", entries_read_key, entries_read) + if (ttl > 0 and entries_read_key_ttl < 0) then + redis.call("EXPIRE", entries_read_key, ttl) + end + end + +-- not needed with singleRegionPEL: local pel_key = string.format("__PEL_{%s}_%s", KEYS[1], ARGV[1]) +-- not needed with singleRegionPEL: local pel_key_ttl = redis.call("TTL", pel_key) + + -- We need to make sure that XREADGROUP moves the CG's last_dlvr_id (to minimize over-processing of messages + -- in case of a failover) + -- We do that by using a helper key (last_dlvr_id_key) which is updated with the maximal message ID returned + -- by XREADGROUP. + -- Before executing any XREADGROUP we need to set the last_dlvr_id of the group to whatever is in last_dlvr_id_key + -- (the block of code below) + -- last_dlvr_id_key is a stream, and we use the most recent entry as the CG's last_dlvr_id. + -- It is a stream because we want to keep this value monotonically increasing, as the stream's head is. + -- Here's an exmaple of how it could break monotonicity if it were a string: + -- R2 XREADGROUP 1-0 (last_dlvr_id_key is 1-0 in R2) + -- R2 XREADGROUP 2-0 (last_dlvr_id_key is 2-0 in R2) + -- R2 XACK 1-0 2-0 (last consecutive ack is 2-0 in R2) + -- R1 XREADGROUP 1-0 (last_dlvr_id_key is 1-0 in R1) + -- R1->R2 + -- Now, R2's last_dlvr_id_key is 1-0 but last consecutive ack os 2-0. If the XREADGROUP script runs on R2 again, + -- XGROUP SETID will raise an error ("ERR This ID is in the initial set of consecutive messages ...") + local last_dlvr_id_key = string.format("__last_dlvr_id_{%s}_%s", KEYS[1], ARGV[1]) + local last_dlvr_id_key_ttl = redis.call("TTL", last_dlvr_id_key) + -- Using redis.pcall because XINFO returns an error if key doesn't exist + local last_dlvr_id_key_xinfo = redis.pcall("XINFO", "STREAM", last_dlvr_id_key) + if last_dlvr_id_key_xinfo['err'] == nil then + -- XINFO[8] is the last-generated-id + redis.call("XGROUP", "SETID", KEYS[1], ARGV[1], last_dlvr_id_key_xinfo[8]) + end + -- ARGV[1] == groupname for current thread ARGV[2] == consumername in group for current thread + local res = redis.call("XREADGROUP", "GROUP", ARGV[1], ARGV[2], "COUNT", 1, "STREAMS", KEYS[1], ">") + + -- if res == false there is no processing to do - no entry is available now + if (res == false) then + return res + end + + -- We always use COUNT 1 but having a loop is mre bulletproof + local now = mstime() + local last_dlvr_id = "" + for i, entry in ipairs(res[1][2]) do + last_dlvr_id = entry[1] +-- not needed with singleRegionPEL: redis.call("XADD", pel_key, last_dlvr_id, "consumer", ARGV[2], "delivery_time", now) + +-- not needed with singleRegionPEL: if (ttl > 0 and pel_key_ttl < 0) then +-- not needed with singleRegionPEL: pel_key_ttl = ttl +-- not needed with singleRegionPEL: redis.call("EXPIRE", pel_key, ttl) +-- not needed with singleRegionPEL: end + + if (entries_read ~= -1) then + entries_read = entries_read + 1 + end + end + + if (entries_read == -1) then + if (xinfo[8] == last_dlvr_id) then + -- CG has caught up with the stream's tip + redis.call("SET", entries_read_key, xlen, "KEEPTTL") + if (ttl > 0 and entries_read_key_ttl < 0) then + redis.call("EXPIRE", entries_read_key, ttl) + end + end + else + redis.call("SET", entries_read_key, entries_read, "KEEPTTL") + if (ttl > 0 and entries_read_key_ttl < 0) then + redis.call("EXPIRE", entries_read_key, ttl) + end + end + + redis.call("XADD", last_dlvr_id_key, "MAXLEN", "1", last_dlvr_id, "f", "v") + if (ttl > 0 and last_dlvr_id_key_ttl < 0) then + redis.call("EXPIRE", last_dlvr_id_key, ttl) + end + return res \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 9b1fe07f7..fbc7cffa1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -13,3 +13,4 @@ include 'demos:roms-permits' include 'demos:roms-vectorizers' include 'demos:roms-vss-movies' include 'demos:roms-vss' +include 'demos:roms-streams'