From 4322ea02e48f208cff7cddc0095517799ddb6bad Mon Sep 17 00:00:00 2001 From: "musab.bozkurt" Date: Sun, 4 Feb 2024 19:59:47 +0300 Subject: [PATCH] Redis changes are implemented --- README.md | 29 ++++++- docker-compose.yml | 14 ++++ pom.xml | 22 +++++ .../api/controller/RedisController.java | 25 ++++++ .../livedataservice/config/CacheConfig.java | 61 ++++++++++++++ .../config/RedissonConfig.java | 22 +++++ .../data/entity/RedisHashData.java | 43 ++++++++++ .../repository/RedisHashDataRepository.java | 8 ++ .../RedisKeyExpiredEventListenerImpl.java | 23 ++++++ .../service/RedisHashService.java | 16 ++++ .../service/RedisTokenStoreService.java | 10 +++ .../service/impl/RedisHashServiceImpl.java | 39 +++++++++ .../impl/RedisTokenStoreServiceImpl.java | 82 +++++++++++++++++++ .../livedataservice/utils/RedisConstants.java | 13 +++ src/main/resources/application.yml | 24 +++++- .../controller/TutorialControllerTest.java | 11 +++ src/test/resources/application.yml | 24 +++++- 17 files changed, 462 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/mb/livedataservice/api/controller/RedisController.java create mode 100644 src/main/java/com/mb/livedataservice/config/CacheConfig.java create mode 100644 src/main/java/com/mb/livedataservice/config/RedissonConfig.java create mode 100644 src/main/java/com/mb/livedataservice/data/entity/RedisHashData.java create mode 100644 src/main/java/com/mb/livedataservice/data/repository/RedisHashDataRepository.java create mode 100644 src/main/java/com/mb/livedataservice/queue/RedisKeyExpiredEventListenerImpl.java create mode 100644 src/main/java/com/mb/livedataservice/service/RedisHashService.java create mode 100644 src/main/java/com/mb/livedataservice/service/RedisTokenStoreService.java create mode 100644 src/main/java/com/mb/livedataservice/service/impl/RedisHashServiceImpl.java create mode 100644 src/main/java/com/mb/livedataservice/service/impl/RedisTokenStoreServiceImpl.java create mode 100644 src/main/java/com/mb/livedataservice/utils/RedisConstants.java diff --git a/README.md b/README.md index c9b29be..e2f6461 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@
  • How To Run And Test Application
  • How To Run And Test Application with Dockerfile (OPTIONAL)
  • How To Run And Test Application with docker-compose.yml (OPTIONAL)
    • +
  • Redis Commands
    • +
  • References
    • @@ -45,7 +47,7 @@ - `Micrometer` dependencies were added to track the logs easily - `Testcontainers` dependencies were added for integration tests - `docker-compose.yml` contains `Grafana`, `Prometheus` and `Zipkin` to track metrics, `Kafka` for event-driven - architecture + architecture, `Redis` for caching - `Actuator`: http://localhost:8080/actuator - `Kafka UI`: http://localhost:9091/ - `Grafana` @@ -117,8 +119,31 @@ ------- +### Redis + +* The following command returns all matched data by `'keyPattern:*'` pattern + * `redis-cli --scan --pattern 'keyPattern:*'` + +* The following command deletes all matched data by `'keyPattern:*'` pattern + * `redis-cli KEYS 'keyPattern:*' | xargs redis-cli DEL` + +* The following command finds `TYPE` in redis with `KEY` + * `TYPE key` -> `TYPE xxx:hashedIdOrSomethingElse` + +* The following commands search by `TYPE` + + * for `"string" TYPE`: `get key` + * for `"hash" TYPE`: `hgetall key` + * for `"list" TYPE`: `lrange key 0 -1` + * for `"set" TYPE`: `smembers key` + * for `"zset" TYPE`: `zrange key 0 -1 withScores` + +* RedisInsight: + ### References - [Metrics Made Easy Via Spring Actuator, Docker, Prometheus, and Grafana](https://www.youtube.com/watch?v=Utv7MWgNTvI) - https://prometheus.io/docs/prometheus/latest/installation/#volumes-bind-mount -- [Spring Boot Rest Controller Unit Test with @WebMvcTest](https://www.bezkoder.com/spring-boot-webmvctest/) \ No newline at end of file +- [Spring Boot Rest Controller Unit Test with @WebMvcTest](https://www.bezkoder.com/spring-boot-webmvctest/) +- [Redis Commands](https://auth0.com/blog/introduction-to-redis-install-cli-commands-and-data-types/) +- [Running RedisInsight using Docker Compose](https://collabnix.com/running-redisinsight-using-docker-compose/) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 31db0ea..23ab8e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,6 +71,20 @@ services: ports: - "9411:9411" + redis: + image: redis + restart: always + container_name: redis + ports: + - "6378:6379" # Default port is 6379 + + redisinsight: + image: redislabs/redisinsight:latest + restart: always + container_name: redisinsight + ports: + - '8001:8001' + volumes: zookeeper_data: driver: local diff --git a/pom.xml b/pom.xml index db24a3f..d9795f9 100644 --- a/pom.xml +++ b/pom.xml @@ -33,6 +33,22 @@ spring-boot-starter-data-jpa + + org.springframework.boot + spring-boot-starter-data-redis + + + + org.springframework.session + spring-session-data-redis + + + + org.redisson + redisson-spring-boot-starter + 3.26.0 + + com.h2database h2 @@ -142,6 +158,12 @@ test + + org.testcontainers + kafka + test + + diff --git a/src/main/java/com/mb/livedataservice/api/controller/RedisController.java b/src/main/java/com/mb/livedataservice/api/controller/RedisController.java new file mode 100644 index 0000000..d155a27 --- /dev/null +++ b/src/main/java/com/mb/livedataservice/api/controller/RedisController.java @@ -0,0 +1,25 @@ +package com.mb.livedataservice.api.controller; + +import com.mb.livedataservice.data.entity.RedisHashData; +import com.mb.livedataservice.service.RedisHashService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RestController; + +@Slf4j +@RestController +@RequiredArgsConstructor +public class RedisController { + + private final RedisHashService redisHashService; + + /** + * Create RedisHashData + */ + @PostMapping("/redis-hash") + public RedisHashData createRedisHashData() { + log.info("Received a request to create RedisHashData. createRedisHashData."); + return redisHashService.save(RedisHashData.builder().destination("hello_world").build()); + } +} \ No newline at end of file diff --git a/src/main/java/com/mb/livedataservice/config/CacheConfig.java b/src/main/java/com/mb/livedataservice/config/CacheConfig.java new file mode 100644 index 0000000..db8cd83 --- /dev/null +++ b/src/main/java/com/mb/livedataservice/config/CacheConfig.java @@ -0,0 +1,61 @@ +package com.mb.livedataservice.config; + +import com.mb.livedataservice.utils.RedisConstants; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.cache.RedisCacheConfiguration; +import org.springframework.data.redis.cache.RedisCacheManager; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisKeyValueAdapter; +import org.springframework.data.redis.core.RedisOperations; +import org.springframework.data.redis.repository.configuration.EnableRedisRepositories; +import org.springframework.session.data.redis.config.ConfigureRedisAction; + +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; + +@Configuration +@EnableCaching +@AutoConfigureAfter(RedisAutoConfiguration.class) +@ConditionalOnClass({RedisOperations.class, RedisConnectionFactory.class, RedisCacheConfiguration.class}) +@EnableRedisRepositories(enableKeyspaceEvents = RedisKeyValueAdapter.EnableKeyspaceEvents.ON_STARTUP) +public class CacheConfig { + + @Bean(name = "cacheManager") + @ConditionalOnMissingBean(name = "cacheManager") + public RedisCacheManager cacheManager(RedisConnectionFactory connectionFactory) { + RedisCacheConfiguration expireIn1Day = RedisCacheConfiguration.defaultCacheConfig().entryTtl(Duration.ofDays(1)); + + Map cacheConfigurations = new HashMap<>(); + + cacheConfigurations.put(RedisConstants.CACHE_KEY, expireIn1Day); + + return RedisCacheManager.RedisCacheManagerBuilder + .fromConnectionFactory(connectionFactory) + .withInitialCacheConfigurations(cacheConfigurations) + .build(); + } + + /* + * If the Redis client is protected, add this config bean. Otherwise, this bean can be removed. + * + * However, if you can run any commands in redis, please run the following command to enable org.springframework.data.redis.core.RedisKeyExpiredEvent + * command -> redis-cli config set notify-keyspace-events xE + * + * notify-keyspace-events should be xE. + * To get the value of notify-keyspace-events run this -> config get "notify-keyspace-events" + * + * This means that Spring Session cannot configure Redis Keyspace events for you. + * To disable the automatic configuration add ConfigureRedisAction.NO_OP as a bean. + * */ + @Bean + public static ConfigureRedisAction configureRedisAction() { + return ConfigureRedisAction.NO_OP; + } +} diff --git a/src/main/java/com/mb/livedataservice/config/RedissonConfig.java b/src/main/java/com/mb/livedataservice/config/RedissonConfig.java new file mode 100644 index 0000000..9cfad78 --- /dev/null +++ b/src/main/java/com/mb/livedataservice/config/RedissonConfig.java @@ -0,0 +1,22 @@ +package com.mb.livedataservice.config; + +import org.redisson.Redisson; +import org.redisson.api.RedissonClient; +import org.redisson.config.Config; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class RedissonConfig { + + @Bean + @ConditionalOnProperty(value = "redisson.enabled", havingValue = "true") + public RedissonClient redissonClient(@Value("${redisson.url}") String address) { + Config config = new Config(); + config.useSingleServer().setAddress(address); + + return Redisson.create(config); + } +} \ No newline at end of file diff --git a/src/main/java/com/mb/livedataservice/data/entity/RedisHashData.java b/src/main/java/com/mb/livedataservice/data/entity/RedisHashData.java new file mode 100644 index 0000000..8bfe830 --- /dev/null +++ b/src/main/java/com/mb/livedataservice/data/entity/RedisHashData.java @@ -0,0 +1,43 @@ +package com.mb.livedataservice.data.entity; + +import jakarta.persistence.Id; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.data.redis.core.RedisHash; +import org.springframework.data.redis.core.TimeToLive; +import org.springframework.data.redis.core.index.Indexed; + +import java.util.UUID; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@RedisHash(value = "RedisHashData") +public class RedisHashData { + + @Id + @Builder.Default + private String id = UUID.randomUUID().toString(); + + @Indexed + private String redisHashCode; + + @Indexed + private String reference; + + private String destination; + + private int count; + + @TimeToLive + @Builder.Default + private long expiration = 60L; + + public RedisHashData(String redisHashCode, String reference) { + this.redisHashCode = redisHashCode; + this.reference = reference; + } +} diff --git a/src/main/java/com/mb/livedataservice/data/repository/RedisHashDataRepository.java b/src/main/java/com/mb/livedataservice/data/repository/RedisHashDataRepository.java new file mode 100644 index 0000000..e2907ae --- /dev/null +++ b/src/main/java/com/mb/livedataservice/data/repository/RedisHashDataRepository.java @@ -0,0 +1,8 @@ +package com.mb.livedataservice.data.repository; + +import com.mb.livedataservice.data.entity.RedisHashData; +import org.springframework.data.repository.CrudRepository; + +public interface RedisHashDataRepository extends CrudRepository { + +} diff --git a/src/main/java/com/mb/livedataservice/queue/RedisKeyExpiredEventListenerImpl.java b/src/main/java/com/mb/livedataservice/queue/RedisKeyExpiredEventListenerImpl.java new file mode 100644 index 0000000..355e25b --- /dev/null +++ b/src/main/java/com/mb/livedataservice/queue/RedisKeyExpiredEventListenerImpl.java @@ -0,0 +1,23 @@ +package com.mb.livedataservice.queue; + +import com.mb.livedataservice.data.entity.RedisHashData; +import com.mb.livedataservice.service.RedisHashService; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.event.EventListener; +import org.springframework.data.redis.core.RedisKeyExpiredEvent; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +@AllArgsConstructor +public class RedisKeyExpiredEventListenerImpl { + + private final RedisHashService redisHashService; + + @EventListener(condition = "#event.keyspace == 'RedisHashData'") + public void redisExpiredKeyEventForRedisHashData(RedisKeyExpiredEvent event) { + log.info("Redis key expired event log. RedisHashData - event:{}", event.toString()); + redisHashService.delete(RedisHashData.builder().id(new String(event.getId())).build()); + } +} diff --git a/src/main/java/com/mb/livedataservice/service/RedisHashService.java b/src/main/java/com/mb/livedataservice/service/RedisHashService.java new file mode 100644 index 0000000..a1000fa --- /dev/null +++ b/src/main/java/com/mb/livedataservice/service/RedisHashService.java @@ -0,0 +1,16 @@ +package com.mb.livedataservice.service; + +import com.mb.livedataservice.data.entity.RedisHashData; + +import java.util.Optional; + +public interface RedisHashService { + + RedisHashData save(RedisHashData redisHashData); + + Optional findById(String id); + + void delete(RedisHashData redisHashData); + + void deleteRedisHashDataById(String redisHashDataId); +} diff --git a/src/main/java/com/mb/livedataservice/service/RedisTokenStoreService.java b/src/main/java/com/mb/livedataservice/service/RedisTokenStoreService.java new file mode 100644 index 0000000..8255e97 --- /dev/null +++ b/src/main/java/com/mb/livedataservice/service/RedisTokenStoreService.java @@ -0,0 +1,10 @@ +package com.mb.livedataservice.service; + +public interface RedisTokenStoreService { + + String getToken(String tokenId, String key); + + void storeToken(String tokenId, String key); + + void deleteToken(String tokenId, String key); +} diff --git a/src/main/java/com/mb/livedataservice/service/impl/RedisHashServiceImpl.java b/src/main/java/com/mb/livedataservice/service/impl/RedisHashServiceImpl.java new file mode 100644 index 0000000..3417910 --- /dev/null +++ b/src/main/java/com/mb/livedataservice/service/impl/RedisHashServiceImpl.java @@ -0,0 +1,39 @@ +package com.mb.livedataservice.service.impl; + +import com.mb.livedataservice.data.entity.RedisHashData; +import com.mb.livedataservice.data.repository.RedisHashDataRepository; +import com.mb.livedataservice.service.RedisHashService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.Optional; + +@Slf4j +@Service +@RequiredArgsConstructor +public class RedisHashServiceImpl implements RedisHashService { + + private final RedisHashDataRepository redisHashDataRepository; + + @Override + public RedisHashData save(RedisHashData redisHashData) { + return redisHashDataRepository.save(redisHashData); + } + + @Override + public Optional findById(String id) { + return redisHashDataRepository.findById(id); + } + + @Override + public void delete(RedisHashData redisHashData) { + redisHashDataRepository.delete(redisHashData); + } + + @Override + public void deleteRedisHashDataById(String redisHashDataId) { + log.info("Deleting RedisHashData by ID: '{}'.", redisHashDataId); + redisHashDataRepository.deleteById(redisHashDataId); + } +} diff --git a/src/main/java/com/mb/livedataservice/service/impl/RedisTokenStoreServiceImpl.java b/src/main/java/com/mb/livedataservice/service/impl/RedisTokenStoreServiceImpl.java new file mode 100644 index 0000000..41a74df --- /dev/null +++ b/src/main/java/com/mb/livedataservice/service/impl/RedisTokenStoreServiceImpl.java @@ -0,0 +1,82 @@ +package com.mb.livedataservice.service.impl; + +import com.mb.livedataservice.service.RedisTokenStoreService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.redisson.api.RLock; +import org.redisson.api.RedissonClient; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(name = "services.token.store", havingValue = "redis") +public class RedisTokenStoreServiceImpl implements RedisTokenStoreService { + + private final RedissonClient redissonClient; + private final StringRedisTemplate stringRedisTemplate; + + // tokenId should be unique for every integration. + @Override + public String getToken(String tokenId, String key) { + String token = stringRedisTemplate.opsForValue().get(tokenId); + if (!StringUtils.hasLength(token)) { + RLock lock = redissonClient.getLock(key); + + boolean control = false; + while (lock.isLocked()) { + try { + TimeUnit.MILLISECONDS.sleep(1000); + control = true; + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + if (control) { + return stringRedisTemplate.opsForValue().get(tokenId); + } + try { + lock.lock(3, TimeUnit.SECONDS); + // Call 3rd party to get token + storeToken(tokenId, token); + } catch (Exception ex) { + log.info("Error occurred while getting and saving 3rd party token exception. Exception: {}", ExceptionUtils.getStackTrace(ex)); + } finally { + if (lock.isLocked()) { + lock.unlock(); + } + } + } + return token; + } + + @Override + public void storeToken(String tokenId, String key) { + stringRedisTemplate.opsForValue().set(tokenId, key, Duration.ofMinutes(30)); + } + + @Override + public void deleteToken(String tokenId, String key) { + RLock lock = redissonClient.getLock(key); + + boolean control = true; + while (lock.isLocked()) { + try { + TimeUnit.MILLISECONDS.sleep(1000); + control = false; + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + if (control) { + stringRedisTemplate.delete(tokenId); + } + } +} diff --git a/src/main/java/com/mb/livedataservice/utils/RedisConstants.java b/src/main/java/com/mb/livedataservice/utils/RedisConstants.java new file mode 100644 index 0000000..8a5d40b --- /dev/null +++ b/src/main/java/com/mb/livedataservice/utils/RedisConstants.java @@ -0,0 +1,13 @@ +package com.mb.livedataservice.utils; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +/** + * Collected constants of general utility. All members of this class are immutable. + */ +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public final class RedisConstants { + + public static final String CACHE_KEY = "cacheKey"; +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 24082a3..fbfa9fc 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,3 +1,6 @@ +REDIS_HOST: ${REDIS_HOST_ENV:localhost} +REDIS_PORT: ${REDIS_PORT_ENV:6378} + server: port: 8080 @@ -15,6 +18,17 @@ spring: driverClassName: org.h2.Driver url: jdbc:h2:mem:${DB_NAME:MB_TEST};DB_CLOSE_DELAY=-1 + data: + redis: + host: ${REDIS_HOST} + port: ${REDIS_PORT} + jedis: + pool: + max-active: 7 + max-idle: 7 + min-idle: 2 + max-wait: -1ms + jpa: database-platform: org.hibernate.dialect.H2Dialect show-sql: ${SHOW_SQL_ENABLED:true} @@ -75,4 +89,12 @@ swagger: version: 2.0 - name: live-data-service-2 url: /v2/api-docs - version: 2.0 \ No newline at end of file + version: 2.0 + +redisson: + enabled: true + url: redis://${REDIS_HOST}:${REDIS_PORT} + +services: + token: + store: redis \ No newline at end of file diff --git a/src/test/java/com/mb/livedataservice/integration_tests/api/controller/TutorialControllerTest.java b/src/test/java/com/mb/livedataservice/integration_tests/api/controller/TutorialControllerTest.java index 8f9de07..2eee14a 100644 --- a/src/test/java/com/mb/livedataservice/integration_tests/api/controller/TutorialControllerTest.java +++ b/src/test/java/com/mb/livedataservice/integration_tests/api/controller/TutorialControllerTest.java @@ -18,9 +18,12 @@ import org.springframework.http.ResponseEntity; import org.springframework.test.annotation.Rollback; import org.springframework.transaction.annotation.Transactional; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.KafkaContainer; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; import java.util.Objects; @@ -35,6 +38,14 @@ class TutorialControllerTest extends BaseUnitTest { @ServiceConnection private static final PostgreSQLContainer postgres = new PostgreSQLContainer<>("postgres:16.1"); + @Container + @ServiceConnection + public static final GenericContainer redis = new GenericContainer(DockerImageName.parse("redis:7.2.4")).withExposedPorts(6379); + + @Container + @ServiceConnection + public static final KafkaContainer kafka = new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:7.5.3")); + @Autowired private TestRestTemplate restTemplate; diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml index c3462d1..e0e6164 100644 --- a/src/test/resources/application.yml +++ b/src/test/resources/application.yml @@ -1,3 +1,6 @@ +REDIS_HOST: ${REDIS_HOST_ENV:localhost} +REDIS_PORT: ${REDIS_PORT_ENV:6378} + spring: jpa: database-platform: org.hibernate.dialect.PostgreSQLDialect @@ -5,6 +8,17 @@ spring: hibernate: ddl-auto: update + data: + redis: + host: ${REDIS_HOST} + port: ${REDIS_PORT} + jedis: + pool: + max-active: 7 + max-idle: 7 + min-idle: 2 + max-wait: -1ms + kafka: consumer: bootstrap-servers: localhost:9092 @@ -27,4 +41,12 @@ swagger: version: 2.0 - name: live-data-service-2 url: /v2/api-docs - version: 2.0 \ No newline at end of file + version: 2.0 + +redisson: + enabled: false + url: redis://${REDIS_HOST}:${REDIS_PORT} + +services: + token: + store: redis \ No newline at end of file