gateway lock component 적용 (#164)

* chore: logback 프로필 분리 gateway, auth, user 모듈 수정 * feat: prometheus, grafana 기능 추가 gateway 사용량 추적 * feat: redisson lock 적용 * feat: auth 모듈 redis 설정 추가
sasaping · Nov 19, 2024 · 71cd2fd · 71cd2fd
1 parent 279ce6f
commit 71cd2fd
Show file tree

Hide file tree

Showing 12 changed files with 207 additions and 96 deletions.
diff --git a/compose-monitoring.yml b/compose-monitoring.yml
@@ -0,0 +1,18 @@
+version: '3.8'
+
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    ports:
+      - "3000:3000"
+    volumes:
+      - ./grafana:/var/lib/grafana
diff --git a/prometheus.yml b/prometheus.yml
@@ -0,0 +1,15 @@
+global:
+  scrape_interval: 15s
+scrape_configs:
+  - job_name: 'eureka-services'
+    eureka_sd_configs:
+      - server: 'http://host.docker.internal:19090/eureka'   # Eureka 서버 주소
+    metrics_path: '/actuator/prometheus'
+    relabel_configs:
+      - source_labels: [ '__meta_eureka_app_name' ]
+        target_label: 'job'
+      - source_labels: [ '__meta_eureka_instance_ip_addr' ]
+        target_label: 'instance'
+      - source_labels: [ '__meta_eureka_app_instance_status' ]
+        action: keep
+        regex: 'UP'  # 서비스 상태가 "UP"인 인스턴스만 스크랩
diff --git a/service/auth/server/build.gradle b/service/auth/server/build.gradle
@@ -39,8 +39,9 @@ dependencies {
     testImplementation 'org.springframework.boot:spring-boot-starter-test'
     testImplementation 'org.springframework.security:spring-security-test'
     testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
-    implementation 'net.logstash.logback:logstash-logback-encoder:8.0'
-
+//    implementation 'net.logstash.logback:logstash-logback-encoder:8.0'
+    implementation 'org.springframework.boot:spring-boot-starter-data-redis'
+
 }
 
 dependencyManagement {

diff --git a/service/auth/server/src/main/resources/application-local.yml b/service/auth/server/src/main/resources/application-local.yml
@@ -6,3 +6,11 @@ eureka:
 jwt:
   secret-key: Zr2PMyKH4UheWy6zscq6Wc/nSLl6L/AJ6b5QvLzWXJg5wzQiGdJncTTOBCxvW8Rkl1T0N+1bqF52Mw2eZvwA0i5zyq+VVbIjPyz+b7DW0Xd2wTnXifwxM12LU2oyXpyLz
   access-token-expire-in: 3600000
+
+spring:
+  data:
+    redis:
+      host: localhost
+      port: 6381
+      username: default
+      password: systempass
diff --git a/service/auth/server/src/main/resources/logback-spring.xml b/service/auth/server/src/main/resources/logback-spring.xml
@@ -1,34 +1,41 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<configuration scan="true" scanPeriod="30 seconds">
+<configuration>
+
   <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
     <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
       <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} springboot-elk [%thread] %-5level %logger{36} - %msg%n
       </pattern>
     </encoder>
   </appender>
 
-  <!-- Logstash로 로그 전송 -->
-  <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
-    <param name="Encoding" value="UTF-8"/>
-    <destination>logstash01:50000</destination>
-
-    <encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
-      <providers>
-        <timestamp>
-          <timeZone>UTC</timeZone>
-        </timestamp>
-        <logLevel/>
-        <threadName/>
-        <loggerName/>
-        <message/>
-        <stackTrace/>
-      </providers>
-    </encoder>
-  </appender>
+  <!-- Logstash appender는 prod 프로파일에서만 사용 -->
+  <springProfile name="prod">
+    <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
+      <param name="Encoding" value="UTF-8"/>
+      <destination>127.0.0.1:50000</destination>
+      <encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
+        <providers>
+          <timestamp>
+            <timeZone>UTC</timeZone>
+          </timestamp>
+          <logLevel/>
+          <threadName/>
+          <loggerName/>
+          <message/>
+          <stackTrace/>
+        </providers>
+      </encoder>
+    </appender>
+    <root level="INFO">
+      <appender-ref ref="CONSOLE"/>
+      <appender-ref ref="LOGSTASH"/>
+    </root>
+  </springProfile>
 
-  <root level="INFO">
-    <appender-ref ref="CONSOLE"/>
-    <appender-ref ref="LOGSTASH"/>
-  </root>
+  <!-- prod가 아닌 환경에서는 CONSOLE appender만 사용 -->
+  <springProfile name="!prod">
+    <root level="INFO">
+      <appender-ref ref="CONSOLE"/>
+    </root>
+  </springProfile>
 
 </configuration>
diff --git a/service/eureka/server/build.gradle b/service/eureka/server/build.gradle
@@ -27,6 +27,8 @@ dependencies {
     implementation 'org.springframework.cloud:spring-cloud-starter-netflix-eureka-server'
     testImplementation 'org.springframework.boot:spring-boot-starter-test'
     testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
+    implementation 'org.springframework.boot:spring-boot-starter-actuator'
+    implementation 'io.micrometer:micrometer-registry-prometheus'
 }
 
 dependencyManagement {

diff --git a/service/eureka/server/src/main/resources/application.yml b/service/eureka/server/src/main/resources/application.yml
@@ -6,3 +6,17 @@ spring:
 
 server:
   port: 19090
+
+management:
+  endpoints:
+    web:
+      exposure:
+        include: prometheus
+  endpoint:
+    health:
+      show-details: always
+    prometheus:
+      enabled: true
+  metrics:
+    tags:
+      application: ${spring.application.name}
diff --git a/service/gateway/server/build.gradle b/service/gateway/server/build.gradle
@@ -33,7 +33,7 @@ dependencies {
     implementation project(':common:domain')
     implementation project(':service:auth:auth_dto')
 
-    implementation 'net.logstash.logback:logstash-logback-encoder:8.0'
+//    implementation 'net.logstash.logback:logstash-logback-encoder:8.0'
     implementation 'org.springframework.boot:spring-boot-starter-web'
     implementation 'org.springframework.boot:spring-boot-starter-aop'
     implementation 'org.springframework.cloud:spring-cloud-starter-gateway'
@@ -46,7 +46,8 @@ dependencies {
     testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
     implementation 'org.springframework.boot:spring-boot-starter-data-redis'
     implementation 'org.redisson:redisson:3.35.0'
-
+    implementation 'org.springframework.boot:spring-boot-starter-actuator'
+    implementation 'io.micrometer:micrometer-registry-prometheus'
 }
 
 dependencyManagement {

diff --git a/.../gateway/server/src/main/java/com/sparta/gateway/server/application/UserQueueService.java b/.../gateway/server/src/main/java/com/sparta/gateway/server/application/UserQueueService.java
@@ -27,29 +27,26 @@ public class UserQueueService {
   @Value("${MAX_ACTIVE_USERS}")
   private long MAX_ACTIVE_USERS;
   private final long INACTIVITY_THRESHOLD = 300;
-  private Long activeUsers;
 
   public Mono<RegisterUserResponse> registerUser(String userId) {
     return reactiveRedisTemplate.opsForZSet()
         .rank(USER_QUEUE_PROCEED_KEY, userId)
         .defaultIfEmpty(-1L)
-        .flatMap(rank -> {
-          if (rank >= 0) {
-            return updateUserActivityTime(userId)
-                .thenReturn(new RegisterUserResponse(0L));
-          }
+        .flatMap(rank -> rank >= 0 ? handleProceedUser(userId) : handleNewUser(userId));
+  }
 
-          return reactiveRedisTemplate.opsForSet().size(USER_ACTIVE_SET_KEY)
-              .flatMap(activeUsers -> {
-                if (activeUsers < MAX_ACTIVE_USERS) {
-                  return addToProceedQueue(userId);
-                } else {
-                  return checkAndAddToQueue(userId);
-                }
-              });
-        });
+  private Mono<RegisterUserResponse> handleProceedUser(String userId) {
+    return updateUserActivityTime(userId)
+        .thenReturn(new RegisterUserResponse(0L));
+  }
+
+  private Mono<RegisterUserResponse> handleNewUser(String userId) {
+    return reactiveRedisTemplate.opsForSet().size(USER_ACTIVE_SET_KEY)
+        .flatMap(activeUsers -> activeUsers < MAX_ACTIVE_USERS ? addToProceedQueue(userId)
+            : checkAndAddToQueue(userId));
   }
 
+
   private Mono<RegisterUserResponse> checkAndAddToQueue(String userId) {
     return reactiveRedisTemplate.opsForZSet().score(USER_QUEUE_WAIT_KEY, userId)
         .defaultIfEmpty(-1.0)
@@ -73,22 +70,42 @@ private Mono<RegisterUserResponse> updateWaitQueueScore(String userId) {
         ;
   }
 
-  private Mono<RegisterUserResponse> addToProceedQueue(String userId) {
+  public Mono<RegisterUserResponse> addToProceedQueue(String userId) {
+    return Mono.create(sink -> {
+      lockComponent.execute(userId, 1000, 1000, () -> {
+        try {
+          addUserToQueue(userId)
+              .doOnSuccess(sink::success)
+              .doOnError(sink::error)
+              .subscribe();
+        } catch (Exception e) {
+          sink.error(e);
+        }
+      });
+    });
+  }
+
+  private Mono<RegisterUserResponse> addUserToQueue(String userId) {
     var unixTime = Instant.now().getEpochSecond();
     return reactiveRedisTemplate.opsForZSet()
         .add(USER_QUEUE_PROCEED_KEY, userId, unixTime)
-        .filter(i -> i)
+        .filter(success -> success)
         .flatMap(success -> {
           if (success) {
-            return reactiveRedisTemplate.opsForSet()
-                .add(USER_ACTIVE_SET_KEY, userId)
-                .map(i -> new RegisterUserResponse(0L));
+            return addToActiveSet(userId);
           } else {
             return checkAndAddToQueue(userId);
           }
         });
   }
 
+  private Mono<RegisterUserResponse> addToActiveSet(String userId) {
+    return reactiveRedisTemplate.opsForSet()
+        .add(USER_ACTIVE_SET_KEY, userId)
+        .map(i -> new RegisterUserResponse(0L));
+  }
+
+
   private Mono<RegisterUserResponse> addToWaitQueue(String userId) {
     var unixTime = Instant.now().getEpochSecond();
     return reactiveRedisTemplate.opsForZSet()

diff --git a/service/gateway/server/src/main/resources/application.yml b/service/gateway/server/src/main/resources/application.yml
@@ -54,3 +54,17 @@ spring:
       discovery:
         locator:
           enabled: true
+
+management:
+  endpoints:
+    web:
+      exposure:
+        include: prometheus
+  endpoint:
+    health:
+      show-details: always
+    prometheus:
+      enabled: true
+  metrics:
+    tags:
+      application: ${spring.application.name}
diff --git a/service/gateway/server/src/main/resources/logback-spring.xml b/service/gateway/server/src/main/resources/logback-spring.xml
@@ -1,34 +1,41 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<configuration scan="true" scanPeriod="30 seconds">
+<configuration>
+
   <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
     <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
       <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} springboot-elk [%thread] %-5level %logger{36} - %msg%n
       </pattern>
     </encoder>
   </appender>
 
-  <!-- Logstash로 로그 전송 -->
-  <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
-    <param name="Encoding" value="UTF-8"/>
-    <destination>logstash01:50000</destination>
-
-    <encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
-      <providers>
-        <timestamp>
-          <timeZone>UTC</timeZone>
-        </timestamp>
-        <logLevel/>
-        <threadName/>
-        <loggerName/>
-        <message/>
-        <stackTrace/>
-      </providers>
-    </encoder>
-  </appender>
+  <!-- Logstash appender는 prod 프로파일에서만 사용 -->
+  <springProfile name="prod">
+    <appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
+      <param name="Encoding" value="UTF-8"/>
+      <destination>127.0.0.1:50000</destination>
+      <encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
+        <providers>
+          <timestamp>
+            <timeZone>UTC</timeZone>
+          </timestamp>
+          <logLevel/>
+          <threadName/>
+          <loggerName/>
+          <message/>
+          <stackTrace/>
+        </providers>
+      </encoder>
+    </appender>
+    <root level="INFO">
+      <appender-ref ref="CONSOLE"/>
+      <appender-ref ref="LOGSTASH"/>
+    </root>
+  </springProfile>
 
-  <root level="INFO">
-    <appender-ref ref="CONSOLE"/>
-    <appender-ref ref="LOGSTASH"/>
-  </root>
+  <!-- prod가 아닌 환경에서는 CONSOLE appender만 사용 -->
+  <springProfile name="!prod">
+    <root level="INFO">
+      <appender-ref ref="CONSOLE"/>
+    </root>
+  </springProfile>
 
 </configuration>