feat/OPS-289 : dataprocessor 서비스 생성 (#59)

ohsoohyuk · web-flow · commit 2edf68811348 · 2025-09-25T15:07:52.000+09:00
diff --git a/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/ai/dto/AiExtractorDto.java b/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/ai/dto/AiExtractorDto.java
@@ -0,0 +1,12 @@
+package org.tuna.zoopzoop.backend.domain.datasource.ai.dto;
+
+import java.time.LocalDate;
+
+public record AiExtractorDto(
+        String title, // 제목
+        LocalDate dataCreatedDate, // 작성일자
+        String content, // ai한테 줘야할 내용
+        String imageUrl, // 썸네일 이미지 url
+        String sources // 출처
+) {
+}
diff --git a/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/ai/prompt/AiPrompt.java b/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/ai/prompt/AiPrompt.java
@@ -0,0 +1,25 @@
+package org.tuna.zoopzoop.backend.domain.datasource.ai.prompt;
+
+public class AiPrompt {
+    public static final String EXTRACTION = """
+        아래 HTML 전문에서 필요한 정보를 JSON 형식으로 추출해 주세요.
+        반환 JSON 구조:
+        {
+          "title": "제목",
+          "datacreatedDate": "작성일자 (YYYY-MM-DD)",
+          "content": "본문 내용",
+          "imageUrl": "썸네일 이미지 URL",
+          "sources": "출판사 이름 or 서비스 이름 or 도메인 이름"
+        }
+
+        HTML 전문:
+        %s
+
+        - 반드시 JSON 형식으로만 출력해 주세요.
+        - 해당정보가 없으면 반드시 빈 문자열로 출력해 주세요.
+        """;
+
+    public static final String SUMMARY_TAG_CATEGORY = """
+        내용 요약, 태그 요약, 카테고리 선정 프롬프트
+        """;
+}
diff --git a/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/ai/service/AiService.java b/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/ai/service/AiService.java
@@ -1,79 +1,22 @@
 package org.tuna.zoopzoop.backend.domain.datasource.ai.service;
 
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.RequiredArgsConstructor;
 import org.springframework.ai.chat.client.ChatClient;
 import org.springframework.stereotype.Service;
-import org.tuna.zoopzoop.backend.domain.datasource.repository.TagRepository;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import org.tuna.zoopzoop.backend.domain.datasource.ai.dto.AiExtractorDto;
+import org.tuna.zoopzoop.backend.domain.datasource.ai.prompt.AiPrompt;
 
 @Service
 @RequiredArgsConstructor
 public class AiService {
     private final ChatClient chatClient;
-    private final TagRepository tagRepository;
-
-    public Set<String> duplicateTag() {
-        Set<String> existingTags = new HashSet<>();
-        existingTags.addAll(tagRepository.findAllTagNames());
-        return existingTags;
-    }
-
-    public Map<String, Object> summarizeAndTag(String text) {
-        Set<String> existingTags = duplicateTag();
-        String tagsForPrompt = String.join(", ", existingTags);
 
-        String prompt = """
-            본문 요약 프롬프트:
-            아래 본문을 무조건 50자 이상, 100자 이하로 요약해주세요.
-            
-            핵심 태그 프롬프트:
-            이미 존재하는 태그 목록은 다음과 같습니다:
-            [%s]
-            
-            본문을 요약하고, 해당 본문과 관련된 태그 3~5개를 생성하세요.
-            - 태그는 반드시 본문과 관련된 것만 선택하세요.
-            - 기존 태그 중 본문과 관련 없는 것은 포함하지 마세요.
-            - 새로운 태그는 본문에 꼭 필요한 경우에만 생성하세요.
-            - 결과는 JSON 형식으로만 출력하세요.
-            
-            본문:
-            %s
-            
-            예시 출력:
-            {
-              "summary": "...",
-              "tags": ["...", "..."]
-            }
-            """.formatted(tagsForPrompt, text);
-
-        String response = chatClient.prompt()
-                .user(prompt)
+    public AiExtractorDto extract(String rawHtml) {
+        AiExtractorDto response = chatClient.prompt()
+                .user(AiPrompt.EXTRACTION.formatted(rawHtml))
                 .call()
-                .content();
-
-        // JSON 시작/끝만 추출
-        int start = response.indexOf("{");
-        int end = response.lastIndexOf("}") + 1;
-        if (start >= 0 && end > start) {
-            response = response.substring(start, end);
-        }
-
-        try {
-            ObjectMapper mapper = new ObjectMapper();
-            Map<String, Object> map = mapper.readValue(response, new TypeReference<Map<String, Object>>() {});
-
-            String summary = (String) map.get("summary");
-            List<String> tags = (List<String>) map.get("tags");
+                .entity(AiExtractorDto.class);
 
-            return map;
-        } catch (Exception e) {
-            throw new RuntimeException("AI 응답 파싱 실패: " + response, e);
-        }
+        return response;
     }
 }
diff --git a/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/controller/CrawlerTestController.java b/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/controller/CrawlerTestController.java
@@ -6,16 +6,18 @@
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 import org.tuna.zoopzoop.backend.domain.datasource.crawler.service.CrawlerManagerService;
+import org.tuna.zoopzoop.backend.domain.datasource.dataprocessor.service.DataProcessorService;
 import org.tuna.zoopzoop.backend.domain.datasource.dto.ArticleData;
 
 @RestController
 @RequestMapping("api/v1")
 @RequiredArgsConstructor
 public class CrawlerTestController {
     private final CrawlerManagerService crawlerManagerService;
+    private final DataProcessorService dataProcessorService;
 
     @GetMapping("/crawl")
     public ArticleData crawl(@RequestParam String url) throws Exception {
-        return crawlerManagerService.extractContent(url);
+        return dataProcessorService.process(url);
     }
 }
diff --git a/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/dto/CrawlerResult.java b/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/dto/CrawlerResult.java
@@ -0,0 +1,10 @@
+package org.tuna.zoopzoop.backend.domain.datasource.crawler.dto;
+
+public record CrawlerResult<T>(
+        CrawlerType type, // SPECIFIC or UNSPECIFIC
+        T data
+) {
+    public enum CrawlerType {
+        SPECIFIC, UNSPECIFIC
+    }
+}
diff --git a/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/dto/SpecificSiteDto.java b/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/dto/SpecificSiteDto.java
@@ -0,0 +1,12 @@
+package org.tuna.zoopzoop.backend.domain.datasource.crawler.dto;
+
+import java.time.LocalDate;
+
+public record SpecificSiteDto(
+        String title, // 제목
+        LocalDate dataCreatedDate, // 작성일자
+        String content, // ai한테 줘야할 내용
+        String imageUrl, // 썸네일 이미지 url
+        String sources // 출처
+) {
+}
diff --git a/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/dto/UnspecificSiteDto.java b/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/dto/UnspecificSiteDto.java
@@ -0,0 +1,6 @@
+package org.tuna.zoopzoop.backend.domain.datasource.crawler.dto;
+
+public record UnspecificSiteDto(
+        String rawHtml // 불특정 사이트의 html 전문
+) {
+}
diff --git a/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/service/Crawler.java b/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/service/Crawler.java
@@ -1,12 +1,12 @@
 package org.tuna.zoopzoop.backend.domain.datasource.crawler.service;
 
 import org.jsoup.nodes.Document;
-import org.tuna.zoopzoop.backend.domain.datasource.dto.ArticleData;
+import org.tuna.zoopzoop.backend.domain.datasource.crawler.dto.CrawlerResult;
 
 import java.time.LocalDate;
 
 public interface Crawler {
     boolean supports(String domain);
-    ArticleData extract(Document doc);
+    CrawlerResult<?> extract(Document doc);
     LocalDate transLocalDate(String rawDate);
 }
diff --git a/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/service/CrawlerManagerService.java b/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/service/CrawlerManagerService.java
@@ -4,7 +4,7 @@
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.springframework.stereotype.Service;
-import org.tuna.zoopzoop.backend.domain.datasource.dto.ArticleData;
+import org.tuna.zoopzoop.backend.domain.datasource.crawler.dto.CrawlerResult;
 
 import java.io.IOException;
 import java.util.List;
@@ -14,7 +14,7 @@
 public class CrawlerManagerService {
     private final List<Crawler> crawlers;
 
-    public ArticleData extractContent(String url) throws IOException {
+    public CrawlerResult<?> extractContent(String url) throws IOException {
         Document doc = Jsoup.connect(url)
                 .userAgent("Mozilla/5.0")
                 .timeout(10000)
diff --git a/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/service/GenericCrawler.java b/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/service/GenericCrawler.java
@@ -4,7 +4,8 @@
 import org.springframework.core.Ordered;
 import org.springframework.core.annotation.Order;
 import org.springframework.stereotype.Component;
-import org.tuna.zoopzoop.backend.domain.datasource.dto.ArticleData;
+import org.tuna.zoopzoop.backend.domain.datasource.crawler.dto.CrawlerResult;
+import org.tuna.zoopzoop.backend.domain.datasource.crawler.dto.UnspecificSiteDto;
 
 import java.time.LocalDate;
 
@@ -17,8 +18,17 @@ public boolean supports(String url) {
     }
 
     @Override
-    public ArticleData extract(Document doc) {
-        return new ArticleData(null, null, null, null, null, doc.outerHtml());
+    public CrawlerResult<?> extract(Document doc) {
+        // 불필요한 태그 제거
+        doc.select("script, style, noscript, iframe, nav, header, footer, form, aside, meta, link").remove();
+
+        // 본문만 가져오기 (HTML)
+        String cleanHtml = doc.body().html();
+
+        return new CrawlerResult<>(
+                CrawlerResult.CrawlerType.UNSPECIFIC,
+                new UnspecificSiteDto(cleanHtml)
+        );
     }
 
     @Override
diff --git a/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/service/NaverNewsCrawler.java b/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/crawler/service/NaverNewsCrawler.java
@@ -4,7 +4,8 @@
 import org.springframework.core.Ordered;
 import org.springframework.core.annotation.Order;
 import org.springframework.stereotype.Component;
-import org.tuna.zoopzoop.backend.domain.datasource.dto.ArticleData;
+import org.tuna.zoopzoop.backend.domain.datasource.crawler.dto.CrawlerResult;
+import org.tuna.zoopzoop.backend.domain.datasource.crawler.dto.SpecificSiteDto;
 
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
@@ -22,7 +23,7 @@ public boolean supports(String domain) {
     }
 
     @Override
-    public ArticleData extract(Document doc) {
+    public CrawlerResult<?> extract(Document doc) {
         // 제목
         String title = doc.selectFirst("h2#title_area").text();
 
@@ -41,7 +42,10 @@ public ArticleData extract(Document doc) {
         // 출처
         String sources = doc.selectFirst("span.media_end_head_top_logo_text").text();
 
-        return new ArticleData(title, dataCreatedDate, content, imageUrl, sources, null);
+        return new CrawlerResult<>(
+                CrawlerResult.CrawlerType.SPECIFIC,
+                new SpecificSiteDto(title, dataCreatedDate, content, imageUrl, sources)
+        );
     }
 
     @Override
diff --git a/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/dataprocessor/service/DataProcessorService.java b/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/dataprocessor/service/DataProcessorService.java
@@ -0,0 +1,48 @@
+package org.tuna.zoopzoop.backend.domain.datasource.dataprocessor.service;
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Service;
+import org.tuna.zoopzoop.backend.domain.datasource.ai.dto.AiExtractorDto;
+import org.tuna.zoopzoop.backend.domain.datasource.ai.service.AiService;
+import org.tuna.zoopzoop.backend.domain.datasource.crawler.dto.CrawlerResult;
+import org.tuna.zoopzoop.backend.domain.datasource.crawler.dto.SpecificSiteDto;
+import org.tuna.zoopzoop.backend.domain.datasource.crawler.dto.UnspecificSiteDto;
+import org.tuna.zoopzoop.backend.domain.datasource.crawler.service.CrawlerManagerService;
+import org.tuna.zoopzoop.backend.domain.datasource.dto.ArticleData;
+
+import java.io.IOException;
+
+@Service
+@RequiredArgsConstructor
+public class DataProcessorService {
+    public final CrawlerManagerService crawlerManagerService;
+    public final AiService aiService;
+
+    public ArticleData process(String url) throws IOException {
+        CrawlerResult<?> result = crawlerManagerService.extractContent(url);
+
+        return switch (result.type()) {
+            case SPECIFIC -> {
+                SpecificSiteDto specificSiteDto = (SpecificSiteDto) result.data();
+                yield new ArticleData(
+                        specificSiteDto.title(),
+                        specificSiteDto.dataCreatedDate(),
+                        specificSiteDto.content(),
+                        specificSiteDto.imageUrl(),
+                        specificSiteDto.sources()
+                );
+            }
+            case UNSPECIFIC -> {
+                UnspecificSiteDto unspecificSiteDto = (UnspecificSiteDto) result.data();
+                AiExtractorDto aiExtractorDto = aiService.extract(unspecificSiteDto.rawHtml());
+                yield new ArticleData(
+                        aiExtractorDto.title(),
+                        aiExtractorDto.dataCreatedDate(),
+                        aiExtractorDto.content(),
+                        aiExtractorDto.imageUrl(),
+                        aiExtractorDto.sources()
+                );
+            }
+        };
+    }
+}
diff --git a/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/dto/ArticleData.java b/src/main/java/org/tuna/zoopzoop/backend/domain/datasource/dto/ArticleData.java
@@ -7,7 +7,6 @@ public record ArticleData(
         LocalDate dataCreatedDate, // 작성일자
         String content, // ai한테 줘야할 내용
         String imageUrl, // 이미지 url
-        String sources, // 출처
-        String rawHtml // GenericCrawler 용
+        String sources // 출처
 ) {
 }
diff --git a/src/main/java/org/tuna/zoopzoop/backend/global/initData/BaseInitData.java b/src/main/java/org/tuna/zoopzoop/backend/global/initData/BaseInitData.java
@@ -54,13 +54,5 @@ public void initTagData() {
         tagRepository.save(tag1);
         tagRepository.save(tag2);
         tagRepository.save(tag3);
-
-        aiService.summarizeAndTag("안녕 내 이름은 오수혁이야");
-        aiService.summarizeAndTag("D2Coding 1.3.2 버전을 릴리즈 합니다. ligature 관련 이슈를 수정하여, ligature 적용/미적용 폰트를 구분하여 배포합니다.\n" +
-                "\n" +
-                "기존 버전은 반드시 삭제후 설치 바랍니다.\n" +
-                "\n");
-
-        aiService.summarizeAndTag("Spring AI는 예외 발생 시 AiClientException, RetryableException 등으로 예외를 포장합니다. Spring의 기본 예외 처리 방식을 활용하면 에러 핸들링을 일관성 있게 구성할 수 있습니다.");
     }
 }
diff --git a/src/test/java/org/tuna/zoopzoop/backend/domain/datasource/service/CrawlerManagerServiceTest.java b/src/test/java/org/tuna/zoopzoop/backend/domain/datasource/service/CrawlerManagerServiceTest.java
