Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,14 @@ public class FileManager {
public PresignedUrlResponse getUploadUrl() {
String uuid = UUID.randomUUID().toString();
Integer expires = 5;
URL url = s3Service.generateUploadUrl("videos", uuid, expires);
URL url = s3Service.generateUploadUrl("videos/"+uuid, expires);
LocalDateTime expiresAt = LocalDateTime.now().plusMinutes(expires);
return new PresignedUrlResponse(url, expiresAt);
}

public PresignedUrlResponse getDownloadUrl(String objectKey) {
Integer expires = 60;
URL url = s3Service.generateDownloadUrl("videos", objectKey, expires);
URL url = s3Service.generateDownloadUrl(objectKey, expires);
LocalDateTime expiresAt = LocalDateTime.now().plusMinutes(expires);
return new PresignedUrlResponse(url, expiresAt);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.back.global.exception.ServiceException;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.*;
Expand All @@ -17,8 +18,12 @@
public class S3Service {
private final S3Presigner presigner;
private final S3Client s3Client;
@Value("${aws.s3.bucket}")
private String bucket;

public URL generateUploadUrl(String objectKey, Integer expireMinutes) {
validateRequest(objectKey);

public URL generateUploadUrl(String bucket, String objectKey, Integer expireMinutes) {
PutObjectRequest request = PutObjectRequest.builder()
.bucket(bucket)
.key(objectKey)
Expand All @@ -33,19 +38,15 @@ public URL generateUploadUrl(String bucket, String objectKey, Integer expireMinu
throw new ServiceException("500", "Presigned URL 생성 실패");
}

URL url = presignedRequest.url();

return url;
return presignedRequest.url();
}

public URL generateUploadUrl(String bucket, String objectKey) {
validateRequest(bucket, objectKey);

return generateUploadUrl(bucket, objectKey, 30);
public URL generateUploadUrl(String objectKey) {
return generateUploadUrl(objectKey, 30);
}

public URL generateDownloadUrl(String bucket, String objectKey, Integer expireHours) {
isExist(bucket, objectKey);
public URL generateDownloadUrl(String objectKey, Integer expireMinutes) {
isExist(objectKey);

GetObjectRequest request = GetObjectRequest.builder()
.bucket(bucket)
Expand All @@ -54,25 +55,22 @@ public URL generateDownloadUrl(String bucket, String objectKey, Integer expireHo

PresignedGetObjectRequest presignedRequest =
presigner.presignGetObject(builder -> builder
.signatureDuration(Duration.ofMinutes(expireHours))
.signatureDuration(Duration.ofMinutes(expireMinutes))
.getObjectRequest(request));

if (presignedRequest == null) {
throw new ServiceException("500", "Presigned URL 생성 실패");
}

URL url = presignedRequest.url();

return url;
return presignedRequest.url();
}

public URL generateDownloadUrl(String bucket, String objectKey) {
validateRequest(bucket, objectKey);

return generateDownloadUrl(bucket, objectKey, 60);
public URL generateDownloadUrl(String objectKey) {
return generateDownloadUrl(objectKey, 60);
}

public void isExist(String bucket, String objectKey) {
public void isExist(String objectKey) {
validateRequest(objectKey);
try {
HeadObjectRequest headRequest = HeadObjectRequest.builder()
.bucket(bucket)
Expand All @@ -87,9 +85,9 @@ public void isExist(String bucket, String objectKey) {
}
}

public void validateRequest(String bucket, String objectKey) {
if (bucket == null || bucket.isEmpty() || objectKey == null || objectKey.isEmpty()) {
throw new ServiceException("400", "버킷 이름과 객체 키는 필수입니다.");
public void validateRequest(String objectKey) {
if (objectKey == null || objectKey.isEmpty()) {
throw new ServiceException("400", "객체 키는 필수입니다.");
}
}
}
28 changes: 13 additions & 15 deletions back/src/main/java/com/back/global/app/S3Config.java
Original file line number Diff line number Diff line change
@@ -1,35 +1,35 @@
package com.back.global.app;

import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.S3Configuration;

import java.net.URI;

@Configuration
public class S3Config {

private static final String ACCESS_KEY = "minioadmin"; // FIXME: 실제 키로 교체
private static final String SECRET_KEY = "minioadmin"; // FIXME: 실제 비밀키로 교체
private static final String ENDPOINT = "http://localhost:9000"; // FIXME: 실제 엔드포인트로 교체
private static final Region REGION = Region.AP_SOUTHEAST_1; // FIXME: 실제 리전으로 교체
@Value("${aws.s3.access-key}")
private String accessKey;

@Value("${aws.s3.secret-key}")
private String secretKey;

@Value("${aws.s3.region}")
private String region;

@Bean
public S3Presigner s3Presigner() {
return S3Presigner.builder()
.credentialsProvider(
StaticCredentialsProvider.create(
AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY)
AwsBasicCredentials.create(accessKey, secretKey)
)
)
.endpointOverride(URI.create(ENDPOINT))
.region(REGION)
.serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build())
.region(Region.of(region))
.build();
}

Expand All @@ -38,12 +38,10 @@ public S3Client s3Client() {
return S3Client.builder()
.credentialsProvider(
StaticCredentialsProvider.create(
AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY)
AwsBasicCredentials.create(accessKey, secretKey)
)
)
.endpointOverride(URI.create(ENDPOINT))
.region(REGION)
.forcePathStyle(true)
.region(Region.of(region))
.build();
}
}
6 changes: 6 additions & 0 deletions back/src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -98,3 +98,9 @@ custom: # ??? ??
secure: false # ??????? false, ??????? true
sameSite: "Lax"
maxAge: "#{60*60*24}" # 24??
aws:
s3:
access-key: ${AWS_ACCESS_KEY_ID}
secret-key: ${AWS_SECRET_ACCESS_KEY}
region: "ap-northeast-2"
bucket: "fivelogic-files-bucket"
22 changes: 8 additions & 14 deletions back/src/test/java/com/back/domain/file/service/S3ServiceTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ public class S3ServiceTest {
@Test
@DisplayName("S3 업로드 URL 생성")
void generateUploadUrlTest() throws MalformedURLException {
String bucket = "test-bucket";
String objectKey = "test-video.mp4";

PresignedPutObjectRequest mocked = mock(PresignedPutObjectRequest.class);
Expand All @@ -47,7 +46,7 @@ void generateUploadUrlTest() throws MalformedURLException {

when(mocked.url()).thenReturn(new URL("http://localhost:8080/upload"));

URL url = s3Service.generateUploadUrl(bucket, objectKey);
URL url = s3Service.generateUploadUrl(objectKey);

assertThat(url).isNotNull();
assertThat(url.toString()).isEqualTo("http://localhost:8080/upload");
Expand All @@ -56,7 +55,6 @@ void generateUploadUrlTest() throws MalformedURLException {
@Test
@DisplayName("S3 다운로드 URL 생성")
void generateDownloadUrlTest() throws MalformedURLException {
String bucket = "test-bucket";
String objectKey = "test-video.mp4";

var mocked = mock(software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest.class);
Expand All @@ -65,7 +63,7 @@ void generateDownloadUrlTest() throws MalformedURLException {

when(mocked.url()).thenReturn(new URL("http://localhost:8080/download"));

URL url = s3Service.generateDownloadUrl(bucket, objectKey);
URL url = s3Service.generateDownloadUrl(objectKey);

assertThat(url).isNotNull();
assertThat(url.toString()).isEqualTo("http://localhost:8080/download");
Expand All @@ -74,13 +72,12 @@ void generateDownloadUrlTest() throws MalformedURLException {
@Test
@DisplayName("업로드 URL 요청의 결과가 null일 경우 예외 발생")
void generateUploadUrl_PresignRequestNull_Test() {
String bucket = "test-bucket";
String objectKey = "test-video.mp4";

when(presigner.presignPutObject(any(Consumer.class))).thenReturn(null);

try {
s3Service.generateUploadUrl(bucket, objectKey);
s3Service.generateUploadUrl(objectKey);
} catch (Exception e) {
assertThat(e).isInstanceOf(ServiceException.class);
}
Expand All @@ -89,13 +86,12 @@ void generateUploadUrl_PresignRequestNull_Test() {
@Test
@DisplayName("다운로드 URL 요청의 결과가 null일 경우 예외 발생")
void generateDownloadUrl_PresignRequestNull_Test() {
String bucket = "test-bucket";
String objectKey = "test-video.mp4";

when(presigner.presignGetObject(any(Consumer.class))).thenReturn(null);

try {
s3Service.generateDownloadUrl(bucket, objectKey);
s3Service.generateDownloadUrl(objectKey);
} catch (Exception e) {
assertThat(e).isInstanceOf(ServiceException.class);
}
Expand All @@ -105,41 +101,39 @@ void generateDownloadUrl_PresignRequestNull_Test() {
@Test
@DisplayName("isExist() - 객체 존재 시 예외를 던지지 않고 정상 완료")
void isExist_objectExists_shouldNotThrowException() {
String bucket = "test-bucket";
String objectKey = "existing.mp4";

HeadObjectResponse mockResponse = mock(HeadObjectResponse.class);
when(s3Client.headObject(any(HeadObjectRequest.class))).thenReturn(mockResponse);

assertDoesNotThrow(() -> s3Service.isExist(bucket, objectKey));
assertDoesNotThrow(() -> s3Service.isExist(objectKey));
}

@Test
@DisplayName("isExist() - 객체 존재하지 않으면 ObjectNotFoundException을 던짐")
void isExist_objectNotExists_shouldThrowObjectNotFoundException() {
String bucket = "test-bucket";
String objectKey = "non-existing.mp4";

doThrow(S3Exception.builder().statusCode(404).build())
.when(s3Client).headObject(any(HeadObjectRequest.class));

assertThrows(ServiceException.class, () ->
s3Service.isExist(bucket, objectKey)
s3Service.isExist(objectKey)
);
}

@Test
@DisplayName("bucket이나 objectKey가 null 혹은 공백인 요청은 예외를 반환한다")
void validateRequest_InvalidInput_Test() {
try {
s3Service.validateRequest(null, "file.mp4");
s3Service.validateRequest("file.mp4");
} catch (Exception e) {
assertThat(e).isInstanceOf(ServiceException.class);
assertThat(e.getMessage()).isEqualTo("400 : 버킷 이름과 객체 키는 필수입니다.");
}

try {
s3Service.validateRequest("bucket", " ");
s3Service.validateRequest("bucket");
} catch (Exception e) {
assertThat(e).isInstanceOf(ServiceException.class);
assertThat(e.getMessage()).isEqualTo("400 : 버킷 이름과 객체 키는 필수입니다.");
Expand Down
22 changes: 12 additions & 10 deletions consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
from kafka.errors import NoBrokersAvailable
import boto3
import urllib.parse
from dotenv import load_dotenv

# .env 파일로부터 환경변수 로드
load_dotenv()

# =========================
# 환경 변수 설정
Expand All @@ -15,12 +19,11 @@
KAFKA_TOPIC = os.getenv("KAFKA_TOPIC", "s3-events")
KAFKA_TRANSCODING_STATUS_TOPIC = os.getenv("KAFKA_TRANSCODING_STATUS_TOPIC", "transcoding-status")
KAFKA_BOOTSTRAP = os.getenv("KAFKA_BOOTSTRAP", "kafka:29092") # Docker 내부 네트워크용
GROUP_ID = os.getenv("GROUP_ID", "minio-consumer")
GROUP_ID = os.getenv("GROUP_ID", "s3-consumer")

# S3 / MinIO
MINIO_ENDPOINT = os.getenv("MINIO_ENDPOINT", "http://minio:9000")
MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY", "minioadmin")
MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY", "minioadmin")
# S3
AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID")
AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")
DOWNLOAD_DIR = os.getenv("DOWNLOAD_DIR", "/downloads")
REUPLOAD_BUCKET = os.getenv("REUPLOAD_BUCKET", "transcoded-videos")

Expand All @@ -32,9 +35,8 @@
# =========================
s3 = boto3.client(
"s3",
endpoint_url=MINIO_ENDPOINT,
aws_access_key_id=MINIO_ACCESS_KEY,
aws_secret_access_key=MINIO_SECRET_KEY
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY
)

# =========================
Expand Down Expand Up @@ -144,7 +146,7 @@ def encode_dash_multi_quality(input_file, output_dir, producer, topic, bucket, k
# =========================
# DASH 폴더 업로드 함수
# =========================
def upload_folder_to_minio(local_folder, bucket_name, s3_prefix=""):
def upload_folder_to_s3(local_folder, bucket_name, s3_prefix=""):
try:
s3.head_bucket(Bucket=bucket_name)
except:
Expand Down Expand Up @@ -217,7 +219,7 @@ def upload_folder_to_minio(local_folder, bucket_name, s3_prefix=""):
)
print(f"DASH 인코딩 완료: {dash_output_dir}")

upload_folder_to_minio(dash_output_dir, bucket, s3_prefix="transcoded/"+object_key_without_ext)
upload_folder_to_s3(dash_output_dir, bucket, s3_prefix="transcoded/"+object_key_without_ext)
else:
print(f"영상 아님, 인코딩 스킵: {download_path}")

Expand Down
41 changes: 0 additions & 41 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,47 +28,6 @@ services:
networks:
- app-network

minio:
image: minio/minio
container_name: minio
ports:
- "9000:9000"
- "9001:9001"
environment:
- MINIO_ROOT_USER=minioadmin
- MINIO_ROOT_PASSWORD=minioadmin
- MINIO_NOTIFY_KAFKA_ENABLE_PRIMARY=on
- MINIO_NOTIFY_KAFKA_BROKERS_PRIMARY=kafka:29092
- MINIO_NOTIFY_KAFKA_TOPIC_PRIMARY=s3-events
- MINIO_NOTIFY_KAFKA_VERSION_PRIMARY=2.7.0
command: server /data --console-address ":9001"
volumes:
- minio-data:/data
depends_on:
- kafka
networks:
- app-network

minio-consumer:
build:
context: .
dockerfile: Dockerfile
container_name: minio-consumer
depends_on:
- kafka
- minio
environment:
- KAFKA_TOPIC=s3-events
- KAFKA_BOOTSTRAP=kafka:29092
- MINIO_ENDPOINT=http://minio:9000
- MINIO_ACCESS_KEY=minioadmin
- MINIO_SECRET_KEY=minioadmin
- DOWNLOAD_DIR=/downloads
volumes:
- ./downloads:/downloads
networks:
- app-network

spring-boot:
image: ghcr.io/prgms-web-devcourse-final-project/fivelogic:latest
container_name: spring-boot
Expand Down
Loading