Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@ jobs:
- name: build image
run: |
docker build \
--build-arg AWS_SECRET_ACCESS_KEY \
--build-arg AWS_ACCESS_KEY_ID \
-t ${ECR_REPOSITORY}:commit_$(git rev-parse --short "$GITHUB_SHA") \
-t ${ECR_REPOSITORY}:${PROD_IMAGE_TAG} \
.
Expand Down
3 changes: 0 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,6 @@ jobs:
key: cache-v1.1

- name: Download data
env:
S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
S3_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY }}
run: |
ln -s ../tests/data/wikipedia2vec.pkl data/wikipedia2vec.pkl
touch data/wikipedia2vec.pkl.vectors.npy
Expand Down
3 changes: 0 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,6 @@ RUN poetry install --only main --no-root --no-interaction --no-ansi
RUN poetry self add poetry-plugin-export
RUN poetry export -f requirements.txt -o requirements.txt

ARG AWS_SECRET_ACCESS_KEY
ARG AWS_ACCESS_KEY_ID

COPY data/ data

RUN mkdir namegraph
Expand Down
17 changes: 5 additions & 12 deletions namegraph/download_from_s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,30 +4,23 @@

import tarfile
import boto3
import botocore
import hydra

from dotenv import load_dotenv
from omegaconf import DictConfig


class S3Downloader:
def __init__(self):
self.s3_client = None
self.region_name = 'us-east-1'
self.bucket = 'prod-name-generator-namegeneratori-inputss3bucket-c26jqo3twfxy'

def get_client(self):
if self.s3_client is None:
load_dotenv()

S3_ACCESS_KEY_ID = os.getenv('S3_ACCESS_KEY_ID')
S3_SECRET_ACCESS_KEY = os.getenv('S3_SECRET_ACCESS_KEY')
REGION_NAME = 'us-east-1'
self.s3_client = boto3.client('s3',
aws_access_key_id=S3_ACCESS_KEY_ID,
aws_secret_access_key=S3_SECRET_ACCESS_KEY,
region_name=REGION_NAME
)

self.s3_client = boto3.client(
's3', region_name=self.region_name, config=botocore.config.Config(signature_version=botocore.UNSIGNED)
)
return self.s3_client

def download_file(self, url, path, override=True):
Expand Down