Merge pull request #20 from JexPY/feature/GetDataFromUrl

Feature/get data from url

Author: JexPY

.gitignore

@@ -8,7 +8,6 @@ certbot/letsecrypt/
 key.json
 docker-compose.yml
 nginx/sites/app.conf
-static/
 __pycache__/
  *.pyc
  .pytest_cache

api/Dockerfile

@@ -9,7 +9,7 @@ WORKDIR $WORKDIR
 RUN apt-get update && apt-get install -y python3-dev build-essential
 
 # Image modifications
-RUN apt-get install -y libtiff5-dev libjpeg62-turbo-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev
+RUN apt-get install -y libtiff5-dev libjpeg62-turbo-dev zlib1g-dev libfreetype6-dev liblcms2-dev libwebp-dev libmagic1
 
 RUN if [ ! -e /lib/libz.so ]; then \
     ln -s /usr/lib/x86_64-linux-gnu/libz.so /lib/ \

api/app/.env-example

@@ -1,6 +1,7 @@
 # SERVER
 API_URL=http://localhost/
-FILE_MANAGER_BEARER_TOKEN=very_secret
+# several separate with ,
+FILE_MANAGER_BEARER_TOKEN=very_secret,any_more_secret_tokens
 # several separate with ,
 CORS_ORIGINS=http://localhost,http://ff.etomer.io
 
@@ -55,12 +56,13 @@ QR_IMAGE_LOCAL_PATH=static/qr/
 
 # IMAGES
 IMAGE_CONVERTING_PREFERED_FORMAT=webp
-THUMBNAIL_MAX_WIDHT=320
+IMAGE_AllOWED_FILE_FORMAT=png,jpeg,jpg,webp
 # make thumbnails or nah True/False
 IMAGE_THUMBNAIL=True
+THUMBNAIL_MAX_WIDHT=320
 # suported:pillow-simd,ffmpeg
 # default:pillow-simd
-IMAGE_OPTIMIZATION_USING=ffmpeg
+IMAGE_OPTIMIZATION_USING=pillow-simd
 
 
 # VIDEOS

api/app/main.py

@@ -11,9 +11,10 @@
 import sys
 
 from services.serveUploadedFiles import handle_upload_image_file, handle_multiple_image_file_uploads, handle_upload_video_file
-from services.serverQrcode import handle_qr_code
+from services.serveQrcode import handle_qr_code
 from services.security.customBearerCheck import validate_token
 from services.storage.local import response_image_file
+from services.serveDataFromUrl import handle_download_data_from_url, handle_multiple_image_file_downloads
 
 load_dotenv()
 app = FastAPI(docs_url=None if os.environ.get('docs_url') == 'None' else '/docs', redoc_url=None if os.environ.get('redoc_url') == 'None' else '/redoc')
@@ -50,6 +51,7 @@ def root(
         result['cpu_average_load'] = os.getloadavg()
     return result
 
+
 # File size validates NGINX
 @app.post("/image", tags=["image"])
 async def upload_image_file(
@@ -64,14 +66,23 @@ async def upload_image_file(
 
 
 @app.post("/images", tags=["image"])
-async def upload_image_files(files: List[UploadFile] = File(...), OAuth2AuthorizationCodeBearer = Depends(validate_token)):
+async def upload_image_files(
+        thumbnail: Optional[str] = Query(
+        os.environ.get('IMAGE_THUMBNAIL'),
+        description='True/False depending your needs',
+        regex='^(True|False)$'
+        ),
+        files: List[UploadFile] = File(...),
+        OAuth2AuthorizationCodeBearer = Depends(validate_token)
+    ):
     fileAmount = len(files)
     if fileAmount > int(os.environ.get('MULTIPLE_FILE_UPLOAD_LIMIT')):
         raise HTTPException(
             status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
             detail='Amount of files must not be more than {}'.format(os.environ.get('MULTIPLE_FILE_UPLOAD_LIMIT'))
     )
-    return handle_multiple_image_file_uploads(files, fileAmount)
+    return handle_multiple_image_file_uploads(files, fileAmount, True if thumbnail == 'True' else False)
+
 
 @app.get("/image", tags=["image"])
 async def get_image(
@@ -88,6 +99,7 @@ async def get_image(
         ):
     return response_image_file(image, image_type)
 
+
 @app.post("/qrImage", tags=["image"])
 async def text_to_generate_qr_image(
     qr_text: str = Query(
@@ -112,4 +124,53 @@ async def upload_video_file(
         ),
     file: UploadFile = File(..., description='Allows mov, mp4, m4a, 3gp, 3g2, mj2'),
     OAuth2AuthorizationCodeBearer = Depends(validate_token)):
-    return handle_upload_video_file(True if optimize == 'True' else False, file)
+    return handle_upload_video_file(True if optimize == 'True' else False, file)
+
+
+@app.get("/imageUrl", tags=["from url"])
+async def image_from_url(
+    image_url: str = Query(
+        None,
+        description = "Pass valid image url to upload",
+        min_length  = 5
+    ),
+    thumbnail: Optional[str] = Query(
+        os.environ.get('IMAGE_THUMBNAIL'),
+        description='True/False depending your needs',
+        regex='^(True|False)$'
+    ),
+    OAuth2AuthorizationCodeBearer = Depends(validate_token)):
+    return handle_download_data_from_url(image_url, True if thumbnail == 'True' else False, file_type='image')
+
+
+@app.get("/imageUrls", tags=["from url"])
+async def images_from_urls(
+    image_urls: List[str] = Query(
+        None,
+        description = "Pass valid image urls to upload",
+        min_length  = 5
+    ),
+    OAuth2AuthorizationCodeBearer = Depends(validate_token)):
+    fileAmount = len(image_urls)
+    if fileAmount > int(os.environ.get('MULTIPLE_FILE_UPLOAD_LIMIT')):
+        raise HTTPException(
+            status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
+            detail='Amount of files must not be more than {}'.format(os.environ.get('MULTIPLE_FILE_UPLOAD_LIMIT'))
+    )
+    return handle_multiple_image_file_downloads(image_urls, fileAmount)
+
+
+@app.get("/videoUrl", tags=["from url"])
+async def video_from_url(
+    video_url: str = Query(
+        None,
+        description = "Pass valid video url to upload",
+        min_length  = 5
+    ),
+    optimize: Optional[str] = Query(
+        os.environ.get('VIDEO_OPTIMIZE'),
+        description='True/False depending your needs default is {}'.format(os.environ.get('VIDEO_OPTIMIZE')),
+        regex='^(True|False)$'
+        ),
+    OAuth2AuthorizationCodeBearer = Depends(validate_token)):
+    return handle_download_data_from_url(video_url, False, True if optimize == 'True' else False, file_type='video')

api/app/services/helpers/detectFileExtension.py

@@ -0,0 +1,12 @@
+import magic
+import mimetypes
+from pathlib import Path
+from fastapi import HTTPException,status
+
+def magic_extensions(file_path: Path):
+    mime = magic.Magic(mime=True)
+    detectedExtension = mimetypes.guess_all_extensions(mime.from_buffer(open(file_path, "rb").read(2048)))
+    if len(detectedExtension) > 0:
+        return detectedExtension[0]
+    else:
+        raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail='File extension not detected')

api/app/services/security/customBearerCheck.py

@@ -1,15 +1,17 @@
 import os
+import json
 import secrets
 from fastapi.security import HTTPBearer,OAuth2AuthorizationCodeBearer,HTTPBasicCredentials
 from fastapi import Depends, HTTPException, status
-
 security = HTTPBearer()
 
 def validate_token(credentials: HTTPBasicCredentials = Depends(security)):
-    correct_token = secrets.compare_digest(credentials.credentials, os.environ.get('FILE_MANAGER_BEARER_TOKEN'))
-    if not (correct_token):
+
+    for eachKey in os.environ.get('FILE_MANAGER_BEARER_TOKEN').split(','):
+        if secrets.compare_digest(credentials.credentials, eachKey):
+            return True
+    else:
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,
             detail="Incorrect token"
         )
-    return True

api/app/services/serveDataFromUrl.py

@@ -0,0 +1,34 @@
+import requests
+import concurrent.futures
+from typing import List
+from fastapi import HTTPException,status,Query
+from .serveUploadedFiles import handle_upload_image_file,handle_upload_video_file
+import sys
+
+def handle_download_data_from_url(url = str, thumbnail = bool, optimize = None, file_type = str):
+    try:
+        get_data = requests.get(url, stream = True)
+        if get_data.status_code == 200:
+            get_data.raw.decode_content = True
+            if file_type == 'image':
+                return handle_upload_image_file(thumbnail, upload_file=None, raw_data_file=get_data)
+            elif file_type =='video':
+                return handle_upload_video_file(optimize, upload_file=None, raw_data_file=get_data)
+        else:
+           raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=f'Unsuccessful request for: {url}')
+    except:
+        raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail=f'Unable to download data from: {url}')
+
+
+def handle_multiple_image_file_downloads(URL_FILES: List[Query], workers: int, thumbnail = bool):
+    # We can use a with statement to ensure threads are cleaned up promptly
+    with concurrent.futures.ThreadPoolExecutor(max_workers = workers) as executor:
+
+        future_to_url = {executor.submit(handle_download_data_from_url, eachFile, thumbnail, file_type = 'image'): eachFile for eachFile in URL_FILES}
+        result = []
+        for future in concurrent.futures.as_completed(future_to_url):
+            try:
+                result.append(future.result())
+            except:
+                raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail='Multiple upload failed')
+        return result

api/app/services/serverQrcode.py renamed to api/app/services/serveQrcode.py

@@ -1,6 +1,5 @@
 import os
 import shutil
-import imghdr
 import _thread
 import copy
 from typing import List
@@ -12,27 +11,40 @@
 from fastapi import UploadFile,HTTPException,status
 from .storage.googleCloud import upload_image_file_to_google_storage,upload_video_file_to_google_storage
 from .storage.s3 import upload_image_file_to_s3_storage,upload_video_file_to_s3_storage
-from ffmpeg import probe
+from .helpers.detectFileExtension import magic_extensions
+import sys
 
-def save_upload_file_tmp(upload_file: UploadFile) -> Path:
+def save_upload_file_tmp(upload_file: None, raw_data_file = None) -> Path:
     try:
-        suffix = Path(upload_file.filename).suffix
-        with NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
-            shutil.copyfileobj(upload_file.file, tmp)
-            tmp_path = Path(tmp.name)
+        if raw_data_file:
+            with NamedTemporaryFile(delete=False, suffix=None) as tmp:
+                    shutil.copyfileobj(raw_data_file.raw, tmp)
+
+        else:
+            with NamedTemporaryFile(delete=False, suffix=None) as tmp:
+                shutil.copyfileobj(upload_file.file, tmp)
+
+        extension = magic_extensions(Path(tmp.name))
+        final_temp_file = tmp.name + extension
+        os.rename(Path(tmp.name), final_temp_file)
+
     except:
         raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail='Impossible to manipulate the file')
     finally:
-        upload_file.file.close()
-    return tmp_path
+        if upload_file:
+            upload_file.file.close()
+        else:
+            raw_data_file.close()
+    return Path(final_temp_file), extension
 
 
-def handle_upload_image_file(thumbnail, upload_file: UploadFile):
+def handle_upload_image_file(thumbnail, upload_file: None, raw_data_file = None):
     try:
-        tmp_path = save_upload_file_tmp(upload_file)
-        if imghdr.what(tmp_path):
+        tmp_path, file_extension = save_upload_file_tmp(upload_file, raw_data_file)
+
+        if file_extension[1:] in os.environ.get('IMAGE_AllOWED_FILE_FORMAT').split(','):
 
-            imagePaths = resize_image(tmp_path, imghdr.what(tmp_path), thumbnail, os.environ.get('IMAGE_CONVERTING_PREFERED_FORMAT'))
+            imagePaths = resize_image(tmp_path, file_extension[1:], thumbnail, os.environ.get('IMAGE_CONVERTING_PREFERED_FORMAT'))
 
             if os.environ.get('PREFERED_STORAGE') == 'google':
                 _thread.start_new_thread(upload_image_file_to_google_storage, (copy.deepcopy(imagePaths),))
@@ -49,18 +61,18 @@ def handle_upload_image_file(thumbnail, upload_file: UploadFile):
                 imagePaths['thumbnail'] = os.environ.get('API_URL') + os.environ.get('IMAGE_THUMBNAIL_LOCAL_PATH') + imagePaths['thumbnail'] if imagePaths.get('thumbnail') else None
 
             imagePaths['storage'] = os.environ.get('PREFERED_STORAGE')
+            imagePaths['file_name'] = imagePaths['original'].split('/')[-1]
             return imagePaths
         else:
             raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail='The file format not supported')
     finally:
         tmp_path.unlink()  # Delete the temp file
 
 
-def handle_multiple_image_file_uploads(FILES: List[UploadFile], workers: int):
+def handle_multiple_image_file_uploads(FILES: List[UploadFile], workers: int, thumbnail: bool):
     # We can use a with statement to ensure threads are cleaned up promptly
     with concurrent.futures.ThreadPoolExecutor(max_workers = workers) as executor:
-        # Start the load operations and mark each future with its FILES thumbnail is default for this function
-        future_to_url = {executor.submit(handle_upload_image_file, True, eachFile): eachFile for eachFile in FILES}
+        future_to_url = {executor.submit(handle_upload_image_file, eachFile, thumbnail): eachFile for eachFile in FILES}
         result = []
         for future in concurrent.futures.as_completed(future_to_url):
             try:
@@ -70,15 +82,12 @@ def handle_multiple_image_file_uploads(FILES: List[UploadFile], workers: int):
         return result
 
 
-def handle_upload_video_file(optimize, upload_file: UploadFile):
+def handle_upload_video_file(optimize, upload_file: None, raw_data_file = None):
+
     try:
-        tmp_path = save_upload_file_tmp(upload_file)
-        videoFileCheck = probe(tmp_path).get('format')
-        if videoFileCheck.get('format_name'):
-            # Checks for video file type also is possible to restict for only mp4 format by checking major_brand
-            # videoFileCheck.get('tags').get('major_brand') == 'mp42'
-            if os.environ.get('VIDEO_AllOWED_FILE_FORMAT') in videoFileCheck.get('format_name').split(','):
+        tmp_path, file_extension = save_upload_file_tmp(upload_file, raw_data_file)
 
+        if file_extension[1:] in os.environ.get('VIDEO_AllOWED_FILE_FORMAT').split(','):
                 videoPaths = video_file_FFMPEG(tmp_path, optimize)
 
                 if os.environ.get('PREFERED_STORAGE') == 'google':
@@ -98,10 +107,9 @@ def handle_upload_video_file(optimize, upload_file: UploadFile):
                     videoPaths['optimized'] = os.environ.get('API_URL') + os.environ.get('VIDEO_OPTIMIZED_LOCAL_PATH') + videoPaths['optimized'] if videoPaths.get('optimized') else None
 
                 videoPaths['storage'] = os.environ.get('PREFERED_STORAGE')
+                videoPaths['file_name'] = videoPaths['original'].split('/')[-1]
 
                 return videoPaths
-            else:
-                raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail='Format not granted')
         else:
             raise HTTPException(status_code=status.HTTP_503_SERVICE_UNAVAILABLE, detail='Not valid format')
     except: