From ed0f9fa50762970b4dbf7570a13acdb7a4e435cc Mon Sep 17 00:00:00 2001 From: Alex Lambson Date: Fri, 21 Mar 2025 01:17:10 -0600 Subject: [PATCH 1/3] feat: Legacy file validation. --- kirovy/constants/__init__.py | 8 +- kirovy/constants/api_codes.py | 9 ++ kirovy/models/cnc_user.py | 56 ++++---- kirovy/services/legacy_upload/__init__.py | 0 kirovy/services/legacy_upload/base.py | 130 ++++++++++++++++++ kirovy/services/legacy_upload/dune_2000.py | 6 + kirovy/services/legacy_upload/westwood.py | 14 ++ kirovy/utils/file_utils.py | 20 +++ kirovy/views/map_upload_views.py | 4 +- kirovy/zip_storage.py | 20 +-- tests/fixtures/file_fixtures.py | 30 +++- tests/fixtures/game_fixtures.py | 16 ++- tests/test_data/valid_dune_map.zip | Bin 0 -> 13756 bytes .../test_backwards_compatibility.py | 14 ++ 14 files changed, 279 insertions(+), 48 deletions(-) create mode 100644 kirovy/services/legacy_upload/__init__.py create mode 100644 kirovy/services/legacy_upload/base.py create mode 100644 kirovy/services/legacy_upload/dune_2000.py create mode 100644 kirovy/services/legacy_upload/westwood.py create mode 100644 tests/test_data/valid_dune_map.zip diff --git a/kirovy/constants/__init__.py b/kirovy/constants/__init__.py index 2162621..29db99e 100644 --- a/kirovy/constants/__init__.py +++ b/kirovy/constants/__init__.py @@ -90,11 +90,17 @@ def is_messiah(cls, user_group: str) -> bool: class MigrationUser: - ID = -1 + CNCNET_ID = -1 USERNAME = "MobileConstructionVehicle_Migrator" GROUP = CncnetUserGroup.USER +class LegacyUploadUser: + CNCNET_ID = -2 + USERNAME = "Spy_ShapeShifting_LegacyUploader" + GROUP = CncnetUserGroup.USER + + class GameSlugs(enum.StrEnum): """The slugs for each game / total conversion mod. diff --git a/kirovy/constants/api_codes.py b/kirovy/constants/api_codes.py index 567a181..995c4de 100644 --- a/kirovy/constants/api_codes.py +++ b/kirovy/constants/api_codes.py @@ -8,3 +8,12 @@ class UploadApiCodes(enum.StrEnum): EMPTY_UPLOAD = "where-file" DUPLICATE_MAP = "duplicate-map" FILE_EXTENSION_NOT_SUPPORTED = "file-extension-not-supported" + + +class LegacyUploadApiCodes(enum.StrEnum): + NOT_A_VALID_ZIP_FILE = "invalid-zipfile" + BAD_ZIP_STRUCTURE = "invalid-zip-structure" + MAP_TOO_LARGE = "map-file-too-large" + NO_VALID_MAP_FILE = "no-valid-map-file" + HASH_MISMATCH = "file-hash-does-not-match-zip-name" + INVALID_FILE_TYPE = "invalid-file-type-in-zip" diff --git a/kirovy/models/cnc_user.py b/kirovy/models/cnc_user.py index 4c22fb1..f531377 100644 --- a/kirovy/models/cnc_user.py +++ b/kirovy/models/cnc_user.py @@ -27,10 +27,10 @@ def get_or_create_migration_user(self) -> "CncUser": :return: The user for running migrations. """ - mcv = self.find_by_cncnet_id(constants.MigrationUser.ID) + mcv = self.find_by_cncnet_id(constants.MigrationUser.CNCNET_ID) if not mcv: mcv = CncUser( - cncnet_id=constants.MigrationUser.ID, + cncnet_id=constants.MigrationUser.CNCNET_ID, username=constants.MigrationUser.USERNAME, group=constants.MigrationUser.GROUP, ) @@ -39,6 +39,30 @@ def get_or_create_migration_user(self) -> "CncUser": return mcv + def get_or_create_legacy_upload_user(self) -> "CncUser": + """Gets or creates a system-user to represent anonymous uploads from a CnCNet client. + + .. warning:: + + This should **only** be used for the legacy upload URLs for clients + that CnCNet doesn't have the source for. + + :return: + User for legacy uploads. + """ + # If we copy and paste this again then it should be DRY'd up. + spy = self.find_by_cncnet_id(constants.LegacyUploadUser.CNCNET_ID) + if not spy: + spy = CncUser( + cncnet_id=constants.LegacyUploadUser.CNCNET_ID, + username=constants.LegacyUploadUser.USERNAME, + group=constants.LegacyUploadUser.GROUP, + ) + spy.save() + spy.refresh_from_db() + + return spy + def get_queryset(self) -> models.QuerySet: """Makes ``CncUser.object.all()`` filter out the system users by default. @@ -61,9 +85,7 @@ class CncUser(AbstractBaseUser): help_text=_("The user ID from the CNCNet ladder API."), ) - username = models.CharField( - null=True, help_text=_("The name from the CNCNet ladder API."), blank=False - ) + username = models.CharField(null=True, help_text=_("The name from the CNCNet ladder API."), blank=False) """:attr: The username for debugging purposes. Don't rely on this field for much else.""" verified_map_uploader = models.BooleanField(null=False, default=False) @@ -77,23 +99,15 @@ class CncUser(AbstractBaseUser): blank=False, ) - is_banned = models.BooleanField( - default=False, help_text="If true, user was banned for some reason." - ) - ban_reason = models.CharField( - default=None, null=True, help_text="If banned, the reason the user was banned." - ) - ban_date = models.DateTimeField( - default=None, null=True, help_text="If banned, when the user was banned." - ) + is_banned = models.BooleanField(default=False, help_text="If true, user was banned for some reason.") + ban_reason = models.CharField(default=None, null=True, help_text="If banned, the reason the user was banned.") + ban_date = models.DateTimeField(default=None, null=True, help_text="If banned, when the user was banned.") ban_expires = models.DateTimeField( default=None, null=True, help_text="If banned, when the ban expires, if temporary.", ) - ban_count = models.IntegerField( - default=0, help_text="How many times this user has been banned." - ) + ban_count = models.IntegerField(default=0, help_text="How many times this user has been banned.") USERNAME_FIELD = "cncnet_id" """:attr: @@ -113,9 +127,7 @@ def can_upload(self) -> bool: :return: True if user can upload maps / mixes / big, or edit their existing uploads. """ - self.refresh_from_db( - fields=["verified_map_uploader", "verified_email", "is_banned"] - ) + self.refresh_from_db(fields=["verified_map_uploader", "verified_email", "is_banned"]) can_upload = self.verified_map_uploader or self.verified_email or self.is_staff return can_upload and not self.is_banned @@ -142,9 +154,7 @@ def create_or_update_from_cncnet(user_dto: CncnetUserInfo) -> "CncUser": :return: The user object in Kirovy's database, updated with the data from CnCNet. """ - kirovy_user: t.Optional[CncUser] = CncUser.objects.filter( - cncnet_id=user_dto.id - ).first() + kirovy_user: t.Optional[CncUser] = CncUser.objects.filter(cncnet_id=user_dto.id).first() if not kirovy_user: kirovy_user = CncUser.objects.create( cncnet_id=user_dto.id, diff --git a/kirovy/services/legacy_upload/__init__.py b/kirovy/services/legacy_upload/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kirovy/services/legacy_upload/base.py b/kirovy/services/legacy_upload/base.py new file mode 100644 index 0000000..1b8fefd --- /dev/null +++ b/kirovy/services/legacy_upload/base.py @@ -0,0 +1,130 @@ +import dataclasses +import pathlib +import zipfile + +from cryptography.utils import cached_property +from django.core.files.base import ContentFile +from django.core.files.uploadedfile import UploadedFile + +from kirovy import typing as t, constants +from kirovy.constants.api_codes import LegacyUploadApiCodes +from kirovy.exceptions import view_exceptions +from kirovy.services.cnc_gen_2_services import CncGen2MapParser +from kirovy.utils import file_utils +from kirovy.utils.file_utils import ByteSized + + +@dataclasses.dataclass +class ExpectedFile: + possible_extensions: t.Set[str] + file_validator: t.Callable[[str, ContentFile, zipfile.ZipInfo], bool] + required: bool = True + """attr: If false, this file is not required to be present.""" + + +class LegacyMapServiceBase: + game_slug: t.ClassVar[constants.GameSlugs] + _file: zipfile.ZipFile + ini_extensions: t.ClassVar[t.Set[str]] + + def __init__(self, file: UploadedFile): + """Initializes the class and runs the validation for the expected files. + + :param file: + The raw uploaded file from the view. + :raises view_exceptions.KirovyValidationError: + Raised if the file is invalid in any way. + """ + if not file_utils.is_zipfile(file): + raise view_exceptions.KirovyValidationError( + detail="Your zipfile is invalid", code=LegacyUploadApiCodes.NOT_A_VALID_ZIP_FILE + ) + + self._file = zipfile.ZipFile(file) + if len(self.expected_files) == 1: + self.single_file_validator() + else: + self.multi_file_validator() + + def multi_file_validator(self): + file_list = self._file.infolist() + min_files = len([x for x in self.expected_files if x.required]) + max_files = len(self.expected_files) + if min_files > len(file_list) > max_files: + raise view_exceptions.KirovyValidationError( + "Incorrect file count", code=LegacyUploadApiCodes.BAD_ZIP_STRUCTURE + ) + + for file_info in file_list: + expected_file = self._get_expected_file_for_extension(file_info) + with self._file.open(file_info, mode="r") as file: + expected_file.file_validator(self._file.filename, ContentFile(file), file_info) + + def single_file_validator(self): + first_file = self._file.infolist()[0] + if pathlib.Path(first_file.filename).suffix not in self.expected_files[0].possible_extensions: + raise view_exceptions.KirovyValidationError( + "Map file was not the first Zip entry.", code=LegacyUploadApiCodes.BAD_ZIP_STRUCTURE + ) + with self._file.open(first_file, mode="r") as map_file: + self.expected_files[0].file_validator(self._file.filename, ContentFile(map_file), first_file) + + @cached_property + def extract_name(self) -> str: + ini_file_info = self._find_ini_file() + fallback = f"legacy_client_upload_{pathlib.Path(self._file.filename).stem}" + with self._file.open(ini_file_info, mode="r") as ini_file: + return CncGen2MapParser(ContentFile(ini_file)).ini.get("Basic", "Name", fallback=fallback) + + def _find_ini_file(self) -> zipfile.ZipInfo: + for file in self._file.infolist(): + if pathlib.Path(file.filename).suffix in self.ini_extensions: + return file + raise view_exceptions.KirovyValidationError( + "No file containing map INI was found.", LegacyUploadApiCodes.NO_VALID_MAP_FILE + ) + + @cached_property + def expected_files(self) -> t.List[ExpectedFile]: + raise NotImplementedError("This Game's map validator hasn't implemented the expectd file structure.") + + def _get_expected_file_for_extension(self, zip_info: zipfile.ZipInfo) -> ExpectedFile: + extension = pathlib.Path(zip_info.filename).suffix + for expected in self.expected_files: + if extension in expected.possible_extensions: + return expected + raise view_exceptions.KirovyValidationError( + "Unexpected file type in zip file", LegacyUploadApiCodes.INVALID_FILE_TYPE + ) + + @staticmethod + def default_map_file_validator(zip_file_name: str, file_content: ContentFile, zip_info: zipfile.ZipInfo): + """Legacy map file generator that works for most Westwood games. + + Won't work for Dune 2000, or Tiberian Dawn. + + :param zip_file_name: + The name of the uploaded zip file. + :param file_content: + The contents of the map file extracted from the zip file. + :param zip_info: + The zip metadata for the extracted map file. + :return: + None + :raises view_exceptions.KirovyValidationError: + Raised for any validation errors. + """ + if ByteSized(zip_info.file_size) > ByteSized(mega=2): + raise view_exceptions.KirovyValidationError( + "Map file larger than expected.", code=LegacyUploadApiCodes.MAP_TOO_LARGE + ) + + # Reaching into the new map parsers is kind of dirty, but this legacy endpoint + # is (hopefully) just to tide us over until we can migrate everyone to the new endpoints. + if not CncGen2MapParser.is_text(file_content): + raise view_exceptions.KirovyValidationError( + "No valid map file.", code=LegacyUploadApiCodes.NO_VALID_MAP_FILE + ) + + if file_utils.hash_file_sha1(file_content) != pathlib.Path(zip_file_name).stem: + raise view_exceptions.KirovyValidationError("Map file checksum differs from Zip name, rejected.") diff --git a/kirovy/services/legacy_upload/dune_2000.py b/kirovy/services/legacy_upload/dune_2000.py new file mode 100644 index 0000000..a71014c --- /dev/null +++ b/kirovy/services/legacy_upload/dune_2000.py @@ -0,0 +1,6 @@ +from kirovy import constants +from kirovy.services.legacy_upload.base import LegacyMapServiceBase + + +class Dune2000LegacyMapService(LegacyMapServiceBase): + game_slug = constants.GameSlugs.dune_2000 diff --git a/kirovy/services/legacy_upload/westwood.py b/kirovy/services/legacy_upload/westwood.py new file mode 100644 index 0000000..49e7fc4 --- /dev/null +++ b/kirovy/services/legacy_upload/westwood.py @@ -0,0 +1,14 @@ +from kirovy import constants, typing as t +from kirovy.services.legacy_upload.base import LegacyMapServiceBase, ExpectedFile + + +class YurisRevengeLegacyMapService(LegacyMapServiceBase): + ini_extensions = {"map", "yro", "yrm"} + game_slug = constants.GameSlugs.yuris_revenge + + def expected_files(self) -> t.List[ExpectedFile]: + return [ + ExpectedFile( + possible_extensions=self.ini_extensions, file_validator=self.default_map_file_validator, required=True + ) + ] diff --git a/kirovy/utils/file_utils.py b/kirovy/utils/file_utils.py index 884259c..434646e 100644 --- a/kirovy/utils/file_utils.py +++ b/kirovy/utils/file_utils.py @@ -1,6 +1,8 @@ import collections import functools import hashlib +import pathlib +import zipfile from django.core.files import File @@ -131,3 +133,21 @@ def __le__(self, other: "ByteSized") -> bool: def __eq__(self, other: "ByteSized") -> bool: return self.total_bytes == other.total_bytes + + +def is_zipfile(file_or_path: File | pathlib.Path) -> bool: + """Checks if a file is a zip file. + + :param file_or_path: + The path to a file, or the file itself, to check. + :returns: + ``True`` if the file is a zip file. + """ + try: + with zipfile.ZipFile(file_or_path, "r") as zf: + # zf.getinfo("") # check if zipfile is valid. + return True + except zipfile.BadZipfile: + return False + except Exception as e: + raise e diff --git a/kirovy/views/map_upload_views.py b/kirovy/views/map_upload_views.py index b263493..63e2185 100644 --- a/kirovy/views/map_upload_views.py +++ b/kirovy/views/map_upload_views.py @@ -15,7 +15,7 @@ from kirovy import typing as t, permissions, exceptions, constants, logging from kirovy.constants.api_codes import UploadApiCodes from kirovy.exceptions.view_exceptions import KirovyValidationError -from kirovy.models import cnc_map, CncGame, CncFileExtension, MapCategory, map_preview +from kirovy.models import cnc_map, CncGame, CncFileExtension, MapCategory, map_preview, CncUser from kirovy.objects.ui_objects import ResultResponseData from kirovy.request import KirovyRequest from kirovy.response import KirovyResponse @@ -378,7 +378,7 @@ def post(self, request: KirovyRequest, format=None) -> KirovyResponse: cnc_game_id=game_id, is_published=False, incomplete_upload=True, - cnc_user=request.user, + cnc_user=CncUser.objects.get_or_create_legacy_upload_user(), parent=None, ) new_map.save() diff --git a/kirovy/zip_storage.py b/kirovy/zip_storage.py index 832f4cd..0f5d378 100644 --- a/kirovy/zip_storage.py +++ b/kirovy/zip_storage.py @@ -10,7 +10,7 @@ class ZipFileStorage(FileSystemStorage): def save(self, name: str, content: File, max_length: int | None = None): - if is_zipfile(content): + if file_utils.is_zipfile(content): return super().save(name, content, max_length) internal_extension = pathlib.Path(name).suffix @@ -21,21 +21,3 @@ def save(self, name: str, content: File, max_length: int | None = None): zf.writestr(internal_filename, content.read()) return super().save(f"{name}.zip", zip_buffer, max_length=max_length) - - -def is_zipfile(file_or_path: File | pathlib.Path) -> bool: - """Checks if a file is a zip file. - - :param file_or_path: - The path to a file, or the file itself, to check. - :returns: - ``True`` if the file is a zip file. - """ - try: - with zipfile.ZipFile(file_or_path, "r") as zf: - # zf.getinfo("") # check if zipfile is valid. - return True - except zipfile.BadZipfile: - return False - except Exception as e: - raise e diff --git a/tests/fixtures/file_fixtures.py b/tests/fixtures/file_fixtures.py index e365145..3a7aee7 100644 --- a/tests/fixtures/file_fixtures.py +++ b/tests/fixtures/file_fixtures.py @@ -7,6 +7,13 @@ import pytest from django.core.files import File +from kirovy.utils import file_utils + + +class ReadOnlyFile(File): + def write(self, s, /): + raise Exception("Don't tamper with the test files.") + @pytest.fixture def test_data_path() -> pathlib.Path: @@ -28,7 +35,7 @@ def _inner(relative_path: t.Union[str, pathlib.Path], read_mode: str = "rb") -> """ full_path = test_data_path / relative_path - return File(open(full_path, read_mode)) + return ReadOnlyFile(open(full_path, read_mode)) return _inner @@ -85,3 +92,24 @@ def file_map_unfair(load_test_file) -> Generator[File, Any, None]: file = load_test_file("totally_fair_map.map") yield file file.close() + + +@pytest.fixture +def file_map_dune2k(load_test_file) -> Generator[File, Any, None]: + """Return a valid zip file for a dune 2k mpa.""" + file = load_test_file("valid_dune_map.zip") + yield file + file.close() + + +@pytest.fixture +def rename_file_for_legacy_upload(): + """Returns a function to rename a file for upload to a legacy endpoint.""" + + def _inner(file: File) -> File: + assert file.name.endswith(".zip"), "Only zip files can be sent to legacy endpoints." + # Uploads to legacy endpoints need the hash name + file.name = file_utils.hash_file_sha1(file) + ".zip" + return file + + return _inner diff --git a/tests/fixtures/game_fixtures.py b/tests/fixtures/game_fixtures.py index 76c11e9..1a4270b 100644 --- a/tests/fixtures/game_fixtures.py +++ b/tests/fixtures/game_fixtures.py @@ -1,5 +1,5 @@ import pytest -from kirovy import typing as t, models as k_models +from kirovy import typing as t, models as k_models, constants @pytest.fixture @@ -11,4 +11,16 @@ def game_yuri(db) -> k_models.CncGame: :return: The game object for Yuri's revenge. """ - return k_models.CncGame.objects.get(slug="yr") + return k_models.CncGame.objects.get(slug=constants.GameSlugs.yuris_revenge) + + +@pytest.fixture +def game_dune2k(db) -> k_models.CncGame: + """Get the Dune 2000 ``CncGame``. + + This fixture depends on the game data migration being run. + + :return: + The game object for Dune 2000. + """ + return k_models.CncGame.objects.get(slug__iexact=constants.GameSlugs.dune_2000) diff --git a/tests/test_data/valid_dune_map.zip b/tests/test_data/valid_dune_map.zip new file mode 100644 index 0000000000000000000000000000000000000000..2046165b4ca44828db45051993e2179ab54039df GIT binary patch literal 13756 zcmb7rXHZj37%mu!QbnXU6%_^Py@X%^J{zEZNGEiVD7_~k6a|zLyGRR&0@9IQLO`k_ zO-iT1zwo1Jv9ZT*!QT@(dg^3Q zhxVE|f&<4d8ZULzU~w?*X=K9jln$OVSGjPCf3qF2?8tw$Dgr@k-?15mABNf4+4=mL zgKpPjS=8h}#3f;2;b@bisHP1~tE%}74Y?ESRv^yQ_)b;hy2v`;m}eetd9`iZIBA1K zJ&4Gd#oM+ITVq95Wz9{AtB$#CuB#*+rc3eOc|*(B#OJ$t%9ux)@&o%IB@Bky>oI4X zObWo3Vdpa9@OGMk>&fo^Xui3O3C>5!R-B5--F6C7}Xz-WP)G?lgf+{`0ph61^pQCI&!Uk{6alyFWpnl!O~OPq+lzpDAuh|hV z#DAW=6_qK#JXm*s)m*OU|8m_n*0Uq9x$8Rf@jS8IW0;b3*LK!W5;eh)aDQ_yHhF05aa*HXjy82N0e)#28fHa6 zTTdDDMuU6mxwf7i8gwZ1_nN+{#+o;iq^iom9l9aLtliDU#j3PEBor1!-$CxR1y>hy zVvGms39==bA6kUQK|iXE9awf6#ZBVC$LDB=SN4>?7ljI^B|X`dL?+b#>%}sdDmfSN z<452cjcY1k$eaSyUb9WWuetS=&Ve%EGJ*OT6vpz7aPK-h$fVsJ#@gt^QHzZsif*nK zx9WvL3nZb1xV~e?M7P3kHOVpbbBAaVnjI55Zy9Z^{W!O;<&*Fq0dT`P*PxX27c~(H zbA_C=c&H--&)9{N2AZ4lW)0X+cj(WxaKW&=-epkQFnc*lws(EKFS_C(y|s;%hHm>V zu{-tQ+hP}4{@U zt@aV_g~5eR)c)=)@0Npz9&K={vUj-6a4u?mV}}|kjl3$q7uLPOj*gk{`VeN}ELIJ= z(EYEF>9b^zW=A8LJTOD~uycHjBG$a7k2MD0j!8zKyeiqRgYO_V{N?fMk4yrWV={XW zjO4R7qCUz8R^iva$C{`i_vXsU3Z;wLJX5utnz`B@5y`YmlM9R4;{jK2b4s{Y9iY%y z6n3LITRv6RJqF2mwT=e4EGk4#y?Tm1yI6k?wQkDtG5v+LEhc7+>DJ?Q?qDw5VOvi@ zOqRkasq9gMHFlH<>&NC`W@WJ^a3DCVH%dR9xm{np;YW(>&^R}FBZ;)Jh!i3-Ppf}T zky2cm<771qJ&yp-914xy^LDN+S5l`nqu;O8w4P)d-@ZXQG58zz`lBOMaUXh$$=o-s zPY1w$ac!OVU|G_$Tr0IZFW#tO;OJ})nEneOmLk7`yZBKtrmFU+s)9W}Js({@j+wCU zQCd-H4wMX2! z7&CzrlvLs^AMi})0^ELxIolsoOv*b6YU(jH!h@8SWvla2(;zRqelRD|=U!Bu6od_( zjWj!I`$-yMi`6)}uDQL1+|`;-?fKCjgH-9Jg5R-vHQ+op5RP*gl)G zpO>D7Ib4&6yw|+SQ&}4EwMS8v<+qc9na=8CwL568rRfjSGzOSIJhb_@V6%IvZAt?z zFGBFl*+LpOaOuVp-)M(Q*KD{Z(%~Ib`G>t-bLAPVf_J4s7u66D+2S$_k|R+-PLv3? z)^waiH13F9K~EfH52rY(z%DGc9Wta44N;$a6jbGZENimTk|#=$wPo(t*TePjUCIq- z1|%=mP_9nC9Fn7OewxEiBeoExMHwhBur10sRgn5keBJQ(I#t!Pz1+9SuI&i%(fSQ!&qjZx3$3w1F%vMh-M7(h2)O+q|+W z%{!3FV^F@p_1?C2!-rN|X0Lb3QkbRgv@&PQZs*7`UGu4t;mS1zEVGw+hGpiItR(e@ zvaF@Qvarmx#ovf+_dlAmL-zF*g@TJo^Zo66=Ni^8;Tv<;Tv8oOLDTVE82?`G!vm6+ z0AQ;$89w~YvV^_hrZSGbwB%U|x7s~0>Uq`7Xr&QLowMH<61?&V^~69M@P*VTmv>=t z>da>{JYii2c6&@v>O8(vf699KOSF;-el6Y>@5;{F2?ZO=gZ0x9PV3fV2dE;Fi4}gV z9Rfl}s>G(u#Ghk8#AQ=rL$7U_a7f%rDgGAUd>)Yf)x)msmnaf`v7<^r3maBmHkEaJ z&Ld7((#zE=I`CB}LoF{l6oX4fvU)CyKkggbZ~DM=Qw(<3(iq^gvfwghVf#pQnX+k2 zRNfHnMPHJ(o=lBI_WpTNE1KH;_km_=t{7&NgxI&HE{jKr{A#TDAxLHqcn+g(N%|tE z&^?I}j1MvGeSmVZxj9fUY!pyTF_lU_A$9T*b7WpB(mYM?t05 z8`$kuRW9BNJ~<>oR~|SsJ;w>2sLth5fmsi4c0NENEwyu$7ZK*zM>k!~N=sixUF0U*B; z+i2L~eCMi~3miU=opnTr&$4{x@K9pppPt^plaVGjWe>Hc5y?&3=Xg)Y6EzhUG<+QL z>^jqOG%ds*mS{S@KX@mC1!MJN29yh zPv-Kv#m9!Nt2sA`sPmCQr|%T3esul&ll{g9cTZM%59v5j#sWekC)VFCpJoe75}z_M z(}45trZ>t5uh7~&wzZ0#t0xtEA7C+u9AY&m_8f%l-8%9tWWP#7Q+CPi2JUgtAKO(I zF3ujmSSRZA^5q_6FMs=B5IcLKJ@ll^@7`z?B3#w9)9Pr17?$&_Y^2xptP;7GXOIS_ zg(017I!s9~ub-VX8vZif=v`UAcrXXJ0v_tZ*=M4b27mMm`*hcuw+|-{zdR3`f~{6b zlNRl)n-?1JsHcG69(7975_t+AqSKQo4L~p)UFi19+JhaAN4$#iREgdwT=_IB|Cxx3 zw|iZ!BxG+b#3%v?U+S`YE0paNaS^gQZ#YFM)<2#TE(wM zyTy+t{A+~=>$B^urBbAB@`(rPWK@EPoh1OFLLPlr_}&I3E+C&}>8ccmcW4RSmkjKj z^4>CT>pdpWp;%cDOEFv{7><#wp&cUj!w%GG!QT8u5V!Nv>;Y>nsuEM5wSI4J1MFp$ zd%kgQtdbq`pC~d0qW9^|wf51AX ztfh$~M@Flpu3b-w+^q$IK*(K@9hAqr(Q5W&;J^5n?d7{37!%h$yMTkn%Pfdzs5pjn zDRcJbQNxG6JYsQM8%XnS$hTbA&u1<{Y#{3E^3Jmzy65r%n_gN~ z_WLLg&|Vl(9YvTDE@|_p?>3E-_yv~`m ze>!M#28lvZp?gj^3GhZ$2KEU1uog6E5U4mcT${!w>WKdgwo!w2MMZv#KrRtg{zPh3 zKNGE9EbNwL!%RpRdrE9_NG#v3=8yjASIZ7XQJIjsRZ-mx;1AupXx6k0&)JBxiGZS< zhR5UGCtE+W!7rBw=}&O|!9po_kM?Eic}mmA9H<+$X?<_#u&M;C$z4zT&YflS%=5tF z_E}En*XeY?cI#UtO#Cq&gTyjG4}t5L9_WvSLnHdZi^M_0kL^N9vfy;j$PmG87JQaI zBhjN~zicE)Nj^cTt1pY*@Jvd}$qV7+?dIu{OLKtRt+|8G3xSSfmoEj+-kjWdV!{_f z+OUcxojJ$`EH(M==z>>Bx*8p8qOsU07|j#R+e?R;$(9=g^|}&Yb_rOdlZzi}>#s%J zWrrLwz!g5AMtAW#fQW9kP|iqLv5#-1Sa#pocZD*M>jo>zTI9S=Nu~Khqz53cM+tFC z2L3S*I(}s{$KUW%MH)MG<)!rG$XZ94o3Pzf;%q&$cPExrG_gf9cToB}u;;JFaw#}8 zJ8{DHXZQMcjAP%@Ez$wOt?g_>78CQZ7B$PVG+vN@ulsr>p*HpgQIcl|avBwqLXx9C zYwB00h2Ju=@?DJnB9&- zozAeU#bb9{C;BVSPjU1NujSkf#PCS%$zZmWkbTgyX(fl-bGuc2SoUDl(Z?NnZCM^F zq!p#0fK*^kVH>=HzSyg*E<(K)q+QoyPTpzk;N71BsLHb6*Bn6+BqTy-;bs#msp? zB8mi3nb}^yW@X{WBuZD$vT5`@wzfP1YQoA0Dt7#JL|NYX+J_UqQsPUqP;IxZ+LXvy zHdFQ-bsdof?t~g z2fxmGF00k{f;`xlBg0I1oU1)n5tlRFH=Y}8Td4t=65dd|87 zKDo3f+XM0ozRIkrBnh{db!frgMg!QDlIT}9?m&Sbwbw_VP8VcRno^~?5SuOH1$NN>4-0*`%N3V9E zY(f<5zg`n{l(y3nTjb1e{L^!97wF{1InEx<`UVKuS6bXJe+H=*Ll_lUe2;|Trg60o z&q*KUo{1K!xUkxSIEC510!yywZih$>qqo>bcwPN=p1_^+hBiG7G6=&nkwCM+uir18 zUTnNDm&jMC5E}2OzlUCqMBt;2wq%m8Q5%(}_pslRF)4^B_*S^;Isc!oojWnFp+PU_ z{%kW?f9NcXzB~e`2mjw#$!HqPOFL*JrEEytYoC8O`BJztA>dFgg%z+re*Ze zezZrQMD3ho#qCNxuYnW0T^O3ndkPRY-sus}pK_74eN0(8&xxtvGVa}k2ce*;L`bu! zK$JSPG5Kyq3a6_VyX}^9@xjV$G<%!QLg>r0V<#Y&2kW)>=G1uf6;rKci|}?D$^9Vs zYvvP0q!az>&u5QntoPWCN0FycK1!jbix+=-VqRER%&nrdOxa&~lI#Vx%VrMvfxi9m z2a z^S#`@!{y8S3=?Y+Oss`uSLGST3nq)k{|3f7?5Z{d ztiClmM!1;)T?qSIhclgg0ojSj@u2Dy>7GlBPLeDVuPcv%n#3aiqEa|#veHE@5liQ3 zYt(GS@4vuEYtx~IzR0<+xJgKklelxbJif)3WKAB{|Ar5A|Sf$v&Y}nJDp!{fi<1vW;sAzn(bG=^T&%!-iKQhJ-bE(oCkr46u zKhWTtQQ1;Qjec%z=Kde&a%Yyx=q`|*9;*;GvYMN>X5NS@sN{K69G$ATNvI#{lwMal za7|Z~w)nl(0hY$BGwjf=j?R8W)%t77yGbiG%Gh|5p{H4!FPc9Te38lz#~z)S5t%2# z`(|@#S9&5&g=xV07PrC|v$mU8)3Owx@5A8-#~T&|=6qCR+jWj zyvznOuBS(A`hS>cbny?+oedq3(gj7RXs%nW8Lk&C|;6wmw7=*DyF{(fLz!u9o@ zJS`HDGd%OlSQD`Z|GVb}TppdG>KEarNQ+i!!e6a!F5BXSbY6Be$v;4>8oGMt2ut>L z8TUE_FXh6DdIVLKW~er+pnv@^7ZoifZ5V`_j2Zcp^6z(?VZ2rIOEqGCPOsT-3-Bh` zGc-y=OgbmUPDJyH=B4?*tiLD==a$<=_o~HTx^Rn9AzvjaD5V zwy>((F8GPK#B))DxJZ?C<7GiN{U;95;DZxGBh>tiP#%2xp?+egcV=?{(|JNtuQTE@ z7I@35Ryu3nKUs2E4;u6iXm;>bf?)l472&kQKFzjezNp<&@qJFAwDGQ-jz|Coq~#FY z_Mi1Hynp)aPf3Na$SZyRG1>lpka=2o!b4xpm{)n1xF&fFXM)Qok#`}wVAB=T(ZRt*` zI7n9T-P>C#OWKXmHl!8f273gwb2l&ldW6^!W)Y(_-oMoo=qYrN(AF_pO##0@OrkPo zNT2jv_jrMmb|TIOQuPF(_5zP$`G)pC0jz*jWlu%AU2zW3z`%#0hy6Lh7l?{N!H{hfO-nl{$M6jnb%X1HRHf)PZ&mb#hPJkmb;V)c?5JI z9xcKS;BWU`K1EnRwfSSV1$Fg{|I|PGV(eDTIdj%g$)*FSI`SvdLIHW) zIbYiT+N;oqi(wmwR=bV~0p6!_u0htP2fX;pw_oJsEs=lK#7xDDE96}Of2O%M2b6ji zIBslb?T3m`R-VkG;`RTK+`1jXI+C8^B*O8PXW|-r%0QCZG0_r>%1#;PrD!Z$# zqf>^yXdk*yScMk#jV2VFSWIL2Jv0qB6ZF(M)c09{%A?(scjJDPxD4>%RXEDdTUQL~ zt}ZHg4U+O5LPT0&SBDV33+7wNQ%6r{!;f-ycGhFLVDFp4oM;k$sIC0wh_?p$gNefx z?}62A1)6U)D@BZ64mC9jKA3=kpEfv|2`ePGt4jsDcEb99FTa2rmXzIkGNGI)41HN9 z9Q01p#d8qcofpufZ?9-Qw#LagTZ$s@_4c9IB4mf9-#;yv4Inu;=eq{}Re3oIXyZV6 zf{s%!rX;m5=7es9?)4DRPNQQJXNOT~rcvBE@h`%M9qT>F)3y6xl?6oy=C80P6+P>% zamDpwFxJiLX~kWA-cDw{Qt=S_O*t`jVH7%U3Ql)C8B(4j%dI#(Z_3l!yGh{n%hZDCPq1ls%<(sD$Jg9lM&|<+7xlLn^rxkm0`=BkcOTdB+@-)Kzxn3abOmc^NNl!? z=o|fv9`fKOFgFgbm z$(g}tu}Fpd)UEs{FI(E%8j5_9dB6Sg10-wQR9 zZ%*bId_8mUf@$(OLyVzj=q6@M{VX6eaww(DQ)fQ3pArnZ>buX8&bLPR*VvNya4a#` z(zy`P=^&31KyNGkaI#5U0gR$w5D;Hu{`vJ$o#GaQ$G;%{?7ENd?4<2=967u>m2PlB z5o6;n&HuVmCWr=MA^++7kC~5t~%tG+UcJj~7kfs&>oDR|` z&`%Y6(77*%aHm)>j5>xJ!aTJ1JEjNzAzsSt`_WGkOyO1Dy~cU?z6H_LNX^HuD^};H z&2S@++@h-MzzoNfSz7FDDK=oqG#~N+qBGe$g9Rc22_9NVzX>^61%`Yo!+0k?1UOW3 zm45X3@jw~LyZ=8ZCzTC*GKO8%p5tLnSJalXZ9<54Pa;}VkcF|kuTF)2z?cImuzNkM|l)4k`)YrK$$9Jt3i zo190VSS?jwc9Qghb?b;z4-67+>xAikN9XP?6xuX~66gZx!2%zTY?+E;LEY`VxT+j3 zFuJeri75yYK^u7+l^~d}5F~;K;7waU{-jM+15$reNHf9$jnuB&IknlI>-U}4O&^`#l-KfvJtH#dj)@2nA@Dy#u1Gu_lfAq7x zgE;Qnsf^Pud?i2NYjTHUvRxrhv8d0DH}?S4*F@XA^rbRKthOXvprQY4o|`3gmzLZE za8z&Sy~yOMxH)q^kia3>@sMYVS}ktq@b`uH$bZ7b8m_(jSc_!(Fb$ zp^*g0T1&eN9QX*GXjg~3C~29l`=YDFt3hLK>03VNaC!=^8ykjJDM+7Bcu%Lxhe$w zzh5NeE0J5rlNAI83!Vb!<5li=a;q*BX%AId%p>#+_(~6!I&a=%j=vYi0U-C<>VXJ} zUd(%Rbo$Y@6h+F{@1im|U-}@v!ktmym-Odg+~E`Gy`Ui3kHFb#rE;)XnRHESGQey| zQyNCvuiom}5_g;&U_Io-x^y@Ucqs9a&8b;P9;L10YA<)$Ge-3qvFQWvB*Jv}v#2Al z4yq3mR5yA4xS!;vygLzd+v@PF*S><9g`D4?{lyp;&E3XRA!+5yRh&Z0(T;zC-N%;_{ru^m z?d9LJ>uy7(+vb7@==b)yNfjD=V(-D9i4)K4ZjPVp#%6fjPr(vB=Twzh-@5i| zhV@^{Mz{A^X?xoK+kOZ%xlB7Gb%ib-1nA%K8dij>4zQ}wR{$|?#w%5PuxD-rQ$vC% z_jpoZeosITasfWO57&`sIPQ|Jl2#yx;yVF#G&EIDh9C)FPJ~sr$vC;Bfi` z1t(Yc=x+gD?Z#BPx*)XvJe;d=5Tizo|PLR{cs;4J!I@ud7JPcUK z>y2nN-GoteCE=KL|L!|I1jX%w+N=?!`~>a$TuSmIv!?k&#vHW9-J7l&9KL=gcHNbn zSw-3y_dEzE^y;;$?1yh?_cnL_^|d7S)>+qw)axqLJ!_b?g>kFs2U&YMF`S@@s8{hQ z%R-;~(0q#^1fM@!&ZFxot9Gbg;%`Z!obLz>BI}g$hp%tXsm{wlDuZH{2 zgcs~La!yoV7$RKwV6K!@Z*U^Kfb&4IhcI&}oYZb@j#=%O zM&oNIh&OfKgE97&zpW9uayHJxL-m%h(axScmDi3`(e(hd+OB>8WEONnOd^aib&3ssxC+>$yEK5z4ItD@@jVKYLx0$p&9ER z9TmLeoJK23s@Y|^5_VW zzr#_BdKnuWu(or>UdiE4fhEgCMw;VCij=ZP_i@v9BKfH|2_*c-Qa2{|6zWHUi}w~F zxbuO1c>SQrj63(i#ntK%xUR8R56tp89y~Z7m5b^Lj*=_1^er7>}Nb=Uxj{sNHDci6z;=er9n4F;|nIMcztX&)OqN~ zYh7=U5aP>FN*Wqd&21?W|uG!Ot>^P zES$z>bSCTPJlLj9!V-#za~?6gz#cAz=dxcfsA2aQvN6sN9u#o!uQgeg8wwkr5(j>H zAMPhY)t8jrJ?47C5VBHwuU5mOt!xo<1-#IR0Xf0xq9ny_`?JSBMU+>y z{#>{q7}>NqAm0EK-%e`@7vA#y$V*^w#B{%w2li<1_MV^sL=4VBWkj&Y`+??et3M~s z2oHtF957Aw&*-TWQr}fuJ%1&*U@L5ah@y@Q625E+!Y-$AA@)@Afg!mw$x86okS1A$w7l zP{RY82k%IP$6ot9DI+tfJmSTaPq>AMYU?yWd%z$3A)*Mdd8<< z%RkNz`{x}tXR6n-D8qoD)~Bq0!ta$$&;osjdmrzIBNuM*5={IDZ)ojPlqVfHK;@t8 z!nUHYXsf1VA$Oe^uES?~(_NofIt8ZbdurDKMW6KEQm%4BJKgqA3!Lh@k3D}(Q8vFq z{$#BZL7qy{LfQWktcZWU;AKImc}TB65t=(EnH&omzo(J}2AV-tI|9?1NEJLvpGej> zSRgkX?}qiq(d;wFvfb|3jFg=*TRzi;bAn%2u6x;k^~G=AK6mSs>SfNU`?^uqfa)3t zKm`q=xSKzbGM{9QFN}><>3uLt{PL-hhr%IcZSFbTpW1dmkd!(~Q|UB4`Q~^M@=XEf zs`w!aZG9b~aSrDlFg@TL!2`l=8e5{S+`^dL1~g+W2zy!pvRLCnJ%tr`<4E*ZQ+}hl zRBS)GJqEBF2>9dPZL(GG9dk!FQ6EPHQS~a!5NSmHc>dSvQF=@Anh0lsbfN(0Ikoe; zsEVg>I9+h%)=5k3RN1+{do?%{7z_!alG(GYUDKoV1nJDCepTA~i4@T=wUfq#?OPxc zx)XC};frYBKqi(`6Jc)B=c)&YI>yO(3J`v65M{33&4+3JB>d=D-*1_m)fahxw`JZY zK|7r9R@;p(w3k>wubO1XRYdJWj=RXtFmg`BxKDJX@sa~x6kt9FhYr{|8pcwgp5-5F zEQRcge~AiMq!Fu4dC7%|@$YPdF7XuFl_smjQ>|^(6zc^l7`XyR%7sdf-0N@-#03tL z+8YHa}<$(;KB z1y93gX|GrjfGTkls6l-SACf`~@>e&%py`43;N8wAOpf%Z6_L)%(!vW@1?7?v4xW7n zFXOo(Lx{L4O%8Rbg;2Y3MvWzq0kkZ42@IMBKBjP>ZUz^5k@*bsa0?xV`^D!l3A$x4 zYrT+n@~nrthdJhMS4abNJz(_auSQ((3{g!6CL8EyVSh**7gDf+E_R}2TXNhmlUh`r z>0i{j4hx<_E1;*9f!k;r zK$W?`2kmSMHDhmh`K=n!g;p$UQO^_M$49hwl7v4N>9X3lzH9}~_S!Kd+xU1FqhoOX zR!8ZO{i(iZ6C=bEE`Mi`vto-gc7?)i+A;S^hM;E5h^smG1g`v4w3CW^Zcwg?B$Z5| zo?rIq;;r9;X}~h$a-dovPY#A+-&fmteXUuDx&*pQ;cP6qiM|uMU~X`^V(mPBGg;w7 zJ$;CPb-z*ct+cy46b;S!b*t?mroB+7a&a ztasehQdYlB^fig;Ft`rD#RqB<@9#(Va@GKYk2j87*joRmvk6L0;M;me&2Sffl*|?@ zw7k}9G7dl6=^L3D2{7Qd+E3Nap!EcPJ3Q8Y$CIIKQY-Nh@MN3h1u`h~`7)se_n`Qg z!hC+8V)kzAjP14!iXht}E#&;UA}=JaUj-X-qbmskBe2y-iUrq+-D5$VB1CosRUTLc(rm2|h+3~AIs zwH&wiOP3(RVq_bu;ypn(%aqI$0$y#t^V#RVlvyZ$B!7EPClMgt>21qajf3Q*IRcQb z8$V^L{a@X5r8!c7I$!ih3hfn>VAm>I{6Bwnu~AL*5&pY_3;11m-(5Kj zpGr>N0C4Z0Na3-@F3obo+C31LdShg4s>B85Md8l=g3%AKb);06be6=?m3kiV=ZqVa zp4q4}b=r9nJlYAt!Xc;lw-pN+$%g_G`xbL%2rA!vqOYME5z4`quLaWk539@3m-(N}L}Gm|u3*lYS_{E+>UPTiswyb!>faj6+UYl!T2wST-lPr6 zXT9Z>8cxkAgR>l{RK4|~UHIU`wiI?y^cIPq`2^8NI?*f1{Yzx>1Y$hYRtn2W`fA~# zOskALblC0LihhyQVKuo~OMgCtryp0nJfUH8_f2{l z#XY7nM#-9RmGWXLgfRsX(d1ot>07?=!eDBRPiD|`rA`+>jqfBl^_%`Bz9dYVu81&^ zWupV#@nfho`DQ82+p~M&7|;L?{#Me8vVj_rxzA*X)_n)}AjVjUrC{3M+xC}%=*9Fm z=XtBStQ!4WH3?wcwh-<}hz;h1L3p9}aQBSw7tqAgpyjZCDo44t58Vi2r^ic`=)5@y z_tC^E+HF3%VGADY^>|kLK}B$Vqx#Z^ZLi0+WoA2Q#X9dskFLfMwOGyu^0_^?bk}Z) zkG}FS^;AI4qQVV=xtCktxXcO;vVj)vz-sMY=4^*qY-ygEEMFM*R*dx&-r}3Je6mWF zt5}2aqvKMyCM5QG=5JjofuF7WKoE+EQK94IO^>*lybZetzEXSqVFE}kRwJ7YviYCt zqrXC<|EWHD_QK@4^566S^8c6m=uyBUr)%z^k33xj`K;WZcmx0S^FCwa9T4X38yFzS zcQriBFE9jn>;4@A%+emm1>#EB}8KQ2*QWe|G<=ss7J+{=5EP iYKV{htFiij_WUzx|5+aUd({1{cmLXRaQqLQ^l421 literal 0 HcmV?d00001 diff --git a/tests/test_views/test_backwards_compatibility.py b/tests/test_views/test_backwards_compatibility.py index f465639..1dc9584 100644 --- a/tests/test_views/test_backwards_compatibility.py +++ b/tests/test_views/test_backwards_compatibility.py @@ -7,6 +7,7 @@ from rest_framework import status from kirovy.models import CncGame +from kirovy.response import KirovyResponse @pytest.mark.parametrize("game_slug", ["td", "ra", "ts", "dta", "yr", "d2"]) @@ -28,3 +29,16 @@ def test_map_download_backwards_compatible( map_from_zip = zip_file.read(f"{map_file.hash_sha1}.map") downloaded_map_hash = hashlib.sha1(map_from_zip).hexdigest() assert downloaded_map_hash == map_file.hash_sha1 + + +def test_map_upload_dune2k_backwards_compatible( + client_anonymous, rename_file_for_legacy_upload, file_map_dune2k_valid, game_dune2k +): + url = "/upload" + file = rename_file_for_legacy_upload(file_map_dune2k_valid) + + response: KirovyResponse = client_anonymous.post( + url, {"file": file, "game": game_dune2k.slug}, format="multipart", content_type=None + ) + + assert response.status_code == status.HTTP_200_OK From 1ee16bf829c9771a21eab01a843a639bc3a4b460 Mon Sep 17 00:00:00 2001 From: Alex Lambson Date: Sat, 22 Mar 2025 14:52:43 -0600 Subject: [PATCH 2/3] - Backwards compatible yuri's revenge upload - Make endpoints verify that `game` actually exists --- kirovy/constants/__init__.py | 2 +- kirovy/constants/api_codes.py | 2 + kirovy/models/cnc_user.py | 3 +- kirovy/models/file_base.py | 9 +- kirovy/services/legacy_upload/__init__.py | 23 +++ kirovy/services/legacy_upload/base.py | 171 +++++++++++++----- kirovy/services/legacy_upload/westwood.py | 9 +- kirovy/views/map_upload_views.py | 44 +++-- .../test_backwards_compatibility.py | 44 ++++- 9 files changed, 234 insertions(+), 73 deletions(-) diff --git a/kirovy/constants/__init__.py b/kirovy/constants/__init__.py index 29db99e..7e79504 100644 --- a/kirovy/constants/__init__.py +++ b/kirovy/constants/__init__.py @@ -101,7 +101,7 @@ class LegacyUploadUser: GROUP = CncnetUserGroup.USER -class GameSlugs(enum.StrEnum): +class GameSlugs(str, enum.Enum): """The slugs for each game / total conversion mod. These **must** be unique. They are in constants because we need them to determine which parser to use diff --git a/kirovy/constants/api_codes.py b/kirovy/constants/api_codes.py index 995c4de..f94f19f 100644 --- a/kirovy/constants/api_codes.py +++ b/kirovy/constants/api_codes.py @@ -3,6 +3,7 @@ class UploadApiCodes(enum.StrEnum): GAME_SLUG_DOES_NOT_EXIST = "game-slug-does-not-exist" + GAME_DOES_NOT_EXIST = "game-does-not-exist" MISSING_GAME_SLUG = "missing-game-slug" FILE_TO_LARGE = "file-too-large" EMPTY_UPLOAD = "where-file" @@ -17,3 +18,4 @@ class LegacyUploadApiCodes(enum.StrEnum): NO_VALID_MAP_FILE = "no-valid-map-file" HASH_MISMATCH = "file-hash-does-not-match-zip-name" INVALID_FILE_TYPE = "invalid-file-type-in-zip" + GAME_NOT_SUPPORTED = "game-not-supported" diff --git a/kirovy/models/cnc_user.py b/kirovy/models/cnc_user.py index f531377..dfcb4cc 100644 --- a/kirovy/models/cnc_user.py +++ b/kirovy/models/cnc_user.py @@ -15,7 +15,8 @@ class CncUserManager(models.Manager): use_in_migrations = True _SYSTEM_CNCNET_IDS = { - constants.MigrationUser.ID, + constants.MigrationUser.CNCNET_ID, + constants.LegacyUploadUser.CNCNET_ID, } def find_by_cncnet_id(self, cncnet_id: int) -> t.Tuple["CncUser"]: diff --git a/kirovy/models/file_base.py b/kirovy/models/file_base.py index 2217530..2ab2b7b 100644 --- a/kirovy/models/file_base.py +++ b/kirovy/models/file_base.py @@ -89,9 +89,12 @@ def validate_file_extension(self, file_extension: game_models.CncFileExtension) def save(self, *args, **kwargs): self.validate_file_extension(self.file_extension) - self.hash_md5 = file_utils.hash_file_md5(self.file) - self.hash_sha512 = file_utils.hash_file_sha512(self.file) - self.hash_sha1 = file_utils.hash_file_sha1(self.file) + if not self.hash_md5: + self.hash_md5 = file_utils.hash_file_md5(self.file) + if not self.hash_sha512: + self.hash_sha512 = file_utils.hash_file_sha512(self.file) + if not self.hash_sha1: + self.hash_sha1 = file_utils.hash_file_sha1(self.file) super().save(*args, **kwargs) @staticmethod diff --git a/kirovy/services/legacy_upload/__init__.py b/kirovy/services/legacy_upload/__init__.py index e69de29..5fa2631 100644 --- a/kirovy/services/legacy_upload/__init__.py +++ b/kirovy/services/legacy_upload/__init__.py @@ -0,0 +1,23 @@ +from kirovy.constants import GameSlugs +from kirovy.constants.api_codes import LegacyUploadApiCodes +from kirovy.exceptions import view_exceptions +from kirovy.services.legacy_upload import westwood, dune_2000 +from kirovy.services.legacy_upload.base import LegacyMapServiceBase +from kirovy import typing as t + + +_GAME_LEGACY_SERVICE_MAP: t.Dict[str, t.Type[LegacyMapServiceBase]] = { + GameSlugs.yuris_revenge.value: westwood.YurisRevengeLegacyMapService, + GameSlugs.dune_2000.value: dune_2000.Dune2000LegacyMapService, +} + + +def get_legacy_service_for_slug(game_slug: str) -> t.Type[LegacyMapServiceBase]: + if service := _GAME_LEGACY_SERVICE_MAP.get(game_slug): + return service + + raise view_exceptions.KirovyValidationError( + "Game not supported on legacy endpoint", + code=LegacyUploadApiCodes.GAME_NOT_SUPPORTED, + additional={"supported": _GAME_LEGACY_SERVICE_MAP.keys()}, + ) diff --git a/kirovy/services/legacy_upload/base.py b/kirovy/services/legacy_upload/base.py index 1b8fefd..e1d8dd3 100644 --- a/kirovy/services/legacy_upload/base.py +++ b/kirovy/services/legacy_upload/base.py @@ -1,4 +1,6 @@ import dataclasses +import functools +import io import pathlib import zipfile @@ -46,6 +48,10 @@ def __init__(self, file: UploadedFile): else: self.multi_file_validator() + @cached_property + def expected_files(self) -> t.List[ExpectedFile]: + raise NotImplementedError("This Game's map validator hasn't implemented the expectd file structure.") + def multi_file_validator(self): file_list = self._file.infolist() min_files = len([x for x in self.expected_files if x.required]) @@ -57,8 +63,7 @@ def multi_file_validator(self): for file_info in file_list: expected_file = self._get_expected_file_for_extension(file_info) - with self._file.open(file_info, mode="r") as file: - expected_file.file_validator(self._file.filename, ContentFile(file), file_info) + expected_file.file_validator(self._file.filename, ContentFile(self._file.read(file_info)), file_info) def single_file_validator(self): first_file = self._file.infolist()[0] @@ -66,29 +71,89 @@ def single_file_validator(self): raise view_exceptions.KirovyValidationError( "Map file was not the first Zip entry.", code=LegacyUploadApiCodes.BAD_ZIP_STRUCTURE ) - with self._file.open(first_file, mode="r") as map_file: - self.expected_files[0].file_validator(self._file.filename, ContentFile(map_file), first_file) + self.expected_files[0].file_validator(self._file.filename, ContentFile(self._file.read(first_file)), first_file) @cached_property - def extract_name(self) -> str: - ini_file_info = self._find_ini_file() - fallback = f"legacy_client_upload_{pathlib.Path(self._file.filename).stem}" - with self._file.open(ini_file_info, mode="r") as ini_file: - return CncGen2MapParser(ContentFile(ini_file)).ini.get("Basic", "Name", fallback=fallback) + def map_sha1_from_filename(self) -> str: + """Legacy upload endpoints need the hash for the map file, not the full zip hash. + + This should only be used after ``__init__`` has completed and the hash from the filename has been verified + to match the internal map file's hash. + """ + return pathlib.Path(self._file.filename).stem + + @cached_property + def file_contents_merged(self) -> io.BytesIO: + """The legacy map DB checks hash by extracting all zip-file contents, appending them, then hashing. + + We will mirror that behavior for legacy endpoints so that the file hashes in the database match + what the legacy clients expect. + + This will not match the logic for the new UI upload endpoints at all, but that's fine. + + .. note:: - def _find_ini_file(self) -> zipfile.ZipInfo: + The order of ``self.expected_files`` will control the order that files + are appended in. This order matter and needs to match the client, or the hahes will not match. + + :return: + All file contents merged into one stream. Order depends on ``self.expected_files`` + """ + output = io.BytesIO() + for expected_file in self.expected_files: + file_info = self._find_file_info_by_extension(expected_file.possible_extensions) + output.write(self._file.read(file_info)) + output.seek(0) + return output + + @cached_property + def map_name(self) -> str: + ini_file_info = self._find_file_info_by_extension(self.ini_extensions) + fallback = f"legacy_client_upload_{self.map_sha1_from_filename}" + ini_file = ContentFile(self._file.read(ini_file_info)) + return CncGen2MapParser(ini_file).ini.get("Basic", "Name", fallback=fallback) + + def _find_file_info_by_extension(self, extensions: t.Set[str]) -> zipfile.ZipInfo: + """Find the zipinfo object for a file by a set of possible file extensions. + + This is meant to be used to find specific files in the zip. + e.g. finding the ``.ini`` file to extract the map name from. + + It's hacky, but filenames and file order within the zip aren't predictable. + + :param extensions: + A set of possible extensions to look for. + e.g. ``{".map", ".yro", ".yrm"}`` to find the map ini for Yuri's Revenge. + :return: + The zipinfo for the matching file in the archive. + :raises view_exceptions.KirovyValidationError: + Raised when no file matching the possible extensions was found in the zip archive. + """ for file in self._file.infolist(): - if pathlib.Path(file.filename).suffix in self.ini_extensions: + if pathlib.Path(file.filename).suffix in extensions: return file raise view_exceptions.KirovyValidationError( - "No file containing map INI was found.", LegacyUploadApiCodes.NO_VALID_MAP_FILE + "No file matching the expected extensions was found", + LegacyUploadApiCodes.NO_VALID_MAP_FILE, + {"expected": extensions}, ) - @cached_property - def expected_files(self) -> t.List[ExpectedFile]: - raise NotImplementedError("This Game's map validator hasn't implemented the expectd file structure.") - def _get_expected_file_for_extension(self, zip_info: zipfile.ZipInfo) -> ExpectedFile: + """Get the ``expected_file`` class instance corresponding to the file in the zipfile. + + This is used to find the validator for a file in the uploaded zip file. + This is hacky, but the order and filenames aren't predictable in uploaded zips, + so it will have to do. + + :param zip_info: + The info for a file in the uploaded zipfile. We will try to find the + :class:`kirovy.services.legacy_upload.base.ExpectedFile` for it. + :return: + The :class:`kirovy.services.legacy_upload.base.ExpectedFile` for this uploaded file. + :raises view_exceptions.KirovyValidationError: + Raised if no corresponding `kirovy.services.legacy_upload.base.ExpectedFile` is found. + This has a security benefit of not allowing non-c&c-map-files to sneak their way in with a legitimate map. + """ extension = pathlib.Path(zip_info.filename).suffix for expected in self.expected_files: if extension in expected.possible_extensions: @@ -97,34 +162,58 @@ def _get_expected_file_for_extension(self, zip_info: zipfile.ZipInfo) -> Expecte "Unexpected file type in zip file", LegacyUploadApiCodes.INVALID_FILE_TYPE ) - @staticmethod - def default_map_file_validator(zip_file_name: str, file_content: ContentFile, zip_info: zipfile.ZipInfo): - """Legacy map file generator that works for most Westwood games. + def processed_zip_file(self) -> ContentFile: + """Returns a file to save to the database. - Won't work for Dune 2000, or Tiberian Dawn. + This file has been processed to match the format that legacy CnCNet clients expect. - :param zip_file_name: - The name of the uploaded zip file. - :param file_content: - The contents of the map file extracted from the zip file. - :param zip_info: - The zip metadata for the extracted map file. :return: - None - :raises view_exceptions.KirovyValidationError: - Raised for any validation errors. + A django-compatible file for a legacy CnCNet-client-compatible zip file. """ - if ByteSized(zip_info.file_size) > ByteSized(mega=2): - raise view_exceptions.KirovyValidationError( - "Map file larger than expected.", code=LegacyUploadApiCodes.MAP_TOO_LARGE - ) + files_info = self._file.infolist() + zip_bytes = io.BytesIO() + processed_zip = zipfile.ZipFile(zip_bytes, mode="w", compresslevel=5, allowZip64=False) + map_hash = pathlib.Path(self._file.filename).stem + + for file_info_original in files_info: + file_name_original = pathlib.Path(file_info_original.filename) + with self._file.open(file_info_original, mode="r") as original_file_data: + # All files must have the map hash as the filename. + processed_zip.writestr( + f"{map_hash}{file_name_original.suffix}", data=original_file_data.read(), compresslevel=4 + ) + + processed_zip.close() + zip_bytes.seek(0) + return ContentFile(zip_bytes.read(), name=self._file.filename) + + +def default_map_file_validator(zip_file_name: str, file_content: ContentFile, zip_info: zipfile.ZipInfo): + """Legacy map file generator that works for most Westwood games. + + Won't work for Dune 2000, or Tiberian Dawn. + + :param zip_file_name: + The name of the uploaded zip file. + :param file_content: + The contents of the map file extracted from the zip file. + For RA2 and YR this will be a ``.map`` file, or one of its aliases like ``.yrm``. + :param zip_info: + The zip metadata for the extracted map file. + :return: + None + :raises view_exceptions.KirovyValidationError: + Raised for any validation errors. + """ + if ByteSized(zip_info.file_size) > ByteSized(mega=2): + raise view_exceptions.KirovyValidationError( + "Map file larger than expected.", code=LegacyUploadApiCodes.MAP_TOO_LARGE + ) - # Reaching into the new map parsers is kind of dirty, but this legacy endpoint - # is (hopefully) just to tide us over until we can migrate everyone to the new endpoints. - if not CncGen2MapParser.is_text(file_content): - raise view_exceptions.KirovyValidationError( - "No valid map file.", code=LegacyUploadApiCodes.NO_VALID_MAP_FILE - ) + # Reaching into the new map parsers is kind of dirty, but this legacy endpoint + # is (hopefully) just to tide us over until we can migrate everyone to the new endpoints. + if not CncGen2MapParser.is_text(file_content): + raise view_exceptions.KirovyValidationError("No valid map file.", code=LegacyUploadApiCodes.NO_VALID_MAP_FILE) - if file_utils.hash_file_sha1(file_content) != pathlib.Path(zip_file_name).stem: - raise view_exceptions.KirovyValidationError("Map file checksum differs from Zip name, rejected.") + if file_utils.hash_file_sha1(file_content) != pathlib.Path(zip_file_name).stem: + raise view_exceptions.KirovyValidationError("Map file checksum differs from Zip name, rejected.") diff --git a/kirovy/services/legacy_upload/westwood.py b/kirovy/services/legacy_upload/westwood.py index 49e7fc4..99e403f 100644 --- a/kirovy/services/legacy_upload/westwood.py +++ b/kirovy/services/legacy_upload/westwood.py @@ -1,14 +1,17 @@ +from functools import cached_property + from kirovy import constants, typing as t -from kirovy.services.legacy_upload.base import LegacyMapServiceBase, ExpectedFile +from kirovy.services.legacy_upload.base import LegacyMapServiceBase, ExpectedFile, default_map_file_validator class YurisRevengeLegacyMapService(LegacyMapServiceBase): - ini_extensions = {"map", "yro", "yrm"} + ini_extensions = {".map", ".yro", ".yrm"} game_slug = constants.GameSlugs.yuris_revenge + @cached_property def expected_files(self) -> t.List[ExpectedFile]: return [ ExpectedFile( - possible_extensions=self.ini_extensions, file_validator=self.default_map_file_validator, required=True + possible_extensions=self.ini_extensions, file_validator=default_map_file_validator, required=True ) ] diff --git a/kirovy/views/map_upload_views.py b/kirovy/views/map_upload_views.py index 63e2185..fc2581b 100644 --- a/kirovy/views/map_upload_views.py +++ b/kirovy/views/map_upload_views.py @@ -5,6 +5,7 @@ from cryptography.utils import cached_property from django.conf import settings from django.core.exceptions import ObjectDoesNotExist +from django.core.files.base import ContentFile, File from django.core.files.uploadedfile import UploadedFile, InMemoryUploadedFile from django.db.models import Q, QuerySet from rest_framework import status, serializers @@ -20,6 +21,7 @@ from kirovy.request import KirovyRequest from kirovy.response import KirovyResponse from kirovy.serializers import cnc_map_serializers +from kirovy.services import legacy_upload from kirovy.services.cnc_gen_2_services import CncGen2MapParser, CncGen2MapSections from kirovy.utils import file_utils @@ -53,7 +55,9 @@ def post(self, request: KirovyRequest, format=None) -> KirovyResponse: # todo: make validation less trash uploaded_file: UploadedFile = request.data["file"] - game_id = self.get_game_id_from_request(request) + game = self.get_game_from_request(request) + if not game: + raise KirovyValidationError(detail="Game doesnot exist", code=UploadApiCodes.GAME_DOES_NOT_EXIST) extension_id = self.get_extension_id_for_upload(uploaded_file) self.verify_file_size_is_allowed(uploaded_file) @@ -65,7 +69,7 @@ def post(self, request: KirovyRequest, format=None) -> KirovyResponse: # Make the map that we will attach the map file too. new_map = cnc_map.CncMap( map_name=map_parser.ini.map_name, - cnc_game_id=game_id, + cnc_game_id=game.id, is_published=False, incomplete_upload=True, cnc_user=request.user, @@ -199,14 +203,14 @@ def user_log_attrs(self) -> t.DictStrAny: } @staticmethod - def _get_file_hashes(uploaded_file: UploadedFile) -> MapHashes: + def _get_file_hashes(uploaded_file: File) -> MapHashes: map_hash_sha512 = file_utils.hash_file_sha512(uploaded_file) map_hash_md5 = file_utils.hash_file_md5(uploaded_file) map_hash_sha1 = file_utils.hash_file_sha1(uploaded_file) # legacy ban list support return MapHashes(md5=map_hash_md5, sha1=map_hash_sha1, sha512=map_hash_sha512) - def get_game_id_from_request(self, request: KirovyRequest) -> str | None: + def get_game_from_request(self, request: KirovyRequest) -> CncGame | None: """Get the game_id from the request. This is a method, rather than a direct lookup, so that the client can use ``game_slug``. @@ -307,15 +311,18 @@ class MapFileUploadView(_BaseMapFileUploadView): permission_classes = [permissions.CanUpload] upload_is_temporary = False - def get_game_id_from_request(self, request: KirovyRequest) -> str | None: - return request.data.get("game_id") + def get_game_from_request(self, request: KirovyRequest) -> CncGame | None: + game_id = request.data.get("game_id") + return CncGame.objects.filter(id=game_id).first() class CncnetClientMapUploadView(_BaseMapFileUploadView): + """DO NOT USE THIS FOR NOW. Use""" + permission_classes = [AllowAny] upload_is_temporary = True - def get_game_id_from_request(self, request: KirovyRequest) -> str | None: + def get_game_from_request(self, request: KirovyRequest) -> CncGame | None: """Get the game ID for a CnCNet client upload. The client currently sends a slug in ``request.data["game"]``. The game table has a unique constraint @@ -347,7 +354,7 @@ def get_game_id_from_request(self, request: KirovyRequest) -> str | None: additional={"attempted_slug": game_slug}, ) - return str(game.id) + return game class CncNetBackwardsCompatibleUploadView(CncnetClientMapUploadView): @@ -362,20 +369,21 @@ class CncNetBackwardsCompatibleUploadView(CncnetClientMapUploadView): def post(self, request: KirovyRequest, format=None) -> KirovyResponse: uploaded_file: UploadedFile = request.data["file"] - game_id = self.get_game_id_from_request(request) + game = self.get_game_from_request(request) extension_id = self.get_extension_id_for_upload(uploaded_file) self.verify_file_size_is_allowed(uploaded_file) - map_hashes = self._get_file_hashes(uploaded_file) - self.verify_file_does_not_exist(map_hashes) + # Will raise validation errors if the upload is invalid + legacy_map_service = legacy_upload.get_legacy_service_for_slug(game.slug.lower())(uploaded_file) - if uploaded_file.name != f"{map_hashes.sha1}.zip": - return KirovyResponse(status=status.HTTP_400_BAD_REQUEST) + # These hashes are for the full zip file and won't match + map_hashes = self._get_file_hashes(ContentFile(legacy_map_service.file_contents_merged.read())) + self.verify_file_does_not_exist(map_hashes) - # Make the map that we will attach the map file too. + # Make the map that we will attach the map file to. new_map = cnc_map.CncMap( - map_name=f"backwards_compatible_{map_hashes.sha1}", - cnc_game_id=game_id, + map_name=legacy_map_service.map_name, + cnc_game=game, is_published=False, incomplete_upload=True, cnc_user=CncUser.objects.get_or_create_legacy_upload_user(), @@ -388,9 +396,9 @@ def post(self, request: KirovyRequest, format=None) -> KirovyResponse: width=-1, height=-1, cnc_map_id=new_map.id, - file=uploaded_file, + file=legacy_map_service.processed_zip_file(), file_extension_id=extension_id, - cnc_game_id=new_map.cnc_game_id, + cnc_game_id=game.id, hash_md5=map_hashes.md5, hash_sha512=map_hashes.sha512, hash_sha1=map_hashes.sha1, diff --git a/tests/test_views/test_backwards_compatibility.py b/tests/test_views/test_backwards_compatibility.py index 1dc9584..eef1a60 100644 --- a/tests/test_views/test_backwards_compatibility.py +++ b/tests/test_views/test_backwards_compatibility.py @@ -1,13 +1,16 @@ import hashlib +import pathlib import zipfile import io import pytest +from django.core.files.base import ContentFile from django.http import FileResponse from rest_framework import status from kirovy.models import CncGame from kirovy.response import KirovyResponse +from kirovy.utils import file_utils @pytest.mark.parametrize("game_slug", ["td", "ra", "ts", "dta", "yr", "d2"]) @@ -31,14 +34,43 @@ def test_map_download_backwards_compatible( assert downloaded_map_hash == map_file.hash_sha1 -def test_map_upload_dune2k_backwards_compatible( - client_anonymous, rename_file_for_legacy_upload, file_map_dune2k_valid, game_dune2k -): +# def test_map_upload_dune2k_backwards_compatible( +# client_anonymous, rename_file_for_legacy_upload, file_map_dune2k_valid, game_dune2k +# ): +# url = "/upload" +# file = rename_file_for_legacy_upload(file_map_dune2k_valid) +# +# response: KirovyResponse = client_anonymous.post( +# url, {"file": file, "game": game_dune2k.slug}, format="multipart", content_type=None +# ) +# +# assert response.status_code == status.HTTP_200_OK + + +def test_map_upload_yuri_backwards_compatible(client_anonymous, file_map_desert, game_yuri): url = "/upload" - file = rename_file_for_legacy_upload(file_map_dune2k_valid) + file_sha1 = file_utils.hash_file_sha1(file_map_desert) + extension = pathlib.Path(file_map_desert.name).suffix + zip_bytes = io.BytesIO() + zip_file = zipfile.ZipFile(zip_bytes, mode="w") + zip_file.writestr(file_map_desert.name, file_map_desert.read()) + zip_file.close() + zip_bytes.seek(0) + upload_file = ContentFile(zip_bytes.read(), f"{file_sha1}.zip") - response: KirovyResponse = client_anonymous.post( - url, {"file": file, "game": game_dune2k.slug}, format="multipart", content_type=None + upload_response: KirovyResponse = client_anonymous.post( + url, {"file": upload_file, "game": game_yuri.slug}, format="multipart", content_type=None ) + assert upload_response.status_code == status.HTTP_200_OK + + response: FileResponse = client_anonymous.get(f"/{game_yuri.slug}/{file_sha1}") assert response.status_code == status.HTTP_200_OK + + file_content_io = io.BytesIO(response.getvalue()) + zip_file = zipfile.ZipFile(file_content_io) + assert zip_file + + map_from_zip = zip_file.read(f"{file_sha1}.map") + downloaded_map_hash = hashlib.sha1(map_from_zip).hexdigest() + assert downloaded_map_hash == file_sha1 From 0bd62ef59c92f5f15cfa6a2996bd8b3ae9d5acc8 Mon Sep 17 00:00:00 2001 From: Alex Lambson Date: Sat, 22 Mar 2025 15:19:30 -0600 Subject: [PATCH 3/3] - typo --- kirovy/services/legacy_upload/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kirovy/services/legacy_upload/base.py b/kirovy/services/legacy_upload/base.py index e1d8dd3..7af4a86 100644 --- a/kirovy/services/legacy_upload/base.py +++ b/kirovy/services/legacy_upload/base.py @@ -189,7 +189,7 @@ def processed_zip_file(self) -> ContentFile: def default_map_file_validator(zip_file_name: str, file_content: ContentFile, zip_info: zipfile.ZipInfo): - """Legacy map file generator that works for most Westwood games. + """Legacy map file validator that works for most Westwood games. Won't work for Dune 2000, or Tiberian Dawn.