Initial commit

Author: tester2024

.gitignore

@@ -0,0 +1,3 @@
+steamcmd/
+.idea/
+.venv/

main.py

@@ -0,0 +1,44 @@
+from quart import Quart, redirect, request, send_file
+
+from services import (steam_proxied)
+from steam import workshop_download, collection_download
+
+app = Quart(__name__)
+
+
+@app.route("/")
+async def index():
+    return redirect("/workshop/")
+
+
+@app.route("/dl-workshop/<int:app_id>/<int:workshop_id>")
+async def download_workshop(app_id, workshop_id):
+    path = await workshop_download(app_id, workshop_id)
+    if path is None:
+        return "Workshop not found"
+    print(path)
+    return await send_file(path)
+
+
+@app.route("/dl-collection/<int:app_id>/<int:workshop_id>")
+async def download_collection(app_id, workshop_id):
+    path = await collection_download(app_id, workshop_id)
+    if path is None:
+        return "Collection not found"
+
+    return await send_file(path)
+
+
+@app.route("/<path:path>")
+async def steam_proxied_wrapper(path):
+    return await steam_proxied(path, "steamcommunity.com",
+                               headers=dict(request.headers),
+                               data=await request.data,
+                               method=request.method,
+                               params=request.args,
+                               cookies=request.cookies,
+                               host=request.host)
+
+
+if __name__ == "__main__":
+    app.run(host='0.0.0.0', debug=True)

requirements.txt

@@ -0,0 +1,5 @@
+Quart~=0.19.5
+aiofiles~=23.2.1
+httpx~=0.27.0
+aiozipstream~=0.4
+async-cache~=1.1.1

services.py

@@ -0,0 +1,130 @@
+from re import compile
+
+from cache import AsyncTTL
+from httpx import AsyncClient
+from quart.wrappers.response import DataBody, Response
+
+host_fixer = compile(r"^http(s?)://[\w.:]+/")
+info = compile(rb"onclick=\"(?:SubscribeItem|SubscribeCollection|SubscribeCollectionItem)"
+               rb"\(\s+'(\d+)',\s+'(\d+)'\s+\);\"")
+item_rex = compile(rb"(<a\s+onclick=\"SubscribeCollectionItem\(\s+'(\d+)',\s+'(\d+)'\s+\);"
+                   rb"\"\s+id=\"SubscribeItemBtn\d+\"\s+class=\"[^\"]*\"\s*>[\s\S]*?</a>)")
+
+
+@AsyncTTL(time_to_live=60, maxsize=40 << 10)
+async def replace_headers(headers: dict, target_host: str) -> dict:
+    if "Referer" in headers:
+        headers["Referer"] = host_fixer.sub("https://%s/" % target_host, headers["Referer"])
+
+    if "Origin" in headers:
+        headers["Origin"] = host_fixer.sub("https://%s/" % target_host, headers["Origin"])
+
+    if "Host" in headers:
+        headers["Host"] = "steamcommunity.com"
+
+    if "x-frame-options" in headers:
+        del headers["x-frame-options"]
+
+    if "content-security-policy" in headers:
+        del headers["content-security-policy"]
+
+    if "content-encoding" in headers:
+        del headers["content-encoding"]
+
+    if "access-control-allow-origin" in headers:
+        del headers["access-control-allow-origin"]
+
+    if "location" in headers:
+        del headers["location"]
+
+    return headers
+
+
+@AsyncTTL(time_to_live=60, maxsize=40 << 10)
+async def fix_body(content, host: str, target_host: str) -> bytes:
+    target_host = target_host.encode()
+    content = content.replace(b"https://%s/" % target_host, b"/")
+    content = content.replace(b"https://%s" % target_host, b"")
+    content = content.replace(target_host, host.encode())
+    return content
+
+
+def try_to_add_download_button(data_body):
+    if b"<a onclick=\"SubscribeItem(" in data_body:
+        index = data_body.find(b"<a onclick=\"SubscribeItem(")
+
+        if index != -1:
+            while chr(data_body[index - 1]).isspace():
+                index -= 1
+
+            data_info = info.search(data_body[index:])
+            if not data_info:
+                return data_body
+
+            workshop_id, app_id = data_info.groups()
+            if not workshop_id or not app_id:
+                return data_body
+
+            index -= 5
+
+            data_body = data_body[:index] + (b"<div><a href=\"/dl-workshop/%s/%s\" class=\"btn_darkred_white_innerfade "
+                                             b"btn_border_2px btn_medium\" style=\"position: relative\"> <div "
+                                             b"class=\"followIcon\"></div> <span class=\"subscribeText\"> "
+                                             b"<div>Download</div> </span> </a> </div>" % (
+                                                 app_id, workshop_id)) + data_body[index:]
+    elif b"<a class=\"general_btn subscribe\" onclick=\"SubscribeCollection(":
+        index = data_body.find(b"<a class=\"general_btn subscribe\" onclick=\"SubscribeCollection(")
+
+        if index != -1:
+            data_info = info.search(data_body[index:])
+            if not data_info:
+                return data_body
+
+            workshop_id, app_id = data_info.groups()
+            if not workshop_id or not app_id:
+                return data_body
+
+            index -= 5
+
+            data_body = data_body[:index] + (b"<a class=\"general_btn subscribe\" "
+                                             b"style=\"background: #640000; color: white;\" "
+                                             b"href=\"/dl-collection/%s/%s\">"
+                                             b"<div class=\"followIcon\"></div> <span "
+                                             b"class=\"subscribeText\">Download Collection</span> </a>" % (
+                                                 app_id, workshop_id)) + data_body[index:]
+
+            data_body = item_rex.sub(lambda m: b"<a style=\"background: #640000; color: white;\""
+                                               b" href=\"/dl-workshop/%s/%s\""
+                                               b" class=\"general_btn subscribe\""
+                                               b"><div class=\"followIcon\"></div></a> %s" % (
+                m.group(3), m.group(2), m.group(1)), data_body)
+
+    return data_body
+
+
+async def steam_proxied(path: str, target_host: str, headers: dict, host: str, **kwargs) -> Response:
+    headers = await replace_headers(headers, target_host)
+
+    headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0"
+
+    async with AsyncClient() as client:
+        response = await client.request(url="https://%s/%s" % (target_host, path),
+                                        headers=headers,
+                                        **kwargs)
+
+        data_body = response.content
+        data_headers = dict(response.headers.items())
+
+        data_body = await fix_body(data_body, host, "community.akamai.steamstatic.com")
+        data_headers = await replace_headers(data_headers, "community.akamai.steamstatic.com")
+
+        data_body = await fix_body(data_body, host, "steamcommunity.com")
+        data_headers = await replace_headers(data_headers, "steamcommunity.com")
+
+        data_body = try_to_add_download_button(data_body)
+
+        data_headers["content-length"] = len(data_body)
+
+        return Response(response=DataBody(data_body),
+                        headers=data_headers,
+                        status=response.status_code)

steam.py

@@ -0,0 +1,157 @@
+from asyncio import run
+from contextlib import suppress
+from os import getcwd
+from pathlib import Path
+from re import compile
+from typing import List
+
+from aiofiles import open as async_open
+from cache import AsyncTTL
+from httpx import AsyncClient
+from zipstream import AioZipStream
+
+from steamcmdwrapper import SteamCMDWrapper, SteamCMDException, SteamCommand
+
+steamcmd_path = Path(getcwd()) / "steamcmd"
+
+steam_path = steamcmd_path / "steam"
+steam_path.mkdir(exist_ok=True, parents=True)
+
+steam = SteamCMDWrapper(steamcmd_path)
+
+with suppress(SteamCMDException):
+    run(steam.install())
+
+workshop_pattern = compile(rb'<a\s+href="https://steamcommunity\.com/sharedfiles/filedetails/\?id=('
+                           rb'\d+)">\s*<div\s+class="workshopItemTitle">([^<]+)</div>\s*</a>')
+collection_title_pattern = compile(rb'<div\s+class=\"workshopItemTitle\">([^<]+)</div>')
+
+
+@AsyncTTL(time_to_live=60, maxsize=40 << 10)
+async def get_collection_items(workshop_id) -> tuple[str, list[tuple[int, str]]]:
+    var = []
+    title = ""
+
+    async with AsyncClient() as client:
+        response = await client.get(f"https://steamcommunity.com/sharedfiles/filedetails/?id={workshop_id}")
+
+        response.raise_for_status()
+
+        content = await response.aread()
+
+        title = collection_title_pattern.search(content)
+        cols = workshop_pattern.findall(content)
+
+        title = title.group(1).decode()
+
+        if not cols or not title:
+            return title, var
+
+        for coll in cols:
+            var.append((int(coll[0]), coll[1].decode()))
+
+    return title, var
+
+
+@AsyncTTL(time_to_live=60, maxsize=40 << 10)
+async def get_workshop_name(workshop_id):
+    title = ""
+
+    async with AsyncClient() as client:
+        response = await client.get(f"https://steamcommunity.com/sharedfiles/filedetails/?id={workshop_id}")
+
+        response.raise_for_status()
+
+        content = await response.aread()
+
+        title = collection_title_pattern.search(content)
+        if not title:
+            return None
+
+        title = title.group(1).decode()
+
+    return title
+
+
+async def workshop_download(app_id, workshop_id, validate=True):
+    workshop_name = await get_workshop_name(workshop_id)
+    if not workshop_name:
+        return None
+
+    workshop_download_path = (steam_path / "steamapps" / "workshop" / "content" / str(app_id) / str(workshop_id))
+    workshop_zip = workshop_download_path.as_posix() + "%s .zip" % workshop_name
+
+    if not workshop_download_path.exists():
+        try:
+            await steam.workshop_update(app_id, workshop_id, steam_path, validate, n_tries=3)
+        except Exception as e:
+            return "an error occured while downloading the item: " + str(e)
+
+    if not workshop_download_path.exists():
+        return None
+
+    items = []
+    add_directory(items, workshop_download_path, "%s %s" % (workshop_id, workshop_name))
+    atomic = AioZipStream(items, chunksize=32768)
+
+    async with async_open(workshop_zip, mode='wb') as z:
+        async for chunk in atomic.stream():
+            await z.write(chunk)
+
+    return workshop_zip
+
+
+def is_empty(item_path: Path):
+    return not any(item_path.iterdir())
+
+
+def add_directory(items, item_path, target_dir):
+    if not item_path.exists():
+        return
+
+    if item_path.is_file():
+        items.append({"file": item_path, "name": target_dir})
+    elif item_path.is_dir():
+        for item in item_path.iterdir():
+            item_name = item.name.replace('\\', '/')
+            add_directory(items, item, f"{target_dir}/{item_name}")
+
+
+async def collection_download(app_id: int, workshop_id: int, validate: bool = True, batch_size: int = 20):
+    game_path = steam_path / "steamapps" / "workshop" / "content" / str(app_id)
+    title, workshop_matches = await get_collection_items(workshop_id)
+
+    if not workshop_matches or not title:
+        return None
+
+    collection_path = game_path / f"{workshop_id} {title}.zip"
+
+    if collection_path.exists() and collection_path.stat().st_size > 0:
+        return collection_path
+
+    items = list()
+
+    while workshop_matches:
+        batch = workshop_matches[:batch_size]
+        workshop_matches = workshop_matches[batch_size:]
+
+        sc = SteamCommand(steam_path)
+        for item_id, work_title in batch:
+            item_path = game_path / str(item_id)
+
+            add_directory(items, item_path, f"{item_id} {work_title}")
+
+            if item_path.exists() and not is_empty(item_path):
+                continue
+
+            sc.workshop_download_item(app_id, item_id, validate)
+
+        if sc.commands:
+            await steam.execute(sc, n_tries=3)
+
+    atomic = AioZipStream(items, chunksize=32768)
+    async with async_open(collection_path, mode='wb') as z:
+        async for chunk in atomic.stream():
+            await z.write(chunk)
+
+    return collection_path

steamcmdwrapper/__init__.py

@@ -0,0 +1,3 @@
+from .exceptions import SteamCMDException, SteamCMDDownloadException, SteamCMDInstallException
+from .steam_wrapper import SteamCMDWrapper
+from .steam_command import SteamCommand

steamcmdwrapper/exceptions.py

@@ -0,0 +1,20 @@
+class SteamCMDException(Exception):
+    def __init__(self, message=None, *args, **kwargs):
+        self.message = message
+        super(SteamCMDException, self).__init__(message, *args)
+
+    def __unicode__(self):
+        return repr(self.message)
+
+    def __str__(self):
+        return repr(self.message)
+
+
+class SteamCMDDownloadException(SteamCMDException):
+    def __init__(self, *args, **kwargs):
+        super(SteamCMDDownloadException, self).__init__(*args, **kwargs)
+
+
+class SteamCMDInstallException(SteamCMDException):
+    def __init__(self, *args, **kwargs):
+        super(SteamCMDInstallException, self).__init__(*args, **kwargs)