Skip to content

Commit 46fdd63

Browse files
KosinkadinkKosinkadink
authored
Merge pull request #9545 from bigcat88/asset-management
[Assets] Initial implementation
2 parents c708d0a + 283cd27 commit 46fdd63

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+8065
-1003
lines changed

‎.github/workflows/test-assets.yml‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
name: Asset System Tests
2+
3+
on:
4+
push:
5+
paths:
6+
- 'app/**'
7+
- 'alembic_db/**'
8+
- 'tests-assets/**'
9+
- '.github/workflows/test-assets.yml'
10+
- 'requirements.txt'
11+
pull_request:
12+
branches: [master]
13+
workflow_dispatch:
14+
15+
permissions:
16+
contents: read
17+
18+
env:
19+
PIP_DISABLE_PIP_VERSION_CHECK: '1'
20+
PYTHONUNBUFFERED: '1'
21+
22+
jobs:
23+
sqlite:
24+
name: SQLite (${{ matrix.sqlite_mode }}) • Python ${{ matrix.python }}
25+
runs-on: ubuntu-latest
26+
timeout-minutes: 40
27+
strategy:
28+
fail-fast: false
29+
matrix:
30+
python: ['3.9', '3.12']
31+
sqlite_mode: ['memory', 'file']
32+
33+
steps:
34+
- uses: actions/checkout@v4
35+
- name: Set up Python
36+
uses: actions/setup-python@v5
37+
with:
38+
python-version: ${{ matrix.python }}
39+
40+
- name: Install dependencies
41+
run: |
42+
python -m pip install -U pip wheel
43+
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
44+
pip install -r requirements.txt
45+
pip install pytest pytest-aiohttp pytest-asyncio
46+
47+
- name: Set deterministic test base dir
48+
id: basedir
49+
shell: bash
50+
run: |
51+
BASE="$RUNNER_TEMP/comfyui-assets-tests-${{ matrix.python }}-${{ matrix.sqlite_mode }}-${{ github.run_id }}-${{ github.run_attempt }}"
52+
echo "ASSETS_TEST_BASE_DIR=$BASE" >> "$GITHUB_ENV"
53+
echo "ASSETS_TEST_LOGS=$BASE/logs" >> "$GITHUB_ENV"
54+
mkdir -p "$BASE/logs"
55+
echo "ASSETS_TEST_BASE_DIR=$BASE"
56+
57+
- name: Set DB URL for SQLite
58+
id: setdb
59+
shell: bash
60+
run: |
61+
if [ "${{ matrix.sqlite_mode }}" = "memory" ]; then
62+
echo "ASSETS_TEST_DB_URL=sqlite+aiosqlite:///:memory:" >> "$GITHUB_ENV"
63+
else
64+
DBFILE="$RUNNER_TEMP/assets-tests.sqlite"
65+
mkdir -p "$(dirname "$DBFILE")"
66+
echo "ASSETS_TEST_DB_URL=sqlite+aiosqlite:///$DBFILE" >> "$GITHUB_ENV"
67+
fi
68+
69+
- name: Run tests
70+
run: python -m pytest tests-assets
71+
72+
- name: Show ComfyUI logs
73+
if: always()
74+
shell: bash
75+
run: |
76+
echo "==== ASSETS_TEST_BASE_DIR: $ASSETS_TEST_BASE_DIR ===="
77+
echo "==== ASSETS_TEST_LOGS: $ASSETS_TEST_LOGS ===="
78+
ls -la "$ASSETS_TEST_LOGS" || true
79+
for f in "$ASSETS_TEST_LOGS"/stdout.log "$ASSETS_TEST_LOGS"/stderr.log; do
80+
if [ -f "$f" ]; then
81+
echo "----- BEGIN $f -----"
82+
sed -n '1,400p' "$f"
83+
echo "----- END $f -----"
84+
fi
85+
done
86+
87+
- name: Upload ComfyUI logs
88+
if: always()
89+
uses: actions/upload-artifact@v4
90+
with:
91+
name: asset-logs-sqlite-${{ matrix.sqlite_mode }}-py${{ matrix.python }}
92+
path: ${{ env.ASSETS_TEST_LOGS }}/*.log
93+
if-no-files-found: warn
94+
95+
postgres:
96+
name: PostgreSQL ${{ matrix.pgsql }} • Python ${{ matrix.python }}
97+
runs-on: ubuntu-latest
98+
timeout-minutes: 40
99+
strategy:
100+
fail-fast: false
101+
matrix:
102+
python: ['3.9', '3.12']
103+
pgsql: ['14', '16']
104+
105+
services:
106+
postgres:
107+
image: postgres:${{ matrix.pgsql }}
108+
env:
109+
POSTGRES_DB: assets
110+
POSTGRES_USER: postgres
111+
POSTGRES_PASSWORD: postgres
112+
ports:
113+
- 5432:5432
114+
options: >-
115+
--health-cmd "pg_isready -U postgres -d assets"
116+
--health-interval 10s
117+
--health-timeout 5s
118+
--health-retries 12
119+
120+
steps:
121+
- uses: actions/checkout@v4
122+
- name: Set up Python
123+
uses: actions/setup-python@v5
124+
with:
125+
python-version: ${{ matrix.python }}
126+
127+
- name: Install dependencies
128+
run: |
129+
python -m pip install -U pip wheel
130+
pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
131+
pip install -r requirements.txt
132+
pip install pytest pytest-aiohttp pytest-asyncio
133+
pip install greenlet psycopg
134+
135+
- name: Set deterministic test base dir
136+
id: basedir
137+
shell: bash
138+
run: |
139+
BASE="$RUNNER_TEMP/comfyui-assets-tests-${{ matrix.python }}-${{ matrix.sqlite_mode }}-${{ github.run_id }}-${{ github.run_attempt }}"
140+
echo "ASSETS_TEST_BASE_DIR=$BASE" >> "$GITHUB_ENV"
141+
echo "ASSETS_TEST_LOGS=$BASE/logs" >> "$GITHUB_ENV"
142+
mkdir -p "$BASE/logs"
143+
echo "ASSETS_TEST_BASE_DIR=$BASE"
144+
145+
- name: Set DB URL for PostgreSQL
146+
shell: bash
147+
run: |
148+
echo "ASSETS_TEST_DB_URL=postgresql+psycopg://postgres:postgres@localhost:5432/assets" >> "$GITHUB_ENV"
149+
150+
- name: Run tests
151+
run: python -m pytest tests-assets
152+
153+
- name: Show ComfyUI logs
154+
if: always()
155+
shell: bash
156+
run: |
157+
echo "==== ASSETS_TEST_BASE_DIR: $ASSETS_TEST_BASE_DIR ===="
158+
echo "==== ASSETS_TEST_LOGS: $ASSETS_TEST_LOGS ===="
159+
ls -la "$ASSETS_TEST_LOGS" || true
160+
for f in "$ASSETS_TEST_LOGS"/stdout.log "$ASSETS_TEST_LOGS"/stderr.log; do
161+
if [ -f "$f" ]; then
162+
echo "----- BEGIN $f -----"
163+
sed -n '1,400p' "$f"
164+
echo "----- END $f -----"
165+
fi
166+
done
167+
168+
- name: Upload ComfyUI logs
169+
if: always()
170+
uses: actions/upload-artifact@v4
171+
with:
172+
name: asset-logs-pgsql-${{ matrix.pgsql }}-py${{ matrix.python }}
173+
path: ${{ env.ASSETS_TEST_LOGS }}/*.log
174+
if-no-files-found: warn

‎alembic_db/versions/0001_assets.py‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
"""initial assets schema
2+
3+
Revision ID: 0001_assets
4+
Revises:
5+
Create Date: 2025-08-20 00:00:00
6+
"""
7+
8+
from alembic import op
9+
import sqlalchemy as sa
10+
from sqlalchemy.dialects import postgresql
11+
12+
revision = "0001_assets"
13+
down_revision = None
14+
branch_labels = None
15+
depends_on = None
16+
17+
18+
def upgrade() -> None:
19+
# ASSETS: content identity
20+
op.create_table(
21+
"assets",
22+
sa.Column("id", sa.String(length=36), primary_key=True),
23+
sa.Column("hash", sa.String(length=256), nullable=True),
24+
sa.Column("size_bytes", sa.BigInteger(), nullable=False, server_default="0"),
25+
sa.Column("mime_type", sa.String(length=255), nullable=True),
26+
sa.Column("created_at", sa.DateTime(timezone=False), nullable=False),
27+
sa.CheckConstraint("size_bytes >= 0", name="ck_assets_size_nonneg"),
28+
)
29+
if op.get_bind().dialect.name == "postgresql":
30+
op.create_index(
31+
"uq_assets_hash_not_null",
32+
"assets",
33+
["hash"],
34+
unique=True,
35+
postgresql_where=sa.text("hash IS NOT NULL"),
36+
)
37+
else:
38+
op.create_index("uq_assets_hash", "assets", ["hash"], unique=True)
39+
op.create_index("ix_assets_mime_type", "assets", ["mime_type"])
40+
41+
# ASSETS_INFO: user-visible references
42+
op.create_table(
43+
"assets_info",
44+
sa.Column("id", sa.String(length=36), primary_key=True),
45+
sa.Column("owner_id", sa.String(length=128), nullable=False, server_default=""),
46+
sa.Column("name", sa.String(length=512), nullable=False),
47+
sa.Column("asset_id", sa.String(length=36), sa.ForeignKey("assets.id", ondelete="RESTRICT"), nullable=False),
48+
sa.Column("preview_id", sa.String(length=36), sa.ForeignKey("assets.id", ondelete="SET NULL"), nullable=True),
49+
sa.Column("user_metadata", sa.JSON(), nullable=True),
50+
sa.Column("created_at", sa.DateTime(timezone=False), nullable=False),
51+
sa.Column("updated_at", sa.DateTime(timezone=False), nullable=False),
52+
sa.Column("last_access_time", sa.DateTime(timezone=False), nullable=False),
53+
sa.UniqueConstraint("asset_id", "owner_id", "name", name="uq_assets_info_asset_owner_name"),
54+
)
55+
op.create_index("ix_assets_info_owner_id", "assets_info", ["owner_id"])
56+
op.create_index("ix_assets_info_asset_id", "assets_info", ["asset_id"])
57+
op.create_index("ix_assets_info_name", "assets_info", ["name"])
58+
op.create_index("ix_assets_info_created_at", "assets_info", ["created_at"])
59+
op.create_index("ix_assets_info_last_access_time", "assets_info", ["last_access_time"])
60+
op.create_index("ix_assets_info_owner_name", "assets_info", ["owner_id", "name"])
61+
62+
# TAGS: normalized tag vocabulary
63+
op.create_table(
64+
"tags",
65+
sa.Column("name", sa.String(length=512), primary_key=True),
66+
sa.Column("tag_type", sa.String(length=32), nullable=False, server_default="user"),
67+
sa.CheckConstraint("name = lower(name)", name="ck_tags_lowercase"),
68+
)
69+
op.create_index("ix_tags_tag_type", "tags", ["tag_type"])
70+
71+
# ASSET_INFO_TAGS: many-to-many for tags on AssetInfo
72+
op.create_table(
73+
"asset_info_tags",
74+
sa.Column("asset_info_id", sa.String(length=36), sa.ForeignKey("assets_info.id", ondelete="CASCADE"), nullable=False),
75+
sa.Column("tag_name", sa.String(length=512), sa.ForeignKey("tags.name", ondelete="RESTRICT"), nullable=False),
76+
sa.Column("origin", sa.String(length=32), nullable=False, server_default="manual"),
77+
sa.Column("added_at", sa.DateTime(timezone=False), nullable=False),
78+
sa.PrimaryKeyConstraint("asset_info_id", "tag_name", name="pk_asset_info_tags"),
79+
)
80+
op.create_index("ix_asset_info_tags_tag_name", "asset_info_tags", ["tag_name"])
81+
op.create_index("ix_asset_info_tags_asset_info_id", "asset_info_tags", ["asset_info_id"])
82+
83+
# ASSET_CACHE_STATE: N:1 local cache rows per Asset
84+
op.create_table(
85+
"asset_cache_state",
86+
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
87+
sa.Column("asset_id", sa.String(length=36), sa.ForeignKey("assets.id", ondelete="CASCADE"), nullable=False),
88+
sa.Column("file_path", sa.Text(), nullable=False), # absolute local path to cached file
89+
sa.Column("mtime_ns", sa.BigInteger(), nullable=True),
90+
sa.Column("needs_verify", sa.Boolean(), nullable=False, server_default=sa.text("false")),
91+
sa.CheckConstraint("(mtime_ns IS NULL) OR (mtime_ns >= 0)", name="ck_acs_mtime_nonneg"),
92+
sa.UniqueConstraint("file_path", name="uq_asset_cache_state_file_path"),
93+
)
94+
op.create_index("ix_asset_cache_state_file_path", "asset_cache_state", ["file_path"])
95+
op.create_index("ix_asset_cache_state_asset_id", "asset_cache_state", ["asset_id"])
96+
97+
# ASSET_INFO_META: typed KV projection of user_metadata for filtering/sorting
98+
op.create_table(
99+
"asset_info_meta",
100+
sa.Column("asset_info_id", sa.String(length=36), sa.ForeignKey("assets_info.id", ondelete="CASCADE"), nullable=False),
101+
sa.Column("key", sa.String(length=256), nullable=False),
102+
sa.Column("ordinal", sa.Integer(), nullable=False, server_default="0"),
103+
sa.Column("val_str", sa.String(length=2048), nullable=True),
104+
sa.Column("val_num", sa.Numeric(38, 10), nullable=True),
105+
sa.Column("val_bool", sa.Boolean(), nullable=True),
106+
sa.Column("val_json", sa.JSON().with_variant(postgresql.JSONB(), 'postgresql'), nullable=True),
107+
sa.PrimaryKeyConstraint("asset_info_id", "key", "ordinal", name="pk_asset_info_meta"),
108+
)
109+
op.create_index("ix_asset_info_meta_key", "asset_info_meta", ["key"])
110+
op.create_index("ix_asset_info_meta_key_val_str", "asset_info_meta", ["key", "val_str"])
111+
op.create_index("ix_asset_info_meta_key_val_num", "asset_info_meta", ["key", "val_num"])
112+
op.create_index("ix_asset_info_meta_key_val_bool", "asset_info_meta", ["key", "val_bool"])
113+
114+
# Tags vocabulary
115+
tags_table = sa.table(
116+
"tags",
117+
sa.column("name", sa.String(length=512)),
118+
sa.column("tag_type", sa.String()),
119+
)
120+
op.bulk_insert(
121+
tags_table,
122+
[
123+
{"name": "models", "tag_type": "system"},
124+
{"name": "input", "tag_type": "system"},
125+
{"name": "output", "tag_type": "system"},
126+
127+
{"name": "configs", "tag_type": "system"},
128+
{"name": "checkpoints", "tag_type": "system"},
129+
{"name": "loras", "tag_type": "system"},
130+
{"name": "vae", "tag_type": "system"},
131+
{"name": "text_encoders", "tag_type": "system"},
132+
{"name": "diffusion_models", "tag_type": "system"},
133+
{"name": "clip_vision", "tag_type": "system"},
134+
{"name": "style_models", "tag_type": "system"},
135+
{"name": "embeddings", "tag_type": "system"},
136+
{"name": "diffusers", "tag_type": "system"},
137+
{"name": "vae_approx", "tag_type": "system"},
138+
{"name": "controlnet", "tag_type": "system"},
139+
{"name": "gligen", "tag_type": "system"},
140+
{"name": "upscale_models", "tag_type": "system"},
141+
{"name": "hypernetworks", "tag_type": "system"},
142+
{"name": "photomaker", "tag_type": "system"},
143+
{"name": "classifiers", "tag_type": "system"},
144+
145+
{"name": "encoder", "tag_type": "system"},
146+
{"name": "decoder", "tag_type": "system"},
147+
148+
{"name": "missing", "tag_type": "system"},
149+
{"name": "rescan", "tag_type": "system"},
150+
],
151+
)
152+
153+
154+
def downgrade() -> None:
155+
op.drop_index("ix_asset_info_meta_key_val_bool", table_name="asset_info_meta")
156+
op.drop_index("ix_asset_info_meta_key_val_num", table_name="asset_info_meta")
157+
op.drop_index("ix_asset_info_meta_key_val_str", table_name="asset_info_meta")
158+
op.drop_index("ix_asset_info_meta_key", table_name="asset_info_meta")
159+
op.drop_table("asset_info_meta")
160+
161+
op.drop_index("ix_asset_cache_state_asset_id", table_name="asset_cache_state")
162+
op.drop_index("ix_asset_cache_state_file_path", table_name="asset_cache_state")
163+
op.drop_constraint("uq_asset_cache_state_file_path", table_name="asset_cache_state")
164+
op.drop_table("asset_cache_state")
165+
166+
op.drop_index("ix_asset_info_tags_asset_info_id", table_name="asset_info_tags")
167+
op.drop_index("ix_asset_info_tags_tag_name", table_name="asset_info_tags")
168+
op.drop_table("asset_info_tags")
169+
170+
op.drop_index("ix_tags_tag_type", table_name="tags")
171+
op.drop_table("tags")
172+
173+
op.drop_constraint("uq_assets_info_asset_owner_name", table_name="assets_info")
174+
op.drop_index("ix_assets_info_owner_name", table_name="assets_info")
175+
op.drop_index("ix_assets_info_last_access_time", table_name="assets_info")
176+
op.drop_index("ix_assets_info_created_at", table_name="assets_info")
177+
op.drop_index("ix_assets_info_name", table_name="assets_info")
178+
op.drop_index("ix_assets_info_asset_id", table_name="assets_info")
179+
op.drop_index("ix_assets_info_owner_id", table_name="assets_info")
180+
op.drop_table("assets_info")
181+
182+
if op.get_bind().dialect.name == "postgresql":
183+
op.drop_index("uq_assets_hash_not_null", table_name="assets")
184+
else:
185+
op.drop_index("uq_assets_hash", table_name="assets")
186+
op.drop_index("ix_assets_mime_type", table_name="assets")
187+
op.drop_table("assets")

0 commit comments

Comments
 (0)