store url response size in database

Author: nepeat

alembic/versions/86a2d08cfade_url_file_size.py

@@ -0,0 +1,28 @@
+"""url file size
+
+Revision ID: 86a2d08cfade
+Revises: c8d3d789bd34
+Create Date: 2022-04-04 04:16:03.267866
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '86a2d08cfade'
+down_revision = 'c8d3d789bd34'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('urls', sa.Column('size', sa.Integer(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('urls', 'size')
+    # ### end Alembic commands ###

placedump/model.py

@@ -52,6 +52,7 @@ def to_dict(self):
 
     url = Column(String, unique=True)
     fetched = Column(DateTime, server_default=func.now())
+    size = Column(Integer)
 
 
 class Pixel(Base):

placedump/tasks/pixels.py

@@ -7,6 +7,7 @@
 
 import httpx
 import numpy as np
+import sentry_sdk
 from async_timeout import timeout
 from b2sdk.exception import TooManyRequests
 from gql import gql
@@ -123,6 +124,7 @@ def download_url(board: int, url: str):
                     insert(URL)
                     .values(
                         url=url,
+                        size=len(data),
                     )
                     .on_conflict_do_nothing(index_elements=["url"])
                 )