Skip to content

Commit 757083d

Browse files
catdevnullcatdevnull
authored
Merge pull request #70 from catdevnull/cursor/add-test-and-fix-scraper-saving-endpoint-1fc6
Add test and fix scraper saving endpoint
2 parents 8a777b6 + 9b25139 commit 757083d

File tree

11 files changed

+1015
-62
lines changed

11 files changed

+1015
-62
lines changed

‎.github/workflows/container.yml‎

Lines changed: 0 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -27,40 +27,3 @@ jobs:
2727
working-directory: sitio
2828
- run: pnpm run lint
2929
working-directory: sitio
30-
31-
oci-sitio:
32-
name: "[amd64] oci:sitio"
33-
needs: check-sitio
34-
runs-on: ubuntu-latest
35-
permissions:
36-
contents: read
37-
packages: write
38-
steps:
39-
- name: Checkout repository
40-
uses: actions/checkout@v4
41-
with:
42-
lfs: true
43-
- name: Set up Docker Buildx
44-
uses: docker/setup-buildx-action@v3
45-
- name: Log in to the Container registry
46-
uses: docker/login-action@v3
47-
with:
48-
registry: ${{ env.REGISTRY }}
49-
username: ${{ github.actor }}
50-
password: ${{ secrets.GITHUB_TOKEN }}
51-
- name: Extract metadata (tags, labels) for Docker
52-
id: meta
53-
uses: docker/metadata-action@v5
54-
with:
55-
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/sitio
56-
- name: Build and push Docker image
57-
uses: docker/build-push-action@v5
58-
with:
59-
context: .
60-
file: docker/sitio.Dockerfile
61-
push: true
62-
tags: ${{ steps.meta.outputs.tags }}
63-
labels: ${{ steps.meta.outputs.labels }}
64-
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/sitio:buildcache
65-
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/sitio:buildcache,mode=max
66-
platforms: linux/amd64

‎pnpm-lock.yaml‎

Lines changed: 381 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎sitio/drizzle/0001_clean_micromacro.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ALTER TABLE "db_scraps" ADD CONSTRAINT "db_scraps_uid_unique" UNIQUE("uid");

‎sitio/drizzle/meta/0001_snapshot.json‎

Lines changed: 354 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,354 @@
1+
{
2+
"id": "c2e27cff-3bf5-46b6-ad9f-de8f9098aefc",
3+
"prevId": "c19e9290-6bed-4226-921b-e3e4f9d00b1b",
4+
"version": "7",
5+
"dialect": "postgresql",
6+
"tables": {
7+
"public.db_cuentas": {
8+
"name": "db_cuentas",
9+
"schema": "",
10+
"columns": {
11+
"id": {
12+
"name": "id",
13+
"type": "text",
14+
"primaryKey": true,
15+
"notNull": true
16+
},
17+
"account_data_json": {
18+
"name": "account_data_json",
19+
"type": "text",
20+
"primaryKey": false,
21+
"notNull": false
22+
}
23+
},
24+
"indexes": {},
25+
"foreignKeys": {},
26+
"compositePrimaryKeys": {},
27+
"uniqueConstraints": {}
28+
},
29+
"public.db_historic_liked_tweets": {
30+
"name": "db_historic_liked_tweets",
31+
"schema": "",
32+
"columns": {
33+
"post_id": {
34+
"name": "post_id",
35+
"type": "text",
36+
"primaryKey": true,
37+
"notNull": true
38+
},
39+
"url": {
40+
"name": "url",
41+
"type": "text",
42+
"primaryKey": false,
43+
"notNull": true
44+
},
45+
"posted_at": {
46+
"name": "posted_at",
47+
"type": "timestamp with time zone",
48+
"primaryKey": false,
49+
"notNull": true
50+
},
51+
"estimated_liked_at": {
52+
"name": "estimated_liked_at",
53+
"type": "timestamp with time zone",
54+
"primaryKey": false,
55+
"notNull": true
56+
}
57+
},
58+
"indexes": {},
59+
"foreignKeys": {},
60+
"compositePrimaryKeys": {},
61+
"uniqueConstraints": {}
62+
},
63+
"public.db_liked_tweets": {
64+
"name": "db_liked_tweets",
65+
"schema": "",
66+
"columns": {
67+
"url": {
68+
"name": "url",
69+
"type": "text",
70+
"primaryKey": true,
71+
"notNull": true
72+
},
73+
"first_seen_at": {
74+
"name": "first_seen_at",
75+
"type": "timestamp with time zone",
76+
"primaryKey": false,
77+
"notNull": true
78+
},
79+
"last_seen_at": {
80+
"name": "last_seen_at",
81+
"type": "timestamp with time zone",
82+
"primaryKey": false,
83+
"notNull": false
84+
},
85+
"text": {
86+
"name": "text",
87+
"type": "text",
88+
"primaryKey": false,
89+
"notNull": false
90+
},
91+
"scrap_id": {
92+
"name": "scrap_id",
93+
"type": "integer",
94+
"primaryKey": false,
95+
"notNull": false
96+
}
97+
},
98+
"indexes": {
99+
"first_seen_at_idx": {
100+
"name": "first_seen_at_idx",
101+
"columns": [
102+
{
103+
"expression": "first_seen_at",
104+
"isExpression": false,
105+
"asc": true,
106+
"nulls": "last"
107+
}
108+
],
109+
"isUnique": false,
110+
"concurrently": false,
111+
"method": "btree",
112+
"with": {}
113+
},
114+
"liked_tweets_last_seen_at_idx": {
115+
"name": "liked_tweets_last_seen_at_idx",
116+
"columns": [
117+
{
118+
"expression": "last_seen_at",
119+
"isExpression": false,
120+
"asc": true,
121+
"nulls": "last"
122+
}
123+
],
124+
"isUnique": false,
125+
"concurrently": false,
126+
"method": "btree",
127+
"with": {}
128+
},
129+
"liked_tweets_scrap_id_idx": {
130+
"name": "liked_tweets_scrap_id_idx",
131+
"columns": [
132+
{
133+
"expression": "scrap_id",
134+
"isExpression": false,
135+
"asc": true,
136+
"nulls": "last"
137+
}
138+
],
139+
"isUnique": false,
140+
"concurrently": false,
141+
"method": "btree",
142+
"with": {}
143+
}
144+
},
145+
"foreignKeys": {},
146+
"compositePrimaryKeys": {},
147+
"uniqueConstraints": {}
148+
},
149+
"public.db_retweets": {
150+
"name": "db_retweets",
151+
"schema": "",
152+
"columns": {
153+
"poster_id": {
154+
"name": "poster_id",
155+
"type": "text",
156+
"primaryKey": false,
157+
"notNull": true
158+
},
159+
"poster_handle": {
160+
"name": "poster_handle",
161+
"type": "text",
162+
"primaryKey": false,
163+
"notNull": false
164+
},
165+
"post_id": {
166+
"name": "post_id",
167+
"type": "text",
168+
"primaryKey": false,
169+
"notNull": true
170+
},
171+
"first_seen_at": {
172+
"name": "first_seen_at",
173+
"type": "timestamp with time zone",
174+
"primaryKey": false,
175+
"notNull": true
176+
},
177+
"retweet_at": {
178+
"name": "retweet_at",
179+
"type": "timestamp with time zone",
180+
"primaryKey": false,
181+
"notNull": true
182+
},
183+
"posted_at": {
184+
"name": "posted_at",
185+
"type": "timestamp with time zone",
186+
"primaryKey": false,
187+
"notNull": true
188+
},
189+
"text": {
190+
"name": "text",
191+
"type": "text",
192+
"primaryKey": false,
193+
"notNull": false
194+
},
195+
"scrap_id": {
196+
"name": "scrap_id",
197+
"type": "integer",
198+
"primaryKey": false,
199+
"notNull": false
200+
}
201+
},
202+
"indexes": {
203+
"retweet_at_idx": {
204+
"name": "retweet_at_idx",
205+
"columns": [
206+
{
207+
"expression": "retweet_at",
208+
"isExpression": false,
209+
"asc": true,
210+
"nulls": "last"
211+
}
212+
],
213+
"isUnique": false,
214+
"concurrently": false,
215+
"method": "btree",
216+
"with": {}
217+
}
218+
},
219+
"foreignKeys": {},
220+
"compositePrimaryKeys": {
221+
"db_retweets_poster_id_post_id_pk": {
222+
"name": "db_retweets_poster_id_post_id_pk",
223+
"columns": [
224+
"poster_id",
225+
"post_id"
226+
]
227+
}
228+
},
229+
"uniqueConstraints": {}
230+
},
231+
"public.db_scraper_tokens": {
232+
"name": "db_scraper_tokens",
233+
"schema": "",
234+
"columns": {
235+
"id": {
236+
"name": "id",
237+
"type": "serial",
238+
"primaryKey": true,
239+
"notNull": true
240+
},
241+
"token": {
242+
"name": "token",
243+
"type": "text",
244+
"primaryKey": false,
245+
"notNull": true
246+
}
247+
},
248+
"indexes": {},
249+
"foreignKeys": {},
250+
"compositePrimaryKeys": {},
251+
"uniqueConstraints": {}
252+
},
253+
"public.db_scraps": {
254+
"name": "db_scraps",
255+
"schema": "",
256+
"columns": {
257+
"id": {
258+
"name": "id",
259+
"type": "serial",
260+
"primaryKey": true,
261+
"notNull": true
262+
},
263+
"uid": {
264+
"name": "uid",
265+
"type": "text",
266+
"primaryKey": false,
267+
"notNull": false
268+
},
269+
"at": {
270+
"name": "at",
271+
"type": "timestamp with time zone",
272+
"primaryKey": false,
273+
"notNull": true
274+
},
275+
"cuenta_id": {
276+
"name": "cuenta_id",
277+
"type": "text",
278+
"primaryKey": false,
279+
"notNull": false
280+
},
281+
"total_tweets_seen": {
282+
"name": "total_tweets_seen",
283+
"type": "integer",
284+
"primaryKey": false,
285+
"notNull": false
286+
}
287+
},
288+
"indexes": {
289+
"db_scraps_finished_at_idx": {
290+
"name": "db_scraps_finished_at_idx",
291+
"columns": [
292+
{
293+
"expression": "at",
294+
"isExpression": false,
295+
"asc": true,
296+
"nulls": "last"
297+
}
298+
],
299+
"isUnique": false,
300+
"concurrently": false,
301+
"method": "btree",
302+
"with": {}
303+
}
304+
},
305+
"foreignKeys": {},
306+
"compositePrimaryKeys": {},
307+
"uniqueConstraints": {
308+
"db_scraps_uid_unique": {
309+
"name": "db_scraps_uid_unique",
310+
"nullsNotDistinct": false,
311+
"columns": [
312+
"uid"
313+
]
314+
}
315+
}
316+
},
317+
"public.db_tweets": {
318+
"name": "db_tweets",
319+
"schema": "",
320+
"columns": {
321+
"id": {
322+
"name": "id",
323+
"type": "text",
324+
"primaryKey": true,
325+
"notNull": true
326+
},
327+
"twitter_scraper_json": {
328+
"name": "twitter_scraper_json",
329+
"type": "jsonb",
330+
"primaryKey": false,
331+
"notNull": true
332+
},
333+
"captured_at": {
334+
"name": "captured_at",
335+
"type": "timestamp with time zone",
336+
"primaryKey": false,
337+
"notNull": true
338+
}
339+
},
340+
"indexes": {},
341+
"foreignKeys": {},
342+
"compositePrimaryKeys": {},
343+
"uniqueConstraints": {}
344+
}
345+
},
346+
"enums": {},
347+
"schemas": {},
348+
"sequences": {},
349+
"_meta": {
350+
"columns": {},
351+
"schemas": {},
352+
"tables": {}
353+
}
354+
}

0 commit comments

Comments
 (0)