Skip to content

Commit 0415119

Browse files
mazen-rmazen-r
committed
Merge branch 'fix-tripadvisor'
2 parents 23cfffc + 7af4c9b commit 0415119

File tree

30 files changed

+8263
-9876
lines changed

30 files changed

+8263
-9876
lines changed

‎domaincom-scraper/results/properties.json‎

Lines changed: 205 additions & 1194 deletions
Large diffs are not rendered by default.

‎domaincom-scraper/results/search.json‎

Lines changed: 1368 additions & 888 deletions
Large diffs are not rendered by default.

‎domaincom-scraper/run.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ async def run():
2424
urls = [
2525
"https://www.domain.com.au/610-399-bourke-street-melbourne-vic-3000-2018835548",
2626
"https://www.domain.com.au/property-profile/308-9-degraves-street-melbourne-vic-3000",
27-
"https://www.domain.com.au/101-29-31-market-street-melbourne-vic-3000-2018799963"
27+
"https://www.domain.com.au/1518-474-flinders-street-melbourne-vic-3000-17773317"
2828
]
2929
)
3030
with open(output.joinpath("properties.json"), "w", encoding="utf-8") as file:

‎domaincom-scraper/test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,9 +134,9 @@ def validate_or_fail(item, validator):
134134
async def test_properties_scraping():
135135
properties_data = await domaincom.scrape_properties(
136136
urls = [
137-
"https://www.domain.com.au/type-10c-250-spencer-street-melbourne-vic-3000-2018950519",
137+
"https://www.domain.com.au/610-399-bourke-street-melbourne-vic-3000-2018835548",
138138
"https://www.domain.com.au/property-profile/308-9-degraves-street-melbourne-vic-3000",
139-
"https://www.domain.com.au/1223-422-collins-street-melbourne-vic-3000-2019894035",
139+
"https://www.domain.com.au/1518-474-flinders-street-melbourne-vic-3000-17773317"
140140
]
141141
)
142142
validator = Validator(property_schema, allow_unknown=True)

‎ebay-scraper/ebay.py‎

Lines changed: 12 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -150,42 +150,26 @@ async def scrape_product(url: str) -> Dict:
150150
def parse_search(result: ScrapeApiResponse) -> List[Dict]:
151151
"""Parse ebay.com search result page for product previews"""
152152
previews = []
153-
# This logic for best-selling items remains the same.
154-
best_selling_boxes = result.selector.xpath(
155-
"//*[*[h2[contains(text(),'Best selling products')]]]//li[contains(@class, 's-item')]"
156-
)
157-
best_selling_html_set = set([b.get() for b in best_selling_boxes])
158-
159-
for box in result.selector.css("ul.srp-results > li.s-item"):
160-
if box.get() in best_selling_html_set:
161-
continue # skip boxes inside the best selling container
162153

154+
for box in result.selector.css("ul.srp-results li"):
163155
css = lambda css: box.css(css).get("").strip() or None
164156
css_all = lambda css: box.css(css).getall()
165157
css_re = lambda css, pattern: box.css(css).re_first(pattern, default="").strip()
166158
css_float = lambda css: float(box.css(css).re_first(r"(\d+\.*\d*)", default="0.0")) if box.css(css) else None
159+
location = box.xpath(".//*[contains(text(),'Located')]/text()").get()
160+
price = css(".s-card__price::text") or css(".s-item__price::text")
161+
url = css("a.su-link::attr(href)")
167162

168-
bids_text = box.xpath(".//span[contains(text(), 'bid')]/text()").get()
169-
bids_count = 0
170-
if bids_text:
171-
bids_match = re.search(r"(\d+)", bids_text)
172-
if bids_match:
173-
bids_count = int(bids_match.group(1))
174-
175-
auction_end = css_re(".s-item__time-end::text", r"\((.+?)\)") or None
176-
if auction_end:
177-
auction_end = dateutil.parser.parse(auction_end.replace("Today", ""))
163+
if price is None:
164+
continue # skip boxes inside the best selling container
178165

179166
item = {
180-
"url": css("a.s-item__link::attr(href)").split("?")[0],
181-
"title": css(".s-item__title span::text"),
182-
"price": css(".s-item__price span::text") or css(".s-item__price::text"),
183-
"shipping": css_float(".s-item__shipping::text"),
184-
"auction_end": auction_end,
185-
"bids": bids_count,
186-
"location": css(".s-item__itemLocation::text"),
187-
"subtitles": css_all(".s-item__subtitle::text"),
188-
"condition": css(".SECONDARY_INFO::text"),
167+
"url": url.split("?")[0] if url else None,
168+
"title": css(".s-card__title span::text"),
169+
"price": css(".s-card__price::text") or css(".s-item__price::text"),
170+
"shipping": box.xpath(".//*[contains(text(),'delivery')]/text()").get(),
171+
"location": location.split("Located in ")[1] if location else None,
172+
"subtitles": css(".s-card__subtitle span::text"),
189173
"photo": css("img::attr(data-src)") or css("img::attr(src)"),
190174
"rating": css_float(".s-item__reviews .clipped::text"),
191175
"rating_count": int(box.css(".s-item__reviews-count span::text").re_first(r"(\d+)", default="0")),

‎ebay-scraper/results/product-with-variants.json‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,15 @@
22
"url": "https://www.ebay.com/itm/393531906094",
33
"id": "393531906094",
44
"price_original": "C $339.96",
5-
"price_converted": "US $247.08",
5+
"price_converted": "US $246.03",
66
"name": "Apple iPhone 11 Pro Max - Unlocked - 64GB / 256GB / 512GB - CA - Very Good",
77
"seller_name": "MobileKlinik",
88
"seller_url": "https://www.ebay.com/str/devicecare",
99
"photos": [
1010
"https://i.ebayimg.com/thumbs/images/g/93cAAOSwvEJgbLW8/s-l500.jpg",
1111
"https://i.ebayimg.com/images/g/93cAAOSwvEJgbLW8/s-l1600.webp"
1212
],
13-
"description_url": "https://itm.ebaydesc.com/itmdesc/393531906094?t=1737651084000&category=9355&seller=mobileklinik&excSoj=1&ver=0&excTrk=1&lsite=2&ittenable=true&domain=ebay.com&descgauge=1&cspheader=1&oneClk=2&secureDesc=1&variationId=662315637172",
13+
"description_url": "https://itm.ebaydesc.com/itmdesc/393531906094?t=1737651084000&category=9355&seller=mobileklinik&excSoj=1&ver=1&excTrk=1&lsite=2&ittenable=true&domain=ebay.com&descgauge=1&cspheader=1&oneClk=2&secureDesc=1&variationId=662315637172",
1414
"features": {
1515
"Condition": "Very Good - Refurbished: The item shows minimal wear and is backed by a one year warranty. It is ... Read moreabout the conditionVery Good - Refurbished: The item shows minimal wear and is backed by a one year warranty. It is fully functional and has been professionally refurbished, inspected and cleaned to very good condition by qualified sellers. The item includes original or new accessories and will come in new generic packaging. See the seller's listing for full details. See all condition definitionsopens in a new window or tab",
1616
"Processor": "Hexa Core",
@@ -35,7 +35,7 @@
3535
"Color": "Midnight Green",
3636
"price_original": 379.96,
3737
"price_original_currency": "CAD",
38-
"price_converted": 276.15,
38+
"price_converted": 274.97,
3939
"price_converted_currency": "USD",
4040
"out_of_stock": true
4141
},
@@ -46,7 +46,7 @@
4646
"Color": "Gold",
4747
"price_original": 379.96,
4848
"price_original_currency": "CAD",
49-
"price_converted": 276.15,
49+
"price_converted": 274.97,
5050
"price_converted_currency": "USD",
5151
"out_of_stock": true
5252
},
@@ -57,7 +57,7 @@
5757
"Color": "Space Gray",
5858
"price_original": 379.96,
5959
"price_original_currency": "CAD",
60-
"price_converted": 276.15,
60+
"price_converted": 274.97,
6161
"price_converted_currency": "USD",
6262
"out_of_stock": true
6363
},
@@ -68,7 +68,7 @@
6868
"Color": "Midnight Green",
6969
"price_original": 359.96,
7070
"price_original_currency": "CAD",
71-
"price_converted": 261.62,
71+
"price_converted": 260.5,
7272
"price_converted_currency": "USD",
7373
"out_of_stock": true
7474
},
@@ -79,7 +79,7 @@
7979
"Color": "Gold",
8080
"price_original": 359.96,
8181
"price_original_currency": "CAD",
82-
"price_converted": 261.62,
82+
"price_converted": 260.5,
8383
"price_converted_currency": "USD",
8484
"out_of_stock": true
8585
},
@@ -90,7 +90,7 @@
9090
"Color": "Space Gray",
9191
"price_original": 359.96,
9292
"price_original_currency": "CAD",
93-
"price_converted": 261.62,
93+
"price_converted": 260.5,
9494
"price_converted_currency": "USD",
9595
"out_of_stock": true
9696
},
@@ -101,7 +101,7 @@
101101
"Color": "Silver",
102102
"price_original": 379.96,
103103
"price_original_currency": "CAD",
104-
"price_converted": 276.15,
104+
"price_converted": 274.97,
105105
"price_converted_currency": "USD",
106106
"out_of_stock": true
107107
},
@@ -112,7 +112,7 @@
112112
"Color": "Midnight Green",
113113
"price_original": 339.96,
114114
"price_original_currency": "CAD",
115-
"price_converted": 247.08,
115+
"price_converted": 246.03,
116116
"price_converted_currency": "USD",
117117
"out_of_stock": true
118118
},
@@ -123,7 +123,7 @@
123123
"Color": "Gold",
124124
"price_original": 339.96,
125125
"price_original_currency": "CAD",
126-
"price_converted": 247.08,
126+
"price_converted": 246.03,
127127
"price_converted_currency": "USD",
128128
"out_of_stock": true
129129
},
@@ -134,7 +134,7 @@
134134
"Color": "Space Gray",
135135
"price_original": 339.96,
136136
"price_original_currency": "CAD",
137-
"price_converted": 247.08,
137+
"price_converted": 246.03,
138138
"price_converted_currency": "USD",
139139
"out_of_stock": true
140140
},
@@ -145,7 +145,7 @@
145145
"Color": "Silver",
146146
"price_original": 359.96,
147147
"price_original_currency": "CAD",
148-
"price_converted": 261.62,
148+
"price_converted": 260.5,
149149
"price_converted_currency": "USD",
150150
"out_of_stock": true
151151
},
@@ -156,7 +156,7 @@
156156
"Color": "Silver",
157157
"price_original": 339.96,
158158
"price_original_currency": "CAD",
159-
"price_converted": 247.08,
159+
"price_converted": 246.03,
160160
"price_converted_currency": "USD",
161161
"out_of_stock": true
162162
}

‎ebay-scraper/results/product.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"url": "https://www.ebay.com/itm/332562282948",
33
"id": "332562282948",
4-
"price_original": "US $13.26",
4+
"price_original": "US $12.49",
55
"price_converted": "",
66
"name": "Kirby 5.5\" Plush Stuffed Doll (KP01) - Kirby Adventure All Star Collection",
77
"seller_name": "auroraborealismarket",
@@ -10,7 +10,7 @@
1010
"https://i.ebayimg.com/thumbs/images/g/ITEAAOSw9p9ajK16/s-l500.jpg",
1111
"https://i.ebayimg.com/images/g/ITEAAOSw9p9ajK16/s-l1600.webp"
1212
],
13-
"description_url": "https://itm.ebaydesc.com/itmdesc/332562282948?t=1735243883000&category=69528&seller=auroraborealismarket&excSoj=1&ver=0&excTrk=1&lsite=0&ittenable=true&domain=ebay.com&descgauge=1&cspheader=1&oneClk=2&secureDesc=1",
13+
"description_url": "https://itm.ebaydesc.com/itmdesc/332562282948?t=1735243883000&category=69528&seller=auroraborealismarket&excSoj=1&ver=1&excTrk=1&lsite=0&ittenable=true&domain=ebay.com&descgauge=1&cspheader=1&oneClk=2&secureDesc=1",
1414
"features": {
1515
"Condition": "New: A brand-new, unused, unopened, undamaged item (including handmade items). See the seller's ... Read moreabout the conditionNew: A brand-new, unused, unopened, undamaged item (including handmade items). See the seller's listing for full details. See all condition definitionsopens in a new window or tab",
1616
"Brand": "unbranded",

0 commit comments

Comments
 (0)