forked from FarhadAliev/Craiglist-web-scraper
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathindex.js
More file actions
92 lines (75 loc) · 2.38 KB
/
index.js
File metadata and controls
92 lines (75 loc) · 2.38 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
const express=require('express');
const app=new express();
const bodyParser=require('body-parser');
const craigslist=require('node-craigslist');
const cors=require('cors');
const morgan=require('morgan');
const axios=require('axios');
const fetch = require('node-fetch');
const cheerio=require('cheerio');
require('dotenv').config();
const PORT=process.env.PORT || 3000;
const HOST=process.env.HOST;
app.use(bodyParser.json());
app.use(cors());
app.use(morgan('tiny'));
app.listen(PORT,HOST, async () => {
console.log(`Server http://${HOST}:${PORT}/`);
});
function getResults(body) {
const $ = cheerio.load(body);
const rows = $('li.result-row');
const results = [];
rows.each(async (index, element) => {
const result = $(element);
const title = result.find('.result-title').text();
const price = $(result.find('.result-price').get(0)).text();
const imageData = result.find('a.result-image').attr('data-ids');
let images = [];
if (imageData) {
const parts = imageData.split(',');
images = parts.map((id) => {
return `https://images.craigslist.org/${id.split(':')[1]}_300x300.jpg`;
});
};
let hood=result.find('.result-hood').text();
if(hood) {
hood=hood.match(/\((.*)\)/)[1];
};
// .result-title.hdrlnk
let url = result.find('.result-title.hdrlnk').attr('href');
results.push({
title,
price,
images,
hood,
url
})
});
return results;
};
app.get('/search/:location/:search_term',(request, response) => {
const {location, search_term} = request.params;
const url = `https://${location}.craigslist.org/search/cta?query=${search_term}&postedToday=1`;
fetch(url)
.then(async docs => {
let body = await docs.text();
const results = getResults(body);
response.json({
results
});
}).catch((error)=>{
console.log(error)
})
});
app.use((request, response, next) => {
const error = new Error('not found');
response.status(404);
next(error);
});
app.use((error, request, response, next) => {
response.status(response.statusCode || 500);
response.json({
message: error.message
});
});