Skip to content

Commit f0b146f

Browse files
orangecodingorangecoding
authored
Adding images to scraping data (#157)
* Fredy now supports pulling the main Image from the listing and send it together with the usual information
1 parent da743c8 commit f0b146f

26 files changed

+1183
-1177
lines changed

‎lib/notification/adapter/mailJet.js‎

Lines changed: 88 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2,68 +2,120 @@ import mailjet from 'node-mailjet';
22
import path from 'path';
33
import fs from 'fs';
44
import Handlebars from 'handlebars';
5+
import fetch from 'node-fetch';
56
import { markdown2Html } from '../../services/markdown.js';
6-
import { getDirName } from '../../utils.js';
7+
import { getDirName, normalizeImageUrl } from '../../utils.js';
8+
79
const __dirname = getDirName();
810
const template = fs.readFileSync(path.resolve(__dirname + '/notification/emailTemplate/template.hbs'), 'utf8');
911
const emailTemplate = Handlebars.compile(template);
10-
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
12+
13+
const guessMime = (url) => {
14+
const lower = url.split('?')[0].toLowerCase();
15+
if (lower.endsWith('.png')) return 'image/png';
16+
if (lower.endsWith('.gif')) return 'image/gif';
17+
return 'image/jpeg';
18+
};
19+
20+
const toBase64 = async (url) => {
21+
try {
22+
const res = await fetch(url);
23+
if (!res.ok) throw new Error(`Fetch failed with status ${res.status} for URL: ${url}`);
24+
const ab = await res.arrayBuffer();
25+
return Buffer.from(ab).toString('base64');
26+
} catch (error) {
27+
console.error(`Error fetching image from ${url}:`, error.message);
28+
throw error;
29+
}
30+
};
31+
32+
const mapListingsWithCid = async (serviceName, jobKey, listings) => {
33+
const out = [];
34+
const attachments = [];
35+
36+
for (let i = 0; i < listings.length; i++) {
37+
const l = listings[i] || {};
38+
const imgUrl = normalizeImageUrl(l.image);
39+
40+
const item = {
41+
title: l.title || '',
42+
link: l.link || '',
43+
address: l.address || '',
44+
size: l.size || '',
45+
price: l.price || '',
46+
serviceName,
47+
jobKey,
48+
hasImage: false,
49+
imageCid: '',
50+
};
51+
52+
if (imgUrl) {
53+
try {
54+
const base64 = await toBase64(imgUrl);
55+
const cid = `listing-${i}`;
56+
attachments.push({
57+
ContentType: guessMime(imgUrl),
58+
Filename: `listing-${i}.${imgUrl.split('.').pop().split('?')[0] || 'jpg'}`,
59+
Base64Content: base64,
60+
ContentID: cid,
61+
});
62+
item.hasImage = true;
63+
item.imageCid = cid;
64+
} catch (error) {
65+
console.warn(`Skipping image for listing ${i} due to error: ${error.message}`);
66+
}
67+
}
68+
69+
out.push(item);
70+
}
71+
72+
return { listings: out, attachments };
73+
};
74+
75+
export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
1176
const { apiPublicKey, apiPrivateKey, receiver, from } = notificationConfig.find(
1277
(adapter) => adapter.id === config.id,
1378
).fields;
79+
1480
const to = receiver
1581
.trim()
1682
.split(',')
17-
.map((r) => ({
18-
Email: r.trim(),
19-
}));
83+
.map((r) => ({ Email: r.trim() }))
84+
.filter((r) => r.Email.length > 0);
85+
86+
const { listings, attachments } = await mapListingsWithCid(serviceName, jobKey, newListings);
87+
88+
const html = emailTemplate({
89+
serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
90+
numberOfListings: listings.length,
91+
listings,
92+
});
93+
2094
return mailjet
2195
.apiConnect(apiPublicKey, apiPrivateKey)
2296
.post('send', { version: 'v3.1' })
2397
.request({
2498
Messages: [
2599
{
26-
From: {
27-
Email: from,
28-
Name: 'Fredy',
29-
},
100+
From: { Email: from, Name: 'Fredy' },
30101
To: to,
31-
Subject: `Fredy found ${newListings.length} new listings for ${serviceName}`,
32-
HTMLPart: emailTemplate({
33-
serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
34-
numberOfListings: newListings.length,
35-
listings: newListings,
36-
}),
102+
Subject: `Fredy found ${listings.length} new listing(s) for ${serviceName}`,
103+
HTMLPart: html,
104+
InlinedAttachments: attachments,
37105
},
38106
],
39107
});
40108
};
109+
41110
export const config = {
42111
id: 'mailjet',
43112
name: 'MailJet',
44113
description: 'MailJet is being used to send new listings via mail.',
45114
readme: markdown2Html('lib/notification/adapter/mailJet.md'),
46115
fields: {
47-
apiPublicKey: {
48-
type: 'text',
49-
label: 'Public Api Key',
50-
description: 'The public api key needed to access this service.',
51-
},
52-
apiPrivateKey: {
53-
type: 'text',
54-
label: 'Private Api Key',
55-
description: 'The private api key needed to access this service.',
56-
},
57-
receiver: {
58-
type: 'email',
59-
label: 'Receiver Email',
60-
description: 'The email address (single one) which Fredy is using to send notifications to.',
61-
},
62-
from: {
63-
type: 'email',
64-
label: 'Sender email',
65-
description:
66-
'The email address from which Fredy send email. Beware, this email address needs to be verified by Sendgrid.',
67-
},
116+
apiPublicKey: { type: 'text', label: 'Public Api Key' },
117+
apiPrivateKey: { type: 'text', label: 'Private Api Key' },
118+
receiver: { type: 'email', label: 'Receiver Email' },
119+
from: { type: 'email', label: 'Sender email' },
68120
},
69121
};

‎lib/notification/adapter/mailJet.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
### MailJet Adapter
22

3-
To use [MailJet](https://mailjet.com), you need to create an account. You'll need to decided from which email address you want Fredy to send from.
3+
To use [MailJet](https://mailjet.com), you need to create an account. You'll need to decide from which email address you want Fredy to send from.
44

55
E.g. if you use yourGmailAccount@gmail.com, you have to add this to MailJet and verify it as well.
66
The given public/private api keys are needed in order to use MailJet with Fredy. Fredy will use the same template, it is using for SendGrid.
77

8-
If this email should be sent to multiple receiver use a comma separator (some@email.com, someOther@email.com).
8+
If this email should be sent to multiple receiver, use a comma separator (some@email.com, someOther@email.com).

‎lib/notification/adapter/ntfy.js‎

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,41 @@
11
import { markdown2Html } from '../../services/markdown.js';
22
import { getJob } from '../../services/storage/jobStorage.js';
33
import fetch from 'node-fetch';
4+
import { normalizeImageUrl } from '../../utils.js';
45

56
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
67
const { priority, server, topic } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
78
const job = getJob(jobKey);
89
const jobName = job == null ? jobKey : job.name;
10+
911
const promises = newListings.map((newListing) => {
1012
const message = `
11-
Address: ${newListing.address}
12-
Size: ${newListing.size.replace(/2m/g, '$m^2$')}
13-
Price: ${newListing.price}
14-
Link: ${newListing.link}`;
15-
return fetch(server, {
13+
Address: ${newListing.address}
14+
Size: ${newListing.size == null ? 'N/A' : newListing.size.replace(/2m/g, '$m^2$')}
15+
Price: ${newListing.price}
16+
Link: ${newListing.link}`;
17+
18+
const headers = {
19+
Title: newListing.title,
20+
Priority: String(priority),
21+
Tags: `${serviceName},${jobName}`,
22+
Click: newListing.link,
23+
};
24+
25+
if (newListing.image && typeof newListing.image === 'string') {
26+
headers.Attach = normalizeImageUrl(newListing.image);
27+
}
28+
29+
return fetch(`${server}/${topic}`, {
1630
method: 'POST',
17-
body: JSON.stringify({
18-
topic: topic,
19-
message: message,
20-
title: newListing.title,
21-
tags: [serviceName, jobName],
22-
priority: parseInt(priority),
23-
click: newListing.link,
24-
}),
31+
headers,
32+
body: message,
2533
});
2634
});
2735

2836
return Promise.all(promises);
2937
};
38+
3039
export const config = {
3140
id: 'ntfy',
3241
name: 'ntfy',

‎lib/notification/adapter/pushover.js‎

Lines changed: 42 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,50 +2,55 @@ import { markdown2Html } from '../../services/markdown.js';
22
import { getJob } from '../../services/storage/jobStorage.js';
33
import fetch from 'node-fetch';
44

5-
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
5+
export const send = async ({ serviceName, newListings, notificationConfig, jobKey }) => {
66
const { token, user, device } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
77
const job = getJob(jobKey);
88
const jobName = job == null ? jobKey : job.name;
9-
const promises = newListings.map((newListing) => {
10-
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
11-
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
12-
return fetch('https://api.pushover.net/1/messages.json', {
13-
method: 'POST',
14-
headers: { 'Content-Type': 'application/json' },
15-
body: JSON.stringify({
16-
token: token,
17-
user: user,
18-
message: message,
19-
device: device,
20-
title: title,
21-
}),
22-
});
23-
});
249

25-
return Promise.all(promises)
26-
.then((responses) => {
27-
// Convert all responses to JSON
28-
return Promise.all(responses.map((response) => response.json()));
29-
})
30-
.then((data) => {
31-
// Check for errors in the data
32-
const error = data
33-
.map((item) => (item.errors != null && item.errors.length > 0 ? item.errors.join(', ') : null))
34-
.filter((err) => err !== null);
10+
const results = await Promise.all(
11+
newListings.map(async (newListing) => {
12+
const title = `${jobName} at ${serviceName}: ${newListing.title}`;
13+
const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`;
3514

36-
if (error.length > 0) {
37-
// Reject with the combined error messages
38-
return Promise.reject(error.join('; '));
15+
const form = new FormData();
16+
form.append('token', token);
17+
form.append('user', user);
18+
form.append('title', title);
19+
form.append('message', message);
20+
if (device) form.append('device', device);
21+
22+
// Try to attach image if available
23+
if (newListing.image && typeof newListing.image === 'string') {
24+
try {
25+
const imgRes = await fetch(newListing.image);
26+
if (imgRes.ok) {
27+
const ab = await imgRes.arrayBuffer();
28+
form.append('attachment', new Blob([ab]), 'image.jpg');
29+
}
30+
} catch {
31+
// fail silently, just skip the image
32+
}
3933
}
4034

41-
return data;
42-
})
43-
.then(() => {
44-
return Promise.resolve();
45-
})
46-
.catch((error) => {
47-
return Promise.reject(error);
48-
});
35+
const res = await fetch('https://api.pushover.net/1/messages.json', {
36+
method: 'POST',
37+
body: form,
38+
});
39+
40+
return res.json();
41+
}),
42+
);
43+
44+
// Collect errors
45+
const errors = results
46+
.map((r) => (r.errors && r.errors.length > 0 ? r.errors.join(', ') : null))
47+
.filter((e) => e !== null);
48+
49+
if (errors.length > 0) {
50+
return Promise.reject(errors.join('; '));
51+
}
52+
53+
return results;
4954
};
5055

5156
export const config = {

‎lib/notification/adapter/sendGrid.js‎

Lines changed: 34 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,53 @@
11
import sgMail from '@sendgrid/mail';
22
import { markdown2Html } from '../../services/markdown.js';
3+
import { normalizeImageUrl } from '../../utils.js';
4+
5+
const mapListings = (serviceName, jobKey, listings) =>
6+
listings.map((l) => {
7+
const image = normalizeImageUrl(l.image);
8+
return {
9+
title: l.title || '',
10+
link: l.link || '',
11+
address: l.address || '',
12+
size: l.size || '',
13+
price: l.price || '',
14+
image,
15+
hasImage: Boolean(image),
16+
// optional plain text snippet
17+
snippet: [l.address, l.price, l.size].filter(Boolean).join(' | '),
18+
serviceName,
19+
jobKey,
20+
};
21+
});
22+
323
export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => {
424
const { apiKey, receiver, from, templateId } = notificationConfig.find((adapter) => adapter.id === config.id).fields;
25+
526
sgMail.setApiKey(apiKey);
27+
28+
const to = receiver
29+
.trim()
30+
.split(',')
31+
.map((r) => r.trim())
32+
.filter(Boolean);
33+
34+
const listings = mapListings(serviceName, jobKey, newListings);
35+
636
const msg = {
737
templateId,
8-
to: receiver
9-
.trim()
10-
.split(',')
11-
.map((r) => r.trim()),
38+
to,
1239
from,
1340
subject: `Job ${jobKey} | Service ${serviceName} found ${newListings.length} new listing(s)`,
1441
dynamic_template_data: {
1542
serviceName: `Job: (${jobKey}) | Service: ${serviceName}`,
1643
numberOfListings: newListings.length,
17-
listings: newListings,
44+
listings,
1845
},
1946
};
47+
2048
return sgMail.send(msg);
2149
};
50+
2251
export const config = {
2352
id: 'sendgrid',
2453
name: 'SendGrid',

0 commit comments

Comments
 (0)