Skip to content

Commit f1b8ff3

Browse files
karlseguinkarlseguin
authored
Merge pull request #58 from lightpanda-io/request_interception
add tests for request interception
2 parents e446421 + 58ea8c6 commit f1b8ff3

File tree

6 files changed

+390
-9
lines changed

6 files changed

+390
-9
lines changed

‎chromedp/ri/main.go‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
// Copyright 2023-2025 Lightpanda (Selecy SAS)
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
package main
15+
16+
import (
17+
"context"
18+
"encoding/base64"
19+
"errors"
20+
"flag"
21+
"fmt"
22+
"io"
23+
"log"
24+
"log/slog"
25+
"os"
26+
"strings"
27+
"time"
28+
29+
"github.com/chromedp/cdproto/cdp"
30+
"github.com/chromedp/cdproto/fetch"
31+
"github.com/chromedp/chromedp"
32+
)
33+
34+
const (
35+
exitOK = 0
36+
exitFail = 1
37+
)
38+
39+
// main starts interruptable context and runs the program.
40+
func main() {
41+
ctx, cancel := context.WithCancel(context.Background())
42+
defer cancel()
43+
44+
err := run(ctx, os.Args, os.Stdout, os.Stderr)
45+
if err != nil {
46+
fmt.Fprintln(os.Stderr, err.Error())
47+
os.Exit(exitFail)
48+
}
49+
50+
os.Exit(exitOK)
51+
}
52+
53+
const (
54+
CdpWSDefault = "ws://127.0.0.1:9222"
55+
)
56+
57+
func run(ctx context.Context, args []string, stdout, stderr io.Writer) error {
58+
// declare runtime flag parameters.
59+
flags := flag.NewFlagSet(args[0], flag.ExitOnError)
60+
flags.SetOutput(stderr)
61+
62+
var (
63+
verbose = flags.Bool("verbose", false, "enable debug log level")
64+
cdpws = flags.String("cdp", env("CDPCLI_WS", CdpWSDefault), "cdp ws to connect")
65+
)
66+
67+
// usage func declaration.
68+
exec := args[0]
69+
flags.Usage = func() {
70+
fmt.Fprintf(stderr, "usage: %s <url>]\n", exec)
71+
fmt.Fprintf(stderr, "chromedp fetch an url and intercept requests.\n")
72+
fmt.Fprintf(stderr, "\nCommand line options:\n")
73+
flags.PrintDefaults()
74+
fmt.Fprintf(stderr, "\nEnvironment vars:\n")
75+
fmt.Fprintf(stderr, "\tCDPCLI_WS\tdefault %s\n", CdpWSDefault)
76+
}
77+
if err := flags.Parse(args[1:]); err != nil {
78+
return err
79+
}
80+
81+
if *verbose {
82+
slog.SetLogLoggerLevel(slog.LevelDebug)
83+
}
84+
85+
args = flags.Args()
86+
if len(args) != 1 {
87+
return errors.New("url is required")
88+
}
89+
url := args[0]
90+
91+
ctx, cancel := chromedp.NewRemoteAllocator(ctx,
92+
*cdpws, chromedp.NoModifyURL,
93+
)
94+
defer cancel()
95+
96+
// build context options
97+
var opts []chromedp.ContextOption
98+
if *verbose {
99+
opts = append(opts, chromedp.WithDebugf(log.Printf))
100+
}
101+
102+
ctx, cancel = chromedp.NewContext(ctx, opts...)
103+
defer cancel()
104+
105+
// ensure the first tab is created
106+
if err := chromedp.Run(ctx); err != nil {
107+
return fmt.Errorf("new tab: %w", err)
108+
}
109+
110+
chromedp.ListenTarget(ctx, func(ev any) {
111+
switch ev := ev.(type) {
112+
case *fetch.EventRequestPaused:
113+
go func() {
114+
url := ev.Request.URL
115+
fmt.Fprintf(os.Stdout, "%s %s\n", ev.RequestID, url)
116+
117+
// alter the response with a new body
118+
if strings.HasSuffix(url, "/reviews.json") {
119+
encoded := base64.StdEncoding.EncodeToString([]byte(`["alter review"]`))
120+
_ = chromedp.Run(ctx,
121+
fetch.FulfillRequest(ev.RequestID, 200).WithBody(encoded),
122+
)
123+
return
124+
}
125+
126+
// by default let the request running.
127+
_ = chromedp.Run(ctx, fetch.ContinueRequest(ev.RequestID))
128+
}()
129+
}
130+
})
131+
132+
if err := chromedp.Run(ctx,
133+
fetch.Enable().WithPatterns(nil),
134+
); err != nil {
135+
log.Fatal(err)
136+
}
137+
138+
err := chromedp.Run(ctx, chromedp.Navigate(url))
139+
if err != nil {
140+
return fmt.Errorf("navigate %s: %w", url, err)
141+
}
142+
143+
var a []*cdp.Node
144+
if err := chromedp.Run(ctx,
145+
chromedp.Nodes(`#product-reviews > div > p`, &a,
146+
chromedp.Populate(1, false,
147+
chromedp.PopulateWait(50*time.Millisecond),
148+
),
149+
),
150+
); err != nil {
151+
return fmt.Errorf("get reviews: %w", err)
152+
}
153+
154+
reviews := make([]string, 0, len(a))
155+
for _, aa := range a {
156+
if len(aa.Children) != 1 {
157+
// should not happen, but it will be catched by the following
158+
// asserts.
159+
continue
160+
}
161+
reviews = append(reviews, aa.Children[0].NodeValue)
162+
}
163+
164+
fmt.Fprintf(os.Stdout, "%v\n", reviews)
165+
166+
if len(reviews) != 1 {
167+
return errors.New("invalid reviews number")
168+
}
169+
if reviews[0] != "alter review" {
170+
return errors.New("invalid reviews title")
171+
}
172+
173+
return nil
174+
}
175+
176+
// env returns the env value corresponding to the key or the default string.
177+
func env(key, dflt string) string {
178+
val, ok := os.LookupEnv(key)
179+
if !ok {
180+
return dflt
181+
}
182+
183+
return val
184+
}

‎playwright/connect.js‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,6 @@ const browserAddress = process.env.BROWSER_ADDRESS ? process.env.BROWSER_ADDRESS
2121
// web serveur url
2222
const baseURL = process.env.BASE_URL ? process.env.BASE_URL : 'http://127.0.0.1:1234';
2323

24-
// measure general time.
25-
const gstart = process.hrtime.bigint();
26-
// store all run durations
27-
let metrics = [];
28-
2924
// Connect to an existing browser
3025
console.log("Connection to browser on " + browserAddress);
3126
const browser = await chromium.connectOverCDP({
@@ -36,7 +31,6 @@ const browser = await chromium.connectOverCDP({
3631
}
3732
});
3833

39-
4034
const context = await browser.newContext({
4135
baseURL: baseURL,
4236
});

‎playwright/request_interception.js‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
// Copyright 2023-2024 Lightpanda (Selecy SAS)
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// Import the Chromium browser into our scraper.
16+
import { chromium } from 'playwright';
17+
18+
// browserAddress
19+
const browserAddress = process.env.BROWSER_ADDRESS ? process.env.BROWSER_ADDRESS : 'ws://127.0.0.1:9222';
20+
21+
// web serveur url
22+
const baseURL = process.env.BASE_URL ? process.env.BASE_URL : 'https://doesnotexist.localhost:9832';
23+
24+
// Connect to an existing browser
25+
console.log("Connection to browser on " + browserAddress);
26+
const browser = await chromium.connectOverCDP({
27+
endpointURL: browserAddress,
28+
logger: {
29+
isEnabled: (name, severity) => true,
30+
log: (name, severity, message, args) => console.log(`${name} ${message}`)
31+
}
32+
});
33+
34+
const context = await browser.newContext({
35+
baseURL: baseURL,
36+
});
37+
38+
const page = await context.newPage();
39+
await page.route('**', async (route, request) => {
40+
const url = request.url();
41+
if (url === 'https://doesnotexist.localhost:9832/nope/') {
42+
return route.continue({
43+
url: "https://httpbin.io/xhr/post",
44+
});
45+
}
46+
if (url === 'https://httpbin.io/post') {
47+
return route.continue({
48+
method: 'POST',
49+
url: 'https://HTTPBIN.io/post',
50+
headers: {'pw-injected': 'great', 'content-type': 'application/x-www-form-urlencoded'},
51+
postData: 'over=9000&tea=keemun',
52+
});
53+
}
54+
55+
console.error("unexpected request: ", url);
56+
return route.abort();
57+
});
58+
await page.goto('/nope/');
59+
60+
await page.waitForSelector('#response', {timeout: 5000});
61+
const response = await page.locator('#response').textContent();
62+
const data = JSON.parse(response);
63+
64+
if (data.url !== 'http://HTTPBIN.io/post') {
65+
console.log(data.url);
66+
throw new Error("Expected URL to be 'http://HTTPBIN.io/post'");
67+
}
68+
69+
if (data.headers['Pw-Injected'] != 'great') {
70+
console.log(data.headers);
71+
throw new Error("Expected 'Pw-Injected: great' header");
72+
}
73+
74+
if (data.headers['Content-Type'] != 'application/x-www-form-urlencoded') {
75+
console.log(data.headers);
76+
throw new Error("Expected 'Content-Type: application/x-www-form-urlencoded' header");
77+
}
78+
79+
if (data.headers['User-Agent'] != 'Lightpanda/1.0') {
80+
console.log(data.headers);
81+
throw new Error("Expected 'User-Agent: Lightpanda/1.0' header");
82+
}
83+
84+
if (Object.keys(data.form).length != 2) {
85+
console.log(data.form);
86+
throw new Error("Expected 2 form field");
87+
}
88+
89+
if (data.form['over'] != '9000') {
90+
console.log(data.form);
91+
throw new Error("Expected form field 'over: 9000'");
92+
}
93+
94+
if (data.form['tea'] != 'keemun') {
95+
console.log(data.form);
96+
throw new Error("Expected form field 'tea: keemun'");
97+
}
98+
99+
await page.close();
100+
await context.close();
101+
102+
// Turn off the browser to clean up after ourselves.
103+
await browser.close();

‎public/campfire-commerce/script.js‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,16 @@
66
const detailsXHR = new XMLHttpRequest();
77
// blocked by https://github.com/lightpanda-io/browsercore/issues/186
88
// detailsXHR.open('GET', 'json/product.json');
9-
detailsXHR.open('GET', document.URL + 'json/product.json');
9+
detailsXHR.open('GET', document.URL + 'json/product.json');
1010
detailsXHR.responseType = 'json';
1111
detailsXHR.onload = function() {
1212
if (this.status === 200) {
1313
updateProductInfo(this.response);
1414
}
1515
};
16+
detailsXHR.onabort = function(err) {
17+
document.getElementById('product-description').innerHTML = 'xhr: aborted';
18+
}
1619
detailsXHR.send();
1720

1821
// use fetch to retrieve reviews.

0 commit comments

Comments
 (0)