Skip to content

Commit 43bff00

Browse files
author
Gary Blankenship
committed
feat(extractors): port lwn, c2_wiki, x_oembed (Batch B)
Ports three more upstream extractors: LWN: matches lwn.net and *.lwn.net (Linux Weekly News); simple semantic HTML site using ArticleText container; handles subscriber-only gates by returning partial content for paywalled articles rather than failing. C2 Wiki: matches c2.com/cgi/wiki and c2.com/wiki/ paths (Ward Cunningham's original Portland Pattern Repository); handles HTML4-era markup; derives article title from the CamelCase URL path segment via word-boundary splitting. CanExtract() gates on the wiki CGI path signature. X OEmbed: matches publish.twitter.com and publish.x.com oEmbed API endpoint hosts; parses JSON oEmbed response for HTML, title, and author fields. URL patterns are disjoint from the twitter.com/x.com entry (which handles main-site HTML); these match only the publish.* API hosts. CanExtract() gates on valid JSON response shape. Registry: inserts three new entries before the Mastodon catch-all (count 16→19); Mastodon catch-all remains the terminal wildcard.
1 parent e17513b commit 43bff00

8 files changed

Lines changed: 1034 additions & 1 deletion

File tree

‎extractors/c2_wiki.go‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package extractors
2+
3+
import (
4+
"regexp"
5+
"strings"
6+
7+
"github.com/PuerkitoBio/goquery"
8+
)
9+
10+
// c2WikiURLRe matches c2.com CGI wiki URLs (both old ?PageName and path-based).
11+
var c2WikiURLRe = regexp.MustCompile(`(?i)c2\.com/(cgi/wiki|wiki/)`)
12+
13+
// c2WikiPageRe extracts the CamelCase page name from the URL query string.
14+
var c2WikiPageRe = regexp.MustCompile(`[?&]([A-Za-z]\w*)`)
15+
16+
// c2WikiCamelCaseRe splits CamelCase titles into words (e.g. "WelcomeVisitors" → "Welcome Visitors").
17+
var c2WikiCamelCaseRe = regexp.MustCompile(`([a-z])([A-Z])`)
18+
19+
// C2WikiExtractor handles C2 (Ward Cunningham's original wiki) page extraction.
20+
//
21+
// Upstream note: the TypeScript extractor uses canExtractAsync() + a JSON API
22+
// fetch from https://c2.com/wiki/remodel/pages/<Title>. The Go framework is
23+
// synchronous, so this extractor performs DOM extraction from the rendered HTML
24+
// that c2.com's CGI serves at the wiki URL.
25+
//
26+
// TypeScript original:
27+
//
28+
// export class C2WikiExtractor extends BaseExtractor {
29+
// canExtract(): boolean { return false; }
30+
// canExtractAsync(): boolean { return this.getPageTitle() !== null; }
31+
// async extractAsync(): Promise<ExtractorResult> { ... JSON API ... }
32+
// }
33+
type C2WikiExtractor struct {
34+
*ExtractorBase
35+
pageTitle string // CamelCase title extracted from URL; empty if not a wiki URL
36+
}
37+
38+
// NewC2WikiExtractor creates a new C2 Wiki extractor.
39+
// The page title is resolved from the URL at construction time to keep
40+
// CanExtract / Extract consistent.
41+
func NewC2WikiExtractor(document *goquery.Document, url string, schemaOrgData any) *C2WikiExtractor {
42+
e := &C2WikiExtractor{
43+
ExtractorBase: NewExtractorBase(document, url, schemaOrgData),
44+
}
45+
e.pageTitle = e.resolvePageTitle(url)
46+
return e
47+
}
48+
49+
// Name returns the extractor identifier.
50+
func (e *C2WikiExtractor) Name() string { return "C2WikiExtractor" }
51+
52+
// CanExtract returns true when the URL matches a C2 wiki page pattern.
53+
func (e *C2WikiExtractor) CanExtract() bool {
54+
return e.pageTitle != ""
55+
}
56+
57+
// Extract returns the structured content for a C2 wiki page.
58+
// Content is extracted from the rendered HTML that c2.com's CGI serves.
59+
func (e *C2WikiExtractor) Extract() *ExtractorResult {
60+
doc := e.GetDocument()
61+
62+
title := c2WikiCamelCaseRe.ReplaceAllString(e.pageTitle, "$1 $2")
63+
64+
// c2.com wiki CGI pages render their body content inside a plain <body>
65+
// that contains <p> tags, <hr> separators, and list elements.
66+
// Extract all body paragraphs and headers, stripping the navigation links.
67+
contentHTML := e.extractBody(doc)
68+
69+
return &ExtractorResult{
70+
Content: contentHTML,
71+
ContentHTML: contentHTML,
72+
Variables: map[string]string{
73+
"title": title,
74+
"site": "C2 Wiki",
75+
},
76+
}
77+
}
78+
79+
// resolvePageTitle parses the CamelCase page name from a C2 wiki URL.
80+
// Returns the page name, or empty string if the URL doesn't match.
81+
func (e *C2WikiExtractor) resolvePageTitle(rawURL string) string {
82+
if !c2WikiURLRe.MatchString(rawURL) {
83+
return ""
84+
}
85+
if m := c2WikiPageRe.FindStringSubmatch(rawURL); m != nil {
86+
return m[1]
87+
}
88+
// Default entry page.
89+
return "WelcomeVisitors"
90+
}
91+
92+
// extractBody pulls readable content from the rendered c2.com CGI HTML.
93+
// Navigation links (single-word lines that are just wiki links) and bare
94+
// horizontal rules are excluded; substantive paragraphs are kept.
95+
func (e *C2WikiExtractor) extractBody(doc *goquery.Document) string {
96+
var parts []string
97+
98+
doc.Find("body").Children().Each(func(_ int, sel *goquery.Selection) {
99+
tag := goquery.NodeName(sel)
100+
switch tag {
101+
case "form", "script", "style":
102+
return
103+
case "hr":
104+
parts = append(parts, "<hr>")
105+
default:
106+
h, err := goquery.OuterHtml(sel)
107+
if err == nil && strings.TrimSpace(h) != "" {
108+
parts = append(parts, h)
109+
}
110+
}
111+
})
112+
113+
return strings.Join(parts, "\n")
114+
}

‎extractors/c2_wiki_test.go‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package extractors
2+
3+
import (
4+
"strings"
5+
"testing"
6+
7+
"github.com/PuerkitoBio/goquery"
8+
"github.com/stretchr/testify/assert"
9+
"github.com/stretchr/testify/require"
10+
)
11+
12+
// C2 Wiki DOM fixtures.
13+
const c2WikiPageHTML = `<html>
14+
<body>
15+
<p>This wiki page discusses <a href="/cgi/wiki?WikiWikiWeb">WikiWikiWeb</a>.</p>
16+
<p>It contains collaborative notes and patterns.</p>
17+
<hr>
18+
<p>Edit this page if you have something to add.</p>
19+
</body>
20+
</html>`
21+
22+
const c2WikiMinimalHTML = `<html><body><p>Welcome.</p></body></html>`
23+
24+
func parseC2WikiDoc(t *testing.T, rawHTML string) *goquery.Document {
25+
t.Helper()
26+
doc, err := goquery.NewDocumentFromReader(strings.NewReader(rawHTML))
27+
require.NoError(t, err)
28+
return doc
29+
}
30+
31+
func TestC2WikiExtractor_CanExtract(t *testing.T) {
32+
t.Parallel()
33+
34+
tests := []struct {
35+
name string
36+
url string
37+
wantCan bool
38+
}{
39+
{"cgi wiki URL with page", "https://c2.com/cgi/wiki?WelcomeVisitors", true},
40+
{"cgi wiki URL no param (default page)", "https://c2.com/cgi/wiki", true},
41+
{"wiki/ path URL", "https://c2.com/wiki/WelcomeVisitors", true},
42+
{"non-wiki c2 URL", "https://c2.com/about", false},
43+
{"unrelated domain", "https://example.com/cgi/wiki?Foo", false},
44+
}
45+
46+
for _, tc := range tests {
47+
t.Run(tc.name, func(t *testing.T) {
48+
t.Parallel()
49+
doc := parseC2WikiDoc(t, c2WikiMinimalHTML)
50+
ext := NewC2WikiExtractor(doc, tc.url, nil)
51+
assert.Equal(t, tc.wantCan, ext.CanExtract())
52+
})
53+
}
54+
}
55+
56+
func TestC2WikiExtractor_Extract_Metadata(t *testing.T) {
57+
t.Parallel()
58+
59+
tests := []struct {
60+
name string
61+
url string
62+
wantTitle string
63+
}{
64+
{
65+
name: "CamelCase page name split into words",
66+
url: "https://c2.com/cgi/wiki?WelcomeVisitors",
67+
wantTitle: "Welcome Visitors",
68+
},
69+
{
70+
name: "single-word page name unchanged",
71+
url: "https://c2.com/cgi/wiki?Refactoring",
72+
wantTitle: "Refactoring",
73+
},
74+
{
75+
name: "default page when no param",
76+
url: "https://c2.com/cgi/wiki",
77+
wantTitle: "Welcome Visitors",
78+
},
79+
}
80+
81+
for _, tc := range tests {
82+
t.Run(tc.name, func(t *testing.T) {
83+
t.Parallel()
84+
doc := parseC2WikiDoc(t, c2WikiPageHTML)
85+
ext := NewC2WikiExtractor(doc, tc.url, nil)
86+
require.True(t, ext.CanExtract())
87+
88+
result := ext.Extract()
89+
require.NotNil(t, result)
90+
91+
assert.Equal(t, tc.wantTitle, result.Variables["title"])
92+
assert.Equal(t, "C2 Wiki", result.Variables["site"])
93+
})
94+
}
95+
}
96+
97+
func TestC2WikiExtractor_Extract_Content(t *testing.T) {
98+
t.Parallel()
99+
100+
doc := parseC2WikiDoc(t, c2WikiPageHTML)
101+
ext := NewC2WikiExtractor(doc, "https://c2.com/cgi/wiki?WikiWikiWeb", nil)
102+
require.True(t, ext.CanExtract())
103+
104+
result := ext.Extract()
105+
require.NotNil(t, result)
106+
107+
assert.Contains(t, result.ContentHTML, "collaborative notes")
108+
assert.Contains(t, result.ContentHTML, "Edit this page")
109+
}
110+
111+
func TestC2WikiExtractor_Name(t *testing.T) {
112+
t.Parallel()
113+
doc := parseC2WikiDoc(t, c2WikiMinimalHTML)
114+
ext := NewC2WikiExtractor(doc, "https://c2.com/cgi/wiki?Foo", nil)
115+
assert.Equal(t, "C2WikiExtractor", ext.Name())
116+
}

0 commit comments

Comments
 (0)