Skip to content

Cleanup RegisteredDomainProcessorTests #124118

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Mar 5, 2025
Merged

Cleanup RegisteredDomainProcessorTests #124118

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
258ac6b
Tidy up this comment
joegallo Mar 5, 2025
94e8ada
Conserve precious characters
joegallo Mar 5, 2025
8fa2f88
There's not really an output document
joegallo Mar 5, 2025
5017c7f
Simplify these tests
joegallo Mar 5, 2025
3579dfb
Add another test
joegallo Mar 5, 2025
cd41af1
Remove unused five arg arity
joegallo Mar 5, 2025
0c4bc9b
Drop the buildEvent helper
joegallo Mar 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 75 additions & 77 deletions ...-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,52 +13,45 @@
import org.elasticsearch.ingest.TestIngestDocument;
import org.elasticsearch.test.ESTestCase;

import java.util.Collections;
import java.util.Map;

import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static java.util.Map.entry;
import static org.hamcrest.Matchers.anEmptyMap;
import static org.hamcrest.Matchers.is;

/**
* Test parsing of an eTLD from a FQDN. The list of eTLDs is maintained here:
* https://github.com/publicsuffix/list/blob/master/public_suffix_list.dat
*
* Effective TLDs (eTLS) are not the same as DNS TLDs. Uses for eTLDs are listed here.
* <p>
* Effective TLDs (eTLDs) are not the same as DNS TLDs. Uses for eTLDs are listed here:
* https://publicsuffix.org/learn/
*/
public class RegisteredDomainProcessorTests extends ESTestCase {
private Map<String, Object> buildEvent(String domain) {
return Map.of("domain", domain);
}

public void testBasic() throws Exception {
testRegisteredDomainProcessor(buildEvent("www.google.com"), "www.google.com", "google.com", "com", "www");
testRegisteredDomainProcessor(buildEvent("google.com"), "google.com", "google.com", "com", null);
testRegisteredDomainProcessor(buildEvent(""), null, null, null, null);
testRegisteredDomainProcessor(buildEvent("."), null, null, null, null);
testRegisteredDomainProcessor(buildEvent("$"), null, null, null, null);
testRegisteredDomainProcessor(buildEvent("foo.bar.baz"), null, null, null, null);
testRegisteredDomainProcessor(buildEvent("www.books.amazon.co.uk"), "www.books.amazon.co.uk", "amazon.co.uk", "co.uk", "www.books");
testRegisteredDomainProcessor("www.google.com", "www.google.com", "google.com", "com", "www");
testRegisteredDomainProcessor("google.com", "google.com", "google.com", "com", null);
testRegisteredDomainProcessor("", null, null, null, null);
testRegisteredDomainProcessor(".", null, null, null, null);
testRegisteredDomainProcessor("$", null, null, null, null);
testRegisteredDomainProcessor("foo.bar.baz", null, null, null, null);
testRegisteredDomainProcessor("www.books.amazon.co.uk", "www.books.amazon.co.uk", "amazon.co.uk", "co.uk", "www.books");
// Verify "com" is returned as the eTLD, for that FQDN or subdomain
testRegisteredDomainProcessor(buildEvent("com"), "com", null, "com", null);
testRegisteredDomainProcessor(buildEvent("example.com"), "example.com", "example.com", "com", null);
testRegisteredDomainProcessor(buildEvent("googleapis.com"), "googleapis.com", "googleapis.com", "com", null);
testRegisteredDomainProcessor("com", "com", null, "com", null);
testRegisteredDomainProcessor("example.com", "example.com", "example.com", "com", null);
testRegisteredDomainProcessor("googleapis.com", "googleapis.com", "googleapis.com", "com", null);
testRegisteredDomainProcessor(
buildEvent("content-autofill.googleapis.com"),
"content-autofill.googleapis.com",
"content-autofill.googleapis.com",
"googleapis.com",
"com",
"content-autofill"
);
// Verify "ssl.fastly.net" is returned as the eTLD, for that FQDN or subdomain
testRegisteredDomainProcessor("global.ssl.fastly.net", "global.ssl.fastly.net", "global.ssl.fastly.net", "ssl.fastly.net", null);
testRegisteredDomainProcessor(
buildEvent("global.ssl.fastly.net"),
"global.ssl.fastly.net",
"global.ssl.fastly.net",
"ssl.fastly.net",
null
);
testRegisteredDomainProcessor(
buildEvent("1.www.global.ssl.fastly.net"),
"1.www.global.ssl.fastly.net",
"1.www.global.ssl.fastly.net",
"global.ssl.fastly.net",
"ssl.fastly.net",
Expand All @@ -67,76 +60,81 @@ public void testBasic() throws Exception {
}

public void testUseRoot() throws Exception {
Map<String, Object> source = buildEvent("www.google.co.uk");

String domainField = "domain";
String registeredDomainField = "registered_domain";
String topLevelDomainField = "top_level_domain";
String subdomainField = "subdomain";

var processor = new RegisteredDomainProcessor(null, null, "domain", "", false);

IngestDocument input = TestIngestDocument.withDefaultVersion(source);
IngestDocument output = processor.execute(input);

String domain = output.getFieldValue(domainField, String.class);
assertThat(domain, equalTo("www.google.co.uk"));
String registeredDomain = output.getFieldValue(registeredDomainField, String.class);
assertThat(registeredDomain, equalTo("google.co.uk"));
String eTLD = output.getFieldValue(topLevelDomainField, String.class);
assertThat(eTLD, equalTo("co.uk"));
String subdomain = output.getFieldValue(subdomainField, String.class);
assertThat(subdomain, equalTo("www"));
IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of("domain", "www.google.co.uk"));
processor.execute(document);
assertThat(
document.getSource(),
is(
Map.ofEntries(
entry("domain", "www.google.co.uk"),
entry("registered_domain", "google.co.uk"),
entry("top_level_domain", "co.uk"),
entry("subdomain", "www")
)
)
);
}

public void testError() throws Exception {
IllegalArgumentException e = expectThrows(
IllegalArgumentException.class,
() -> testRegisteredDomainProcessor(buildEvent("foo.bar.baz"), null, null, null, null, false)
);
assertThat(e.getMessage(), containsString("unable to set domain information for document"));
e = expectThrows(
IllegalArgumentException.class,
() -> testRegisteredDomainProcessor(buildEvent("$"), null, null, null, null, false)
);
assertThat(e.getMessage(), containsString("unable to set domain information for document"));
var processor = new RegisteredDomainProcessor(null, null, "domain", "", false);

{
IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of("domain", "foo.bar.baz"));
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(document));
assertThat(e.getMessage(), is("unable to set domain information for document"));
assertThat(document.getSource(), is(Map.of("domain", "foo.bar.baz")));
}

{
IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of("domain", "$"));
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(document));
assertThat(e.getMessage(), is("unable to set domain information for document"));
assertThat(document.getSource(), is(Map.of("domain", "$")));
}
}

private void testRegisteredDomainProcessor(
Map<String, Object> source,
String expectedDomain,
String expectedRegisteredDomain,
String expectedETLD,
String expectedSubdomain
) throws Exception {
testRegisteredDomainProcessor(source, expectedDomain, expectedRegisteredDomain, expectedETLD, expectedSubdomain, true);
public void testIgnoreMissing() throws Exception {
{
var processor = new RegisteredDomainProcessor(null, null, "domain", "", false);
IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of());
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(document));
assertThat(e.getMessage(), is("field [domain] not present as part of path [domain]"));
assertThat(document.getSource(), is(anEmptyMap()));
}

{
var processor = new RegisteredDomainProcessor(null, null, "domain", "", true);
IngestDocument document = TestIngestDocument.withDefaultVersion(Collections.singletonMap("domain", null));
processor.execute(document);
assertThat(document.getSource(), is(Collections.singletonMap("domain", null)));
}
}

private void testRegisteredDomainProcessor(
Map<String, Object> source,
String fqdn,
String expectedDomain,
String expectedRegisteredDomain,
String expectedETLD,
String expectedSubdomain,
boolean ignoreMissing
String expectedSubdomain
) throws Exception {
String domainField = "url.domain";
String registeredDomainField = "url.registered_domain";
String topLevelDomainField = "url.top_level_domain";
String subdomainField = "url.subdomain";

var processor = new RegisteredDomainProcessor(null, null, "domain", "url", ignoreMissing);
var processor = new RegisteredDomainProcessor(null, null, "domain", "url", true);

IngestDocument input = TestIngestDocument.withDefaultVersion(source);
IngestDocument output = processor.execute(input);
IngestDocument document = TestIngestDocument.withDefaultVersion(Map.of("domain", fqdn));
processor.execute(document);

String domain = output.getFieldValue(domainField, String.class, expectedDomain == null);
assertThat(domain, equalTo(expectedDomain));
String registeredDomain = output.getFieldValue(registeredDomainField, String.class, expectedRegisteredDomain == null);
assertThat(registeredDomain, equalTo(expectedRegisteredDomain));
String eTLD = output.getFieldValue(topLevelDomainField, String.class, expectedETLD == null);
assertThat(eTLD, equalTo(expectedETLD));
String subdomain = output.getFieldValue(subdomainField, String.class, expectedSubdomain == null);
assertThat(subdomain, equalTo(expectedSubdomain));
String domain = document.getFieldValue(domainField, String.class, expectedDomain == null);
assertThat(domain, is(expectedDomain));
String registeredDomain = document.getFieldValue(registeredDomainField, String.class, expectedRegisteredDomain == null);
assertThat(registeredDomain, is(expectedRegisteredDomain));
String eTLD = document.getFieldValue(topLevelDomainField, String.class, expectedETLD == null);
assertThat(eTLD, is(expectedETLD));
String subdomain = document.getFieldValue(subdomainField, String.class, expectedSubdomain == null);
assertThat(subdomain, is(expectedSubdomain));
}
}