Add SDK documentation on README + normalize (sub)domains accidentally reported as URL

.mocharc.json

@@ -1,5 +1,8 @@
 {
     "$schema": "https://json.schemastore.org/mocharc.json",
     "require": "tsx",
-    "spec": "test/**/*.ts"
+    "spec": [
+        "test/**/*.ts",
+        "src/**/*.test.ts"
+    ]
 }

README.md

@@ -1,21 +1,69 @@
 # SEAL Intel SDK
 
+SDK for interacting with SEAL Intel to block or trust web content (domains, IPs, URLs). 
+
 ## Usage
 
 ### Setup
 
 ```typescript
+import { OpenCTIClient } from "@security-alliance/opencti-client";
+import { generateIdentityId } from "@security-alliance/opencti-client/stix";
 import { WebContentClient } from "@security-alliance/seal-intel-sdk";
 
-// initialize the SDK with your API key
-const client = new WebContentClient("https://sealisac.org", "your-api-key");
+// Create an OpenCTI client
+const opencti = new OpenCTIClient("https://sealisac.org", "your-api-key");
+
+// Generate your organization's identity ID (you create this, it's not provided by SEAL)
+const YOUR_IDENTITY_ID = generateIdentityId({
+  name: "Your Organization Name",
+  identity_class: "organization",
+});
+
+// Initialize the SDK
+const client = new WebContentClient(opencti, YOUR_IDENTITY_ID);
 ```
 
-### Functions
+### Block or Trust Content
 
 ```typescript
-client.blockWebContent(content: WebContent, creator: Identifier<'identity'>): Promise<Indicator>;
-client.unblockWebContent(content: WebContent): Promise<Indicator | undefined>;
-client.trustWebContent(content: WebContent, creator: Identifier<'identity'>): Promise<Observable>;
-client.untrustWebContent(content: WebContent): Promise<Observable | undefined>;
+// Block a domain
+await client.blockWebContent({ type: "domain-name", value: "phishing-site.com" });
+
+// Block a subdomain
+await client.blockWebContent({ type: "domain-name", value: "scam.github.io" });
+
+// Trust a domain
+await client.trustWebContent({ type: "domain-name", value: "example.com" });
+
+// Check status
+const status = await client.getWebContentStatus({ type: "domain-name", value: "phishing-site.com" });
+// Returns: { status: "blocked" | "trusted" | "unknown", actor?: Identifier<'identity'> }
+
+// Unblock or untrust
+await client.unblockWebContent({ type: "domain-name", value: "phishing-site.com" });
+await client.untrustWebContent({ type: "domain-name", value: "example.com" });
 ```
+
+### Domain vs URL Blocking
+
+Prefer reporting domains or subdomains. Most platforms do not support URL blocking yet:
+- URL blocking support varies by platform
+- Some platforms only allow URL blocking for whitelisted domains
+- Outcome for reporting URLs is not guaranteed
+
+Domains and subdomains are supported across most platforms.
+
+**When reporting:**
+- Report the subdomain when only it is malicious: `scam.github.io`, `phishing.medium.com`
+- Report the full domain when the entire domain is malicious: `phishing-site.com`
+- Do not report parent domains when only a subdomain is malicious (e.g., `github.io`, `medium.com`)
+
+### Available Content Types
+
+```typescript
+{ type: "url", value: "https://example.com/path" }
+{ type: "domain-name", value: "example.com" }
+{ type: "ipv4-addr", value: "192.168.1.1" }
+{ type: "ipv6-addr", value: "2001:0db8:85a3:0000:0000:8a2e:0370:7334" }
+```

src/index.ts

@@ -1 +1,2 @@
 export * from "./web-content/client.js";
+export { normalizeWebContent } from "./utils/normalize-web-content.js";

src/utils/normalize-web-content.test.ts

@@ -0,0 +1,83 @@
+import assert from "node:assert";
+import { describe, it } from "node:test";
+import { normalizeWebContent } from "./normalize-web-content.js";
+import { WebContent } from "../web-content/types.js";
+
+describe("normalizeWebContent", () => {
+    it("should keep non-URL types unchanged", () => {
+        const cases: WebContent[] = [
+            { type: "domain-name", value: "example.com" },
+            { type: "domain-name", value: "sub.example.com" },
+            { type: "ipv4-addr", value: "192.168.1.1" },
+            { type: "ipv6-addr", value: "2001:0db8:85a3::8a2e:0370:7334" },
+        ];
+
+        cases.forEach(content => {
+            assert.deepEqual(normalizeWebContent(content), content);
+        });
+    });
+
+    it("should convert http(s) URLs without paths to domain-name", () => {
+        const cases = [
+            { url: "https://example.com", expected: "example.com" },
+            { url: "https://example.com/", expected: "example.com" },
+            { url: "http://example.com", expected: "example.com" },
+            { url: "https://sub.example.com", expected: "sub.example.com" },
+            { url: "https://example.com:8080", expected: "example.com" },
+        ];
+
+        cases.forEach(({ url, expected }) => {
+            const result = normalizeWebContent({ type: "url", value: url });
+            assert.equal(result.type, "domain-name");
+            assert.equal(result.value, expected);
+        });
+    });
+
+    it("should keep URLs with paths unchanged", () => {
+        const urls = [
+            "https://llama.airdrop-defi.io/claim",
+            "http://example.com/path/to/resource",
+            "https://example.com/path?query=value",
+            "https://example.com/path#section",
+            "https://example.com:8080/path",
+        ];
+
+        urls.forEach(url => {
+            const content: WebContent = { type: "url", value: url };
+            assert.deepEqual(normalizeWebContent(content), content);
+        });
+    });
+
+    it("should keep non-http(s) URLs unchanged", () => {
+        const urls = [
+            "ipfs://QmXyz123",
+            "ftp://files.example.com/file.txt",
+            "custom://resource",
+        ];
+
+        urls.forEach(url => {
+            const content: WebContent = { type: "url", value: url };
+            assert.deepEqual(normalizeWebContent(content), content);
+        });
+    });
+
+    it("should not normalize non-ICANN hostnames", () => {
+        const urls = [
+            "https://192.168.1.1/path",  // IP address
+            "https://localhost/path",     // localhost
+            "https://test.invalid/path",  // special use TLD
+        ];
+
+        urls.forEach(url => {
+            const content: WebContent = { type: "url", value: url };
+            assert.deepEqual(normalizeWebContent(content), content);
+        });
+    });
+
+    it("should preserve subdomains when normalizing", () => {
+        const result = normalizeWebContent({ type: "url", value: "https://phishing.github.io" });
+        assert.equal(result.type, "domain-name");
+        assert.equal(result.value, "phishing.github.io");
+    });
+});
+

src/utils/normalize-web-content.ts

@@ -0,0 +1,48 @@
+import { parse } from "tldts";
+import { WebContent } from "../web-content/types.js";
+
+/**
+ * Normalizes web content to handle cases where a domain is accidentally reported as a URL.
+ * Only converts http/https URLs to domain-name when the URL has no path (empty or just "/").
+ * URLs with actual paths are kept as-is since the user likely wants to block that specific URL.
+ * Preserves the full hostname including subdomains to avoid blocking legitimate parent domains.
+ * 
+ * Examples:
+ * - "https://example.com" -> { type: "domain-name", value: "example.com" }
+ * - "https://example.com/" -> { type: "domain-name", value: "example.com" }
+ * - "https://phishing.github.io" -> { type: "domain-name", value: "phishing.github.io" } (keeps subdomain)
+ * - "https://example.com/path" -> { type: "url", value: "https://example.com/path" } (kept as URL)
+ * - "https://llama.airdrop-defi.io/claim" -> { type: "url", value: "https://llama.airdrop-defi.io/claim" } (kept as URL)
+ * - "ipfs://xyz" -> { type: "url", value: "ipfs://xyz" } (kept as URL)
+ */
+export const normalizeWebContent = (content: WebContent): WebContent => {
+    if (content.type !== "url") {
+        return content;
+    }
+
+    try {
+        const url = new URL(content.value);
+
+        if (url.protocol === "http:" || url.protocol === "https:") {
+            // Only normalize if path is empty or just "/"
+            const hasPath = url.pathname !== "" && url.pathname !== "/";
+
+            if (!hasPath) {
+                const hostname = url.hostname;
+                const parsed = parse(hostname);
+
+                // isIcann = true for ICANN-managed public domains (.com, .org, .io, etc.)
+                // Excludes special use TLDs (.local, .invalid), localhost, and IP addresses
+                // Use hostname (not domain) to preserve subdomains and avoid blocking legitimate parent domains
+                if (parsed.isIcann && parsed.hostname) {
+                    return { type: "domain-name", value: parsed.hostname };
+                }
+            }
+        }
+    } catch {
+        // If URL parsing fails, return as-is
+    }
+
+    return content;
+};
+

src/web-content/client.ts

@@ -7,6 +7,7 @@ import {
 } from "@security-alliance/opencti-client";
 import { generateIndicatorId, generateLabelId, MARKING_TLP_CLEAR } from "@security-alliance/opencti-client/stix";
 import { Identifier } from "@security-alliance/stix/2.1";
+import { normalizeWebContent } from "../utils/normalize-web-content.js";
 import {
     ALLOWLISTED_DOMAIN_LABEL,
     BLOCKLISTED_DOMAIN_LABEL,
@@ -192,6 +193,8 @@ export class WebContentClient {
     }
 
     public async getWebContentStatus(content: WebContent): Promise<WebContentStatus> {
+        content = normalizeWebContent(content);
+
         const [observable, indicator] = await Promise.all([
             this.client.stixCyberObservable({ id: generateObservableIdForWebContent(content) }),
             this.client.indicator({ id: generateIndicatorId({ pattern: generatePatternForWebContent(content) }) }),
@@ -216,6 +219,7 @@ export class WebContentClient {
     }
 
     public async blockWebContent(content: WebContent, creator?: Identifier<"identity">): Promise<Indicator_All> {
+        content = normalizeWebContent(content);
         creator ??= this.defaultIdentity;
 
         const observable = await this.createOrUpdateObservable(content, {
@@ -238,6 +242,7 @@ export class WebContentClient {
         content: WebContent,
         creator?: Identifier<"identity">,
     ): Promise<Indicator_All | undefined> {
+        content = normalizeWebContent(content);
         creator ??= this.defaultIdentity;
 
         const observable = await this.client.stixCyberObservable({ id: generateObservableIdForWebContent(content) });
@@ -267,6 +272,7 @@ export class WebContentClient {
         content: WebContent,
         creator?: Identifier<"identity">,
     ): Promise<StixCyberObservable_All> {
+        content = normalizeWebContent(content);
         creator ??= this.defaultIdentity;
 
         await this.unblockWebContent(content, creator);
@@ -282,6 +288,7 @@ export class WebContentClient {
         content: WebContent,
         creator?: Identifier<"identity">,
     ): Promise<StixCyberObservable_All | undefined> {
+        content = normalizeWebContent(content);
         creator ??= this.defaultIdentity;
 
         const observable = await this.client.stixCyberObservable({ id: generateObservableIdForWebContent(content) });