Merge pull request #160 from DomainTools/IDEV-2061-implement-noh

IDEV-2061: Implement noh

Author: jbabac

domaintools/api.py

@@ -1096,3 +1096,20 @@ def domaindiscovery(self, **kwargs) -> FeedsResults:
             cls=FeedsResults,
             **kwargs,
         )
+
+    def noh(self, **kwargs) -> FeedsResults:
+        """Returns back list of the newly observed hostnames feed"""
+        validate_feeds_parameters(kwargs)
+        endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
+        source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
+        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+            # headers param is allowed only in Feed API and CSV format
+            kwargs.pop("headers", None)
+
+        return self._results(
+            f"newly-observed-hosts-feed-({source})",
+            f"v1/{endpoint}/noh/",
+            response_path=(),
+            cls=FeedsResults,
+            **kwargs,
+        )

domaintools/cli/commands/feeds.py

@@ -314,3 +314,83 @@ def feeds_domaindiscovery(
     ),
 ):
     DTCLICommand.run(name=c.FEEDS_DOMAINDISCOVERY, params=ctx.params)
+
+
+@dt_cli.command(
+    name=c.FEEDS_NOH,
+    help=get_cli_helptext_by_name(command_name=c.FEEDS_NOH),
+)
+def feeds_noh(
+    ctx: typer.Context,
+    user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
+    key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
+    creds_file: str = typer.Option(
+        "~/.dtapi",
+        "-c",
+        "--credfile",
+        help="Optional file with API username and API key, one per line.",
+    ),
+    no_verify_ssl: bool = typer.Option(
+        False,
+        "--no-verify-ssl",
+        help="Skip verification of SSL certificate when making HTTPs API calls",
+    ),
+    no_sign_api_key: bool = typer.Option(
+        False,
+        "--no-sign-api-key",
+        help="Skip signing of api key",
+    ),
+    header_authentication: bool = typer.Option(
+        True,
+        "--no-header-auth",
+        help="Don't use header authentication",
+    ),
+    output_format: str = typer.Option(
+        "jsonl",
+        "-f",
+        "--format",
+        help=f"Output format in [{OutputFormat.JSONL.value}, {OutputFormat.CSV.value}]",
+        callback=DTCLICommand.validate_feeds_format_input,
+    ),
+    endpoint: str = typer.Option(
+        Endpoint.FEED.value,
+        "-e",
+        "--endpoint",
+        help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]",
+        callback=DTCLICommand.validate_endpoint_input,
+    ),
+    sessionID: str = typer.Option(
+        None,
+        "--session-id",
+        help="Unique identifier for the session",
+    ),
+    after: str = typer.Option(
+        None,
+        "--after",
+        help="Start of the time window, relative to the current time in seconds, for which data will be provided",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    before: str = typer.Option(
+        None,
+        "--before",
+        help="The end of the query window in seconds, relative to the current time, inclusive",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    domain: str = typer.Option(
+        None,
+        "-d",
+        "--domain",
+        help="A string value used to filter feed results",
+    ),
+    headers: bool = typer.Option(
+        False,
+        "--headers",
+        help="Adds a header to the first line of response when text/csv is set in header parameters",
+    ),
+    top: str = typer.Option(
+        None,
+        "--top",
+        help="Number of results to return in the response payload. This is ignored in download endpoint",
+    ),
+):
+    DTCLICommand.run(name=c.FEEDS_NOH, params=ctx.params)

domaintools/cli/constants.py

@@ -42,5 +42,6 @@
 # feeds
 FEEDS_NAD = "nad"
 FEEDS_NOD = "nod"
+FEEDS_NOH = "noh"
 FEEDS_DOMAINRDAP = "domainrdap"
 FEEDS_DOMAINDISCOVERY = "domaindiscovery"

domaintools/cli/utils.py

@@ -63,6 +63,7 @@ def _iris_investigate_helptext():
     c.IRIS_DETECT_IGNORED_DOMAINS: "Returns back a list of ignored domains in Iris Detect based on the provided filters.",
     c.FEEDS_NAD: "Returns back newly active domains feed.",
     c.FEEDS_NOD: "Returns back newly observed domains feed.",
+    c.FEEDS_NOH: "Returns back newly observed hosts feed.",
     c.FEEDS_DOMAINRDAP: "Returns changes to global domain registration information, populated by the Registration Data Access Protocol (RDAP).",
     c.FEEDS_DOMAINDISCOVERY: "Returns new domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties.",
 }

domaintools/constants.py

@@ -28,6 +28,8 @@ class OutputFormat(Enum):
     "newly-active-domains-feed-(s3)",
     "newly-observed-domains-feed-(api)",
     "newly-observed-domains-feed-(s3)",
+    "newly-observed-hosts-feed-(api)",
+    "newly-observed-hosts-feed-(s3)",
     "domain-registration-data-access-protocol-feed-(api)",
     "domain-registration-data-access-protocol-feed-(s3)",
     "real-time-domain-discovery-feed-(api)",

requirements/common.txt

@@ -1 +1 @@
-httpx==0.22.0
+httpx