Skip to content

Commit 763a9b1

Browse files
jbabacjbabac
authored
Merge pull request #145 from DomainTools/IDEV-2011-add-support-for-passing-api-key-via-header
IDEV-2011: add support for passing api key via header
2 parents e3e76ac + d263f96 commit 763a9b1

File tree

10 files changed

+503
-75
lines changed

10 files changed

+503
-75
lines changed

‎README.md‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,5 +211,28 @@ API_KEY
211211
Python Version Support Policy
212212
===================
213213

214-
Please see the [supported versions](https://github.com/DomainTools/python_api/raw/main/PYTHON_SUPPORT.md) document
214+
Please see the [supported versions](https://github.com/DomainTools/python_api/raw/main/PYTHON_SUPPORT.md) document
215215
for the DomainTools Python support policy.
216+
217+
218+
Real-Time Threat Intelligence Feeds
219+
===================
220+
221+
Real-Time Threat Intelligence Feeds provide data on the different stages of the domain lifecycle: from first-observed in the wild, to newly re-activated after a period of quiet. Access current feed data in real-time or retrieve historical feed data through separate APIs.
222+
223+
Custom parameters aside from the common `GET` Request parameters:
224+
- `endpoint` (choose either `download` or `feed` API endpoint - default is `feed`)
225+
```python
226+
api = API(USERNAME, KEY)
227+
api.nod(endpoint="feed", **kwargs)
228+
```
229+
- `header_authentication`: by default, we're using API Header Authentication. Set this False if you want to use API Key and Secret Authentication. Apparently, you can't use API Header Authentication for `download` endpoints so you need to set this to `False` when calling `download` API endpoints.
230+
```python
231+
api = API(USERNAME, KEY)
232+
api.nod(header_authentication=False, **kwargs)
233+
```
234+
- `output_format`: (choose either `csv` or `jsonl` - default is `jsonl`). Cannot be used in `domainrdap` feeds. Additionally, `csv` is not available for `download` endpoints.
235+
```python
236+
api = API(USERNAME, KEY)
237+
api.nod(output_format="csv", **kwargs)
238+
```

‎domaintools/api.py‎

Lines changed: 28 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from hmac import new as hmac
44
import re
55

6-
from domaintools.constants import Endpoint, ENDPOINT_TO_SOURCE_MAP, OutputFormat
6+
from domaintools.constants import Endpoint, ENDPOINT_TO_SOURCE_MAP, FEEDS_PRODUCTS_LIST, OutputFormat
77
from domaintools._version import current as version
88
from domaintools.results import (
99
GroupedIterable,
@@ -125,14 +125,18 @@ def _results(self, product, path, cls=Results, **kwargs):
125125
uri = "/".join((self._rest_api_url, path.lstrip("/")))
126126
parameters = self.default_parameters.copy()
127127
parameters["api_username"] = self.username
128-
self.handle_api_key(path, parameters)
128+
header_authentication = kwargs.pop("header_authentication", True) # Used only by Real-Time Threat Intelligence Feeds endpoints for now
129+
self.handle_api_key(product, path, parameters, header_authentication)
129130
parameters.update({key: str(value).lower() if value in (True, False) else value for key, value in kwargs.items() if value is not None})
130131

131132
return cls(self, product, uri, **parameters)
132133

133-
def handle_api_key(self, path, parameters):
134+
def handle_api_key(self, product, path, parameters, header_authentication):
134135
if self.https and not self.always_sign_api_key:
135-
parameters["api_key"] = self.key
136+
if product in FEEDS_PRODUCTS_LIST and header_authentication:
137+
parameters["X-Api-Key"] = self.key
138+
else:
139+
parameters["api_key"] = self.key
136140
else:
137141
if self.key_sign_hash and self.key_sign_hash in AVAILABLE_KEY_SIGN_HASHES:
138142
signing_hash = eval(self.key_sign_hash)
@@ -1063,28 +1067,32 @@ def iris_detect_ignored_domains(
10631067

10641068
def nod(self, **kwargs):
10651069
"""Returns back list of the newly observed domains feed"""
1066-
sessionID = kwargs.get("sessionID")
1067-
after = kwargs.get("after")
1068-
if not (sessionID or after):
1069-
raise ValueError("sessionID or after (can be both) must be defined")
1070+
validate_feeds_parameters(kwargs)
1071+
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
1072+
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint)
1073+
if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
1074+
# headers param is allowed only in Feed API and CSV format
1075+
kwargs.pop("headers", None)
10701076

10711077
return self._results(
1072-
"newly-observed-domains-feed-(api)",
1073-
"v1/feed/nod/",
1078+
f"newly-observed-domains-feed-({source.value})",
1079+
f"v1/{endpoint}/nod/",
10741080
response_path=(),
10751081
**kwargs,
10761082
)
10771083

10781084
def nad(self, **kwargs):
10791085
"""Returns back list of the newly active domains feed"""
1080-
sessionID = kwargs.get("sessionID")
1081-
after = kwargs.get("after")
1082-
if not (sessionID or after):
1083-
raise ValueError("sessionID or after (can be both) must be defined")
1086+
validate_feeds_parameters(kwargs)
1087+
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
1088+
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
1089+
if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
1090+
# headers param is allowed only in Feed API and CSV format
1091+
kwargs.pop("headers", None)
10841092

10851093
return self._results(
1086-
"newly-active-domains-feed-(api)",
1087-
"v1/feed/nad/",
1094+
f"newly-active-domains-feed-({source})",
1095+
f"v1/{endpoint}/nad/",
10881096
response_path=(),
10891097
**kwargs,
10901098
)
@@ -1093,10 +1101,10 @@ def domainrdap(self, **kwargs):
10931101
"""Returns changes to global domain registration information, populated by the Registration Data Access Protocol (RDAP)"""
10941102
validate_feeds_parameters(kwargs)
10951103
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
1096-
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint)
1104+
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
10971105

10981106
return self._results(
1099-
f"domain-registration-data-access-protocol-feed-({source.value})",
1107+
f"domain-registration-data-access-protocol-feed-({source})",
11001108
f"v1/{endpoint}/domainrdap/",
11011109
response_path=(),
11021110
**kwargs,
@@ -1106,13 +1114,13 @@ def domaindiscovery(self, **kwargs):
11061114
"""Returns new domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties"""
11071115
validate_feeds_parameters(kwargs)
11081116
endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
1109-
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint)
1117+
source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
11101118
if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
11111119
# headers param is allowed only in Feed API and CSV format
11121120
kwargs.pop("headers", None)
11131121

11141122
return self._results(
1115-
f"real-time-domain-discovery-feed-({source.value})",
1123+
f"real-time-domain-discovery-feed-({source})",
11161124
f"v1/{endpoint}/domaindiscovery/",
11171125
response_path=(),
11181126
**kwargs,

‎domaintools/base_results.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from datetime import datetime
1010
from httpx import Client
1111

12-
from domaintools.constants import OutputFormat, HEADER_ACCEPT_KEY_CSV_FORMAT
12+
from domaintools.constants import FEEDS_PRODUCTS_LIST, OutputFormat, HEADER_ACCEPT_KEY_CSV_FORMAT
1313
from domaintools.exceptions import (
1414
BadRequestException,
1515
InternalServerErrorException,
@@ -20,7 +20,6 @@
2020
IncompleteResponseException,
2121
RequestUriTooLongException,
2222
)
23-
from domaintools.utils import get_feeds_products_list
2423

2524

2625
try: # pragma: no cover
@@ -93,7 +92,7 @@ def _make_request(self):
9392
patch_data = self.kwargs.copy()
9493
patch_data.update(self.api.extra_request_params)
9594
return session.patch(url=self.url, json=patch_data)
96-
elif self.product in get_feeds_products_list():
95+
elif self.product in FEEDS_PRODUCTS_LIST:
9796
parameters = deepcopy(self.kwargs)
9897
parameters.pop("output_format", None)
9998
parameters.pop(
@@ -104,6 +103,10 @@ def _make_request(self):
104103
parameters["headers"] = int(bool(self.kwargs.get("headers", False)))
105104
headers["accept"] = HEADER_ACCEPT_KEY_CSV_FORMAT
106105

106+
header_api_key = parameters.pop("X-Api-Key", None)
107+
if header_api_key:
108+
headers["X-Api-Key"] = header_api_key
109+
107110
return session.get(url=self.url, params=parameters, headers=headers, **self.api.extra_request_params)
108111
else:
109112
return session.get(url=self.url, params=self.kwargs, **self.api.extra_request_params)
@@ -135,8 +138,7 @@ def data(self):
135138
self.setStatus(results.status_code, results)
136139
if (
137140
self.kwargs.get("format", "json") == "json"
138-
and self.product
139-
not in get_feeds_products_list() # Special handling of feeds products' data to preserve the result in jsonline format
141+
and self.product not in FEEDS_PRODUCTS_LIST # Special handling of feeds products' data to preserve the result in jsonline format
140142
):
141143
self._data = results.json()
142144
else:
@@ -153,7 +155,7 @@ def data(self):
153155
return self._data
154156

155157
def check_limit_exceeded(self):
156-
if self.kwargs.get("format", "json") == "json" and self.product not in get_feeds_products_list():
158+
if self.kwargs.get("format", "json") == "json" and self.product not in FEEDS_PRODUCTS_LIST:
157159
if "response" in self._data and "limit_exceeded" in self._data["response"] and self._data["response"]["limit_exceeded"] is True:
158160
return True, self._data["response"]["message"]
159161
# TODO: handle html, xml response errors better.

‎domaintools/cli/api.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,16 @@
88
from typing import Optional, Dict, Tuple
99
from rich.progress import Progress, SpinnerColumn, TextColumn
1010

11-
from domaintools.constants import Endpoint, OutputFormat
1211
from domaintools.api import API
13-
from domaintools.exceptions import ServiceException
12+
from domaintools.constants import Endpoint, OutputFormat
1413
from domaintools.cli.utils import get_file_extension
14+
from domaintools.exceptions import ServiceException
15+
from domaintools._version import current as version
1516

1617

1718
class DTCLICommand:
1819
API_SUCCESS_STATUS = 200
19-
APP_PARTNER_NAME = "python_wrapper_cli_2.0.0"
20+
APP_PARTNER_NAME = f"python_wrapper_cli_{version}"
2021

2122
@staticmethod
2223
def print_api_version(value: bool):

‎domaintools/cli/commands/feeds.py‎

Lines changed: 74 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import sys
21
import typer
32

43

@@ -23,20 +22,6 @@ def feeds_nad(
2322
"--credfile",
2423
help="Optional file with API username and API key, one per line.",
2524
),
26-
rate_limit: bool = typer.Option(
27-
False,
28-
"-l",
29-
"--rate-limit",
30-
help="Rate limit API calls against the API based on per minute limits.",
31-
),
32-
format: str = typer.Option(
33-
"json",
34-
"-f",
35-
"--format",
36-
help="Output format in {'list', 'json', 'xml', 'html'}",
37-
callback=DTCLICommand.validate_format_input,
38-
),
39-
out_file: typer.FileTextWrite = typer.Option(sys.stdout, "-o", "--out-file", help="Output file (defaults to stdout)"),
4025
no_verify_ssl: bool = typer.Option(
4126
False,
4227
"--no-verify-ssl",
@@ -47,6 +32,25 @@ def feeds_nad(
4732
"--no-sign-api-key",
4833
help="Skip signing of api key",
4934
),
35+
header_authentication: bool = typer.Option(
36+
True,
37+
"--no-header-auth",
38+
help="Don't use header authentication",
39+
),
40+
output_format: str = typer.Option(
41+
"jsonl",
42+
"-f",
43+
"--format",
44+
help=f"Output format in [{OutputFormat.JSONL.value}, {OutputFormat.CSV.value}]",
45+
callback=DTCLICommand.validate_feeds_format_input,
46+
),
47+
endpoint: str = typer.Option(
48+
Endpoint.FEED.value,
49+
"-e",
50+
"--endpoint",
51+
help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]",
52+
callback=DTCLICommand.validate_endpoint_input,
53+
),
5054
sessionID: str = typer.Option(
5155
None,
5256
"--session-id",
@@ -56,17 +60,29 @@ def feeds_nad(
5660
None,
5761
"--after",
5862
help="Start of the time window, relative to the current time in seconds, for which data will be provided",
63+
callback=DTCLICommand.validate_after_or_before_input,
64+
),
65+
before: str = typer.Option(
66+
None,
67+
"--before",
68+
help="The end of the query window in seconds, relative to the current time, inclusive",
69+
callback=DTCLICommand.validate_after_or_before_input,
5970
),
6071
domain: str = typer.Option(
6172
None,
6273
"-d",
6374
"--domain",
6475
help="A string value used to filter feed results",
6576
),
77+
headers: bool = typer.Option(
78+
False,
79+
"--headers",
80+
help="Adds a header to the first line of response when text/csv is set in header parameters",
81+
),
6682
top: str = typer.Option(
6783
None,
6884
"--top",
69-
help="Number of results to return in the response payload",
85+
help="Number of results to return in the response payload. This is ignored in download endpoint",
7086
),
7187
):
7288
DTCLICommand.run(name=c.FEEDS_NAD, params=ctx.params)
@@ -86,20 +102,6 @@ def feeds_nod(
86102
"--credfile",
87103
help="Optional file with API username and API key, one per line.",
88104
),
89-
rate_limit: bool = typer.Option(
90-
False,
91-
"-l",
92-
"--rate-limit",
93-
help="Rate limit API calls against the API based on per minute limits.",
94-
),
95-
format: str = typer.Option(
96-
"json",
97-
"-f",
98-
"--format",
99-
help="Output format in {'list', 'json', 'xml', 'html'}",
100-
callback=DTCLICommand.validate_format_input,
101-
),
102-
out_file: typer.FileTextWrite = typer.Option(sys.stdout, "-o", "--out-file", help="Output file (defaults to stdout)"),
103105
no_verify_ssl: bool = typer.Option(
104106
False,
105107
"--no-verify-ssl",
@@ -110,6 +112,25 @@ def feeds_nod(
110112
"--no-sign-api-key",
111113
help="Skip signing of api key",
112114
),
115+
header_authentication: bool = typer.Option(
116+
True,
117+
"--no-header-auth",
118+
help="Don't use header authentication",
119+
),
120+
output_format: str = typer.Option(
121+
"jsonl",
122+
"-f",
123+
"--format",
124+
help=f"Output format in [{OutputFormat.JSONL.value}, {OutputFormat.CSV.value}]",
125+
callback=DTCLICommand.validate_feeds_format_input,
126+
),
127+
endpoint: str = typer.Option(
128+
Endpoint.FEED.value,
129+
"-e",
130+
"--endpoint",
131+
help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]",
132+
callback=DTCLICommand.validate_endpoint_input,
133+
),
113134
sessionID: str = typer.Option(
114135
None,
115136
"--session-id",
@@ -119,17 +140,29 @@ def feeds_nod(
119140
None,
120141
"--after",
121142
help="Start of the time window, relative to the current time in seconds, for which data will be provided",
143+
callback=DTCLICommand.validate_after_or_before_input,
144+
),
145+
before: str = typer.Option(
146+
None,
147+
"--before",
148+
help="The end of the query window in seconds, relative to the current time, inclusive",
149+
callback=DTCLICommand.validate_after_or_before_input,
122150
),
123151
domain: str = typer.Option(
124152
None,
125153
"-d",
126154
"--domain",
127155
help="A string value used to filter feed results",
128156
),
157+
headers: bool = typer.Option(
158+
False,
159+
"--headers",
160+
help="Adds a header to the first line of response when text/csv is set in header parameters",
161+
),
129162
top: str = typer.Option(
130163
None,
131164
"--top",
132-
help="Number of results to return in the response payload",
165+
help="Number of results to return in the response payload. This is ignored in download endpoint",
133166
),
134167
):
135168
DTCLICommand.run(name=c.FEEDS_NOD, params=ctx.params)
@@ -159,6 +192,11 @@ def feeds_domainrdap(
159192
"--no-sign-api-key",
160193
help="Skip signing of api key",
161194
),
195+
header_authentication: bool = typer.Option(
196+
True,
197+
"--no-header-auth",
198+
help="Don't use header authentication",
199+
),
162200
endpoint: str = typer.Option(
163201
Endpoint.FEED.value,
164202
"-e",
@@ -222,6 +260,11 @@ def feeds_domaindiscovery(
222260
"--no-sign-api-key",
223261
help="Skip signing of api key",
224262
),
263+
header_authentication: bool = typer.Option(
264+
True,
265+
"--no-header-auth",
266+
help="Don't use header authentication",
267+
),
225268
output_format: str = typer.Option(
226269
"jsonl",
227270
"-f",

0 commit comments

Comments
 (0)