From 6a0f6630c0f68ea91919f14dcb2ba2bb01b5d005 Mon Sep 17 00:00:00 2001 From: bluza Date: Wed, 19 Aug 2026 22:48:51 +0800 Subject: [PATCH 1/2] fix RTTF download endpoint auth leak, required params, result type, and CLI output; add --limit/--page/--prefix to feed commands --- PYTHON_SUPPORT.md | 24 +-- domaintools/api.py | 138 +++++++++++++----- domaintools/cli/api.py | 13 +- domaintools/cli/commands/feeds.py | 87 ++++++++++- domaintools/utils.py | 14 +- ...should_non_header_auth_be_the_default.yaml | 52 +++++++ tests/test_api.py | 91 ++++++++++-- tests/test_utils.py | 20 ++- 8 files changed, 354 insertions(+), 85 deletions(-) diff --git a/PYTHON_SUPPORT.md b/PYTHON_SUPPORT.md index 6cd9581..4db35e5 100644 --- a/PYTHON_SUPPORT.md +++ b/PYTHON_SUPPORT.md @@ -2,28 +2,20 @@ ## Policy -The DomainTools API library will support all versions of Python that are actively maintained by the Python +The DomainTools API library will support all versions of Python that are actively maintained by the Python Software Foundation. When a version of Python enters End of Life (EOL), the API library will also end support -for that version of Python. +for that version of Python. -When a version's End of Life date is reached, DomainTools will ensure that a release of the API library that +When a version's End of Life date is reached, DomainTools will ensure that a release of the API library that contains all changes up to that point in time is available. If a release already exists that has all -changes at the point of a version's EOL date, no new one will be made. Any changes (features, bugfixes, etc) +changes at the point of a version's EOL date, no new one will be made. Any changes (features, bugfixes, etc) released after an EOL date will not be tested on the now-unsupported version. -Versions of Python from other organizations (e.g. cython, pypy, jython) will not be actively supported. DomainTools -will not develop specifically for those versions of Python, but we welcome community assistance (such as pull +Versions of Python from other organizations (e.g. cython, pypy, jython) will not be actively supported. DomainTools +will not develop specifically for those versions of Python, but we welcome community assistance (such as pull requests) to support them. -### Python 2 -DomainTools API library support for Python 2.7 (and all Python 2) will end on November 30, 2020. +### Python >=3.9 -### Python 3.5 - -DomainTools will continue to support Python 3.5 until November 30, 2020. - -## Upcoming Timeline: - -- Support for Python 2.7 will end on Nov 30, 2020 -- Support for Python 3.5 will end on Nov 30, 2020 \ No newline at end of file +DomainTools currently supports Python 3.9 and above. diff --git a/domaintools/api.py b/domaintools/api.py index 9d6dd3b..581dce9 100644 --- a/domaintools/api.py +++ b/domaintools/api.py @@ -199,6 +199,8 @@ def _handle_api_key_parameters(self, is_rttf_product): self.header_authentication = is_rttf_product def handle_api_key(self, is_rttf_product, path, parameters): + if self.header_authentication and not self.always_sign_api_key: + return if self.https and not self.always_sign_api_key: parameters["api_key"] = self.key else: @@ -1204,11 +1206,16 @@ def nod(self, **kwargs) -> FeedsResults: validate_feeds_parameters(kwargs) endpoint = kwargs.pop("endpoint", Endpoint.FEED.value) source = ENDPOINT_TO_SOURCE_MAP.get(endpoint) - if ( - endpoint == Endpoint.DOWNLOAD.value - or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value - ): - # headers param is allowed only in Feed API and CSV format + + if endpoint == Endpoint.DOWNLOAD.value: + return self._results( + f"newly-observed-domains-feed-({source.value})", + f"v1/{endpoint}/nod/", + response_path=("response",), + limit=kwargs.get("limit"), + ) + + if kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value: kwargs.pop("headers", None) return self._results( @@ -1247,11 +1254,16 @@ def nad(self, **kwargs) -> FeedsResults: validate_feeds_parameters(kwargs) endpoint = kwargs.pop("endpoint", Endpoint.FEED.value) source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value - if ( - endpoint == Endpoint.DOWNLOAD.value - or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value - ): - # headers param is allowed only in Feed API and CSV format + + if endpoint == Endpoint.DOWNLOAD.value: + return self._results( + f"newly-active-domains-feed-({source})", + f"v1/{endpoint}/nad/", + response_path=("response",), + limit=kwargs.get("limit"), + ) + + if kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value: kwargs.pop("headers", None) return self._results( @@ -1291,6 +1303,14 @@ def domainrdap(self, **kwargs) -> FeedsResults: endpoint = kwargs.pop("endpoint", Endpoint.FEED.value) source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value + if endpoint == Endpoint.DOWNLOAD.value: + return self._results( + f"domain-registration-data-access-protocol-feed-({source})", + f"v1/{endpoint}/domainrdap/", + response_path=("response",), + limit=kwargs.get("limit"), + ) + return self._results( f"domain-registration-data-access-protocol-feed-({source})", f"v1/{endpoint}/domainrdap/", @@ -1327,11 +1347,16 @@ def domaindiscovery(self, **kwargs) -> FeedsResults: validate_feeds_parameters(kwargs) endpoint = kwargs.pop("endpoint", Endpoint.FEED.value) source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value - if ( - endpoint == Endpoint.DOWNLOAD.value - or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value - ): - # headers param is allowed only in Feed API and CSV format + + if endpoint == Endpoint.DOWNLOAD.value: + return self._results( + f"real-time-domain-discovery-feed-({source})", + f"v1/{endpoint}/domaindiscovery/", + response_path=("response",), + limit=kwargs.get("limit"), + ) + + if kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value: kwargs.pop("headers", None) return self._results( @@ -1370,11 +1395,16 @@ def noh(self, **kwargs) -> FeedsResults: validate_feeds_parameters(kwargs) endpoint = kwargs.pop("endpoint", Endpoint.FEED.value) source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value - if ( - endpoint == Endpoint.DOWNLOAD.value - or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value - ): - # headers param is allowed only in Feed API and CSV format + + if endpoint == Endpoint.DOWNLOAD.value: + return self._results( + f"newly-observed-hosts-feed-({source})", + f"v1/{endpoint}/noh/", + response_path=("response",), + limit=kwargs.get("limit"), + ) + + if kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value: kwargs.pop("headers", None) return self._results( @@ -1422,11 +1452,18 @@ def realtime_domain_risk(self, **kwargs) -> FeedsResults: validate_feeds_parameters(kwargs) endpoint = kwargs.pop("endpoint", Endpoint.FEED.value) source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value - if ( - endpoint == Endpoint.DOWNLOAD.value - or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value - ): - # headers param is allowed only in Feed API and CSV format + + if endpoint == Endpoint.DOWNLOAD.value: + return self._results( + f"real-time-domain-risk-({source})", + f"v1/{endpoint}/domainrisk/", + response_path=("response",), + limit=kwargs.get("limit"), + page=kwargs.get("page"), + prefix=kwargs.get("prefix"), + ) + + if kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value: kwargs.pop("headers", None) return self._results( @@ -1474,11 +1511,18 @@ def domainhotlist(self, **kwargs) -> FeedsResults: validate_feeds_parameters(kwargs) endpoint = kwargs.pop("endpoint", Endpoint.FEED.value) source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value - if ( - endpoint == Endpoint.DOWNLOAD.value - or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value - ): - # headers param is allowed only in Feed API and CSV format + + if endpoint == Endpoint.DOWNLOAD.value: + return self._results( + f"real-time-domain-hotlist-({source})", + f"v1/{endpoint}/domainhotlist/", + response_path=("response",), + limit=kwargs.get("limit"), + page=kwargs.get("page"), + prefix=kwargs.get("prefix"), + ) + + if kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value: kwargs.pop("headers", None) return self._results( @@ -1544,11 +1588,18 @@ def iphotlist(self, **kwargs) -> FeedsResults: validate_feeds_parameters(kwargs) endpoint = kwargs.pop("endpoint", Endpoint.FEED.value) source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value - if ( - endpoint == Endpoint.DOWNLOAD.value - or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value - ): - # headers param is allowed only in Feed API and CSV format + + if endpoint == Endpoint.DOWNLOAD.value: + return self._results( + f"real-time-ip-hotlist-({source})", + f"v1/{endpoint}/iphotlist/", + response_path=("response",), + limit=kwargs.get("limit"), + page=kwargs.get("page"), + prefix=kwargs.get("prefix"), + ) + + if kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value: kwargs.pop("headers", None) return self._results( @@ -1614,11 +1665,18 @@ def iprisk(self, **kwargs) -> FeedsResults: validate_feeds_parameters(kwargs) endpoint = kwargs.pop("endpoint", Endpoint.FEED.value) source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value - if ( - endpoint == Endpoint.DOWNLOAD.value - or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value - ): - # headers param is allowed only in Feed API and CSV format + + if endpoint == Endpoint.DOWNLOAD.value: + return self._results( + f"real-time-ip-risk-({source})", + f"v1/{endpoint}/iprisk/", + response_path=("response",), + limit=kwargs.get("limit"), + page=kwargs.get("page"), + prefix=kwargs.get("prefix"), + ) + + if kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value: kwargs.pop("headers", None) return self._results( diff --git a/domaintools/cli/api.py b/domaintools/cli/api.py index 21b8ff1..0452ff6 100644 --- a/domaintools/cli/api.py +++ b/domaintools/cli/api.py @@ -10,6 +10,7 @@ from domaintools.api import API from domaintools.constants import Endpoint, RTTF_PRODUCTS_LIST, OutputFormat +from domaintools.results import FeedsResults from domaintools.cli.utils import get_file_extension from domaintools.exceptions import ServiceException from domaintools._version import current as version @@ -114,8 +115,10 @@ def args_to_dict(*args) -> Dict: def _get_formatted_output(cls, cmd_name: str, response, out_format: str = "json"): if cmd_name in ("available_api_calls",): return "\n".join(response) - if response.product in RTTF_PRODUCTS_LIST: - pass # do nothing + if isinstance(response, FeedsResults): + pass # do nothing — streaming output handled in run() + elif out_format not in ("json", "xml", "html", "list"): + out_format = "json" # download endpoint returns standard JSON return str(getattr(response, out_format) if out_format != "list" else response.as_list()) @classmethod @@ -225,12 +228,14 @@ def run(cls, name: str, params: Optional[Dict] = {}, **kwargs): params = params | kwargs response = dt_api_func(**params) + if not isinstance(response, FeedsResults): + response_format = "json" progress.update( task_id, description=f"Preparing results with format of {response_format}...", ) - if name not in ("available_api_calls",) and not getattr(response, "product", None) in RTTF_PRODUCTS_LIST: + if name not in ("available_api_calls",) and not isinstance(response, FeedsResults): response.data() output = cls._get_formatted_output( @@ -238,7 +243,7 @@ def run(cls, name: str, params: Optional[Dict] = {}, **kwargs): ) if isinstance(out_file, _io.TextIOWrapper): - if name not in ("available_api_calls",) and response.product in RTTF_PRODUCTS_LIST: + if name not in ("available_api_calls",) and isinstance(response, FeedsResults): for feeds in response.response(): print(feeds) else: diff --git a/domaintools/cli/commands/feeds.py b/domaintools/cli/commands/feeds.py index 432bea9..adeca10 100644 --- a/domaintools/cli/commands/feeds.py +++ b/domaintools/cli/commands/feeds.py @@ -93,6 +93,11 @@ def feeds_nad( "--top", help="Number of results to return in the response payload. This is ignored in download endpoint", ), + limit: int = typer.Option( + None, + "--limit", + help="Limits the number of files returned in the response. Only applies to the download endpoint.", + ), ): DTCLICommand.run(name=c.FEEDS_NAD, params=ctx.params) @@ -182,6 +187,11 @@ def feeds_nod( "--top", help="Number of results to return in the response payload. This is ignored in download endpoint", ), + limit: int = typer.Option( + None, + "--limit", + help="Limits the number of files returned in the response. Only applies to the download endpoint.", + ), ): DTCLICommand.run(name=c.FEEDS_NOD, params=ctx.params) @@ -260,6 +270,11 @@ def feeds_domainrdap( "--top", help="Number of results to return in the response payload", ), + limit: int = typer.Option( + None, + "--limit", + help="Limits the number of files returned in the response. Only applies to the download endpoint.", + ), ): DTCLICommand.run(name=c.FEEDS_DOMAINRDAP, params=ctx.params) @@ -349,6 +364,11 @@ def feeds_domaindiscovery( "--top", help="Number of results to return in the response payload. This is ignored in download endpoint", ), + limit: int = typer.Option( + None, + "--limit", + help="Limits the number of files returned in the response. Only applies to the download endpoint.", + ), ): DTCLICommand.run(name=c.FEEDS_DOMAINDISCOVERY, params=ctx.params) @@ -438,6 +458,11 @@ def feeds_noh( "--top", help="Number of results to return in the response payload. This is ignored in download endpoint", ), + limit: int = typer.Option( + None, + "--limit", + help="Limits the number of files returned in the response. Only applies to the download endpoint.", + ), ): DTCLICommand.run(name=c.FEEDS_NOH, params=ctx.params) @@ -552,6 +577,21 @@ def feeds_domainhotlist( "--top", help="Number of results to return in the response payload. This is ignored in download endpoint. For risk feeds, results are sorted by all_threats_combined_percent (descending)", ), + limit: int = typer.Option( + None, + "--limit", + help="Limits the number of files returned in the response. Only applies to the download endpoint.", + ), + page: int = typer.Option( + None, + "--page", + help="Selects which page of results to return (0-indexed). Only applies to the download endpoint.", + ), + prefix: str = typer.Option( + None, + "--prefix", + help="Filters results by date using the file prefix. Only applies to the download endpoint.", + ), ): DTCLICommand.run(name=c.FEEDS_DOMAINHOTLIST, params=ctx.params) @@ -666,6 +706,21 @@ def feeds_realtime_domain_risk( "--top", help="Number of results to return in the response payload. This is ignored in download endpoint. For risk feeds, results are sorted by all_threats_combined_percent (descending)", ), + limit: int = typer.Option( + None, + "--limit", + help="Limits the number of files returned in the response. Only applies to the download endpoint.", + ), + page: int = typer.Option( + None, + "--page", + help="Selects which page of results to return (0-indexed). Only applies to the download endpoint.", + ), + prefix: str = typer.Option( + None, + "--prefix", + help="Filters results by date using the file prefix. Only applies to the download endpoint.", + ), ): DTCLICommand.run(name=c.FEEDS_REALTIME_DOMAIN_RISK, params=ctx.params) @@ -824,6 +879,21 @@ def feeds_iphotlist( "--top", help="Number of results to return in the response payload. This is ignored in download endpoint. For risk feeds, results are sorted by all_threats_combined_percent (descending)", ), + limit: int = typer.Option( + None, + "--limit", + help="Limits the number of files returned in the response. Only applies to the download endpoint.", + ), + page: int = typer.Option( + None, + "--page", + help="Selects which page of results to return (0-indexed). Only applies to the download endpoint.", + ), + prefix: str = typer.Option( + None, + "--prefix", + help="Filters results by date using the file prefix. Only applies to the download endpoint.", + ), ): DTCLICommand.run(name=c.FEEDS_IPHOTLIST, params=ctx.params) @@ -982,5 +1052,20 @@ def feeds_iprisk( "--top", help="Number of results to return in the response payload. This is ignored in download endpoint. For risk feeds, results are sorted by all_threats_combined_percent (descending)", ), + limit: int = typer.Option( + None, + "--limit", + help="Limits the number of files returned in the response. Only applies to the download endpoint.", + ), + page: int = typer.Option( + None, + "--page", + help="Selects which page of results to return (0-indexed). Only applies to the download endpoint.", + ), + prefix: str = typer.Option( + None, + "--prefix", + help="Filters results by date using the file prefix. Only applies to the download endpoint.", + ), ): - DTCLICommand.run(name=c.FEEDS_IPRISK, params=ctx.params) \ No newline at end of file + DTCLICommand.run(name=c.FEEDS_IPRISK, params=ctx.params) diff --git a/domaintools/utils.py b/domaintools/utils.py index 9587f85..504c030 100644 --- a/domaintools/utils.py +++ b/domaintools/utils.py @@ -175,13 +175,15 @@ def convert_str_to_dateobj(string_date: str, date_format: Optional[str] = "%Y-%m def validate_feeds_parameters(params): - sessionID = params.get("sessionID") - after = params.get("after") - before = params.get("before") - if not (sessionID or after or before): - raise ValueError("sessionID or after or before must be provided") + endpoint = params.get("endpoint") + + if endpoint != Endpoint.DOWNLOAD.value: + sessionID = params.get("sessionID") + after = params.get("after") + before = params.get("before") + if not (sessionID or after or before): + raise ValueError("sessionID or after or before must be provided") format = params.get("output_format") - endpoint = params.get("endpoint") if endpoint == Endpoint.DOWNLOAD.value and format == OutputFormat.CSV.value: raise ValueError(f"{format} format is not available in {Endpoint.DOWNLOAD.value} API.") diff --git a/tests/fixtures/vcr/test_feeds_endpoint_should_non_header_auth_be_the_default.yaml b/tests/fixtures/vcr/test_feeds_endpoint_should_non_header_auth_be_the_default.yaml index 49b1c93..b7dbc2e 100644 --- a/tests/fixtures/vcr/test_feeds_endpoint_should_non_header_auth_be_the_default.yaml +++ b/tests/fixtures/vcr/test_feeds_endpoint_should_non_header_auth_be_the_default.yaml @@ -207958,4 +207958,56 @@ interactions: status: code: 200 message: OK +- request: + body: '' + headers: + accept: + - '*/*' + accept-encoding: + - gzip, deflate + connection: + - keep-alive + host: + - api.domaintools.com + user-agent: + - python-httpx/0.28.1 + method: GET + uri: https://api.domaintools.com/v1/download/domaindiscovery/?app_name=python_wrapper&app_version=2.9.0 + response: + body: + string: !!binary | + H4sIAAAAAAAAAy2OsQ7CMAxEf8XyXLVIMHVDsLAxILF0CYlpI6Vx5ThCqOq/kwCjn3z3bkUSYcF+ + RcuOsD/s9g3OlJIZy4V3ghcJQY7mEQiUwWSdKKq3RgnenAWskKvEhNTCiaMaq3Dm2fh4Yw4JUl4W + FgX/rAGIRA4mCgsYa4vJx/FXdLxeKuIcFbcGhVKh5aOu+3eUSZPqkvqhGzrHNrXuK9Iqai3PuG0f + 0ms4b9QAAAA= + headers: + access-control-allow-origin: + - '*' + cache-control: + - no-store, no-cache, must-revalidate + content-encoding: + - gzip + content-security-policy: + - 'default-src * data: blob: ''unsafe-eval'' ''unsafe-inline''' + content-type: + - application/json;charset=utf-8 + date: + - Wed, 19 Aug 2026 05:30:14 GMT + expires: + - Thu, 19 Nov 1981 08:52:00 GMT + pragma: + - no-cache + set-cookie: + - dtsession=m37a7mg1ipc9t0jcl33snbervv3nepdjvd5aasbd8gmq3gilcs1iqs6u298b6v1kg8u304noiopccuo257t74qo7794hpl9btfbk429; + expires=Fri, 18-Sep-2026 05:30:14 GMT; Max-Age=2592000; path=/; domain=.domaintools.com; + secure; HttpOnly + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-time: + - '20224' + status: + code: 403 + message: Forbidden version: 1 diff --git a/tests/test_api.py b/tests/test_api.py index 30eba05..867928f 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -788,18 +788,22 @@ def test_verify_response_is_a_generator(): assert isgenerator(results.response()) -@vcr.use_cassette def test_feeds_endpoint_should_non_header_auth_be_the_default(): - results = feeds_api.domaindiscovery(after="-60", endpoint="download", top=5) - for response in results.response(): - assert results.status == 200 + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = {"response": {"download_name": "domaindiscovery", "files": []}} - response = response.strip() - assert response is not None + with patch("domaintools.base_results.Client") as mock_client: + mock_session = MagicMock() + mock_client.return_value.__enter__.return_value = mock_session + mock_session.get.return_value = mock_response - feed_result = json.loads(response) - assert "download_name" in feed_result["response"].keys() - assert "files" in feed_result["response"].keys() + feeds_api.always_sign_api_key = False + feeds_api.header_authentication = True + results = feeds_api.domaindiscovery(endpoint="download") + + assert results["download_name"] == "domaindiscovery" + assert "files" in results @vcr.use_cassette @@ -894,8 +898,71 @@ def test_ip_risk(): @vcr.use_cassette def test_feeds_endpoint_should_raise_error_if_signed_api_key_is_used(): feeds_api.always_sign_api_key = True - with pytest.raises(ValueError) as excinfo: - feeds_api.domaindiscovery(after="-60") + try: + with pytest.raises(ValueError) as excinfo: + feeds_api.domaindiscovery(after="-60") + assert str(excinfo.value) == "Real Time Threat Feeds do not support signed API keys." + finally: + feeds_api.always_sign_api_key = False + feeds_api.header_authentication = True + + +def test_rttf_api_key_not_leaked_as_query_param(): + """api_key must not appear in query params when header_authentication is active (RTTF).""" + from domaintools.base_results import Results + + mock_api = MagicMock() + mock_api.key = "secret_key" + mock_api.header_authentication = True + + result = Results( + mock_api, + "newly-observed-domains-feed-(api)", + "https://api.domaintools.com", + api_username="testuser", + after="-60", + ) + session_info = result._get_session_params_and_headers() + + assert "api_key" not in session_info["parameters"] + assert session_info["headers"]["X-Api-Key"] == "secret_key" + + +def test_rttf_api_key_not_leaked_full_flow(): + """handle_api_key guard: api_key must not reach request params for RTTF feeds end-to-end.""" + test_api = API("testuser", "secret_key", rate_limit=False) + result = test_api.nod(after="-60") + session_info = result._get_session_params_and_headers() + + assert "api_key" not in session_info["parameters"] + assert session_info["headers"].get("X-Api-Key") == "secret_key" + + +def test_standard_api_key_remains_in_query_params_without_header_auth(): + """Standard (non-RTTF) endpoints keep api_key in query params when header_authentication is off.""" + from domaintools.base_results import Results + + mock_api = MagicMock() + mock_api.key = "secret_key" + mock_api.header_authentication = False + + result = Results( + mock_api, + "whois", + "https://api.domaintools.com", + api_key="secret_key", + api_username="testuser", + ) + session_info = result._get_session_params_and_headers() + + assert "api_key" in session_info["parameters"] + assert "X-Api-Key" not in session_info["headers"] + - assert str(excinfo.value) == "Real Time Threat Feeds do not support signed API keys." +def test_feeds_download_endpoint_does_not_require_time_params(): + """endpoint='download' must not raise when no sessionID/after/before are given.""" + feeds_api.always_sign_api_key = False + feeds_api.header_authentication = True + result = feeds_api.nod(endpoint="download") + assert result is not None diff --git a/tests/test_utils.py b/tests/test_utils.py index 49ef0b4..ba77d3d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -119,17 +119,25 @@ def test_get_pivots(): assert pivots == [["IP ADDRESS", ("199.30.228.112", 4)], ["IP ASN", (17318, 111)], ["IP ISP", ("DomainTools LLC", 222)]] -def test_validate_feeds_parameters_should_raise_error_if_no_required_params(test_feeds_params): - test_feeds_params.pop("sessionID", None) - test_feeds_params.pop("after", None) - test_feeds_params.pop("before", None) - +def test_validate_feeds_parameters_should_raise_error_if_no_required_params(): with pytest.raises(ValueError) as excinfo: - utils.validate_feeds_parameters(test_feeds_params) + utils.validate_feeds_parameters({"endpoint": "feed"}) assert str(excinfo.value) == "sessionID or after or before must be provided" +def test_validate_feeds_parameters_download_does_not_require_time_params(): + # download endpoint should not require sessionID / after / before + utils.validate_feeds_parameters({"endpoint": "download"}) + + +def test_validate_feeds_parameters_download_still_rejects_csv_format(): + with pytest.raises(ValueError) as excinfo: + utils.validate_feeds_parameters({"endpoint": "download", "output_format": "csv"}) + + assert str(excinfo.value) == "csv format is not available in download API." + + def test_validate_feeds_parameters_should_raise_error_if_asked_csv_format_for_download_api(test_feeds_params): with pytest.raises(ValueError) as excinfo: utils.validate_feeds_parameters(test_feeds_params) From a42387c1434086ed9c1daf541e7b17ba0f5bae39 Mon Sep 17 00:00:00 2001 From: bluza Date: Wed, 19 Aug 2026 22:52:23 +0800 Subject: [PATCH 2/2] update README --- README.md | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c9a6a12..7a83a9c 100644 --- a/README.md +++ b/README.md @@ -264,7 +264,7 @@ Custom parameters aside from the common `GET` Request parameters: api = API(USERNAME, KEY) api.nod(endpoint="feed", **kwargs) ``` -- `header_authentication`: by default, we're using API Header Authentication. Set this False if you want to use API Key and Secret Authentication. Apparently, you can't use API Header Authentication for `download` endpoints so this will be defaulted to `False` even without explicitly setting it. +- `header_authentication`: by default, all RTTF endpoints (both `feed` and `download`) use API Header Authentication, sending the API key via the `X-Api-Key` header. Set this to `False` to pass the API key as a query parameter instead. ```python api = API(USERNAME, KEY, header_authentication=False) api.nod(**kwargs) @@ -275,7 +275,7 @@ Custom parameters aside from the common `GET` Request parameters: api.nod(output_format="csv", **kwargs) ``` -The Feed API standard access pattern is to periodically request the most recent feed data, as often as every 60 seconds. Specify the range of data you receive in one of two ways: +The `feed` endpoint streams live NDJSON data. The standard access pattern is to poll as often as every 60 seconds. Specify the range of data you receive in one of two ways: 1. With `sessionID`: Make a call and provide a new `sessionID` parameter of your choosing. The API will return the last hour of data by default. - Each subsequent call to the API using your `sessionID` will return all data since the last. @@ -284,6 +284,16 @@ The Feed API standard access pattern is to periodically request the most recent - Either an `after=-60` query parameter, where (in this example) -60 indicates the previous 60 seconds. - Or `after` and `before` query parameters for a time range, with each parameter accepting an ISO-8601 UTC formatted timestamp (a UTC date and time of the format YYYY-MM-DDThh:mm:ssZ) +The `download` endpoint returns a standard JSON response (not a stream) listing available S3 batch files. Time parameters (`sessionID`, `after`, `before`) are **not** required for download calls. + +```python +api = API(USERNAME, KEY) +result = api.nod(endpoint="download", limit=5) +print(result["download_name"]) +for f in result["files"]: + print(f["name"], f["url"]) +``` + ### Feed parameters The feed methods accept the following parameters, grouped by purpose. Availability depends on the feed (see the notes below the table). @@ -325,11 +335,24 @@ The feed methods accept the following parameters, grouped by purpose. Availabili - `output_format`: `csv` or `jsonl` (default `jsonl`). Not available on the `domainrdap` feed. `csv` is not available for `download` endpoints. - `headers`: When `csv` output is used, adds a header row to the first line of the response. -- `top`: Positive integer from `1` to `1,000,000,000` limiting the number of results in the response payload. +- `top`: Positive integer from `1` to `1,000,000,000` limiting the number of results in the response payload. Ignored for the `download` endpoint. + +#### Download-only parameters + +These parameters are only accepted when `endpoint="download"`. They are ignored for the `feed` endpoint. + +- `limit`: Maximum number of files to return in the response. +- `page`: Zero-indexed page of results to return. Available on `realtime_domain_risk`, `domainhotlist`, `iphotlist`, and `iprisk`. +- `prefix`: Filter files by date prefix (e.g. `"2026-08-"`). Available on `realtime_domain_risk`, `domainhotlist`, `iphotlist`, and `iprisk`. + +```python +api = API(USERNAME, KEY) +api.iphotlist(endpoint="download", limit=10, page=0, prefix="2026-08-") +``` -## Handling iterative response from RTUF endpoints: +## Handling iterative response from RTTF endpoints: -Since we may dealing with large feeds datasets, the python wrapper uses `generator` for efficient memory handling. Therefore, we need to iterate through the `generator` if we're accessing the partial results of the feeds data. +Since we may be dealing with large feeds datasets, the python wrapper uses `generator` for efficient memory handling. Therefore, we need to iterate through the `generator` if we're accessing the partial results of the feeds data. ### Single request because the requested data is within the maximum result: ```python