From cc613a39996e6f00a15d85b32a2cdb397f7e99f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Wed, 20 Nov 2024 10:40:50 +0100 Subject: [PATCH 1/6] docs: bump the docusaurus plugin --- Makefile | 4 +- src/apify/_actor.py | 5 +- src/apify/_configuration.py | 2 + src/apify/_models.py | 12 +- src/apify/_platform_event_manager.py | 23 +- src/apify/_proxy_configuration.py | 5 +- src/apify/_utils.py | 28 + .../_apify_storage_client.py | 2 + website/build_api_reference.sh | 35 -- website/docusaurus.config.js | 27 +- website/package-lock.json | 529 ++++++++++++++---- website/package.json | 5 +- website/pydoc-markdown.yml | 14 - website/pydoc-markdown/__init__.py | 0 website/pydoc-markdown/generate_ast.py | 46 -- .../google_docstring_processor.py | 183 ------ website/transformDocs.js | 382 ------------- 17 files changed, 514 insertions(+), 788 deletions(-) delete mode 100644 website/pydoc-markdown.yml delete mode 100644 website/pydoc-markdown/__init__.py delete mode 100644 website/pydoc-markdown/generate_ast.py delete mode 100644 website/pydoc-markdown/google_docstring_processor.py delete mode 100644 website/transformDocs.js diff --git a/Makefile b/Makefile index 96cb6298..78e0cc0e 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ build-api-reference: cd website && poetry run ./build_api_reference.sh build-docs: - cd website && npm clean-install && npm run build + cd website && npm clean-install && poetry run npm run build run-docs: build-api-reference - cd website && npm clean-install && npm run start + cd website && npm clean-install && poetry run npm run start diff --git a/src/apify/_actor.py b/src/apify/_actor.py index f60a99df..57bffca4 100644 --- a/src/apify/_actor.py +++ b/src/apify/_actor.py @@ -22,7 +22,7 @@ from apify._models import ActorRun from apify._platform_event_manager import EventManager, LocalEventManager, PlatformEventManager from apify._proxy_configuration import ProxyConfiguration -from apify._utils import get_system_info, is_running_in_ipython +from apify._utils import get_system_info, is_running_in_ipython, docs_group, docs_name from apify.apify_storage_client import ApifyStorageClient from apify.log import _configure_logging, logger from apify.storages import Dataset, KeyValueStore, RequestQueue @@ -38,7 +38,8 @@ MainReturnType = TypeVar('MainReturnType') - +@docs_name('Actor') +@docs_group('Classes') class _ActorType: """The class of `Actor`. Only make a new instance if you're absolutely sure you need to.""" diff --git a/src/apify/_configuration.py b/src/apify/_configuration.py index ab249284..40532763 100644 --- a/src/apify/_configuration.py +++ b/src/apify/_configuration.py @@ -11,7 +11,9 @@ from crawlee._utils.urls import validate_http_url from crawlee.configuration import Configuration as CrawleeConfiguration +from apify._utils import docs_group +@docs_group('Classes') class Configuration(CrawleeConfiguration): """A class for specifying the configuration of an Actor. diff --git a/src/apify/_models.py b/src/apify/_models.py index 5963ec9a..1f57ab33 100644 --- a/src/apify/_models.py +++ b/src/apify/_models.py @@ -10,7 +10,9 @@ from crawlee._utils.models import timedelta_ms from crawlee._utils.urls import validate_http_url +from apify._utils import docs_group +@docs_group('Data structures') class Webhook(BaseModel): __model_config__ = ConfigDict(populate_by_name=True) @@ -28,13 +30,13 @@ class Webhook(BaseModel): Field(description='Template for the payload sent by the webook'), ] = None - +@docs_group('Data structures') class ActorRunMeta(BaseModel): __model_config__ = ConfigDict(populate_by_name=True) origin: Annotated[MetaOrigin, Field()] - +@docs_group('Data structures') class ActorRunStats(BaseModel): __model_config__ = ConfigDict(populate_by_name=True) @@ -54,7 +56,7 @@ class ActorRunStats(BaseModel): metamorph: Annotated[int | None, Field(alias='metamorph')] = None compute_units: Annotated[float, Field(alias='computeUnits')] - +@docs_group('Data structures') class ActorRunOptions(BaseModel): __model_config__ = ConfigDict(populate_by_name=True) @@ -63,7 +65,7 @@ class ActorRunOptions(BaseModel): memory_mbytes: Annotated[int, Field(alias='memoryMbytes')] disk_mbytes: Annotated[int, Field(alias='diskMbytes')] - +@docs_group('Data structures') class ActorRunUsage(BaseModel): __model_config__ = ConfigDict(populate_by_name=True) @@ -80,7 +82,7 @@ class ActorRunUsage(BaseModel): proxy_residential_transfer_gbytes: Annotated[float | None, Field(alias='PROXY_RESIDENTIAL_TRANSFER_GBYTES')] = None proxy_serps: Annotated[float | None, Field(alias='PROXY_SERPS')] = None - +@docs_group('Data structures') class ActorRun(BaseModel): __model_config__ = ConfigDict(populate_by_name=True) diff --git a/src/apify/_platform_event_manager.py b/src/apify/_platform_event_manager.py index afbf2c5d..9de6cfdb 100644 --- a/src/apify/_platform_event_manager.py +++ b/src/apify/_platform_event_manager.py @@ -21,6 +21,8 @@ from apify.log import logger +from apify._utils import docs_group + if TYPE_CHECKING: from types import TracebackType @@ -29,12 +31,12 @@ __all__ = ['EventManager', 'LocalEventManager', 'PlatformEventManager'] - +@docs_group('Data structures') class PersistStateEvent(BaseModel): name: Literal[Event.PERSIST_STATE] data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))] - +@docs_group('Data structures') class SystemInfoEventData(BaseModel): mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')] mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')] @@ -60,27 +62,27 @@ def to_crawlee_format(self) -> EventSystemInfoData: } ) - +@docs_group('Data structures') class SystemInfoEvent(BaseModel): name: Literal[Event.SYSTEM_INFO] data: SystemInfoEventData - +@docs_group('Data structures') class MigratingEvent(BaseModel): name: Literal[Event.MIGRATING] data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)] - +@docs_group('Data structures') class AbortingEvent(BaseModel): name: Literal[Event.ABORTING] data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)] - +@docs_group('Data structures') class ExitEvent(BaseModel): name: Literal[Event.EXIT] data: Annotated[EventExitData, Field(default_factory=EventExitData)] - +@docs_group('Data structures') class EventWithoutData(BaseModel): name: Literal[ Event.SESSION_RETIRED, @@ -92,12 +94,12 @@ class EventWithoutData(BaseModel): ] data: Any = None - +@docs_group('Data structures') class DeprecatedEvent(BaseModel): name: Literal['cpuInfo'] data: Annotated[dict[str, Any], Field(default_factory=dict)] - +@docs_group('Data structures') class UnknownEvent(BaseModel): name: str data: Annotated[dict[str, Any], Field(default_factory=dict)] @@ -123,8 +125,7 @@ class UnknownEvent(BaseModel): UnknownEvent, ] ) - - +@docs_group('Classes') class PlatformEventManager(EventManager): """A class for managing Actor events. diff --git a/src/apify/_proxy_configuration.py b/src/apify/_proxy_configuration.py index 214fb4c9..e0bf2977 100644 --- a/src/apify/_proxy_configuration.py +++ b/src/apify/_proxy_configuration.py @@ -18,6 +18,8 @@ from apify._configuration import Configuration from apify.log import logger +from apify._utils import docs_group + if TYPE_CHECKING: from apify_client import ApifyClientAsync from crawlee import Request @@ -68,6 +70,7 @@ def _check( raise ValueError(f'{error_str} does not match pattern {pattern.pattern!r}') +@docs_group('Classes') @dataclass class ProxyInfo(CrawleeProxyInfo): """Provides information about a proxy connection that is used for requests.""" @@ -86,7 +89,7 @@ class ProxyInfo(CrawleeProxyInfo): This parameter is optional, by default, the proxy uses all available proxy servers from all countries. """ - +@docs_group('Classes') class ProxyConfiguration(CrawleeProxyConfiguration): """Configures a connection to a proxy server with the provided options. diff --git a/src/apify/_utils.py b/src/apify/_utils.py index 254b7c36..4577a351 100644 --- a/src/apify/_utils.py +++ b/src/apify/_utils.py @@ -1,5 +1,7 @@ from __future__ import annotations +from typing import Callable, Literal + import builtins import sys from importlib import metadata @@ -24,3 +26,29 @@ def get_system_info() -> dict: def is_running_in_ipython() -> bool: return getattr(builtins, '__IPYTHON__', False) + +GroupName = Literal['Classes', 'Abstract classes', 'Data structures', 'Errors', 'Functions'] + +def docs_group(group_name: GroupName) -> Callable: # noqa: ARG001 + """Decorator to mark symbols for rendering and grouping in documentation. + + This decorator is used purely for documentation purposes and does not alter the behavior + of the decorated callable. + """ + + def wrapper(func: Callable) -> Callable: + return func + + return wrapper + +def docs_name(symbol_name: str) -> Callable: + """Decorator for renaming symbols in documentation. This changes the rendered name of the symbol only in the web documentation. + + This decorator is used purely for documentation purposes and does not alter the behavior + of the decorated callable. + """ + + def wrapper(func: Callable) -> Callable: + return func + + return wrapper diff --git a/src/apify/apify_storage_client/_apify_storage_client.py b/src/apify/apify_storage_client/_apify_storage_client.py index 1153e95d..b436e376 100644 --- a/src/apify/apify_storage_client/_apify_storage_client.py +++ b/src/apify/apify_storage_client/_apify_storage_client.py @@ -12,7 +12,9 @@ from apify.apify_storage_client._request_queue_client import RequestQueueClient from apify.apify_storage_client._request_queue_collection_client import RequestQueueCollectionClient +from apify._utils import docs_group +@docs_group('Classes') class ApifyStorageClient(BaseStorageClient): """A storage client implementation based on the Apify platform storage.""" diff --git a/website/build_api_reference.sh b/website/build_api_reference.sh index fc2f982c..134fd6c2 100755 --- a/website/build_api_reference.sh +++ b/website/build_api_reference.sh @@ -1,39 +1,4 @@ #!/bin/bash -# On macOS, sed requires a space between -i and '' to specify no backup should be done -# On Linux, sed requires no space between -i and '' to specify no backup should be done -sed_no_backup() { - if [[ $(uname) = "Darwin" ]]; then - sed -i '' "$@" - else - sed -i'' "$@" - fi -} - -# Create docspec dump of this package's source code through pydoc-markdown -python ./pydoc-markdown/generate_ast.py > docspec-dump.jsonl -sed_no_backup "s#${PWD}/..#REPO_ROOT_PLACEHOLDER#g" docspec-dump.jsonl - -# Create docpec dump from the right version of the apify-shared package -apify_shared_version=$(python -c "import apify_shared; print(apify_shared.__version__)") -apify_shared_tempdir=$(realpath "$(mktemp -d)") -git clone --quiet https://github.com/apify/apify-shared-python.git "${apify_shared_tempdir}" -cp ./pydoc-markdown.yml "${apify_shared_tempdir}/pydoc-markdown.yml" -sed_no_backup "s#search_path: \[../src\]#search_path: \[./src\]#g" "${apify_shared_tempdir}/pydoc-markdown.yml" - -( - cd "${apify_shared_tempdir}"; - git checkout --quiet "v${apify_shared_version}"; - pydoc-markdown --quiet --dump > ./apify-shared-docspec-dump.jsonl -) - -cp "${apify_shared_tempdir}/apify-shared-docspec-dump.jsonl" . -sed_no_backup "s#${apify_shared_tempdir}#REPO_ROOT_PLACEHOLDER#g" apify-shared-docspec-dump.jsonl - -rm -rf "${apify_shared_tempdir}" - # Generate import shortcuts from the modules python generate_module_shortcuts.py - -# Transform the docpec dumps into Typedoc-compatible docs tree -node transformDocs.js diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 9655b566..f0e400ed 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -1,7 +1,20 @@ -/* eslint-disable global-require,import/no-extraneous-dependencies */ +const path = require('path'); + const { config } = require('@apify/docs-theme'); + const { externalLinkProcessor } = require('./tools/utils/externalLink'); -const { groupSort } = require('./transformDocs.js'); + +const GROUP_ORDER = [ + 'Classes', + 'Data structures', +]; + +const groupSort = (g1, g2) => { + if (GROUP_ORDER.includes(g1) && GROUP_ORDER.includes(g2)) { + return GROUP_ORDER.indexOf(g1) - GROUP_ORDER.indexOf(g2); + } + return g1.localeCompare(g2); +}; const { absoluteUrl } = config; @@ -83,10 +96,18 @@ module.exports = { typedocOptions: { excludeExternals: false, }, - pathToCurrentVersionTypedocJSON: `${__dirname}/api-typedoc-generated.json`, sortSidebar: groupSort, routeBasePath: 'reference', python: true, + pythonOptions: { + pythonModulePath: path.join(__dirname, '../src/apify'), + moduleShortcutsPath: path.join(__dirname, '/module_shortcuts.json'), + }, + // reexports: [ + // { + // url: 'http://localhost:3000/python/api/class/Dataset', + // }, + // ], }, ], ...config.plugins, diff --git a/website/package-lock.json b/website/package-lock.json index 9727622e..595c50b0 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -7,7 +7,7 @@ "name": "apify-sdk-python", "dependencies": { "@apify/docs-theme": "^1.0.132", - "@apify/docusaurus-plugin-typedoc-api": "^4.2.6", + "@apify/docusaurus-plugin-typedoc-api": "^4.3.1", "@docusaurus/core": "^3.5.2", "@docusaurus/plugin-client-redirects": "^3.5.2", "@docusaurus/preset-classic": "^3.5.2", @@ -324,9 +324,9 @@ } }, "node_modules/@apify/docusaurus-plugin-typedoc-api": { - "version": "4.2.6", - "resolved": "https://registry.npmjs.org/@apify/docusaurus-plugin-typedoc-api/-/docusaurus-plugin-typedoc-api-4.2.6.tgz", - "integrity": "sha512-M/rpRqbHZjL49xXtvdxdh/M7jSSR5lb3mjKERzKKnX6i92B5BRetL7Eb4qHre9QPQVijgzILhlhTi8otvr0LAw==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@apify/docusaurus-plugin-typedoc-api/-/docusaurus-plugin-typedoc-api-4.3.1.tgz", + "integrity": "sha512-tRi+Kly2tJurx9cie7bZQjZizzg1ogf22140+8mNvtkX2P3tC6UB/KcWgZ006SXcv69T07UDyVd+3D8hTdaibw==", "license": "MIT", "dependencies": { "@docusaurus/plugin-content-docs": "^3.5.2", @@ -334,9 +334,10 @@ "@docusaurus/utils": "^3.5.2", "@types/react": "^18.3.11", "@vscode/codicons": "^0.0.35", + "html-entities": "2.3.2", "marked": "^9.1.6", "marked-smartypants": "^1.1.5", - "typedoc": "^0.25.7", + "typedoc": "^0.26.11", "zx": "^8.1.4" }, "engines": { @@ -349,6 +350,12 @@ "typescript": "^5.0.0" } }, + "node_modules/@apify/docusaurus-plugin-typedoc-api/node_modules/html-entities": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.3.2.tgz", + "integrity": "sha512-c3Ab/url5ksaT0WyleslpBEthOzWhrjQbg75y7XUsfSzi3Dgzt0l8w5e7DylRn15MTlMMD58dTfzddNS2kcAjQ==", + "license": "MIT" + }, "node_modules/@apify/eslint-config": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@apify/eslint-config/-/eslint-config-0.4.0.tgz", @@ -4550,6 +4557,75 @@ "integrity": "sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==", "dev": true }, + "node_modules/@shikijs/core": { + "version": "1.23.1", + "resolved": "https://registry.npmjs.org/@shikijs/core/-/core-1.23.1.tgz", + "integrity": "sha512-NuOVgwcHgVC6jBVH5V7iblziw6iQbWWHrj5IlZI3Fqu2yx9awH7OIQkXIcsHsUmY19ckwSgUMgrqExEyP5A0TA==", + "license": "MIT", + "dependencies": { + "@shikijs/engine-javascript": "1.23.1", + "@shikijs/engine-oniguruma": "1.23.1", + "@shikijs/types": "1.23.1", + "@shikijs/vscode-textmate": "^9.3.0", + "@types/hast": "^3.0.4", + "hast-util-to-html": "^9.0.3" + } + }, + "node_modules/@shikijs/core/node_modules/@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@shikijs/engine-javascript": { + "version": "1.23.1", + "resolved": "https://registry.npmjs.org/@shikijs/engine-javascript/-/engine-javascript-1.23.1.tgz", + "integrity": "sha512-i/LdEwT5k3FVu07SiApRFwRcSJs5QM9+tod5vYCPig1Ywi8GR30zcujbxGQFJHwYD7A5BUqagi8o5KS+LEVgBg==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "1.23.1", + "@shikijs/vscode-textmate": "^9.3.0", + "oniguruma-to-es": "0.4.1" + } + }, + "node_modules/@shikijs/engine-oniguruma": { + "version": "1.23.1", + "resolved": "https://registry.npmjs.org/@shikijs/engine-oniguruma/-/engine-oniguruma-1.23.1.tgz", + "integrity": "sha512-KQ+lgeJJ5m2ISbUZudLR1qHeH3MnSs2mjFg7bnencgs5jDVPeJ2NVDJ3N5ZHbcTsOIh0qIueyAJnwg7lg7kwXQ==", + "license": "MIT", + "dependencies": { + "@shikijs/types": "1.23.1", + "@shikijs/vscode-textmate": "^9.3.0" + } + }, + "node_modules/@shikijs/types": { + "version": "1.23.1", + "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-1.23.1.tgz", + "integrity": "sha512-98A5hGyEhzzAgQh2dAeHKrWW4HfCMeoFER2z16p5eJ+vmPeF6lZ/elEne6/UCU551F/WqkopqRsr1l2Yu6+A0g==", + "license": "MIT", + "dependencies": { + "@shikijs/vscode-textmate": "^9.3.0", + "@types/hast": "^3.0.4" + } + }, + "node_modules/@shikijs/types/node_modules/@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@shikijs/vscode-textmate": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@shikijs/vscode-textmate/-/vscode-textmate-9.3.0.tgz", + "integrity": "sha512-jn7/7ky30idSkd/O5yDBfAnVt+JJpepofP/POZ1iMOxK59cOfqIgg/Dj0eFsjOTMw+4ycJN0uhZH/Eb0bs/EUA==", + "license": "MIT" + }, "node_modules/@sideway/address": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.5.tgz", @@ -5906,12 +5982,6 @@ "node": ">=8" } }, - "node_modules/ansi-sequence-parser": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ansi-sequence-parser/-/ansi-sequence-parser-1.1.1.tgz", - "integrity": "sha512-vJXt3yiaUL4UU546s3rPXlsry/RnM730G1+HkpKE012AN0sx1eOrxSu95oKDIonskeLTijMgqWZ3uDEe3NFvyg==", - "license": "MIT" - }, "node_modules/ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -8157,6 +8227,12 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, + "node_modules/emoji-regex-xs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex-xs/-/emoji-regex-xs-1.0.0.tgz", + "integrity": "sha512-LRlerrMYoIDrT6jgpeZ2YYl/L8EulRTt5hQcYjy5AInh7HWXKimpqx68aknBFpGL2+/IcogTcaydJEgaTmOpDg==", + "license": "MIT" + }, "node_modules/emojilib": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/emojilib/-/emojilib-2.4.0.tgz", @@ -10538,6 +10614,74 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/hast-util-to-html": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.3.tgz", + "integrity": "sha512-M17uBDzMJ9RPCqLMO92gNNUDuBSq10a25SDBI08iCCxmorf4Yy6sYHK57n9WAbRAAaU+DuR4W6GN9K4DFZesYg==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-whitespace": "^3.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "stringify-entities": "^4.0.0", + "zwitch": "^2.0.4" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-to-html/node_modules/@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/hast-util-to-html/node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, + "node_modules/hast-util-to-html/node_modules/comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/hast-util-to-html/node_modules/property-information": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.5.0.tgz", + "integrity": "sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/hast-util-to-html/node_modules/space-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", + "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/hast-util-to-jsx-runtime": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.0.tgz", @@ -12166,7 +12310,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", - "dev": true, "dependencies": { "uc.micro": "^2.0.0" } @@ -12318,7 +12461,6 @@ "version": "14.1.0", "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", - "dev": true, "dependencies": { "argparse": "^2.0.1", "entities": "^4.4.0", @@ -13312,8 +13454,7 @@ "node_modules/mdurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", - "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", - "dev": true + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==" }, "node_modules/media-typer": { "version": "0.3.0", @@ -15687,6 +15828,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/oniguruma-to-es": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/oniguruma-to-es/-/oniguruma-to-es-0.4.1.tgz", + "integrity": "sha512-rNcEohFz095QKGRovP/yqPIKc+nP+Sjs4YTHMv33nMePGKrq/r2eu9Yh4646M5XluGJsUnmwoXuiXE69KDs+fQ==", + "license": "MIT", + "dependencies": { + "emoji-regex-xs": "^1.0.0", + "regex": "^5.0.0", + "regex-recursion": "^4.2.1" + } + }, "node_modules/open": { "version": "8.4.2", "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", @@ -17631,7 +17783,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", - "dev": true, "engines": { "node": ">=6" } @@ -18210,6 +18361,30 @@ "@babel/runtime": "^7.8.4" } }, + "node_modules/regex": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/regex/-/regex-5.0.2.tgz", + "integrity": "sha512-/pczGbKIQgfTMRV0XjABvc5RzLqQmwqxLHdQao2RTXPk+pmTXB2P0IaUHYdYyk412YLwUIkaeMd5T+RzVgTqnQ==", + "license": "MIT", + "dependencies": { + "regex-utilities": "^2.3.0" + } + }, + "node_modules/regex-recursion": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/regex-recursion/-/regex-recursion-4.2.1.tgz", + "integrity": "sha512-QHNZyZAeKdndD1G3bKAbBEKOSSK4KOHQrAJ01N1LJeb0SoH4DJIeFhp0uUpETgONifS4+P3sOgoA1dhzgrQvhA==", + "license": "MIT", + "dependencies": { + "regex-utilities": "^2.3.0" + } + }, + "node_modules/regex-utilities": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/regex-utilities/-/regex-utilities-2.3.0.tgz", + "integrity": "sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==", + "license": "MIT" + }, "node_modules/regexp.prototype.flags": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz", @@ -19251,15 +19426,26 @@ } }, "node_modules/shiki": { - "version": "0.14.7", - "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.14.7.tgz", - "integrity": "sha512-dNPAPrxSc87ua2sKJ3H5dQ/6ZaY8RNnaAqK+t0eG7p0Soi2ydiqbGOTaZCqaYvA/uZYfS1LJnemt3Q+mSfcPCg==", + "version": "1.23.1", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-1.23.1.tgz", + "integrity": "sha512-8kxV9TH4pXgdKGxNOkrSMydn1Xf6It8lsle0fiqxf7a1149K1WGtdOu3Zb91T5r1JpvRPxqxU3C2XdZZXQnrig==", "license": "MIT", "dependencies": { - "ansi-sequence-parser": "^1.1.0", - "jsonc-parser": "^3.2.0", - "vscode-oniguruma": "^1.7.0", - "vscode-textmate": "^8.0.0" + "@shikijs/core": "1.23.1", + "@shikijs/engine-javascript": "1.23.1", + "@shikijs/engine-oniguruma": "1.23.1", + "@shikijs/types": "1.23.1", + "@shikijs/vscode-textmate": "^9.3.0", + "@types/hast": "^3.0.4" + } + }, + "node_modules/shiki/node_modules/@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" } }, "node_modules/side-channel": { @@ -20188,24 +20374,25 @@ } }, "node_modules/typedoc": { - "version": "0.25.13", - "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.25.13.tgz", - "integrity": "sha512-pQqiwiJ+Z4pigfOnnysObszLiU3mVLWAExSPf+Mu06G/qsc3wzbuM56SZQvONhHLncLUhYzOVkjFFpFfL5AzhQ==", + "version": "0.26.11", + "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.26.11.tgz", + "integrity": "sha512-sFEgRRtrcDl2FxVP58Ze++ZK2UQAEvtvvH8rRlig1Ja3o7dDaMHmaBfvJmdGnNEFaLTpQsN8dpvZaTqJSu/Ugw==", "license": "Apache-2.0", "dependencies": { "lunr": "^2.3.9", - "marked": "^4.3.0", - "minimatch": "^9.0.3", - "shiki": "^0.14.7" + "markdown-it": "^14.1.0", + "minimatch": "^9.0.5", + "shiki": "^1.16.2", + "yaml": "^2.5.1" }, "bin": { "typedoc": "bin/typedoc" }, "engines": { - "node": ">= 16" + "node": ">= 18" }, "peerDependencies": { - "typescript": "4.6.x || 4.7.x || 4.8.x || 4.9.x || 5.0.x || 5.1.x || 5.2.x || 5.3.x || 5.4.x" + "typescript": "4.6.x || 4.7.x || 4.8.x || 4.9.x || 5.0.x || 5.1.x || 5.2.x || 5.3.x || 5.4.x || 5.5.x || 5.6.x" } }, "node_modules/typedoc/node_modules/brace-expansion": { @@ -20217,18 +20404,6 @@ "balanced-match": "^1.0.0" } }, - "node_modules/typedoc/node_modules/marked": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", - "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", - "license": "MIT", - "bin": { - "marked": "bin/marked.js" - }, - "engines": { - "node": ">= 12" - } - }, "node_modules/typedoc/node_modules/minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", @@ -20244,6 +20419,18 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/typedoc/node_modules/yaml": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.6.1.tgz", + "integrity": "sha512-7r0XPzioN/Q9kXBro/XPnA6kznR73DHq+GXh5ON7ZozRO6aMjbmiBuKste2wslTFkC5d1dw0GooOCepZXJ2SAg==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/typescript": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", @@ -20260,8 +20447,7 @@ "node_modules/uc.micro": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", - "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", - "dev": true + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==" }, "node_modules/unbox-primitive": { "version": "1.0.2", @@ -20788,18 +20974,6 @@ "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", "license": "MIT" }, - "node_modules/vscode-oniguruma": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz", - "integrity": "sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==", - "license": "MIT" - }, - "node_modules/vscode-textmate": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-8.0.0.tgz", - "integrity": "sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==", - "license": "MIT" - }, "node_modules/watchpack": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz", @@ -21840,19 +22014,27 @@ } }, "@apify/docusaurus-plugin-typedoc-api": { - "version": "4.2.6", - "resolved": "https://registry.npmjs.org/@apify/docusaurus-plugin-typedoc-api/-/docusaurus-plugin-typedoc-api-4.2.6.tgz", - "integrity": "sha512-M/rpRqbHZjL49xXtvdxdh/M7jSSR5lb3mjKERzKKnX6i92B5BRetL7Eb4qHre9QPQVijgzILhlhTi8otvr0LAw==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@apify/docusaurus-plugin-typedoc-api/-/docusaurus-plugin-typedoc-api-4.3.1.tgz", + "integrity": "sha512-tRi+Kly2tJurx9cie7bZQjZizzg1ogf22140+8mNvtkX2P3tC6UB/KcWgZ006SXcv69T07UDyVd+3D8hTdaibw==", "requires": { "@docusaurus/plugin-content-docs": "^3.5.2", "@docusaurus/types": "^3.5.2", "@docusaurus/utils": "^3.5.2", "@types/react": "^18.3.11", "@vscode/codicons": "^0.0.35", + "html-entities": "2.3.2", "marked": "^9.1.6", "marked-smartypants": "^1.1.5", - "typedoc": "^0.25.7", + "typedoc": "^0.26.11", "zx": "^8.1.4" + }, + "dependencies": { + "html-entities": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.3.2.tgz", + "integrity": "sha512-c3Ab/url5ksaT0WyleslpBEthOzWhrjQbg75y7XUsfSzi3Dgzt0l8w5e7DylRn15MTlMMD58dTfzddNS2kcAjQ==" + } } }, "@apify/eslint-config": { @@ -24425,6 +24607,72 @@ "integrity": "sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==", "dev": true }, + "@shikijs/core": { + "version": "1.23.1", + "resolved": "https://registry.npmjs.org/@shikijs/core/-/core-1.23.1.tgz", + "integrity": "sha512-NuOVgwcHgVC6jBVH5V7iblziw6iQbWWHrj5IlZI3Fqu2yx9awH7OIQkXIcsHsUmY19ckwSgUMgrqExEyP5A0TA==", + "requires": { + "@shikijs/engine-javascript": "1.23.1", + "@shikijs/engine-oniguruma": "1.23.1", + "@shikijs/types": "1.23.1", + "@shikijs/vscode-textmate": "^9.3.0", + "@types/hast": "^3.0.4", + "hast-util-to-html": "^9.0.3" + }, + "dependencies": { + "@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "requires": { + "@types/unist": "*" + } + } + } + }, + "@shikijs/engine-javascript": { + "version": "1.23.1", + "resolved": "https://registry.npmjs.org/@shikijs/engine-javascript/-/engine-javascript-1.23.1.tgz", + "integrity": "sha512-i/LdEwT5k3FVu07SiApRFwRcSJs5QM9+tod5vYCPig1Ywi8GR30zcujbxGQFJHwYD7A5BUqagi8o5KS+LEVgBg==", + "requires": { + "@shikijs/types": "1.23.1", + "@shikijs/vscode-textmate": "^9.3.0", + "oniguruma-to-es": "0.4.1" + } + }, + "@shikijs/engine-oniguruma": { + "version": "1.23.1", + "resolved": "https://registry.npmjs.org/@shikijs/engine-oniguruma/-/engine-oniguruma-1.23.1.tgz", + "integrity": "sha512-KQ+lgeJJ5m2ISbUZudLR1qHeH3MnSs2mjFg7bnencgs5jDVPeJ2NVDJ3N5ZHbcTsOIh0qIueyAJnwg7lg7kwXQ==", + "requires": { + "@shikijs/types": "1.23.1", + "@shikijs/vscode-textmate": "^9.3.0" + } + }, + "@shikijs/types": { + "version": "1.23.1", + "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-1.23.1.tgz", + "integrity": "sha512-98A5hGyEhzzAgQh2dAeHKrWW4HfCMeoFER2z16p5eJ+vmPeF6lZ/elEne6/UCU551F/WqkopqRsr1l2Yu6+A0g==", + "requires": { + "@shikijs/vscode-textmate": "^9.3.0", + "@types/hast": "^3.0.4" + }, + "dependencies": { + "@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "requires": { + "@types/unist": "*" + } + } + } + }, + "@shikijs/vscode-textmate": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/@shikijs/vscode-textmate/-/vscode-textmate-9.3.0.tgz", + "integrity": "sha512-jn7/7ky30idSkd/O5yDBfAnVt+JJpepofP/POZ1iMOxK59cOfqIgg/Dj0eFsjOTMw+4ycJN0uhZH/Eb0bs/EUA==" + }, "@sideway/address": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.5.tgz", @@ -25447,11 +25695,6 @@ "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" }, - "ansi-sequence-parser": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ansi-sequence-parser/-/ansi-sequence-parser-1.1.1.tgz", - "integrity": "sha512-vJXt3yiaUL4UU546s3rPXlsry/RnM730G1+HkpKE012AN0sx1eOrxSu95oKDIonskeLTijMgqWZ3uDEe3NFvyg==" - }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -26945,6 +27188,11 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, + "emoji-regex-xs": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex-xs/-/emoji-regex-xs-1.0.0.tgz", + "integrity": "sha512-LRlerrMYoIDrT6jgpeZ2YYl/L8EulRTt5hQcYjy5AInh7HWXKimpqx68aknBFpGL2+/IcogTcaydJEgaTmOpDg==" + }, "emojilib": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/emojilib/-/emojilib-2.4.0.tgz", @@ -28655,6 +28903,54 @@ } } }, + "hast-util-to-html": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.3.tgz", + "integrity": "sha512-M17uBDzMJ9RPCqLMO92gNNUDuBSq10a25SDBI08iCCxmorf4Yy6sYHK57n9WAbRAAaU+DuR4W6GN9K4DFZesYg==", + "requires": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-whitespace": "^3.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "property-information": "^6.0.0", + "space-separated-tokens": "^2.0.0", + "stringify-entities": "^4.0.0", + "zwitch": "^2.0.4" + }, + "dependencies": { + "@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "requires": { + "@types/unist": "*" + } + }, + "@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==" + }, + "comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==" + }, + "property-information": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-6.5.0.tgz", + "integrity": "sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig==" + }, + "space-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", + "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==" + } + } + }, "hast-util-to-jsx-runtime": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.0.tgz", @@ -29770,7 +30066,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-5.0.0.tgz", "integrity": "sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==", - "dev": true, "requires": { "uc.micro": "^2.0.0" } @@ -29881,7 +30176,6 @@ "version": "14.1.0", "resolved": "https://registry.npmjs.org/markdown-it/-/markdown-it-14.1.0.tgz", "integrity": "sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==", - "dev": true, "requires": { "argparse": "^2.0.1", "entities": "^4.4.0", @@ -30559,8 +30853,7 @@ "mdurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-2.0.0.tgz", - "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==", - "dev": true + "integrity": "sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==" }, "media-typer": { "version": "0.3.0", @@ -31832,6 +32125,16 @@ "mimic-fn": "^2.1.0" } }, + "oniguruma-to-es": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/oniguruma-to-es/-/oniguruma-to-es-0.4.1.tgz", + "integrity": "sha512-rNcEohFz095QKGRovP/yqPIKc+nP+Sjs4YTHMv33nMePGKrq/r2eu9Yh4646M5XluGJsUnmwoXuiXE69KDs+fQ==", + "requires": { + "emoji-regex-xs": "^1.0.0", + "regex": "^5.0.0", + "regex-recursion": "^4.2.1" + } + }, "open": { "version": "8.4.2", "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", @@ -32943,8 +33246,7 @@ "punycode.js": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", - "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", - "dev": true + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==" }, "pupa": { "version": "3.1.0", @@ -33356,6 +33658,27 @@ "@babel/runtime": "^7.8.4" } }, + "regex": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/regex/-/regex-5.0.2.tgz", + "integrity": "sha512-/pczGbKIQgfTMRV0XjABvc5RzLqQmwqxLHdQao2RTXPk+pmTXB2P0IaUHYdYyk412YLwUIkaeMd5T+RzVgTqnQ==", + "requires": { + "regex-utilities": "^2.3.0" + } + }, + "regex-recursion": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/regex-recursion/-/regex-recursion-4.2.1.tgz", + "integrity": "sha512-QHNZyZAeKdndD1G3bKAbBEKOSSK4KOHQrAJ01N1LJeb0SoH4DJIeFhp0uUpETgONifS4+P3sOgoA1dhzgrQvhA==", + "requires": { + "regex-utilities": "^2.3.0" + } + }, + "regex-utilities": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/regex-utilities/-/regex-utilities-2.3.0.tgz", + "integrity": "sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==" + }, "regexp.prototype.flags": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz", @@ -34122,14 +34445,26 @@ } }, "shiki": { - "version": "0.14.7", - "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.14.7.tgz", - "integrity": "sha512-dNPAPrxSc87ua2sKJ3H5dQ/6ZaY8RNnaAqK+t0eG7p0Soi2ydiqbGOTaZCqaYvA/uZYfS1LJnemt3Q+mSfcPCg==", + "version": "1.23.1", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-1.23.1.tgz", + "integrity": "sha512-8kxV9TH4pXgdKGxNOkrSMydn1Xf6It8lsle0fiqxf7a1149K1WGtdOu3Zb91T5r1JpvRPxqxU3C2XdZZXQnrig==", "requires": { - "ansi-sequence-parser": "^1.1.0", - "jsonc-parser": "^3.2.0", - "vscode-oniguruma": "^1.7.0", - "vscode-textmate": "^8.0.0" + "@shikijs/core": "1.23.1", + "@shikijs/engine-javascript": "1.23.1", + "@shikijs/engine-oniguruma": "1.23.1", + "@shikijs/types": "1.23.1", + "@shikijs/vscode-textmate": "^9.3.0", + "@types/hast": "^3.0.4" + }, + "dependencies": { + "@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "requires": { + "@types/unist": "*" + } + } } }, "side-channel": { @@ -34786,14 +35121,15 @@ } }, "typedoc": { - "version": "0.25.13", - "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.25.13.tgz", - "integrity": "sha512-pQqiwiJ+Z4pigfOnnysObszLiU3mVLWAExSPf+Mu06G/qsc3wzbuM56SZQvONhHLncLUhYzOVkjFFpFfL5AzhQ==", + "version": "0.26.11", + "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.26.11.tgz", + "integrity": "sha512-sFEgRRtrcDl2FxVP58Ze++ZK2UQAEvtvvH8rRlig1Ja3o7dDaMHmaBfvJmdGnNEFaLTpQsN8dpvZaTqJSu/Ugw==", "requires": { "lunr": "^2.3.9", - "marked": "^4.3.0", - "minimatch": "^9.0.3", - "shiki": "^0.14.7" + "markdown-it": "^14.1.0", + "minimatch": "^9.0.5", + "shiki": "^1.16.2", + "yaml": "^2.5.1" }, "dependencies": { "brace-expansion": { @@ -34804,11 +35140,6 @@ "balanced-match": "^1.0.0" } }, - "marked": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", - "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==" - }, "minimatch": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", @@ -34816,6 +35147,11 @@ "requires": { "brace-expansion": "^2.0.1" } + }, + "yaml": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.6.1.tgz", + "integrity": "sha512-7r0XPzioN/Q9kXBro/XPnA6kznR73DHq+GXh5ON7ZozRO6aMjbmiBuKste2wslTFkC5d1dw0GooOCepZXJ2SAg==" } } }, @@ -34828,8 +35164,7 @@ "uc.micro": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/uc.micro/-/uc.micro-2.1.0.tgz", - "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==", - "dev": true + "integrity": "sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==" }, "unbox-primitive": { "version": "1.0.2", @@ -35192,16 +35527,6 @@ } } }, - "vscode-oniguruma": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz", - "integrity": "sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==" - }, - "vscode-textmate": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-8.0.0.tgz", - "integrity": "sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==" - }, "watchpack": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz", diff --git a/website/package.json b/website/package.json index aa00dd61..b87db6c5 100644 --- a/website/package.json +++ b/website/package.json @@ -22,7 +22,7 @@ }, "dependencies": { "@apify/docs-theme": "^1.0.132", - "@apify/docusaurus-plugin-typedoc-api": "^4.2.6", + "@apify/docusaurus-plugin-typedoc-api": "^4.3.1", "@docusaurus/core": "^3.5.2", "@docusaurus/plugin-client-redirects": "^3.5.2", "@docusaurus/preset-classic": "^3.5.2", @@ -48,5 +48,6 @@ "path-browserify": "^1.0.1", "prettier": "^2.8.4", "rimraf": "^4.1.3" - } + }, + "packageManager": "yarn@1.22.21+sha1.1959a18351b811cdeedbd484a8f86c3cc3bbaf72" } diff --git a/website/pydoc-markdown.yml b/website/pydoc-markdown.yml deleted file mode 100644 index 200eebb4..00000000 --- a/website/pydoc-markdown.yml +++ /dev/null @@ -1,14 +0,0 @@ -loaders: - - type: python - search_path: [../src] -processors: - - type: filter - skip_empty_modules: true - documented_only: false - - type: crossref -renderer: - type: docusaurus - docs_base_path: docs - relative_output_path: reference - relative_sidebar_path: sidebar.json - sidebar_top_level_label: null diff --git a/website/pydoc-markdown/__init__.py b/website/pydoc-markdown/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/website/pydoc-markdown/generate_ast.py b/website/pydoc-markdown/generate_ast.py deleted file mode 100644 index 3eb9ff07..00000000 --- a/website/pydoc-markdown/generate_ast.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -Replaces the default pydoc-markdown shell script with a custom Python script calling the pydoc-markdown API directly. - -This script generates an AST from the Python source code in the `src` directory and prints it as a JSON object. -""" - -from pydoc_markdown.interfaces import Context -from pydoc_markdown.contrib.loaders.python import PythonLoader -from pydoc_markdown.contrib.processors.filter import FilterProcessor -from pydoc_markdown.contrib.processors.crossref import CrossrefProcessor -from pydoc_markdown.contrib.renderers.markdown import MarkdownReferenceResolver -from google_docstring_processor import ApifyGoogleProcessor -from docspec import dump_module - -import json -import os - -project_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../../src') - -context = Context(directory='.') -loader = PythonLoader(search_path=[project_path]) -filter = FilterProcessor( - documented_only=False, - skip_empty_modules=False, -) -crossref = CrossrefProcessor() -google = ApifyGoogleProcessor() - -loader.init(context) -filter.init(context) -google.init(context) -crossref.init(context) - -processors = [filter, google, crossref] - -dump = [] - -modules = list(loader.load()) - -for processor in processors: - processor.process(modules, None) - -for module in modules: - dump.append(dump_module(module)) - -print(json.dumps(dump, indent=4)) diff --git a/website/pydoc-markdown/google_docstring_processor.py b/website/pydoc-markdown/google_docstring_processor.py deleted file mode 100644 index 0e01b303..00000000 --- a/website/pydoc-markdown/google_docstring_processor.py +++ /dev/null @@ -1,183 +0,0 @@ -# -*- coding: utf8 -*- -# Copyright (c) 2019 Niklas Rosenstein -# !!! Modified 2024 Jindřich Bär -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import dataclasses -import re -import typing as t - -import docspec - -from pydoc_markdown.contrib.processors.sphinx import generate_sections_markdown -from pydoc_markdown.interfaces import Processor, Resolver - -import json - - -@dataclasses.dataclass -class ApifyGoogleProcessor(Processor): - """ - This class implements the preprocessor for Google and PEP 257 docstrings. It converts - docstrings formatted in the Google docstyle to Markdown syntax. - - References: - - * https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html - * https://www.python.org/dev/peps/pep-0257/ - - Example: - - ``` - Attributes: - module_level_variable1 (int): Module level variables may be documented in - either the ``Attributes`` section of the module docstring, or in an - inline docstring immediately following the variable. - - Either form is acceptable, but the two should not be mixed. Choose - one convention to document module level variables and be consistent - with it. - - Todo: - * For module TODOs - * You have to also use ``sphinx.ext.todo`` extension - ``` - - Renders as: - - Attributes: - module_level_variable1 (int): Module level variables may be documented in - either the ``Attributes`` section of the module docstring, or in an - inline docstring immediately following the variable. - - Either form is acceptable, but the two should not be mixed. Choose - one convention to document module level variables and be consistent - with it. - - Todo: - * For module TODOs - * You have to also use ``sphinx.ext.todo`` extension - - @doc:fmt:google - """ - - _param_res = [ - re.compile(r"^(?P\S+):\s+(?P.+)$"), - re.compile(r"^(?P\S+)\s+\((?P[^)]+)\):\s+(?P.+)$"), - re.compile(r"^(?P\S+)\s+--\s+(?P.+)$"), - re.compile(r"^(?P\S+)\s+\{\[(?P\S+)\]\}\s+--\s+(?P.+)$"), - re.compile(r"^(?P\S+)\s+\{(?P\S+)\}\s+--\s+(?P.+)$"), - ] - - _keywords_map = { - "Args:": "Arguments", - "Arguments:": "Arguments", - "Attributes:": "Attributes", - "Example:": "Example", - "Examples:": "Examples", - "Keyword Args:": "Arguments", - "Keyword Arguments:": "Arguments", - "Methods:": "Methods", - "Note:": "Notes", - "Notes:": "Notes", - "Other Parameters:": "Arguments", - "Parameters:": "Arguments", - "Return:": "Returns", - "Returns:": "Returns", - "Raises:": "Raises", - "References:": "References", - "See Also:": "See Also", - "Todo:": "Todo", - "Warning:": "Warnings", - "Warnings:": "Warnings", - "Warns:": "Warns", - "Yield:": "Yields", - "Yields:": "Yields", - } - - def check_docstring_format(self, docstring: str) -> bool: - for section_name in self._keywords_map: - if section_name in docstring: - return True - return False - - def process(self, modules: t.List[docspec.Module], resolver: t.Optional[Resolver]) -> None: - docspec.visit(modules, self._process) - - def _process(self, node: docspec.ApiObject): - if not node.docstring: - return - - lines = [] - sections = [] - current_lines: t.List[str] = [] - in_codeblock = False - keyword = None - multiline_argument_offset = -1 - - def _commit(): - if keyword: - sections.append({keyword: list(current_lines)}) - else: - lines.extend(current_lines) - current_lines.clear() - - for line in node.docstring.content.split("\n"): - multiline_argument_offset += 1 - if line.lstrip().startswith("```"): - in_codeblock = not in_codeblock - current_lines.append(line) - continue - - if in_codeblock: - current_lines.append(line) - continue - - line = line.strip() - if line in self._keywords_map: - _commit() - keyword = self._keywords_map[line] - continue - - if keyword is None: - lines.append(line) - continue - - for param_re in self._param_res: - param_match = param_re.match(line) - if param_match: - current_lines.append(param_match.groupdict()) - multiline_argument_offset = 0 - break - - if not param_match: - if multiline_argument_offset == 1: - current_lines[-1]["desc"] += "\n" + line - multiline_argument_offset = 0 - else: - current_lines.append(line) - - _commit() - node.docstring.content = json.dumps({ - "text": "\n".join(lines), - "sections": sections, - }, indent=None) - - diff --git a/website/transformDocs.js b/website/transformDocs.js deleted file mode 100644 index 71971f34..00000000 --- a/website/transformDocs.js +++ /dev/null @@ -1,382 +0,0 @@ -/* eslint-disable */ - -const fs = require('fs'); -const { spawnSync } = require('child_process'); - -const moduleShortcuts = require('./module_shortcuts.json'); - -const REPO_ROOT_PLACEHOLDER = 'REPO_ROOT_PLACEHOLDER'; - -const APIFY_CLIENT_REPO_URL = 'https://github.com/apify/apify-client-python'; -const APIFY_SDK_REPO_URL = 'https://github.com/apify/apify-sdk-python'; -const APIFY_SHARED_REPO_URL = 'https://github.com/apify/apify-shared-python'; - -const REPO_URL_PER_PACKAGE = { - 'apify': APIFY_SDK_REPO_URL, - 'apify_client': APIFY_CLIENT_REPO_URL, - 'apify_shared': APIFY_SHARED_REPO_URL, -}; - -// For each package, get the installed version, and set the tag to the corresponding version -const TAG_PER_PACKAGE = {}; -for (const pkg of ['apify', 'apify_client', 'apify_shared']) { - const spawnResult = spawnSync('python', ['-c', `import ${pkg}; print(${pkg}.__version__)`]); - if (spawnResult.status === 0) { - TAG_PER_PACKAGE[pkg] = `v${spawnResult.stdout.toString().trim()}`; - } -} - -// For the current package, set the tag to 'master' -const thisPackagePyprojectToml = fs.readFileSync('../pyproject.toml', 'utf8'); -const thisPackageName = thisPackagePyprojectToml.match(/^name = "(.+)"$/m)[1]; -TAG_PER_PACKAGE[thisPackageName] = 'master'; - - -// Taken from https://github.com/TypeStrong/typedoc/blob/v0.23.24/src/lib/models/reflections/kind.ts, modified -const TYPEDOC_KINDS = { - 'class': { - kind: 128, - kindString: 'Class', - }, - 'function': { - kind: 2048, - kindString: 'Method', - }, - 'data': { - kind: 1024, - kindString: 'Property', - }, - 'enum': { - kind: 8, - kindString: 'Enumeration', - }, - 'enumValue': { - kind: 16, - kindString: 'Enumeration Member', - }, -} - -const GROUP_ORDER = [ - 'Classes', - 'Data structures', - 'Scrapy Integration', - 'Constants', - 'Enumeration Members' -]; - -const groupSort = (g1, g2) => { - if(GROUP_ORDER.includes(g1) && GROUP_ORDER.includes(g2)){ - return GROUP_ORDER.indexOf(g1) - GROUP_ORDER.indexOf(g2) - } - return g1.localeCompare(g2); -}; - -function getGroupName(object) { - const groupPredicates = { - 'Scrapy integration': (x) => ['ApifyScheduler', 'ActorDatasetPushPipeline', 'ApifyHttpProxyMiddleware', 'apply_apify_settings'].includes(x.name), - 'Data structures': (x) => ['BaseModel', 'TypedDict'].some(base => x?.bases?.includes(base)) || x?.decorations?.some(d => d.name === 'dataclass'), - 'Errors': (x) => x.name.toLowerCase().includes('error'), - 'Classes': (x) => x.kindString === 'Class', - 'Main Clients': (x) => ['ApifyClient', 'ApifyClientAsync'].includes(x.name), - 'Async Resource Clients': (x) => x.name.toLowerCase().includes('async'), - 'Resource Clients': (x) => x.kindString === 'Class' && x.name.toLowerCase().includes('client'), - 'Methods': (x) => x.kindString === 'Method', - 'Constructors': (x) => x.kindString === 'Constructor', - 'Properties': (x) => x.kindString === 'Property', - 'Constants': (x) => x.kindString === 'Enumeration', - 'Enumeration members': (x) => x.kindString === 'Enumeration Member', - }; - - const [group] = Object.entries(groupPredicates).find( - ([_, predicate]) => predicate(object) - ) ?? ['Other']; - - return group; -} - -// Strips the Optional[] type from the type string, and replaces generic types with just the main type -function getBaseType(type) { - return type?.replace(/Optional\[(.*)\]/g, '$1').replace('ListPage[Dict]', 'ListPage'); -} - -// Returns whether a type is a custom class, or a primitive type -function isCustomClass(type) { - return !['dict', 'list', 'str', 'int', 'float', 'bool'].includes(type.toLowerCase()); -} - -// Infer the Typedoc type from the docspec type -function inferTypedocType(docspecType) { - const typeWithoutOptional = getBaseType(docspecType); - if (!typeWithoutOptional) { - return undefined; - } - - // Typically, if a type is a custom class, it will be a reference in Typedoc - return isCustomClass(typeWithoutOptional) ? { - type: 'reference', - name: docspecType - } : { - type: 'intrinsic', - name: docspecType, - } -} - -// Sorts the groups of a Typedoc member, and sorts the children of each group -function sortChildren(typedocMember) { - for (let group of typedocMember.groups) { - group.children - .sort((a, b) => { - const firstName = typedocMember.children.find(x => x.id === a).name; - const secondName = typedocMember.children.find(x => x.id === b).name; - return firstName.localeCompare(secondName); - }); - } - typedocMember.groups.sort((a, b) => groupSort(a.title, b.title)); -} - -// Objects with decorators named 'ignore_docs' or with empty docstrings will be ignored -function isHidden(member) { - return member.decorations?.some(d => d.name === 'ignore_docs') - || member.name === 'ignore_docs'; -} - -// Each object in the Typedoc structure has an unique ID, -// we'll just increment it for each object we convert -let oid = 1; - -// Converts a docspec object to a Typedoc object, including all its children -function convertObject(obj, parent, module) { - const rootModuleName = module.name.split('.')[0]; - for (let member of obj.members ?? []) { - let typedocKind = TYPEDOC_KINDS[member.type]; - - if(member.bases?.includes('Enum')) { - typedocKind = TYPEDOC_KINDS['enum']; - } - - if (member.decorations?.some(d => d.name === 'dualproperty')) { - typedocKind = TYPEDOC_KINDS['data']; - } - - let typedocType = inferTypedocType(member.datatype); - - if(parent.kindString === 'Enumeration') { - typedocKind = TYPEDOC_KINDS['enumValue']; - typedocType = { - type: 'literal', - value: member.value, - } - } - - if(member.type in TYPEDOC_KINDS && !isHidden(member)) { - // Get the URL of the member in GitHub - const repoBaseUrl = `${REPO_URL_PER_PACKAGE[rootModuleName]}/blob/${TAG_PER_PACKAGE[rootModuleName]}`; - const filePathInRepo = member.location.filename.replace(REPO_ROOT_PLACEHOLDER, ''); - const fileGitHubUrl = member.location.filename.replace(REPO_ROOT_PLACEHOLDER, repoBaseUrl); - const memberGitHubUrl = `${fileGitHubUrl}#L${member.location.lineno}`; - - // Get the module name of the member, and check if it has a shortcut (reexport from an ancestor module) - const fullName = `${module.name}.${member.name}`; - let moduleName = module.name; - if (fullName in moduleShortcuts) { - moduleName = moduleShortcuts[fullName].replace(`.${member.name}`, ''); - } - - if(member.name === 'Actor' || (member.name.endsWith('Client') && !member.name.endsWith('StorageClient')) || member.name === 'ListPage') { - continue; - } - - if (member.name === '_ActorType') { - member.name = 'Actor'; - } - - let docstring = { text: member.docstring?.content ?? '' }; - try { - docstring = JSON.parse(docstring.text); - - docstring.args = docstring.sections.find((section) => Object.keys(section)[0] === 'Arguments')['Arguments'] ?? []; - - docstring.args = docstring.args.reduce((acc, arg) => { - acc[arg.param] = arg.desc; - return acc; - }, {}); - - docstring.returns = docstring.sections.find((section) => Object.keys(section)[0] === 'Returns')['Returns'] ?? []; - - docstring.returns = docstring.returns.join('\n'); - } catch { - // Do nothing - } - - // Create the Typedoc member object - let typedocMember = { - id: oid++, - name: member.name, - module: moduleName, // This is an extension to the original Typedoc structure, to support showing where the member is exported from - ...typedocKind, - flags: {}, - bases: member.bases, - comment: member.docstring ? { - summary: [{ - kind: 'text', - text: docstring.text, - }], - } : undefined, - type: typedocType, - children: [], - groups: [], - sources: [{ - filename: filePathInRepo, - line: member.location.lineno, - character: 1, - url: memberGitHubUrl, - }], - }; - - if(!GROUP_ORDER.includes(getGroupName(typedocMember)) && parent.kindString === 'Project'){ - continue; - } - - if(typedocMember.kindString === 'Method') { - typedocMember.signatures = [{ - id: oid++, - name: member.name, - modifiers: member.modifiers ?? [], - kind: 4096, - kindString: 'Call signature', - flags: {}, - comment: docstring.text ? { - summary: [{ - kind: 'text', - text: docstring?.text, - }], - blockTags: docstring?.returns ? [ - { tag: '@returns', content: [{ kind: 'text', text: docstring.returns }] }, - ] : undefined, - } : undefined, - type: inferTypedocType(member.return_type), - parameters: member.args.filter((arg) => (arg.name !== 'self' && arg.name !== 'cls')).map((arg) => ({ - id: oid++, - name: arg.name, - kind: 32768, - kindString: 'Parameter', - flags: { - isOptional: arg.datatype?.includes('Optional') ? 'true' : undefined, - 'keyword-only': arg.type === 'KEYWORD_ONLY' ? 'true' : undefined, - }, - type: inferTypedocType(arg.datatype), - comment: docstring.args?.[arg.name] ? { - summary: [{ - kind: 'text', - text: docstring.args[arg.name] - }] - } : undefined, - defaultValue: arg.default_value, - })), - }]; - } - - if(typedocMember.name === '__init__') { - typedocMember.kind = 512; - typedocMember.kindString = 'Constructor'; - } - - convertObject(member, typedocMember, module); - - const groupName = getGroupName(typedocMember); - - const group = parent.groups.find((g) => g.title === groupName); - if (group) { - group.children.push(typedocMember.id); - } else { - parent.groups.push({ - title: groupName, - children: [typedocMember.id], - }); - } - - sortChildren(typedocMember); - parent.children.push(typedocMember); - } - } -} - -function main() { - // Root object of the Typedoc structure - const typedocApiReference = { - 'id': 0, - 'name': 'apify-client', - 'kind': 1, - 'kindString': 'Project', - 'flags': {}, - 'originalName': '', - 'children': [], - 'groups': [], - 'sources': [ - { - 'fileName': 'src/index.ts', - 'line': 1, - 'character': 0, - 'url': `http://example.com/blob/123456/src/dummy.py`, - } - ] - }; - - // Load the docspec dump files of this module and of apify-shared - const thisPackageDocspecDump = fs.readFileSync('docspec-dump.jsonl', 'utf8'); - const thisPackageModules = JSON.parse(thisPackageDocspecDump) - - const apifySharedDocspecDump = fs.readFileSync('apify-shared-docspec-dump.jsonl', 'utf8'); - const apifySharedModules = apifySharedDocspecDump.split('\n').filter((line) => line !== ''); - - // Convert all the modules, store them in the root object - for (const module of thisPackageModules) { - convertObject(module, typedocApiReference, module); - }; - - // Recursively fix references (collect names->ids of all the named entities and then inject those in the reference objects) - const namesToIds = {}; - function collectIds(obj) { - for (const child of obj.children ?? []) { - namesToIds[child.name] = child.id; - collectIds(child); - } - } - collectIds(typedocApiReference); - - function fixRefs(obj) { - for (const child of obj.children ?? []) { - if (child.type?.type === 'reference') { - child.type.id = namesToIds[child.type.name]; - } - if (child.signatures) { - for (const sig of child.signatures) { - for (const param of sig.parameters ?? []) { - if (param.type?.type === 'reference') { - param.type.id = namesToIds[param.type.name]; - } - } - if (sig.type?.type === 'reference') { - sig.type.id = namesToIds[sig.type.name]; - } - } - } - fixRefs(child); - } - } - fixRefs(typedocApiReference); - - // Sort the children of the root object - sortChildren(typedocApiReference); - - // Write the Typedoc structure to the output file - fs.writeFileSync('./api-typedoc-generated.json', JSON.stringify(typedocApiReference, null, 4)); -} - -if (require.main === module) { - main(); -} - -module.exports = { - groupSort, -} From cfe0816591176384a506f9f36ebc33bec514cb26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Wed, 20 Nov 2024 10:46:11 +0100 Subject: [PATCH 2/6] docs: reexport storages documentation from `crawlee-python` --- website/docusaurus.config.js | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index f0e400ed..52519c7e 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -28,6 +28,7 @@ module.exports = { projectName: 'apify-sdk-python', scripts: ['/js/custom.js'], favicon: 'img/favicon.ico', + githubHost: 'github.com', onBrokenLinks: /** @type {import('@docusaurus/types').ReportingSeverity} */ ('warn'), onBrokenMarkdownLinks: @@ -103,11 +104,20 @@ module.exports = { pythonModulePath: path.join(__dirname, '../src/apify'), moduleShortcutsPath: path.join(__dirname, '/module_shortcuts.json'), }, - // reexports: [ - // { - // url: 'http://localhost:3000/python/api/class/Dataset', - // }, - // ], + reexports: [ + { + url: 'https://crawlee.dev/python/api/class/Dataset', + group: 'Classes', + }, + { + url: 'https://crawlee.dev/python/api/class/KeyValueStore', + group: 'Classes', + }, + { + url: 'https://crawlee.dev/python/api/class/RequestQueue', + group: 'Classes', + }, + ], }, ], ...config.plugins, From 7b2c453ee75ef04831f25e169c57328da509a126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Wed, 20 Nov 2024 10:58:24 +0100 Subject: [PATCH 3/6] chore: run formatter --- src/apify/_actor.py | 1 + src/apify/_configuration.py | 1 + src/apify/_models.py | 6 ++++++ src/apify/_platform_event_manager.py | 11 +++++++++++ src/apify/_proxy_configuration.py | 1 + src/apify/_utils.py | 3 +++ .../apify_storage_client/_apify_storage_client.py | 1 + 7 files changed, 24 insertions(+) diff --git a/src/apify/_actor.py b/src/apify/_actor.py index 57bffca4..41c599a0 100644 --- a/src/apify/_actor.py +++ b/src/apify/_actor.py @@ -38,6 +38,7 @@ MainReturnType = TypeVar('MainReturnType') + @docs_name('Actor') @docs_group('Classes') class _ActorType: diff --git a/src/apify/_configuration.py b/src/apify/_configuration.py index 40532763..306e9d63 100644 --- a/src/apify/_configuration.py +++ b/src/apify/_configuration.py @@ -13,6 +13,7 @@ from apify._utils import docs_group + @docs_group('Classes') class Configuration(CrawleeConfiguration): """A class for specifying the configuration of an Actor. diff --git a/src/apify/_models.py b/src/apify/_models.py index 1f57ab33..d4450790 100644 --- a/src/apify/_models.py +++ b/src/apify/_models.py @@ -12,6 +12,7 @@ from apify._utils import docs_group + @docs_group('Data structures') class Webhook(BaseModel): __model_config__ = ConfigDict(populate_by_name=True) @@ -30,12 +31,14 @@ class Webhook(BaseModel): Field(description='Template for the payload sent by the webook'), ] = None + @docs_group('Data structures') class ActorRunMeta(BaseModel): __model_config__ = ConfigDict(populate_by_name=True) origin: Annotated[MetaOrigin, Field()] + @docs_group('Data structures') class ActorRunStats(BaseModel): __model_config__ = ConfigDict(populate_by_name=True) @@ -56,6 +59,7 @@ class ActorRunStats(BaseModel): metamorph: Annotated[int | None, Field(alias='metamorph')] = None compute_units: Annotated[float, Field(alias='computeUnits')] + @docs_group('Data structures') class ActorRunOptions(BaseModel): __model_config__ = ConfigDict(populate_by_name=True) @@ -65,6 +69,7 @@ class ActorRunOptions(BaseModel): memory_mbytes: Annotated[int, Field(alias='memoryMbytes')] disk_mbytes: Annotated[int, Field(alias='diskMbytes')] + @docs_group('Data structures') class ActorRunUsage(BaseModel): __model_config__ = ConfigDict(populate_by_name=True) @@ -82,6 +87,7 @@ class ActorRunUsage(BaseModel): proxy_residential_transfer_gbytes: Annotated[float | None, Field(alias='PROXY_RESIDENTIAL_TRANSFER_GBYTES')] = None proxy_serps: Annotated[float | None, Field(alias='PROXY_SERPS')] = None + @docs_group('Data structures') class ActorRun(BaseModel): __model_config__ = ConfigDict(populate_by_name=True) diff --git a/src/apify/_platform_event_manager.py b/src/apify/_platform_event_manager.py index 9de6cfdb..a7af4772 100644 --- a/src/apify/_platform_event_manager.py +++ b/src/apify/_platform_event_manager.py @@ -31,11 +31,13 @@ __all__ = ['EventManager', 'LocalEventManager', 'PlatformEventManager'] + @docs_group('Data structures') class PersistStateEvent(BaseModel): name: Literal[Event.PERSIST_STATE] data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))] + @docs_group('Data structures') class SystemInfoEventData(BaseModel): mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')] @@ -62,26 +64,31 @@ def to_crawlee_format(self) -> EventSystemInfoData: } ) + @docs_group('Data structures') class SystemInfoEvent(BaseModel): name: Literal[Event.SYSTEM_INFO] data: SystemInfoEventData + @docs_group('Data structures') class MigratingEvent(BaseModel): name: Literal[Event.MIGRATING] data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)] + @docs_group('Data structures') class AbortingEvent(BaseModel): name: Literal[Event.ABORTING] data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)] + @docs_group('Data structures') class ExitEvent(BaseModel): name: Literal[Event.EXIT] data: Annotated[EventExitData, Field(default_factory=EventExitData)] + @docs_group('Data structures') class EventWithoutData(BaseModel): name: Literal[ @@ -94,11 +101,13 @@ class EventWithoutData(BaseModel): ] data: Any = None + @docs_group('Data structures') class DeprecatedEvent(BaseModel): name: Literal['cpuInfo'] data: Annotated[dict[str, Any], Field(default_factory=dict)] + @docs_group('Data structures') class UnknownEvent(BaseModel): name: str @@ -125,6 +134,8 @@ class UnknownEvent(BaseModel): UnknownEvent, ] ) + + @docs_group('Classes') class PlatformEventManager(EventManager): """A class for managing Actor events. diff --git a/src/apify/_proxy_configuration.py b/src/apify/_proxy_configuration.py index e0bf2977..07cf1f26 100644 --- a/src/apify/_proxy_configuration.py +++ b/src/apify/_proxy_configuration.py @@ -89,6 +89,7 @@ class ProxyInfo(CrawleeProxyInfo): This parameter is optional, by default, the proxy uses all available proxy servers from all countries. """ + @docs_group('Classes') class ProxyConfiguration(CrawleeProxyConfiguration): """Configures a connection to a proxy server with the provided options. diff --git a/src/apify/_utils.py b/src/apify/_utils.py index 4577a351..c4ac0df9 100644 --- a/src/apify/_utils.py +++ b/src/apify/_utils.py @@ -27,8 +27,10 @@ def get_system_info() -> dict: def is_running_in_ipython() -> bool: return getattr(builtins, '__IPYTHON__', False) + GroupName = Literal['Classes', 'Abstract classes', 'Data structures', 'Errors', 'Functions'] + def docs_group(group_name: GroupName) -> Callable: # noqa: ARG001 """Decorator to mark symbols for rendering and grouping in documentation. @@ -41,6 +43,7 @@ def wrapper(func: Callable) -> Callable: return wrapper + def docs_name(symbol_name: str) -> Callable: """Decorator for renaming symbols in documentation. This changes the rendered name of the symbol only in the web documentation. diff --git a/src/apify/apify_storage_client/_apify_storage_client.py b/src/apify/apify_storage_client/_apify_storage_client.py index b436e376..a8f550d9 100644 --- a/src/apify/apify_storage_client/_apify_storage_client.py +++ b/src/apify/apify_storage_client/_apify_storage_client.py @@ -14,6 +14,7 @@ from apify._utils import docs_group + @docs_group('Classes') class ApifyStorageClient(BaseStorageClient): """A storage client implementation based on the Apify platform storage.""" From 93dffc4b0b621692270331bf36be3b44e99100d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Wed, 20 Nov 2024 11:07:48 +0100 Subject: [PATCH 4/6] chore: run ruff --- src/apify/_actor.py | 2 +- src/apify/_platform_event_manager.py | 3 +-- src/apify/_proxy_configuration.py | 3 +-- src/apify/_utils.py | 9 +++++---- src/apify/apify_storage_client/_apify_storage_client.py | 3 +-- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/apify/_actor.py b/src/apify/_actor.py index 41c599a0..4920c593 100644 --- a/src/apify/_actor.py +++ b/src/apify/_actor.py @@ -22,7 +22,7 @@ from apify._models import ActorRun from apify._platform_event_manager import EventManager, LocalEventManager, PlatformEventManager from apify._proxy_configuration import ProxyConfiguration -from apify._utils import get_system_info, is_running_in_ipython, docs_group, docs_name +from apify._utils import docs_group, docs_name, get_system_info, is_running_in_ipython from apify.apify_storage_client import ApifyStorageClient from apify.log import _configure_logging, logger from apify.storages import Dataset, KeyValueStore, RequestQueue diff --git a/src/apify/_platform_event_manager.py b/src/apify/_platform_event_manager.py index a7af4772..c0c8b67b 100644 --- a/src/apify/_platform_event_manager.py +++ b/src/apify/_platform_event_manager.py @@ -19,9 +19,8 @@ EventSystemInfoData, ) -from apify.log import logger - from apify._utils import docs_group +from apify.log import logger if TYPE_CHECKING: from types import TracebackType diff --git a/src/apify/_proxy_configuration.py b/src/apify/_proxy_configuration.py index 07cf1f26..6f4195d0 100644 --- a/src/apify/_proxy_configuration.py +++ b/src/apify/_proxy_configuration.py @@ -16,9 +16,8 @@ from crawlee.proxy_configuration import _NewUrlFunction from apify._configuration import Configuration -from apify.log import logger - from apify._utils import docs_group +from apify.log import logger if TYPE_CHECKING: from apify_client import ApifyClientAsync diff --git a/src/apify/_utils.py b/src/apify/_utils.py index c4ac0df9..fc278072 100644 --- a/src/apify/_utils.py +++ b/src/apify/_utils.py @@ -1,10 +1,9 @@ from __future__ import annotations -from typing import Callable, Literal - import builtins import sys from importlib import metadata +from typing import Callable, Literal def get_system_info() -> dict: @@ -44,8 +43,10 @@ def wrapper(func: Callable) -> Callable: return wrapper -def docs_name(symbol_name: str) -> Callable: - """Decorator for renaming symbols in documentation. This changes the rendered name of the symbol only in the web documentation. +def docs_name(symbol_name: str) -> Callable: # noqa: ARG001 + """Decorator for renaming symbols in documentation. + + This changes the rendered name of the symbol only in the rendered web documentation. This decorator is used purely for documentation purposes and does not alter the behavior of the decorated callable. diff --git a/src/apify/apify_storage_client/_apify_storage_client.py b/src/apify/apify_storage_client/_apify_storage_client.py index a8f550d9..45689c60 100644 --- a/src/apify/apify_storage_client/_apify_storage_client.py +++ b/src/apify/apify_storage_client/_apify_storage_client.py @@ -5,6 +5,7 @@ from crawlee.base_storage_client import BaseStorageClient from apify._configuration import Configuration +from apify._utils import docs_group from apify.apify_storage_client._dataset_client import DatasetClient from apify.apify_storage_client._dataset_collection_client import DatasetCollectionClient from apify.apify_storage_client._key_value_store_client import KeyValueStoreClient @@ -12,8 +13,6 @@ from apify.apify_storage_client._request_queue_client import RequestQueueClient from apify.apify_storage_client._request_queue_collection_client import RequestQueueCollectionClient -from apify._utils import docs_group - @docs_group('Classes') class ApifyStorageClient(BaseStorageClient): From 0d03380bcf0e88151e6baf7c0836014d9d9e998e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Wed, 20 Nov 2024 11:15:28 +0100 Subject: [PATCH 5/6] chore: run ruff some more --- src/apify/_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apify/_utils.py b/src/apify/_utils.py index fc278072..0179c414 100644 --- a/src/apify/_utils.py +++ b/src/apify/_utils.py @@ -43,7 +43,7 @@ def wrapper(func: Callable) -> Callable: return wrapper -def docs_name(symbol_name: str) -> Callable: # noqa: ARG001 +def docs_name(symbol_name: str) -> Callable: # noqa: ARG001 """Decorator for renaming symbols in documentation. This changes the rendered name of the symbol only in the rendered web documentation. From a28f569454b9687b09d59ea0da8caf32cf3036ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Wed, 20 Nov 2024 11:21:56 +0100 Subject: [PATCH 6/6] chore: remove traces of `yarn` --- website/package-lock.json | 6 ++++-- website/package.json | 3 +-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/website/package-lock.json b/website/package-lock.json index 595c50b0..d4cb6f38 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -12167,7 +12167,8 @@ "node_modules/jsonc-parser": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", - "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==" + "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", + "dev": true }, "node_modules/jsonfile": { "version": "6.1.0", @@ -29958,7 +29959,8 @@ "jsonc-parser": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", - "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==" + "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", + "dev": true }, "jsonfile": { "version": "6.1.0", diff --git a/website/package.json b/website/package.json index b87db6c5..efe35280 100644 --- a/website/package.json +++ b/website/package.json @@ -48,6 +48,5 @@ "path-browserify": "^1.0.1", "prettier": "^2.8.4", "rimraf": "^4.1.3" - }, - "packageManager": "yarn@1.22.21+sha1.1959a18351b811cdeedbd484a8f86c3cc3bbaf72" + } }