docs: hotfix API reference docs (#282)

Slightly bends the `transformDocs.js` script to accommodate the new structure of the project better.

The next Python docs steps are documented in apify/crawlee-python#324

Closes #270

Author: barjin

src/apify/_crypto.py

@@ -114,6 +114,7 @@ def private_decrypt(
     return decipher_bytes.decode('utf-8')
 
 
+@ignore_docs
 def load_private_key(private_key_file_base64: str, private_key_password: str) -> rsa.RSAPrivateKey:
     private_key = serialization.load_pem_private_key(
         base64.b64decode(private_key_file_base64.encode('utf-8')),
@@ -133,6 +134,7 @@ def _load_public_key(public_key_file_base64: str) -> rsa.RSAPublicKey:
     return public_key
 
 
+@ignore_docs
 def decrypt_input_secrets(private_key: rsa.RSAPrivateKey, input_data: Any) -> Any:
     """Decrypt input secrets."""
     if not isinstance(input_data, dict):

src/apify/_platform_event_manager.py

@@ -8,7 +8,6 @@
 from pydantic import BaseModel, Discriminator, Field, TypeAdapter
 from typing_extensions import Self, Unpack, override
 
-from apify_shared.utils import ignore_docs
 from crawlee.events._event_manager import EventManager, EventManagerOptions
 from crawlee.events._local_event_manager import LocalEventManager
 from crawlee.events._types import (
@@ -126,7 +125,6 @@ class UnknownEvent(BaseModel):
 )
 
 
-@ignore_docs
 class PlatformEventManager(EventManager):
     """A class for managing Actor events.
 

src/apify/_proxy_configuration.py

@@ -27,6 +27,7 @@
 SESSION_ID_MAX_LENGTH = 50
 
 
+@ignore_docs
 def is_url(url: str) -> bool:
     """Check if the given string is a valid URL."""
     try:

src/apify/log.py

@@ -3,6 +3,7 @@
 import logging
 from typing import TYPE_CHECKING
 
+from apify_shared.utils import ignore_docs
 from crawlee._log_config import CrawleeLogFormatter, configure_logger, get_configured_log_level
 
 if TYPE_CHECKING:
@@ -15,6 +16,7 @@
 logger = logging.getLogger(logger_name)
 
 
+@ignore_docs
 class ActorLogFormatter(CrawleeLogFormatter):  # noqa: D101 Inherited from parent class
     pass
 

src/apify/scrapy/requests.py

@@ -4,6 +4,8 @@
 import pickle
 from typing import Any, cast
 
+from apify_shared.utils import ignore_docs
+
 try:
     from scrapy import Request, Spider
     from scrapy.http.headers import Headers
@@ -28,6 +30,7 @@ def _is_request_produced_by_middleware(scrapy_request: Request) -> bool:
     return bool(scrapy_request.meta.get('redirect_times')) or bool(scrapy_request.meta.get('retry_times'))
 
 
+@ignore_docs
 def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest | None:
     """Convert a Scrapy request to an Apify request.
 
@@ -98,6 +101,7 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest
     return apify_request
 
 
+@ignore_docs
 def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request:
     """Convert an Apify request to a Scrapy request.
 

src/apify/scrapy/utils.py

@@ -4,6 +4,8 @@
 from base64 import b64encode
 from urllib.parse import unquote
 
+from apify_shared.utils import ignore_docs
+
 try:
     from scrapy.settings import Settings  # noqa: TCH002
     from scrapy.utils.project import get_project_settings
@@ -18,13 +20,15 @@
 nested_event_loop: asyncio.AbstractEventLoop = asyncio.new_event_loop()
 
 
+@ignore_docs
 def get_basic_auth_header(username: str, password: str, auth_encoding: str = 'latin-1') -> bytes:
     """Generate a basic authentication header for the given username and password."""
     string = f'{unquote(username)}:{unquote(password)}'
     user_pass = to_bytes(string, encoding=auth_encoding)
     return b'Basic ' + b64encode(user_pass)
 
 
+@ignore_docs
 def get_running_event_loop_id() -> int:
     """Get the ID of the currently running event loop.
 

website/pydoc-markdown.yml

@@ -4,6 +4,7 @@ loaders:
 processors:
   - type: filter
     skip_empty_modules: true
+    documented_only: false
   - type: crossref
 renderer:
   type: docusaurus

website/transformDocs.js

@@ -57,15 +57,9 @@ const TYPEDOC_KINDS = {
 }
 
 const GROUP_ORDER = [
-    'Main Classes',
-    'Main Clients',
-    'Resource Clients',
-    'Async Resource Clients',
-    'Helper Classes',
-    'Errors',
-    'Constructors',
-    'Methods',
-    'Properties',
+    'Classes',
+    'Data structures',
+    'Scrapy Integration',
     'Constants',
     'Enumeration Members'
 ];
@@ -79,22 +73,23 @@ const groupSort = (g1, g2) => {
 
 function getGroupName(object) {
     const groupPredicates = {
+        'Scrapy integration': (x) => ['ApifyScheduler', 'ActorDatasetPushPipeline', 'ApifyHttpProxyMiddleware', 'apply_apify_settings'].includes(x.name),
+        'Data structures': (x) => ['BaseModel', 'TypedDict'].some(base => x?.bases?.includes(base)) || x?.decorations?.some(d => d.name === 'dataclass'),
         'Errors': (x) => x.name.toLowerCase().includes('error'),
-        'Main Classes': (x) => ['Actor', 'Dataset', 'KeyValueStore', 'RequestQueue'].includes(x.name),
+        'Classes': (x) => x.kindString === 'Class',
         'Main Clients': (x) => ['ApifyClient', 'ApifyClientAsync'].includes(x.name),
         'Async Resource Clients': (x) => x.name.toLowerCase().includes('async'),
         'Resource Clients': (x) => x.kindString === 'Class' && x.name.toLowerCase().includes('client'),
-        'Helper Classes': (x) => x.kindString === 'Class',
         'Methods': (x) => x.kindString === 'Method',
         'Constructors': (x) => x.kindString === 'Constructor',
         'Properties': (x) => x.kindString === 'Property',
         'Constants': (x) => x.kindString === 'Enumeration',
-        'Enumeration Members': (x) => x.kindString === 'Enumeration Member',
+        'Enumeration members': (x) => x.kindString === 'Enumeration Member',
     };
 
     const [group] = Object.entries(groupPredicates).find(
         ([_, predicate]) => predicate(object)
-    );
+    ) ?? ['Other'];
 
     return group;
 }
@@ -162,7 +157,8 @@ function extractArgsAndReturns(docstring) {
 
 // Objects with decorators named 'ignore_docs' or with empty docstrings will be ignored
 function isHidden(member) {
-    return member.decorations?.some(d => d.name === 'ignore_docs') || member.name === 'ignore_docs' || !member.docstring?.content;
+    return member.decorations?.some(d => d.name === 'ignore_docs') 
+        || member.name === 'ignore_docs';
 }
 
 // Each object in the Typedoc structure has an unique ID,
@@ -207,13 +203,22 @@ function convertObject(obj, parent, module) {
                 moduleName = moduleShortcuts[fullName].replace(`.${member.name}`, '');
             }
 
+            if(member.name === 'Actor' || (member.name.endsWith('Client') && !member.name.endsWith('StorageClient')) || member.name === 'ListPage') {
+                continue;
+            }
+
+            if (member.name === '_ActorType') {
+                member.name = 'Actor';
+            }
+
             // Create the Typedoc member object
             let typedocMember = {
                 id: oid++,
                 name: member.name,
                 module: moduleName, // This is an extension to the original Typedoc structure, to support showing where the member is exported from
                 ...typedocKind,
                 flags: {},
+                bases: member.bases,
                 comment: member.docstring ? {
                     summary: [{
                         kind: 'text',
@@ -231,6 +236,10 @@ function convertObject(obj, parent, module) {
                 }],
             };
 
+            if(!GROUP_ORDER.includes(getGroupName(typedocMember)) && parent.kindString === 'Project'){
+                continue;
+            }
+
             if(typedocMember.kindString === 'Method') {
                 const { parameters, returns } = extractArgsAndReturns(member.docstring?.content ?? '');