Skip to content

Commit a3f8c6e

Browse files
vdusekvdusek
committed
Package structure update
1 parent b6e8a5f commit a3f8c6e

File tree

12 files changed

+161
-126
lines changed

12 files changed

+161
-126
lines changed

‎src/apify/_actor.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@
3030
from apify._consts import EVENT_LISTENERS_TIMEOUT
3131
from apify._crypto import decrypt_input_secrets, load_private_key
3232
from apify._models import ActorRun
33-
from apify._platform_event_manager import EventManager, LocalEventManager, PlatformEventManager
3433
from apify._proxy_configuration import ProxyConfiguration
3534
from apify._utils import docs_group, docs_name, get_system_info, is_running_in_ipython
35+
from apify.events import ApifyEventManager, EventManager, LocalEventManager
3636
from apify.log import _configure_logging, logger
3737
from apify.storage_clients import ApifyStorageClient
3838
from apify.storages import Dataset, KeyValueStore, RequestQueue
@@ -130,7 +130,7 @@ def __init__(
130130

131131
# Set the event manager based on whether the Actor is running on the platform or locally.
132132
self._event_manager = (
133-
PlatformEventManager(
133+
ApifyEventManager(
134134
config=self._configuration,
135135
persist_state_interval=self._configuration.persist_state_interval,
136136
)

‎src/apify/events/__init__.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
from crawlee.events import EventManager, LocalEventManager
2+
3+
from ._apify_event_manager import ApifyEventManager
4+
5+
__all__ = ['ApifyEventManager', 'EventManager', 'LocalEventManager']

‎src/apify/_platform_event_manager.py‎ renamed to ‎src/apify/events/_apify_event_manager.py‎

Lines changed: 8 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -1,126 +1,34 @@
11
from __future__ import annotations
22

33
import asyncio
4-
from datetime import datetime
5-
from typing import TYPE_CHECKING, Annotated, Any, Literal
4+
from typing import TYPE_CHECKING, Annotated
65

76
import websockets.asyncio.client
8-
from pydantic import BaseModel, Discriminator, Field, TypeAdapter
7+
from pydantic import Discriminator, TypeAdapter
98
from typing_extensions import Self, Unpack, override
109

11-
from crawlee.events._event_manager import EventManager, EventManagerOptions
12-
from crawlee.events._local_event_manager import LocalEventManager
13-
from crawlee.events._types import (
14-
Event,
15-
EventAbortingData,
16-
EventExitData,
17-
EventMigratingData,
18-
EventPersistStateData,
19-
EventSystemInfoData,
20-
)
10+
from crawlee.events import EventManager
11+
from crawlee.events._types import Event, EventPersistStateData
2112

2213
from apify._utils import docs_group
14+
from apify.events._types import DeprecatedEvent, EventMessage, SystemInfoEventData, UnknownEvent
2315
from apify.log import logger
2416

2517
if TYPE_CHECKING:
2618
from types import TracebackType
2719

28-
from apify._configuration import Configuration
29-
30-
__all__ = ['EventManager', 'LocalEventManager', 'PlatformEventManager']
31-
32-
33-
@docs_group('Event data')
34-
class SystemInfoEventData(BaseModel):
35-
mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
36-
mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
37-
mem_max_bytes: Annotated[float, Field(alias='memMaxBytes')]
38-
cpu_avg_usage: Annotated[float, Field(alias='cpuAvgUsage')]
39-
cpu_max_usage: Annotated[float, Field(alias='cpuMaxUsage')]
40-
cpu_current_usage: Annotated[float, Field(alias='cpuCurrentUsage')]
41-
is_cpu_overloaded: Annotated[bool, Field(alias='isCpuOverloaded')]
42-
created_at: Annotated[datetime, Field(alias='createdAt')]
43-
44-
def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
45-
return EventSystemInfoData.model_validate(
46-
{
47-
'cpu_info': {
48-
'used_ratio': (self.cpu_current_usage / 100) / dedicated_cpus,
49-
'created_at': self.created_at,
50-
},
51-
'memory_info': {
52-
'total_size': self.mem_max_bytes,
53-
'current_size': self.mem_current_bytes,
54-
'created_at': self.created_at,
55-
},
56-
}
57-
)
58-
59-
60-
@docs_group('Events')
61-
class PersistStateEvent(BaseModel):
62-
name: Literal[Event.PERSIST_STATE]
63-
data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))]
64-
65-
66-
@docs_group('Events')
67-
class SystemInfoEvent(BaseModel):
68-
name: Literal[Event.SYSTEM_INFO]
69-
data: SystemInfoEventData
70-
71-
72-
@docs_group('Events')
73-
class MigratingEvent(BaseModel):
74-
name: Literal[Event.MIGRATING]
75-
data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)]
76-
77-
78-
@docs_group('Events')
79-
class AbortingEvent(BaseModel):
80-
name: Literal[Event.ABORTING]
81-
data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)]
82-
83-
84-
@docs_group('Events')
85-
class ExitEvent(BaseModel):
86-
name: Literal[Event.EXIT]
87-
data: Annotated[EventExitData, Field(default_factory=EventExitData)]
88-
89-
90-
@docs_group('Events')
91-
class EventWithoutData(BaseModel):
92-
name: Literal[
93-
Event.SESSION_RETIRED,
94-
Event.BROWSER_LAUNCHED,
95-
Event.BROWSER_RETIRED,
96-
Event.BROWSER_CLOSED,
97-
Event.PAGE_CREATED,
98-
Event.PAGE_CLOSED,
99-
]
100-
data: Any = None
101-
102-
103-
@docs_group('Events')
104-
class DeprecatedEvent(BaseModel):
105-
name: Literal['cpuInfo']
106-
data: Annotated[dict[str, Any], Field(default_factory=dict)]
107-
108-
109-
@docs_group('Events')
110-
class UnknownEvent(BaseModel):
111-
name: str
112-
data: Annotated[dict[str, Any], Field(default_factory=dict)]
20+
from crawlee.events._event_manager import EventManagerOptions
11321

22+
from apify._configuration import Configuration
11423

115-
EventMessage = PersistStateEvent | SystemInfoEvent | MigratingEvent | AbortingEvent | ExitEvent | EventWithoutData
11624

11725
event_data_adapter = TypeAdapter[EventMessage | DeprecatedEvent | UnknownEvent](
11826
Annotated[EventMessage, Discriminator('name')] | DeprecatedEvent | UnknownEvent
11927
)
12028

12129

12230
@docs_group('Event managers')
123-
class PlatformEventManager(EventManager):
31+
class ApifyEventManager(EventManager):
12432
"""A class for managing Actor events.
12533
12634
You shouldn't use this class directly,

‎src/apify/events/_types.py‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
from __future__ import annotations
2+
3+
from datetime import datetime
4+
from typing import Annotated, Any, Literal
5+
6+
from pydantic import BaseModel, Field
7+
8+
from crawlee.events._types import (
9+
Event,
10+
EventAbortingData,
11+
EventExitData,
12+
EventMigratingData,
13+
EventPersistStateData,
14+
EventSystemInfoData,
15+
)
16+
17+
from apify._utils import docs_group
18+
19+
20+
@docs_group('Event data')
21+
class SystemInfoEventData(BaseModel):
22+
mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
23+
mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
24+
mem_max_bytes: Annotated[float, Field(alias='memMaxBytes')]
25+
cpu_avg_usage: Annotated[float, Field(alias='cpuAvgUsage')]
26+
cpu_max_usage: Annotated[float, Field(alias='cpuMaxUsage')]
27+
cpu_current_usage: Annotated[float, Field(alias='cpuCurrentUsage')]
28+
is_cpu_overloaded: Annotated[bool, Field(alias='isCpuOverloaded')]
29+
created_at: Annotated[datetime, Field(alias='createdAt')]
30+
31+
def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
32+
return EventSystemInfoData.model_validate(
33+
{
34+
'cpu_info': {
35+
'used_ratio': (self.cpu_current_usage / 100) / dedicated_cpus,
36+
'created_at': self.created_at,
37+
},
38+
'memory_info': {
39+
'total_size': self.mem_max_bytes,
40+
'current_size': self.mem_current_bytes,
41+
'created_at': self.created_at,
42+
},
43+
}
44+
)
45+
46+
47+
@docs_group('Events')
48+
class PersistStateEvent(BaseModel):
49+
name: Literal[Event.PERSIST_STATE]
50+
data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))]
51+
52+
53+
@docs_group('Events')
54+
class SystemInfoEvent(BaseModel):
55+
name: Literal[Event.SYSTEM_INFO]
56+
data: SystemInfoEventData
57+
58+
59+
@docs_group('Events')
60+
class MigratingEvent(BaseModel):
61+
name: Literal[Event.MIGRATING]
62+
data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)]
63+
64+
65+
@docs_group('Events')
66+
class AbortingEvent(BaseModel):
67+
name: Literal[Event.ABORTING]
68+
data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)]
69+
70+
71+
@docs_group('Events')
72+
class ExitEvent(BaseModel):
73+
name: Literal[Event.EXIT]
74+
data: Annotated[EventExitData, Field(default_factory=EventExitData)]
75+
76+
77+
@docs_group('Events')
78+
class EventWithoutData(BaseModel):
79+
name: Literal[
80+
Event.SESSION_RETIRED,
81+
Event.BROWSER_LAUNCHED,
82+
Event.BROWSER_RETIRED,
83+
Event.BROWSER_CLOSED,
84+
Event.PAGE_CREATED,
85+
Event.PAGE_CLOSED,
86+
]
87+
data: Any = None
88+
89+
90+
@docs_group('Events')
91+
class DeprecatedEvent(BaseModel):
92+
name: Literal['cpuInfo']
93+
data: Annotated[dict[str, Any], Field(default_factory=dict)]
94+
95+
96+
@docs_group('Events')
97+
class UnknownEvent(BaseModel):
98+
name: str
99+
data: Annotated[dict[str, Any], Field(default_factory=dict)]
100+
101+
102+
EventMessage = PersistStateEvent | SystemInfoEvent | MigratingEvent | AbortingEvent | ExitEvent | EventWithoutData

‎src/apify/events/py.typed‎

Whitespace-only changes.

‎src/apify/request_loaders/__init__.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
from crawlee.request_loaders import (
2+
RequestList,
3+
RequestLoader,
4+
RequestManager,
5+
RequestManagerTandem,
6+
SitemapRequestLoader,
7+
)
8+
9+
from ._apify_request_list import ApifyRequestList
10+
11+
__all__ = [
12+
'ApifyRequestList',
13+
'RequestList',
14+
'RequestLoader',
15+
'RequestManager',
16+
'RequestManagerTandem',
17+
'SitemapRequestLoader',
18+
]

‎src/apify/storages/_request_list.py‎ renamed to ‎src/apify/request_loaders/_apify_request_list.py‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
from crawlee._types import HttpMethod
1212
from crawlee.http_clients import HttpClient, HttpxHttpClient
13-
from crawlee.request_loaders import RequestList as CrawleeRequestList
13+
from crawlee.request_loaders import RequestList
1414

1515
from apify import Request
1616
from apify._utils import docs_group
@@ -39,7 +39,7 @@ class _SimpleUrlInput(_RequestDetails):
3939

4040

4141
@docs_group('Request loaders')
42-
class RequestList(CrawleeRequestList):
42+
class ApifyRequestList(RequestList):
4343
"""Extends crawlee RequestList.
4444
4545
Method open is used to create RequestList from actor's requestListSources input.
@@ -50,7 +50,7 @@ async def open(
5050
name: str | None = None,
5151
request_list_sources_input: list[dict[str, Any]] | None = None,
5252
http_client: HttpClient | None = None,
53-
) -> RequestList:
53+
) -> ApifyRequestList:
5454
"""Initialize a new instance from request list source input.
5555
5656
Args:
@@ -74,12 +74,12 @@ async def open(
7474
```
7575
"""
7676
request_list_sources_input = request_list_sources_input or []
77-
return await RequestList._create_request_list(name, request_list_sources_input, http_client)
77+
return await ApifyRequestList._create_request_list(name, request_list_sources_input, http_client)
7878

7979
@staticmethod
8080
async def _create_request_list(
8181
name: str | None, request_list_sources_input: list[dict[str, Any]], http_client: HttpClient | None
82-
) -> RequestList:
82+
) -> ApifyRequestList:
8383
if not http_client:
8484
http_client = HttpxHttpClient()
8585

@@ -88,10 +88,12 @@ async def _create_request_list(
8888
simple_url_inputs = [url_input for url_input in url_inputs if isinstance(url_input, _SimpleUrlInput)]
8989
remote_url_inputs = [url_input for url_input in url_inputs if isinstance(url_input, _RequestsFromUrlInput)]
9090

91-
simple_url_requests = RequestList._create_requests_from_input(simple_url_inputs)
92-
remote_url_requests = await RequestList._fetch_requests_from_url(remote_url_inputs, http_client=http_client)
91+
simple_url_requests = ApifyRequestList._create_requests_from_input(simple_url_inputs)
92+
remote_url_requests = await ApifyRequestList._fetch_requests_from_url(
93+
remote_url_inputs, http_client=http_client
94+
)
9395

94-
return RequestList(name=name, requests=simple_url_requests + remote_url_requests)
96+
return ApifyRequestList(name=name, requests=simple_url_requests + remote_url_requests)
9597

9698
@staticmethod
9799
def _create_requests_from_input(simple_url_inputs: list[_SimpleUrlInput]) -> list[Request]:

‎src/apify/request_loaders/py.typed‎

Whitespace-only changes.

‎src/apify/storages/__init__.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
11
from crawlee.storages import Dataset, KeyValueStore, RequestQueue
22

3-
from ._request_list import RequestList
4-
5-
__all__ = ['Dataset', 'KeyValueStore', 'RequestList', 'RequestQueue']
3+
__all__ = ['Dataset', 'KeyValueStore', 'RequestQueue']

0 commit comments

Comments
 (0)