Skip to content

Commit af8cab7

Browse files
committed
address feedback
1 parent 790666f commit af8cab7

File tree

5 files changed

+11
-12
lines changed

5 files changed

+11
-12
lines changed

src/crawlee/_autoscaling/snapshotter.py

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ def __init__(
9292
self._reserve_memory_ratio = reserve_memory_ratio
9393
self._memory_warning_cooldown_period = memory_warning_cooldown_period
9494
self._client_rate_limit_error_retry_count = client_rate_limit_error_retry_count
95-
self._max_memory_size = max_memory_size or self._get_default_max_memory_size(
95+
self._max_memory_size = max_memory_size or self._determine_max_memory_size(
9696
config.memory_mbytes, self._available_memory_ratio
9797
)
9898
self._cpu_snapshots = self._get_sorted_list_by_created_at(list[CpuSnapshot]())
@@ -113,8 +113,12 @@ def _get_sorted_list_by_created_at(input_list: list[T]) -> SortedList[T]:
113113
return SortedList(input_list, key=attrgetter('created_at'))
114114

115115
@staticmethod
116-
def _get_default_max_memory_size(memory_mbytes: int | None, available_memory_ratio: float) -> ByteSize:
117-
"""If `memory_mbytes` is not provided, calculates the default `max_memory_size` based on the system memory."""
116+
def _determine_max_memory_size(memory_mbytes: int | None, available_memory_ratio: float) -> ByteSize:
117+
"""Determine the maximum memory size for the current run.
118+
119+
If `memory_mbytes` is provided, it uses that value. Otherwise, it calculates the default `max_memory_size`
120+
as a proportion of the system's total available memory based on `available_memory_ratio`.
121+
"""
118122
if memory_mbytes:
119123
return ByteSize.from_mb(memory_mbytes)
120124

src/crawlee/events/_event_manager.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,6 @@ def __init__(
6868
6969
Args:
7070
persist_state_interval: Interval between emitted `PersistState` events to maintain state persistence.
71-
The default value is taken from the `Configuration`.
7271
close_timeout: Optional timeout for canceling pending event listeners if they exceed this duration.
7372
"""
7473
self._persist_state_interval = persist_state_interval

src/crawlee/events/_local_event_manager.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,11 @@ def __init__(
3333
) -> None:
3434
"""A default constructor.
3535
36-
In the most cases you should use the `from_config` constructor to create an new instance based on
36+
In most cases, you should use the `from_config` constructor to create a new instance based on
3737
the provided configuration.
3838
3939
Args:
40-
system_info_interval: Interval at which `SystemInfo` events are emitted. The default value is taken
41-
from the `Configuration`.
40+
system_info_interval: Interval at which `SystemInfo` events are emitted.
4241
event_manager_options: Additional options for the parent class.
4342
"""
4443
self._system_info_interval = system_info_interval

src/crawlee/storage_clients/_memory/_memory_storage_client.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ def __init__(
6868
) -> None:
6969
"""A default constructor.
7070
71-
In the most cases you should use the `from_config` constructor to create an new instance based on
71+
In most cases, you should use the `from_config` constructor to create a new instance based on
7272
the provided configuration.
7373
7474
Args:

tests/unit/crawlers/_basic/test_basic_crawler.py

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1005,10 +1005,7 @@ async def handler(context: BasicCrawlingContext) -> None:
10051005

10061006
async def test_sets_services() -> None:
10071007
custom_configuration = Configuration()
1008-
custom_event_manager = LocalEventManager(
1009-
system_info_interval=timedelta(seconds=3),
1010-
persist_state_interval=timedelta(seconds=30),
1011-
)
1008+
custom_event_manager = LocalEventManager.from_config(custom_configuration)
10121009
custom_storage_client = MemoryStorageClient.from_config(custom_configuration)
10131010

10141011
crawler = BasicCrawler(

0 commit comments

Comments
 (0)