From 0f950affc6f3d958cd3b5ce808c0cb3c51769af2 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 20 Jul 2026 21:09:42 +0200 Subject: [PATCH] docs: Fix stale and inaccurate documentation --- README.md | 2 +- docs/introduction/03_adding_more_urls.mdx | 2 +- src/crawlee/_types.py | 16 +++++++++------- src/crawlee/storages/_dataset.py | 10 +++++----- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 961c434a47..6fc5b9ef67 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Here are some practical examples to help you get started with different types of ### BeautifulSoupCrawler -The [`BeautifulSoupCrawler`](https://crawlee.dev/python/api/class/BeautifulSoupCrawler) downloads web pages using an HTTP library and provides HTML-parsed content to the user. By default it uses [`HttpxHttpClient`](https://crawlee.dev/python/api/class/HttpxHttpClient) for HTTP communication and [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) for parsing HTML. It is ideal for projects that require efficient extraction of data from HTML content. This crawler has very good performance since it does not use a browser. However, if you need to execute client-side JavaScript, to get your content, this is not going to be enough and you will need to use [`PlaywrightCrawler`](https://crawlee.dev/python/api/class/PlaywrightCrawler). Also if you want to use this crawler, make sure you install `crawlee` with `beautifulsoup` extra. +The [`BeautifulSoupCrawler`](https://crawlee.dev/python/api/class/BeautifulSoupCrawler) downloads web pages using an HTTP library and provides HTML-parsed content to the user. By default it uses [`ImpitHttpClient`](https://crawlee.dev/python/api/class/ImpitHttpClient) for HTTP communication and [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) for parsing HTML. It is ideal for projects that require efficient extraction of data from HTML content. This crawler has very good performance since it does not use a browser. However, if you need to execute client-side JavaScript, to get your content, this is not going to be enough and you will need to use [`PlaywrightCrawler`](https://crawlee.dev/python/api/class/PlaywrightCrawler). Also if you want to use this crawler, make sure you install `crawlee` with `beautifulsoup` extra. ```python import asyncio diff --git a/docs/introduction/03_adding_more_urls.mdx b/docs/introduction/03_adding_more_urls.mdx index 7583e3494e..bfe2337d07 100644 --- a/docs/introduction/03_adding_more_urls.mdx +++ b/docs/introduction/03_adding_more_urls.mdx @@ -46,7 +46,7 @@ When you're just testing your code or when your crawler could potentially find m crawler = BeautifulSoupCrawler(max_requests_per_crawl=10) ``` -This means that no new requests will be started after the 20th request is finished. The actual number of processed requests might be a little higher thanks to parallelization, because the running requests won't be forcefully aborted. It's not even possible in most cases. +This means that no new requests will be started after the 10th request is finished. The actual number of processed requests might be a little higher thanks to parallelization, because the running requests won't be forcefully aborted. It's not even possible in most cases. ## Finding new links diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index 5f959f29ff..ffeb92094c 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -151,17 +151,19 @@ class EnqueueLinksKwargs(TypedDict): """Base URL to be used for relative URLs.""" strategy: NotRequired[EnqueueStrategy] - """Enqueue strategy to be used for determining which links to extract and enqueue. + """Enqueue strategy that determines which URLs are enqueued based on their relation to the page the crawler + is currently on. With `enqueue_links` it filters the links extracted from the page. With `add_requests` it + filters the explicitly provided requests. Options: - all: Enqueue every link encountered, regardless of the target domain. Use this option to ensure that all - links, including those leading to external websites, are followed. - same-domain: Enqueue links that share the same domain name as the current page, including any subdomains. + all: Enqueue every URL encountered, regardless of the target domain. Use this option to ensure that all + URLs, including those leading to external websites, are followed. + same-domain: Enqueue URLs that share the same domain name as the current page, including any subdomains. This strategy is ideal for crawling within the same top-level domain while still allowing for subdomain exploration. - same-hostname: Enqueue links only if they match the exact hostname of the current page. This is the default - behavior and restricts the crawl to the current hostname, excluding subdomains. - same-origin: Enqueue links that share the same origin as the current page. The origin is defined by the + same-hostname: Enqueue URLs only if they match the exact hostname of the current page. This is the default + for `enqueue_links` and restricts the crawl to the current hostname, excluding subdomains. + same-origin: Enqueue URLs that share the same origin as the current page. The origin is defined by the combination of protocol, domain, and port, ensuring a strict scope for the crawl. """ diff --git a/src/crawlee/storages/_dataset.py b/src/crawlee/storages/_dataset.py index 67c201cfd4..8dbb47bda3 100644 --- a/src/crawlee/storages/_dataset.py +++ b/src/crawlee/storages/_dataset.py @@ -137,13 +137,13 @@ async def purge(self) -> None: async def push_data(self, data: Sequence[Mapping[str, JsonSerializable]] | Mapping[str, JsonSerializable]) -> None: """Store an object or an array of objects to the dataset. - The size of the data is limited by the receiving API and therefore `push_data()` will only - allow objects whose JSON representation is smaller than 9MB. When an array is passed, - none of the included objects may be larger than 9MB, but the array itself may be of any size. + Local storage clients impose no size limit on the pushed data. Some remote storage clients do, + however. For example, when running on the Apify platform, the receiving API only accepts objects + whose JSON representation is smaller than 9MB, and each item of a passed array is subject to the + same per-item limit (the array itself may be of any size). Args: - data: A JSON serializable data structure to be stored in the dataset. The JSON representation - of each item must be smaller than 9MB. + data: A JSON serializable data structure to be stored in the dataset. """ await self._client.push_data(data=data)