Index

A

aacquire() (pyfetcher.ratelimit.limiter.DomainRateLimiter method) aclose() (pyfetcher.fetch.service.FetchService method) (pyfetcher.transports.aiohttp.AiohttpTransport method) (pyfetcher.transports.curl_cffi.CurlCffiTransport method) (pyfetcher.transports.httpx.HttpxTransport method) acquire() (pyfetcher.ratelimit.limiter.DomainRateLimiter method) add() (pyfetcher.crawler.spider.Router method) add_url() (pyfetcher.crawler.frontier.Frontier method) add_urls() (pyfetcher.crawler.frontier.Frontier method) adownload_to_path() (pyfetcher.download.service.DownloadService method) afetch() (in module pyfetcher.fetch.functions) (pyfetcher.fetch.service.FetchService method) (pyfetcher.transports.aiohttp.AiohttpTransport method) (pyfetcher.transports.base.AsyncTransport method) (pyfetcher.transports.curl_cffi.CurlCffiTransport method) (pyfetcher.transports.httpx.HttpxTransport method)

afetch_many() (in module pyfetcher.fetch.functions) (pyfetcher.fetch.service.FetchService method) AiohttpTransport (class in pyfetcher.transports.aiohttp) ArticleMeta (class in pyfetcher.extractors.article) ArticleResult (class in pyfetcher.mcp.models) astream() (in module pyfetcher.fetch.functions) (pyfetcher.fetch.service.FetchService method) (pyfetcher.transports.aiohttp.AiohttpTransport method) (pyfetcher.transports.base.AsyncTransport method) (pyfetcher.transports.curl_cffi.CurlCffiTransport method) (pyfetcher.transports.httpx.HttpxTransport method) AsyncTransport (class in pyfetcher.transports.base)

B

BatchFetchRequest (class in pyfetcher.contracts.request) BatchFetchResponse (class in pyfetcher.contracts.response) BatchItemResponse (class in pyfetcher.contracts.response) BrowserHeaderProvider (class in pyfetcher.headers.browser) BrowserProfile (class in pyfetcher.headers.profiles)

build() (pyfetcher.headers.base.HeaderProvider method) (pyfetcher.headers.browser.BrowserHeaderProvider method) (pyfetcher.headers.rotating.RotatingHeaderProvider method) (pyfetcher.headers.static.StaticHeaderProvider method) build_batch_request() (in module pyfetcher.fetch.batch) build_seed_urls() (in module pyfetcher.crawler.discovery)

C

calculate_poll_interval() (in module pyfetcher.crawler.feeds) Channels (class in pyfetcher.events.channels) check_robots() (pyfetcher.crawler.politeness.PolitenessEnforcer method) close() (pyfetcher.fetch.service.FetchService method) (pyfetcher.transports.cloudscraper.CloudscraperTransport method) (pyfetcher.transports.curl_cffi.CurlCffiTransport method) (pyfetcher.transports.httpx.HttpxTransport method)

CloudscraperTransport (class in pyfetcher.transports.cloudscraper) collect_bytes() (in module pyfetcher.fetch.stream) collect_stream_bytes() (in module pyfetcher.fetch.stream) compute_entry_hash() (in module pyfetcher.crawler.feeds) CURL_CFFI_TARGETS (in module pyfetcher.transports.curl_cffi) CurlCffiTransport (class in pyfetcher.transports.curl_cffi)

D

default() (pyfetcher.crawler.spider.Router method) delete() (pyfetcher.store.client.ObjectStoreClient method) DirectDownloader (class in pyfetcher.downloaders.direct) discover_sitemaps_from_robots() (in module pyfetcher.crawler.discovery) discover_urls_from_sitemap() (in module pyfetcher.crawler.discovery) DomainRateLimiter (class in pyfetcher.ratelimit.limiter) download() (pyfetcher.downloaders.direct.DirectDownloader method) (pyfetcher.downloaders.gallerydl.GalleryDlDownloader method) (pyfetcher.downloaders.ytdlp.YtdlpDownloader method)

download_bytes() (pyfetcher.store.client.ObjectStoreClient method) DownloaderProtocol (class in pyfetcher.downloaders.base) DownloadProgress (class in pyfetcher.downloaders.base) DownloadResult (class in pyfetcher.downloaders.base) (class in pyfetcher.mcp.models) DownloadService (class in pyfetcher.download.service)

E

ensure_bucket() (pyfetcher.store.client.ObjectStoreClient method) EventListener (class in pyfetcher.events.listener) extract_article_metadata() (in module pyfetcher.extractors.article) extract_article_text() (in module pyfetcher.extractors.content) extract_attrs() (in module pyfetcher.scrape.selectors) extract_basic_html_metadata() (in module pyfetcher.metadata.html) extract_extruct_metadata() (in module pyfetcher.metadata.extruct) extract_forms() (in module pyfetcher.scrape.forms) extract_hostname() (pyfetcher.crawler.politeness.PolitenessEnforcer method)

extract_info() (pyfetcher.downloaders.direct.DirectDownloader method) (pyfetcher.downloaders.gallerydl.GalleryDlDownloader method) (pyfetcher.downloaders.ytdlp.YtdlpDownloader method) extract_links() (in module pyfetcher.scrape.links) extract_media_metadata() (in module pyfetcher.extractors.media_meta) extract_open_graph_metadata() (in module pyfetcher.metadata.opengraph) extract_readable_text() (in module pyfetcher.scrape.content) extract_table() (in module pyfetcher.scrape.selectors) extract_text() (in module pyfetcher.scrape.selectors)

F

FaviconLink (class in pyfetcher.metadata.models) FeedEntry (class in pyfetcher.crawler.feeds) FeedPollResult (class in pyfetcher.crawler.feeds) fetch() (in module pyfetcher.fetch.functions) (pyfetcher.fetch.service.FetchService method) (pyfetcher.transports.base.SyncTransport method) (pyfetcher.transports.cloudscraper.CloudscraperTransport method) (pyfetcher.transports.curl_cffi.CurlCffiTransport method) (pyfetcher.transports.httpx.HttpxTransport method)

FetchRequest (class in pyfetcher.contracts.request) FetchResponse (class in pyfetcher.contracts.response) FetchResult (class in pyfetcher.mcp.models) FetchService (class in pyfetcher.fetch.service) FormData (class in pyfetcher.scrape.forms) FormField (class in pyfetcher.scrape.forms) FormInfo (class in pyfetcher.mcp.models) FormsResult (class in pyfetcher.mcp.models) Frontier (class in pyfetcher.crawler.frontier)

G

GalleryDlDownloader (class in pyfetcher.downloaders.gallerydl) get_best_browser_headers() (in module pyfetcher.headers.browser)

get_crawl_delay() (pyfetcher.crawler.politeness.PolitenessEnforcer method) get_default_fetch_service() (in module pyfetcher.fetch.functions) get_profile() (in module pyfetcher.headers.profiles)

H

handle() (pyfetcher.crawler.spider.Spider method) HeaderProvider (class in pyfetcher.headers.base) HeadersResult (class in pyfetcher.mcp.models)

host (pyfetcher.contracts.url.URL property) html_to_markdown() (in module pyfetcher.extractors.convert) html_to_plaintext() (in module pyfetcher.extractors.convert) HttpxTransport (class in pyfetcher.transports.httpx)

I

interval (pyfetcher.ratelimit.limiter.RateLimitPolicy property)

is_allowed() (in module pyfetcher.scrape.robots) is_seen() (pyfetcher.crawler.dedup.URLDeduplicator method)

L

LinkInfo (class in pyfetcher.mcp.models) (class in pyfetcher.scrape.links)

LinksResult (class in pyfetcher.mcp.models) list_keys() (pyfetcher.store.client.ObjectStoreClient method) list_profiles() (in module pyfetcher.headers.profiles)

M

mark_seen() (pyfetcher.crawler.dedup.URLDeduplicator method) MediaInfo (class in pyfetcher.downloaders.base) MediaResource (class in pyfetcher.contracts.resource) MetadataResult (class in pyfetcher.mcp.models) model_config (pyfetcher.config.PyfetcherConfig attribute) (pyfetcher.contracts.policy.PoolPolicy attribute) (pyfetcher.contracts.policy.RetryPolicy attribute) (pyfetcher.contracts.policy.StreamPolicy attribute) (pyfetcher.contracts.policy.TimeoutPolicy attribute) (pyfetcher.contracts.request.BatchFetchRequest attribute) (pyfetcher.contracts.request.FetchRequest attribute) (pyfetcher.contracts.resource.MediaResource attribute) (pyfetcher.contracts.resource.WebPage attribute) (pyfetcher.contracts.resource.WebResource attribute) (pyfetcher.contracts.response.BatchFetchResponse attribute) (pyfetcher.contracts.response.BatchItemResponse attribute) (pyfetcher.contracts.response.FetchResponse attribute) (pyfetcher.contracts.response.StreamChunk attribute) (pyfetcher.contracts.url.URL attribute) (pyfetcher.mcp.models.ArticleResult attribute) (pyfetcher.mcp.models.DownloadResult attribute) (pyfetcher.mcp.models.FetchResult attribute) (pyfetcher.mcp.models.FormInfo attribute) (pyfetcher.mcp.models.FormsResult attribute) (pyfetcher.mcp.models.HeadersResult attribute) (pyfetcher.mcp.models.LinkInfo attribute) (pyfetcher.mcp.models.LinksResult attribute) (pyfetcher.mcp.models.MetadataResult attribute) (pyfetcher.mcp.models.ProfileInfo attribute) (pyfetcher.mcp.models.RobotsResult attribute) (pyfetcher.mcp.models.ScrapeResult attribute) (pyfetcher.mcp.models.SitemapResult attribute) (pyfetcher.mcp.models.TableResult attribute) (pyfetcher.metadata.models.FaviconLink attribute) (pyfetcher.metadata.models.OpenGraphMetadata attribute) (pyfetcher.metadata.models.PageMetadata attribute) module pyfetcher.config pyfetcher.contracts.policy pyfetcher.contracts.request pyfetcher.contracts.resource pyfetcher.contracts.response pyfetcher.contracts.url pyfetcher.crawler.dedup pyfetcher.crawler.discovery pyfetcher.crawler.feeds pyfetcher.crawler.frontier pyfetcher.crawler.politeness pyfetcher.crawler.spider pyfetcher.download.service pyfetcher.downloaders.base pyfetcher.downloaders.direct pyfetcher.downloaders.gallerydl pyfetcher.downloaders.ytdlp pyfetcher.events.channels pyfetcher.events.listener pyfetcher.extractors.article pyfetcher.extractors.content pyfetcher.extractors.convert pyfetcher.extractors.media_meta pyfetcher.fetch.batch pyfetcher.fetch.functions pyfetcher.fetch.service pyfetcher.fetch.stream pyfetcher.headers.base pyfetcher.headers.browser pyfetcher.headers.profiles pyfetcher.headers.rotating pyfetcher.headers.static pyfetcher.headers.ua pyfetcher.mcp.models pyfetcher.metadata.extruct pyfetcher.metadata.html pyfetcher.metadata.models pyfetcher.metadata.opengraph pyfetcher.ratelimit.limiter pyfetcher.scrape.content pyfetcher.scrape.forms pyfetcher.scrape.links pyfetcher.scrape.robots pyfetcher.scrape.selectors pyfetcher.scrape.sitemap pyfetcher.store.client pyfetcher.transports.aiohttp pyfetcher.transports.base pyfetcher.transports.cloudscraper pyfetcher.transports.curl_cffi pyfetcher.transports.httpx

N

normalize_url() (in module pyfetcher.crawler.dedup)

O

ObjectStoreClient (class in pyfetcher.store.client) ok (pyfetcher.contracts.response.FetchResponse property)

on() (pyfetcher.events.listener.EventListener method) OpenGraphMetadata (class in pyfetcher.metadata.models)

P

PageMetadata (class in pyfetcher.metadata.models) parse_feed() (in module pyfetcher.crawler.feeds) parse_robots_txt() (in module pyfetcher.scrape.robots) parse_sitemap() (in module pyfetcher.scrape.sitemap) path (pyfetcher.contracts.url.URL property) path_segments (pyfetcher.contracts.url.URL property) PolitenessEnforcer (class in pyfetcher.crawler.politeness) PoolPolicy (class in pyfetcher.contracts.policy) port (pyfetcher.contracts.url.URL property) presigned_get_url() (pyfetcher.store.client.ObjectStoreClient method) profile (pyfetcher.headers.browser.BrowserHeaderProvider property) ProfileInfo (class in pyfetcher.mcp.models) profiles (pyfetcher.headers.rotating.RotatingHeaderProvider property) pyfetcher.config module pyfetcher.contracts.policy module pyfetcher.contracts.request module pyfetcher.contracts.resource module pyfetcher.contracts.response module pyfetcher.contracts.url module pyfetcher.crawler.dedup module pyfetcher.crawler.discovery module pyfetcher.crawler.feeds module pyfetcher.crawler.frontier module pyfetcher.crawler.politeness module pyfetcher.crawler.spider module pyfetcher.download.service module pyfetcher.downloaders.base module pyfetcher.downloaders.direct module pyfetcher.downloaders.gallerydl module pyfetcher.downloaders.ytdlp module pyfetcher.events.channels module pyfetcher.events.listener module pyfetcher.extractors.article module pyfetcher.extractors.content module pyfetcher.extractors.convert module pyfetcher.extractors.media_meta module

pyfetcher.fetch.batch module pyfetcher.fetch.functions module pyfetcher.fetch.service module pyfetcher.fetch.stream module pyfetcher.headers.base module pyfetcher.headers.browser module pyfetcher.headers.profiles module pyfetcher.headers.rotating module pyfetcher.headers.static module pyfetcher.headers.ua module pyfetcher.mcp.models module pyfetcher.metadata.extruct module pyfetcher.metadata.html module pyfetcher.metadata.models module pyfetcher.metadata.opengraph module pyfetcher.ratelimit.limiter module pyfetcher.scrape.content module pyfetcher.scrape.forms module pyfetcher.scrape.links module pyfetcher.scrape.robots module pyfetcher.scrape.selectors module pyfetcher.scrape.sitemap module pyfetcher.store.client module pyfetcher.transports.aiohttp module pyfetcher.transports.base module pyfetcher.transports.cloudscraper module pyfetcher.transports.curl_cffi module pyfetcher.transports.httpx module PyfetcherConfig (class in pyfetcher.config)

Q

query (pyfetcher.contracts.url.URL property)

query_params (pyfetcher.contracts.url.URL property)

R

random_profile() (in module pyfetcher.headers.ua) random_user_agent() (in module pyfetcher.headers.ua) RateLimitPolicy (class in pyfetcher.ratelimit.limiter) reset() (pyfetcher.ratelimit.limiter.DomainRateLimiter method) resolve() (pyfetcher.crawler.spider.Router method)

RetryPolicy (class in pyfetcher.contracts.policy) RobotsResult (class in pyfetcher.mcp.models) RobotsRules (class in pyfetcher.scrape.robots) RotatingHeaderProvider (class in pyfetcher.headers.rotating) Router (class in pyfetcher.crawler.spider)

S

scheme (pyfetcher.contracts.url.URL property) ScrapeResult (class in pyfetcher.mcp.models) select() (in module pyfetcher.scrape.selectors) select_one() (in module pyfetcher.scrape.selectors) SitemapEntry (class in pyfetcher.scrape.sitemap) SitemapResult (class in pyfetcher.mcp.models) Spider (class in pyfetcher.crawler.spider)

SpiderResult (class in pyfetcher.crawler.spider) start() (pyfetcher.events.listener.EventListener method) StaticHeaderProvider (class in pyfetcher.headers.static) stop() (pyfetcher.events.listener.EventListener method) StreamChunk (class in pyfetcher.contracts.response) StreamPolicy (class in pyfetcher.contracts.policy) SyncTransport (class in pyfetcher.transports.base)

T

TableResult (class in pyfetcher.mcp.models) TimeoutPolicy (class in pyfetcher.contracts.policy)

to_dict() (pyfetcher.scrape.forms.FormData method) to_headers() (pyfetcher.headers.profiles.BrowserProfile method)

U

unicode_string() (pyfetcher.contracts.url.URL method) upload_bytes() (pyfetcher.store.client.ObjectStoreClient method) upload_file() (pyfetcher.store.client.ObjectStoreClient method)

URL (class in pyfetcher.contracts.url) url_hash() (in module pyfetcher.crawler.dedup) URLDeduplicator (class in pyfetcher.crawler.dedup) user_agents_for_browser() (in module pyfetcher.headers.ua)

W

wait_for_host() (pyfetcher.crawler.politeness.PolitenessEnforcer method)

WebPage (class in pyfetcher.contracts.resource) WebResource (class in pyfetcher.contracts.resource)

Y

YtdlpDownloader (class in pyfetcher.downloaders.ytdlp)