Skip to content

proxy

__all__ = ['ProxyWebScraperDriver'] module-attribute

ProxyWebScraperDriver

Bases: BaseWebScraperDriver

Source code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py 
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
@define
class ProxyWebScraperDriver(BaseWebScraperDriver):
    proxies: dict = field(kw_only=True, metadata={"serializable": False})
    params: dict = field(default=Factory(dict), kw_only=True, metadata={"serializable": True})
    _extract_page: Callable[[ProxyWebScraperDriver, str], TextArtifact] = field(
        default=Factory(lambda: lambda _, page: TextArtifact(page)),
        kw_only=True,
        alias="extract_page",
        metadata={"serializable": False},
    )

    def fetch_url(self, url: str) -> str:
        response = requests.get(url, proxies=self.proxies, **self.params)

        return response.text

    def extract_page(self, page: str) -> TextArtifact:
        return self._extract_page(self, page)

_extract_page = field(default=Factory(lambda: lambda _, page: TextArtifact(page)), kw_only=True, alias='extract_page', metadata={'serializable': False}) class-attribute instance-attribute

params = field(default=Factory(dict), kw_only=True, metadata={'serializable': True}) class-attribute instance-attribute

proxies = field(kw_only=True, metadata={'serializable': False}) class-attribute instance-attribute

extract_page(page)

Source code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py 
28
29
def extract_page(self, page: str) -> TextArtifact:
    return self._extract_page(self, page)

fetch_url(url)

Source code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py 
23
24
25
26
def fetch_url(self, url: str) -> str:
    response = requests.get(url, proxies=self.proxies, **self.params)

    return response.text