Skip to content

proxy

__all__ = ['ProxyWebScraperDriver'] module-attribute

ProxyWebScraperDriver

Bases: BaseWebScraperDriver

Source code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py
@define
class ProxyWebScraperDriver(BaseWebScraperDriver):
    proxies: dict = field(kw_only=True, metadata={"serializable": False})
    params: dict = field(default=Factory(dict), kw_only=True, metadata={"serializable": True})

    def fetch_url(self, url: str) -> str:
        response = requests.get(url, proxies=self.proxies, **self.params)

        return response.text

    def extract_page(self, page: str) -> TextArtifact:
        return TextArtifact(page)

params: dict = field(default=Factory(dict), kw_only=True, metadata={'serializable': True}) class-attribute instance-attribute

proxies: dict = field(kw_only=True, metadata={'serializable': False}) class-attribute instance-attribute

extract_page(page)

Source code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py
def extract_page(self, page: str) -> TextArtifact:
    return TextArtifact(page)

fetch_url(url)

Source code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py
def fetch_url(self, url: str) -> str:
    response = requests.get(url, proxies=self.proxies, **self.params)

    return response.text