Skip to content

proxy_web_scraper_driver

ProxyWebScraperDriver

Bases: BaseWebScraperDriver

Source code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py 
10
11
12
13
14
15
16
17
@define
class ProxyWebScraperDriver(BaseWebScraperDriver):
    proxies: dict = field(kw_only=True, metadata={"serializable": False})
    params: dict = field(default=Factory(dict), kw_only=True, metadata={"serializable": True})

    def scrape_url(self, url: str) -> TextArtifact:
        response = requests.get(url, proxies=self.proxies, **self.params)
        return TextArtifact(response.text)

params: dict = field(default=Factory(dict), kw_only=True, metadata={'serializable': True}) class-attribute instance-attribute

proxies: dict = field(kw_only=True, metadata={'serializable': False}) class-attribute instance-attribute

scrape_url(url)

Source code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py 
15
16
17
def scrape_url(self, url: str) -> TextArtifact:
    response = requests.get(url, proxies=self.proxies, **self.params)
    return TextArtifact(response.text)