Bases: BaseWebScraperDriver
Source code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py
| @define
class ProxyWebScraperDriver(BaseWebScraperDriver):
proxies: dict = field(kw_only=True, metadata={"serializable": False})
params: dict = field(default=Factory(dict), kw_only=True, metadata={"serializable": True})
def fetch_url(self, url: str) -> str:
response = requests.get(url, proxies=self.proxies, **self.params)
return response.text
def extract_page(self, page: str) -> TextArtifact:
return TextArtifact(page)
|
params: dict = field(default=Factory(dict), kw_only=True, metadata={'serializable': True})
class-attribute
instance-attribute
proxies: dict = field(kw_only=True, metadata={'serializable': False})
class-attribute
instance-attribute
Source code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py
| def extract_page(self, page: str) -> TextArtifact:
return TextArtifact(page)
|
fetch_url(url)
Source code in griptape/drivers/web_scraper/proxy_web_scraper_driver.py
| def fetch_url(self, url: str) -> str:
response = requests.get(url, proxies=self.proxies, **self.params)
return response.text
|