Skip to content

base_web_scraper_driver

BaseWebScraperDriver

Bases: ABC

Source code in griptape/drivers/web_scraper/base_web_scraper_driver.py 
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
class BaseWebScraperDriver(ABC):
    def scrape_url(self, url: str) -> TextArtifact:
        source = self.fetch_url(url)

        return self.extract_page(source)

    @abstractmethod
    def fetch_url(self, url: str) -> str:
        pass

    @abstractmethod
    def extract_page(self, page: str) -> TextArtifact:
        pass

extract_page(page) abstractmethod

Source code in griptape/drivers/web_scraper/base_web_scraper_driver.py 
16
17
18
@abstractmethod
def extract_page(self, page: str) -> TextArtifact:
    pass

fetch_url(url) abstractmethod

Source code in griptape/drivers/web_scraper/base_web_scraper_driver.py 
12
13
14
@abstractmethod
def fetch_url(self, url: str) -> str:
    pass

scrape_url(url)

Source code in griptape/drivers/web_scraper/base_web_scraper_driver.py 
 7
 8
 9
10
def scrape_url(self, url: str) -> TextArtifact:
    source = self.fetch_url(url)

    return self.extract_page(source)