We will be building and deploying a python script to scrape sites which will end up looking like this:

run:

python3.12 -m venv .venv && . .venv/bin/activate

then

pip install -r requirements.txt

then

playwright install

finally:

python spiders/google_job_hunt.py

importreimportscrapyfromscrapy.crawlerimportCrawlerProcessfromscrapy.selectorimportSelectorclassGoogleSpider(scrapy.Spider):
name='google_spider'allowed_domains= ['www.google.com']
custom_settings= {
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
"DOWNLOAD_HANDLERS": {
"https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
"http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
}
}
def__init__(self, domain, stop, user_agent, *args, **kwargs):
super().__init__(*args, **kwargs)
self.domain=domainself.stop=int(stop)
self.custom_settings['USER_AGENT'] =user_agentself.start_urls= [
f'https://www.google.com/search?q=intitle%3A%28%22Data+Scientist%22+OR+%22Data+Engineer%22+OR+%22Machine+Learning%22+OR+%22Data+Analyst%22+OR+%22Software+Engineer%22%29+Remote+-%22Director%22+-%22Principal%22+-%22Staff%22+-%22Frontend%22+-%22Front+End%22+-%22Full+Stack%22+site%3A{self.domain}%2F%2A+after%3A2023-03-27']
self.urls_collected= []
@classmethoddeffrom_crawler(cls, crawler, *args, **kwargs):
returnsuper().from_crawler(crawler, *args, **kwargs)
defstart_requests(self):
yieldscrapy.Request(self.start_urls[0], meta={"playwright": True,
"playwright_include_page": True})
asyncdefget_page_info(self, page):
foriinrange(10):
val=page.viewport_size["height"]
awaitpage.mouse.wheel(0, val)
awaitpage.wait_for_timeout(1000)
text=awaitpage.content()
selector=Selector(text=text)
urls= []
forrowinselector.xpath("//div[contains(@class, 'kCrYT')]"):
text=row.xpath(".//h3//text()").get()
url=row.xpath(".//a/@href").get()
ifurl:
urls.append({text: url})
print(urls)
self.urls_collected+=urlsreturnurlsasyncdefparse(self, response):
page=response.meta['playwright_page']
urls=awaitself.get_page_info(page)
found=Truewhilefound:
try:
element=page.get_by_text("Next")
print(element, "parsing next page")
awaitelement.click()
more_urls=awaitself.get_page_info(page)
urls+=more_urlsexcept:
found=Falsereturnurlsdefmain(domain, stop, user_agent):
process=CrawlerProcess()
process.crawl(GoogleSpider, domain=domain, stop=stop, user_agent=user_agent)
process.start()
if__name__=='__main__':
domain='jobs.lever.co'stop=25user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'user_agent2="Opera/9.80 (Windows NT 5.1; U; MRA 5.5 (build 02842); ru) Presto/2.7.62 Version/11.00"user_agent3="Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)"main(domain=domain, stop=stop, user_agent=user_agent3)