Node.js based scraper using headless chrome
$ npm install @jonstuebe/scraper- Scrape top ecommerce sites (Amazon, Walmart, Target)
- Return basic product information (title, price, image, description)
- Easy to use API to scrape any website
Simply require the package and initialize with a url and pass a callback function to receive the data.
constScraper=require("@jonstuebe/scraper");// run inside of an async function(async()=>{constdata=awaitScraper.scrapeAndDetect("http://www.amazon.com/gp/product/B00X4WHP5E/");console.log(data);})();importScraperfrom"@jonstuebe/scraper";// run inside of an async function(async()=>{constdata=awaitScraper("http://www.amazon.com/gp/product/B00X4WHP5E/");console.log(data);})();importScraperfrom"@jonstuebe/scraper";Scraper("http://www.amazon.com/gp/product/B00X4WHP5E/").then(data=>{console.log(data);});If you are going to be running the scraper a number of times in succession, it's recommended to share the same chromium instance for each sequential/parallel scrape.
importpuppeteerfrom"puppeteer";importScraperfrom"@jonstuebe/scraper";// run inside of an async function(async()=>{constbrowser=awaitpuppeteer.launch();letproducts=["https://www.target.com/p/corinna-angle-leg-side-table-wood-threshold-8482/-/A-53496420","https://www.target.com/p/glasgow-metal-end-table-black-project-62-8482/-/A-52343433"];letproductsData=[];for(constproductofproducts){constproductData=awaitScraper(product,browser);productsData.push(productData);}awaitbrowser.close();// make sure and close the browser otherwise the instances will continue to run in the backround on your machineconsole.table(productsData);})();If you want to emulate a device, pass in a puppeteer device as the third agument:
importpuppeteerfrom"puppeteer";importScraperfrom"@jonstuebe/scraper";// run inside of an async function(async()=>{constdata=awaitScraper("http://www.amazon.com/gp/product/B00X4WHP5E/",null,puppeteer.devices["iPhone SE"]);console.log(data);})();constScraper=require("@jonstuebe/scraper");(async()=>{constsite={name: "npm",hosts: ["www.npmjs.com"],scrape: asyncpage=>{constname=awaitScraper.getText("div.content-column > h1 > a",page);constversion=awaitScraper.getText("div.sidebar > ul:nth-child(2) > li:nth-child(2) > strong",page);constauthor=awaitScraper.getText("div.sidebar > ul:nth-child(2) > li.last-publisher > a > span",page);return{
name,
version,
author
};}};constdata=awaitScraper.scrape("https://www.npmjs.com/package/lodash",site);console.log(data);})();If you want to add any sites, or just have an idea or feature, go ahead and fork this repo and send me a pull request. I'll be happy to take a look when I can and get back to you.
For any and all issues/bugs, please post a description and code sample to reproduce the problem on the issues page.