From a08942acacd0ce0d413f7c30be80dea73dea3f2d Mon Sep 17 00:00:00 2001 From: ayushim Date: Thu, 29 Feb 2024 21:22:54 -0400 Subject: [PATCH 1/2] Initial Commit --- .../java/us/codecraft/webmagic/SiteTest.java | 23 ++++++++ .../webmagic/selector/AndSelectorTest.java | 59 +++++++++++++++++++ .../webmagic/selector/CssSelectorTest.java | 40 +++++++++++++ .../webmagic/selector/OrSelectorTest.java | 45 ++++++++++++++ .../downloader/selenium/WebDriverPool.java | 12 +++- .../webmagic/downloader/SeleniumTest.java | 7 ++- 6 files changed, 181 insertions(+), 5 deletions(-) create mode 100644 webmagic-core/src/test/java/us/codecraft/webmagic/selector/AndSelectorTest.java create mode 100644 webmagic-core/src/test/java/us/codecraft/webmagic/selector/CssSelectorTest.java create mode 100644 webmagic-core/src/test/java/us/codecraft/webmagic/selector/OrSelectorTest.java diff --git a/webmagic-core/src/test/java/us/codecraft/webmagic/SiteTest.java b/webmagic-core/src/test/java/us/codecraft/webmagic/SiteTest.java index 783b82ddc..0a583492d 100644 --- a/webmagic-core/src/test/java/us/codecraft/webmagic/SiteTest.java +++ b/webmagic-core/src/test/java/us/codecraft/webmagic/SiteTest.java @@ -1,8 +1,13 @@ package us.codecraft.webmagic; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.junit.Test; @@ -14,4 +19,22 @@ public void test() { assertEquals(StandardCharsets.UTF_8.name(), site.getDefaultCharset()); } + @Test + public void addCookieTest(){ + Site site=Site.me().setDefaultCharset(StandardCharsets.UTF_8.name()); + site.addCookie("cookieDefault","cookie-webmagicDefault"); + String domain="example.com"; + site.addCookie(domain, "cookie", "cookie-webmagic"); + site.addCookie(domain, "cookieCopy", "cookie-webmagicCopy"); + Map> allCookies = site.getAllCookies(); + List domains=new ArrayList<>(); + for(String key : allCookies.keySet()){ + domains.add(key); + } + assertTrue(allCookies.containsKey(domain)); + assertTrue(allCookies.get(domain).containsKey("cookie")); + assertEquals("cookie-webmagic", allCookies.get(domain).get("cookie")); + assertEquals("cookie-webmagicCopy", allCookies.get(domain).get("cookieCopy")); + assertEquals(1, domains.size()); + } } diff --git a/webmagic-core/src/test/java/us/codecraft/webmagic/selector/AndSelectorTest.java b/webmagic-core/src/test/java/us/codecraft/webmagic/selector/AndSelectorTest.java new file mode 100644 index 000000000..59885ebd1 --- /dev/null +++ b/webmagic-core/src/test/java/us/codecraft/webmagic/selector/AndSelectorTest.java @@ -0,0 +1,59 @@ +package us.codecraft.webmagic.selector; + +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +public class AndSelectorTest { + + @Test + public void testSelectList() { + String htmlContent = "\n" + + "\n" + + "\n" + + " \n" + + " \n" + + " HTML with XPath\n" + + "\n" + + "\n" + + "
\n" + + "
Item 1
\n" + + "
Item 2
\n" + + "
\n" + + "\n" + + ""; + List selectors = new ArrayList(); + selectors.add(new CssSelector("div")); + selectors.add(new XpathSelector("//div[@class='item1']")); + AndSelector andSelector = new AndSelector(selectors); + List result = andSelector.selectList(htmlContent); + assertEquals("
\n Item 1\n
", result.get(0)); + } + + @Test + public void testSelectList_NoResults() { + String htmlContent = "\n" + + "\n" + + "\n" + + " \n" + + " \n" + + " HTML with XPath\n" + + "\n" + + "\n" + + "
\n" + + "
Item 1
\n" + + "
Item 2
\n" + + "
\n" + + "\n" + + ""; + List selectors = new ArrayList(); + selectors.add(new CssSelector("div")); + selectors.add(new XpathSelector("//div[@class='item']")); + AndSelector andSelector = new AndSelector(selectors); + List result = andSelector.selectList(htmlContent); + assertEquals(0, result.size()); + } +} diff --git a/webmagic-core/src/test/java/us/codecraft/webmagic/selector/CssSelectorTest.java b/webmagic-core/src/test/java/us/codecraft/webmagic/selector/CssSelectorTest.java new file mode 100644 index 000000000..cc7076178 --- /dev/null +++ b/webmagic-core/src/test/java/us/codecraft/webmagic/selector/CssSelectorTest.java @@ -0,0 +1,40 @@ +package us.codecraft.webmagic.selector; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.junit.Test; +import java.util.List; +import static org.junit.Assert.*; + + +public class CssSelectorTest { + + @Test + public void testSelectElement() { + CssSelector cssSelector=new CssSelector("div"); + String htmlContent = "Dummy Page
Hello World!
"; + // Parse the HTML content + Document doc = Jsoup.parse(htmlContent); + // Select the dummy div element by its ID + Element dummyElement = doc.getElementById("dummyDiv"); + Element resultElement = cssSelector.selectElement(dummyElement); + assertNotNull(resultElement); + } + + @Test + public void testSelectList() { + CssSelector cssSelector=new CssSelector("div"); + String htmlContent = "Dummy Page
Hello World!
"; + // Parse the HTML content + Document doc = Jsoup.parse(htmlContent); + // Select the dummy div element by its ID + Element dummyElement = doc.getElementById("dummyDiv"); + List result = cssSelector.selectList(dummyElement); + assertEquals(1, result.size()); + assertEquals("
\n Hello World!\n
", result.get(0)); + } + + +} diff --git a/webmagic-core/src/test/java/us/codecraft/webmagic/selector/OrSelectorTest.java b/webmagic-core/src/test/java/us/codecraft/webmagic/selector/OrSelectorTest.java new file mode 100644 index 000000000..91b697df7 --- /dev/null +++ b/webmagic-core/src/test/java/us/codecraft/webmagic/selector/OrSelectorTest.java @@ -0,0 +1,45 @@ +package us.codecraft.webmagic.selector; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + +public class OrSelectorTest { + @Test + public void testSelectList() { + String htmlContent = "\n" + + "\n" + + "\n" + + " \n" + + " \n" + + " HTML with XPath\n" + + "\n" + + "\n" + + "
\n" + + "
Item 1
\n" + + "
Item 2
\n" + + "
\n" + + "\n" + + ""; + String expectedResult = "[\n" + + " \n" + + " \n" + + " HTML with XPath\n" + + ",
\n" + + " Item 1\n" + + "
,
\n" + + " Item 2\n" + + "
]"; + List selectors = new ArrayList(); + selectors.add(new CssSelector("head")); + selectors.add(new XpathSelector("//div[@class='item1']")); + selectors.add(new XpathSelector("//div[@class='item2']")); + OrSelector orSelector = new OrSelector(selectors); + List result = orSelector.selectList(htmlContent); + assertEquals(expectedResult, result.toString()); + } +} diff --git a/webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java b/webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java index e1d9dd039..ab4cd5f72 100644 --- a/webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java +++ b/webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/WebDriverPool.java @@ -2,7 +2,9 @@ import org.openqa.selenium.WebDriver; import org.openqa.selenium.chrome.ChromeDriver; +import org.openqa.selenium.chrome.ChromeOptions; import org.openqa.selenium.firefox.FirefoxDriver; +import org.openqa.selenium.firefox.FirefoxOptions; import org.openqa.selenium.phantomjs.PhantomJSDriver; import org.openqa.selenium.phantomjs.PhantomJSDriverService; import org.openqa.selenium.remote.DesiredCapabilities; @@ -73,7 +75,7 @@ public void configure() throws IOException { // Prepare capabilities sCaps = new DesiredCapabilities(); - sCaps.setJavascriptEnabled(true); + sCaps.setCapability("javascriptEnabled", true); sCaps.setCapability("takesScreenshot", false); String driver = sConfig.getProperty("driver", DRIVER_PHANTOMJS); @@ -134,9 +136,13 @@ public void configure() throws IOException { sCaps.setBrowserName("phantomjs"); mDriver = new RemoteWebDriver(new URL(driver), sCaps); } else if (driver.equals(DRIVER_FIREFOX)) { - mDriver = new FirefoxDriver(sCaps); + FirefoxOptions options = new FirefoxOptions(); + options.merge(sCaps); + mDriver = new FirefoxDriver(options); } else if (driver.equals(DRIVER_CHROME)) { - mDriver = new ChromeDriver(sCaps); + ChromeOptions options=new ChromeOptions(); + options.merge(sCaps); + mDriver = new ChromeDriver(options); } else if (driver.equals(DRIVER_PHANTOMJS)) { mDriver = new PhantomJSDriver(sCaps); } diff --git a/webmagic-selenium/src/test/java/us/codecraft/webmagic/downloader/SeleniumTest.java b/webmagic-selenium/src/test/java/us/codecraft/webmagic/downloader/SeleniumTest.java index b7bcd80b3..3c4a933a7 100644 --- a/webmagic-selenium/src/test/java/us/codecraft/webmagic/downloader/SeleniumTest.java +++ b/webmagic-selenium/src/test/java/us/codecraft/webmagic/downloader/SeleniumTest.java @@ -6,6 +6,7 @@ import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; import org.openqa.selenium.chrome.ChromeDriver; +import org.openqa.selenium.chrome.ChromeOptions; import org.openqa.selenium.remote.DesiredCapabilities; import java.util.Arrays; @@ -29,10 +30,12 @@ public void testSelenium() { Map preferences = new HashMap(); preferences.put("profile.default_content_settings", contentSettings); - DesiredCapabilities caps = DesiredCapabilities.chrome(); + DesiredCapabilities caps = new DesiredCapabilities(); caps.setCapability("chrome.prefs", preferences); caps.setCapability("chrome.switches", Arrays.asList("--user-data-dir=/Users/yihua/temp/chrome")); - WebDriver webDriver = new ChromeDriver(caps); + ChromeOptions options=new ChromeOptions(); + options.merge(caps); + WebDriver webDriver = new ChromeDriver(options); webDriver.get("http://huaban.com/"); WebElement webElement = webDriver.findElement(By.xpath("/html")); System.out.println(webElement.getAttribute("outerHTML")); From 636442834ba175277cd52f086e64a17b10875a8d Mon Sep 17 00:00:00 2001 From: ayushim Date: Fri, 1 Mar 2024 21:44:03 -0400 Subject: [PATCH 2/2] Assignment 1 Submission --- .../java/us/codecraft/webmagic/SiteTest.java | 18 +++++++++--------- .../downloader/HttpClientDownloaderTest.java | 9 +++++++++ .../webmagic/selector/CssSelectorTest.java | 19 +++++++++---------- .../webmagic/selector/OrSelectorTest.java | 1 - 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/webmagic-core/src/test/java/us/codecraft/webmagic/SiteTest.java b/webmagic-core/src/test/java/us/codecraft/webmagic/SiteTest.java index 0a583492d..47c4fcc14 100644 --- a/webmagic-core/src/test/java/us/codecraft/webmagic/SiteTest.java +++ b/webmagic-core/src/test/java/us/codecraft/webmagic/SiteTest.java @@ -1,7 +1,6 @@ package us.codecraft.webmagic; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.nio.charset.StandardCharsets; @@ -23,18 +22,19 @@ public void test() { public void addCookieTest(){ Site site=Site.me().setDefaultCharset(StandardCharsets.UTF_8.name()); site.addCookie("cookieDefault","cookie-webmagicDefault"); - String domain="example.com"; - site.addCookie(domain, "cookie", "cookie-webmagic"); - site.addCookie(domain, "cookieCopy", "cookie-webmagicCopy"); + String firstDomain="example.com"; + String secondDomain="exampleCopy.com"; + site.addCookie(firstDomain, "cookie", "cookie-webmagic"); + site.addCookie(firstDomain, "cookieCopy", "cookie-webmagicCopy"); + site.addCookie(secondDomain, "cookie", "cookie-webmagic"); Map> allCookies = site.getAllCookies(); List domains=new ArrayList<>(); for(String key : allCookies.keySet()){ domains.add(key); } - assertTrue(allCookies.containsKey(domain)); - assertTrue(allCookies.get(domain).containsKey("cookie")); - assertEquals("cookie-webmagic", allCookies.get(domain).get("cookie")); - assertEquals("cookie-webmagicCopy", allCookies.get(domain).get("cookieCopy")); - assertEquals(1, domains.size()); + assertEquals("cookie-webmagic", allCookies.get(firstDomain).get("cookie")); + assertEquals("cookie-webmagicCopy", allCookies.get(firstDomain).get("cookieCopy")); + assertEquals("cookie-webmagic", allCookies.get(secondDomain).get("cookie")); + assertEquals(2, domains.size()); } } diff --git a/webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java b/webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java index 780ca7529..1ff7b4dd7 100644 --- a/webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java +++ b/webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java @@ -40,6 +40,7 @@ import static com.github.dreamhead.moco.Moco.with; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; /** @@ -333,5 +334,13 @@ public void run() throws Exception { }); } + @Test + public void test_no_task_download(){ + Request request = new Request(); + request.setUrl("http://127.0.0.1:13423/"); + HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); + assertThrows(NullPointerException.class, () -> httpClientDownloader.download(request,null)); + } + } diff --git a/webmagic-core/src/test/java/us/codecraft/webmagic/selector/CssSelectorTest.java b/webmagic-core/src/test/java/us/codecraft/webmagic/selector/CssSelectorTest.java index cc7076178..8b1ace903 100644 --- a/webmagic-core/src/test/java/us/codecraft/webmagic/selector/CssSelectorTest.java +++ b/webmagic-core/src/test/java/us/codecraft/webmagic/selector/CssSelectorTest.java @@ -5,36 +5,35 @@ import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + import java.util.List; import static org.junit.Assert.*; - public class CssSelectorTest { @Test public void testSelectElement() { - CssSelector cssSelector=new CssSelector("div"); + CssSelector cssSelector = new CssSelector("div"); String htmlContent = "Dummy Page
Hello World!
"; - // Parse the HTML content Document doc = Jsoup.parse(htmlContent); - // Select the dummy div element by its ID Element dummyElement = doc.getElementById("dummyDiv"); Element resultElement = cssSelector.selectElement(dummyElement); assertNotNull(resultElement); } - @Test + @Test public void testSelectList() { - CssSelector cssSelector=new CssSelector("div"); + CssSelector cssSelector = new CssSelector("div"); String htmlContent = "Dummy Page
Hello World!
"; - // Parse the HTML content Document doc = Jsoup.parse(htmlContent); - // Select the dummy div element by its ID Element dummyElement = doc.getElementById("dummyDiv"); List result = cssSelector.selectList(dummyElement); assertEquals(1, result.size()); - assertEquals("
\n Hello World!\n
", result.get(0)); + assertEquals("[
\n Hello World!\n
]", result.toString()); } - } diff --git a/webmagic-core/src/test/java/us/codecraft/webmagic/selector/OrSelectorTest.java b/webmagic-core/src/test/java/us/codecraft/webmagic/selector/OrSelectorTest.java index 91b697df7..24d87647c 100644 --- a/webmagic-core/src/test/java/us/codecraft/webmagic/selector/OrSelectorTest.java +++ b/webmagic-core/src/test/java/us/codecraft/webmagic/selector/OrSelectorTest.java @@ -1,7 +1,6 @@ package us.codecraft.webmagic.selector; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import java.util.ArrayList; import java.util.List;