Search-Engine/CrawlerThread.java at master · SHossamM/Search-Engine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
importjava.io.*;
importjava.net.MalformedURLException;
importjava.net.URL;
importjava.util.*;
importjava.util.concurrent.ConcurrentLinkedQueue;
importjava.util.logging.Level;
importjava.util.logging.Logger;

importorg.apache.commons.io.FileUtils;
importorg.jsoup.*;
importorg.jsoup.nodes.Document;
importorg.jsoup.select.Elements;


publicclassCrawlerThreadimplementsRunnable {
finalStringuserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1";
finalstaticintdomainMax = 20;
DataBasedB;

publicConcurrentLinkedQueue<Url> urlsQueue; //refernce to that of controller's
publicConcurrentLinkedQueue<OutgoingLinks> outgoingLinks;
publicConcurrentLinkedQueue<Integer> visitedLinks;
ConcurrentLinkedQueue<Integer> notVerified;
ConcurrentLinkedQueue<Integer> notRobotallowed;
ConcurrentLinkedQueue<Integer> notParseable;

CrawlerThread(ConcurrentLinkedQueue<Url> urlsQueue,
ConcurrentLinkedQueue<OutgoingLinks> outgoingLinks,
ConcurrentLinkedQueue<Integer> visitedLinks,
ConcurrentLinkedQueue<Integer> notVerified,
ConcurrentLinkedQueue<Integer> notRobotallowed,
ConcurrentLinkedQueue<Integer> notParseable) {
this.urlsQueue = urlsQueue;
this.outgoingLinks = outgoingLinks;
this.visitedLinks = visitedLinks;
this.notVerified = notVerified;
this.notRobotallowed = notRobotallowed;
this.notParseable = notParseable;
 }

/**
 * if it is html it download it as a document and adds it to a queue
 *
 * @param url
 * @return null if not html document
 */
DocumentgetHTMLDocument(Urlurl) {
try {
Connectionconn = Jsoup.connect(url.getUrl()).userAgent(userAgent);
// Connection.Response response;
// response = conn.url(url.toString()).timeout(10000).execute();
// if(response.contentType()!=null)
// {
// if(!response.contentType().contains("text/html"))
// return null;
// }
Documentdoc = conn.get();
Filef = newFile("./pages/" + Integer.toString(url.getId()) + ".html");
FileUtils.writeStringToFile(f, doc.outerHtml(), "UTF-8");
//System.out.println(Thread.currentThread().getName() + " marked " + url.getId() + " as visited and downloaded its web content");
returndoc;
 } catch (IOException | NullPointerExceptione) { //-->ask whether null pointer returns null or not
returnnull;
 }
 }

/**
 * retrieves all document in a vector of strings
 */
privateList<String> getLinks(Documentdoc, Stringdomain) {
List<String> result = newLinkedList<>();
Elementslinks = doc.select("a[href]"); // get all elements with href
StringextractedUrl;
intdomainCount = 0;
for (org.jsoup.nodes.Elementlink : links) {
extractedUrl = link.attr("abs:href").toLowerCase();
if (extractedUrl.contains("#") || extractedUrl.isEmpty()) { //empty | # refers to current document too
continue;
 }
if (extractedUrl.contains(domain)) {
if (domainCount > domainMax) {
continue;
 }
domainCount++;
 }
result.add(extractedUrl); // removing <a tag and href attribute
 }
//System.out.println(Thread.currentThread().getName() + " Extracted Links");
returnresult;
 }

/**
 * pops a link from queue
 */

voidcrawl() throwsMalformedURLException {
//get / pop a url from queue
Urlurl;
Documentdoc;

while ((url = urlsQueue.poll()) != null) {

//Get a url to crawl

System.out.println(Thread.currentThread().getName() + " popped " + url.getId() + " : " + url.getUrl() + " from the urlsQueue");

if (url.verifyUrl() != null) {//check1 that the url is verified

if (url.getUrl().contains("#")) {//check2 removing hashes as they mean same page
url.setUrl(url.getUrl().substring(0, url.getUrl().indexOf('#')));
 }
Stringhost = (newURL(url.getUrl())).getHost().toLowerCase();//get the host of this url which is the domain name i.e http://codeproject.com return codeproject.com

if (Robot.isRobotAllowed(url.getUrl(), userAgent, host)) {//check3 RobotAllowed
//System.out.println(Thread.currentThread().getName() + " trying to download " + url.getUrl());
doc = getHTMLDocument(url); //MARK the link as visited IN DB
if (doc != null) {//null if not html //check5 html document?
//extract links from this document
//System.out.println(Thread.currentThread().getName() + " Adding links of: " + url.getUrl());
outgoingLinks.add(newOutgoingLinks(url.getId(), getLinks(doc, host)));
visitedLinks.add(url.getId());
//System.out.println(Thread.currentThread().getName() + " Added links of: " + url.getUrl());
 } else {
//System.out.println(url.getUrl() + " is not parseable(not html) or not reachable" + Thread.currentThread().getName());
notParseable.add(url.getId());
 }
 } else {
//System.out.println(url.getUrl() + " is robot disallowed " + Thread.currentThread().getName());
notRobotallowed.add(url.getId());
 }
 } else {
//System.out.println(url.getUrl() + " is unverified by " + Thread.currentThread().getName());
notVerified.add(url.getId());
 }
 }
 }


@Override
publicvoidrun() {
try {
crawl();
System.out.println(Thread.currentThread().getName() + " exiting");
 } catch (MalformedURLExceptionex) {
Logger.getLogger(CrawlerThread.class.getName()).log(Level.SEVERE, null, ex);
System.out.println(Thread.currentThread().getName() + " broken");
 }
 }
}