ipProxy

使用goQuery根据模板自动采集IP，验证采集完毕验证代理的访问速度，使用筛选合格的IP作为请求的代理

测试demo，如有问题提交issues

一、goQuery库：

1.简述

据官方描述，goQuery实现了与jQuery相似的DOM操作功能。与jQuery不同的是，jQuery返回的是完整的DOM树，而goQuery返回的是DOM节点。goQuery底层由golang标准库net/html实现，解析器要求文档必须是 UTF-8 编码，使用者应按需转换文档编码。goQuery-readme

2.主要方法

2.1 Document： 返回要被操作的HTML文档

// Document represents an HTML document to be manipulated. Unlike jQuery, which// is loaded as part of a DOM document, and thus acts upon its containing// document, GoQuery doesn't know which HTML document to act upon. So it needs// to be told, and that's what the Document class is for. It holds the root// document node to manipulate, and can make selections on this document.typeDocumentstruct {
*SelectionUrl*url.URLrootNode*html.Node
}

2.2 Selection： 符合指定条件的节点。

// Selection represents a collection of nodes matching some criteria. The// initial Selection can be created by using Document.Find, and then// manipulated using the jQuery-like chainable syntax and methods.typeSelectionstruct {
Nodes []*html.Nodedocument*DocumentprevSel*Selection
}

2.3文档操作函数：

Eq()
Index()
Last()
Slice()
Get()
······

二、采集代理IP：

1.代理IP池

1. 高可用全球免费代理IP库
2. 西刺免费代理IP
3. 全网IP代理
4. 快代理

2.goQuery采集：

不多说，上代码：

import"github.com/PuerkitoBio/goquery"

导入goquery库

//采集代理返回的参数typeproxyResultstruct {
Ipstring`json:"ip"`//ipPortint`json:port`//端口Agreementstring`json:agreement`//请求协议Anonymousstring`json:anonymous`//透明度Regionstring`json:region`//地区Speedstring`json:"speed"`//响应速度Sourcestring`json:"source"`//来源（采集资源站）Verificationstring`json:"verification"`//验证时间
}
//采集代理所需的参数typeproxyParametstruct {
ipIndexint`json:"ipIndex"`//ip下标portIndexint`json:"portIndex"`//端口下标agreementIndexint`json:"agreementIndex"`//请求协议下标anonymousIndexint`json:"anonymousIndex"`//透明度下标regionIndexint`json:"regionIndex"`//地区下标speedIndexint`json:"speedIndex"`//响应速度下标sourceIndexint`json:"sourceIndex"`//来源（采集资源站）下标verificationIndexint`json:"verificationIndex"`//验证时间下标
}

创建两个结构体，proxyResult保存返回的数据，proxyParamet 用来携带传入参数

funcCollectionResources(targetUrlstring, parameproxyParamet)[]proxyResult {
//用来存储采集结果proxyList:= []proxyResult{}
//请求目标站点方式（GET/POST）resp, err:=http.Get(targetUrl)
//请求失败则输出日志ifresp.StatusCode!=200||err!=nil {
log.Pr("spider", "请求出错", err)
}
//添加随机User-Agentresp.Header.Add("User-Agent", random.RandomUseragent())
//返回目标站html文档doc, err:=goquery.NewDocumentFromReader(resp.Body)
//因为大部分都是以表格方式展示，所以这里就直接抓取tbody的内容doc.Find("tbody tr").Each(func(iint, selection*goquery.Selection) {
//golang不支持自动类型转换，这里手动转换拼接ip:=selection.Find("td:nth-child("+strconv.Itoa(parame.ipIndex) +")").Text()
port, err:=strconv.Atoi(selection.Find("td:nth-child("+strconv.Itoa(parame.portIndex) +")").Text())
agreement:=selection.Find("td:nth-child("+strconv.Itoa(parame.agreementIndex) +")").Text()
anonymous:=selection.Find("td:nth-child("+strconv.Itoa(parame.anonymousIndex) +")").Text()
region:=selection.Find("td:nth-child("+strconv.Itoa(parame.regionIndex) +")").Text()
speedString:=selection.Find("td:nth-child("+strconv.Itoa(parame.speedIndex) +")").Text()
//有的代理池会携带上单位：秒，加上这段代码可以去掉。可加可不加，按需设置speed:=strings.TrimRight(speedString, "秒")
verification:=selection.Find("td:nth-child("+strconv.Itoa(parame.verificationIndex) +")").Text()
iferr!=nil {
log.Pr("spider", "数据转换出错", err)
}
proxyList=append(proxyList, proxyResult{Ip: ip,
Port: port,
Agreement: agreement,
Anonymous: anonymous,
Region: region,
Speed: speed,
Source: targetUrl,
Verification: verification}, )
})
//采集完毕之后清理deferresp.Body.Close()
returnproxyList }

调用：

parame:=proxyParamet{
ipIndex: 1,
portIndex: 2,
agreementIndex: 4,
anonymousIndex: 3,
regionIndex: 5,
speedIndex: 6,
verificationIndex: 7,
}
CollectionResources("https://www.kuaidaili.com/free/", parame)

parame的参数对应着表格标题的下标（从1开始）

设置代理

funcStartRequestProxy(addressstring, proxyIpInfo [] proxyResult) string {
proxyAddr:="协议+地址+端口 /如：https://127.0.0.1:8080"url:=addresscli:=newHttpClient(proxyAddr)
data, _:=httpGET(cli, url)
returnstring(data)
}
funcnewHttpClient(proxyAddrstring) *http.Client {
proxy, err:=url.Parse(proxyAddr)
iferr!=nil {
returnnil
}
netTransport:=&http.Transport{
Proxy: http.ProxyURL(proxy),
Dial: func(netw, addrstring) (net.Conn, error) {
c, err:=net.DialTimeout(netw, addr, time.Second*time.Duration(10))
iferr!=nil {
returnnil, err
}
returnc, nil
},
MaxIdleConnsPerHost: 10, //每个host最大空闲连接ResponseHeaderTimeout: time.Second*time.Duration(5), //数据收发5秒超时
}
return&http.Client{
Timeout: time.Second*10,
Transport: netTransport,
}
}
funchttpGET(client*http.Client, urlstring) (body []byte, errerror) {
rsp, err:=client.Get(url)
iferr!=nil {
return
}
deferrsp.Body.Close()
ifrsp.StatusCode!=http.StatusOK||err!=nil {
err=fmt.Errorf("HTTP GET Code=%v, URI=%v, err=%v", rsp.StatusCode, url, err)
log.Pr("HttpGet", "Request error", err)
return
}
returnioutil.ReadAll(rsp.Body)
}

运行结果：

注：goquery的部分介绍引用了go语言中文网