简单使用IP代理池和用户代理池的爬虫
import re
import random
import urllib.request as urlreq
import urllib.error as urlerr
#用户代理池
uapools = [
"Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393"
]
#ip代理池
ipools = []
#获取用户代理
def get_ua(uapools):
thisua = random.choice(uapools)
header = ("User-Agent", thisua)
url_opener = urlreq.build_opener()
url_opener.addheaders = [header]
urlreq.install_opener(url_opener)
#获取ip池，这里从西刺获取首页IP保存到列表中
def get_ipools(ipurl):
get_ua(uapools)
data = urlreq.urlopen(ipurl).read().decode("utf-8","ignore")
pat = "/>.*?(.*?) .*?(.*?) "
ret = re.compile(pat, re.S).findall(data)
# print(ret)
for i in ret:
ips = i[0] + ":" + i[1]
ipools.append(ips)
return ipools
#解析糗事百科的文章
def get_article(data):
pat = '.*?(.*?).*?'
rst = re.compile(pat, re.S).findall(data)
print(rst)
# down_file(rst, i)
def get_html(urlweb):
for i in range(1, 6): #爬取前五页文章
while 1:
try:
page = urlweb + str(i)
thisua = random.choice(uapools)
header = ("User-Agent", thisua) #构建用户代理
ip = random.choice(ipools)
print("当前使用的ip为" + ip)
proxy = urlreq.ProxyHandler({"http": ip}) #构建IP代理
url_opener = urlreq.build_opener(proxy, urlreq.HTTPHandler) #添加IP代理头
url_opener.addheaders = [header] #添加用户代理头
urlreq.install_opener(url_opener) #设为全局变量
data = urlreq.urlopen(page).read().decode("utf-8","ignore")
except Exception as e:
print(e)
ipools.remove(ip) #爬取失败时，从IP池中删除IP，重新爬取文章
continue
get_article(data) #解析文章
break #完成一页的爬取
if __name__ == "__main__":
ipurl = "https://www.xicidaili.com/nn/"
ipools = get_ipools(ipurl) #获取ip池
urlweb = "https://www.qiushibaike.com/text/page/"
get_html(urlweb)另外有需要云服务器可以了解下创新互联cdcxhl.cn，海内外云服务器15元起步，三天无理由+7*72小时售后在线，公司持有idc许可证，提供“云服务器、裸金属服务器、高防服务器、香港服务器、美国服务器、虚拟主机、免备案服务器”等云主机租用服务以及企业上云的综合解决方案，具有“安全稳定、简单易用、服务可用性高、性价比高”等特点与优势，专为企业上云打造定制，能够满足用户丰富、多元化的应用场景需求。