import requests |
import random |
import easygui as e |
import tkinter as tk |
from tkinter import messagebox |
from urllib.parse import urlencode |
from bs4 import BeautifulSoup |
''' |
爬取百度结果信息主要的问题是在通过百度关键词搜索进行爬虫的过程中, |
经常会遇到进入百度安全验证页面的情况。 |
随着访问次数的的增加,进入安全验证页面的概率也随之增大, |
且这种情况并不能通过请求头加入 Accept 和引入随机 UA 来解决 |
(代理是否能解决这一问题未得到充分验证,以我的尝试来看并不能有效解决)。 |
⚪代码有待改进,如果发现问题可以留言。 |
''' |
|
a = "1+1等于几" |
IsTurn = True |
root = tk.Tk() |
root.withdraw() |
while IsTurn: |
question_word = e.enterbox( "请输入您要查询的内容:\n默认为“1+1等于几”" , "content" ) |
if question_word = = "" or question_word = = None : |
question_word = a |
|
resultLEN = 2000 |
data = { 'wd' :question_word , 'ie' : 'utf-8' } |
data = urlencode(data) |
url = "http://www.baidu.com/s?wd=" + '?' + data + "&usm=3&rsv_idx=2&rsv_page=1" |
QUESTION = "QUESTION : " + question_word |
URL = "\nURL : " + url |
headers = { |
'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36 Edg/83.0.478.50' , |
'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' |
} |
|
try : |
r = requests.get(url,headers = headers); |
r.encoding = r.apparent_encoding; |
#r.raise_for_status(); |
STR = BeautifulSoup(r.text, "html.parser" ) |
div = STR ( 'div' ) |
STATUS = "\nSTATUS : " + str (r.status_code) |
|
if div ! = "": |
for members in div: |
try : |
if "result" in members.get( 'class' ): |
STATE = "\n\n获取成功!数据如下:" |
break |
elif "timeout" in members.get( 'class' ): |
STATE = "\n\n获取超时!数据如下:" |
break |
else : |
STATE = "\n\n获取失败!数据如下:" |
break |
except : |
e.exceptionbox( "爬取失败" , "错误" ) |
|
TEXT = "\n\nHTML :\n " + r.text[:resultLEN] |
DIV = "\n\nDIV :\n" + str (div) |
content = QUESTION + URL + STATUS + STATE + TEXT + DIV |
e.textbox( "获取完毕" , "Content Result" , |
text = content, |
codebox = True ) |
else : |
messagebox.showwarning( "404" , "爬取失败" ) |
except : |
e.exceptionbox( "爬取失败" , "错误" ) |
IsTurn = messagebox.askretrycancel( "Ask Retry" , "是否重试?" ) |