AI时代，找源码已成为过去式，纪念我过去的十年0(回)
75天前
还有人吗2(回)
93天前会python真的可以为所欲为0(回)
573天前
这里还有人吗2(回)
758天前这里还有人吗0(回)
758天前
每天面对着电脑屏幕，敲打键盘。我所面对的并不只是代码，而是一种生活方式。0(回)
983天前import time |
import requests |
from bs4 import BeautifulSoup |
import re |
import csv |
def get_links(url): |
# 发送GET请求并获取页面内容 |
response = requests.get(url) |
# 解析HTML |
soup = BeautifulSoup(response.text, 'html.parser') |
# 提取所有链接 |
links = [] |
for link in soup.find_all('a'): |
href = link.get('href') |
if href and href.startswith('http'): |
links.append(href) |
save_data(href, "", f"data_url.csv") |
return links |
def scrape_data(url): |
# 发送GET请求获取网页内容 |
response = requests.get(url) |
# 获取网页内容的编码 |
# encoding = response.encoding |
encoding = 'gbk' |
# 解析网页内容 |
soup = BeautifulSoup(response.text, 'html.parser') |
# 剔除标签并获取文本内容 |
text = soup.get_text() |
# 去除多余的空格和换行符 |
text = re.sub(r'\s+', ' ', text).strip() |
# 解码unicode字符 |
decoded_text = text.encode(encoding, errors='ignore').decode(encoding) |
return decoded_text |
def save_data(url, data, filename): |
with open(filename, 'a', newline='') as file: |
writer = csv.writer(file) |
writer.writerow([url, data]) # 写入数据行 |
print(f"保存成功: {filename}") |
def crawl(start_url, depth=2): |
visited = set() # 存储已访问的链接 |
queue = [(start_url, 0)] # 使用队列实现广度优先搜索，同时记录深度 |
while queue: |
url, level = queue.pop(0) # 从队列中取出一个链接 |
if level > depth: # 达到指定深度则停止 |
break |
try: |
if url not in visited: |
visited.add(url) |
print(f"正在爬取: {url}") |
# 爬取数据 |
data = scrape_data(url) |
# 保存数据 |
save_data(url, data, f"data_{level}.csv") |
# 获取链接 |
links = get_links(url) |
# 将链接添加到队列中 |
queue.extend([(link, level + 1) for link in links]) |
# time.sleep(1) |
except Exception as e: |
print(f"发生异常: {str(e)}，跳过链接: {url}") |
continue |
start_url = 'https://www.ferrari.com/zh-CN/news' |
crawl(start_url, depth=2) |
