爬取并提取网页的所有超链接

[python]代码库

import re
import urllib.request

url = 'https://yuncode.net/list/python'

# set the headers
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'}

# create the request object and set the headers
req = urllib.request.Request(url, headers=headers)

# get the response
res = urllib.request.urlopen(req)

# get the response text
html = res.read()

# use re.findall to get all the link
links = re.findall(r'(?<=href=")[^"]+', html.decode('utf-8'))

# print the links
for link in links:
    print(link)