lil_DengZi's Notes

爬虫实例

1分钟阅读

title:warning
collapse:open
 
有时效性

案例

python爬虫解析案例

Python 爬虫解析案例

学习爬虫时，案例最好能稳定运行。下面这个示例不依赖外部网站，用标准库解析本地 HTML，适合先练“解析数据”的基本流程。

from html.parser import HTMLParser
from http.server import BaseHTTPRequestHandler, HTTPServer
from threading import Thread
from urllib.request import urlopen
 
 
class LinkParser(HTMLParser):
    def __init__(self):
        super().__init__()
        self.links = []
 
    def handle_starttag(self, tag, attrs):
        if tag == "a":
            attrs_dict = dict(attrs)
            if "href" in attrs_dict:
                self.links.append(attrs_dict["href"])
 
 
class Handler(BaseHTTPRequestHandler):
    def do_GET(self):
        body = b"""
        <html>
          <body>
            <a href="https://example.com/python">Python</a>
            <a href="/local">Local</a>
          </body>
        </html>
        """
        self.send_response(200)
        self.send_header("Content-Type", "text/html")
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        self.wfile.write(body)
 
    def log_message(self, format, *args):
        return
 
 
def main():
    server = HTTPServer(("127.0.0.1", 0), Handler)
    thread = Thread(target=server.serve_forever, daemon=True)
    thread.start()
 
    try:
        url = f"http://127.0.0.1:{server.server_port}/"
        with urlopen(url, timeout=3) as response:
            html = response.read().decode("utf-8")
 
        parser = LinkParser()
        parser.feed(html)
        print(parser.links)
    finally:
        server.shutdown()
        thread.join()
 
 
if __name__ == "__main__":
    main()
指向原始笔记的链接

re实例

爬取豆瓣top250

import csv  
import re  
import requests  
  
url = "https://movie.douban.com/top250"  
  
header = {  
"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0"  
}  
  
resp = requests.get(url, headers=header)  
page_content = resp.text  
  
obj = re.compile(r'<li>.*?<div class="item">.*?<span class="title">(?P<name>.*?)'  
             r'</span>.*?<p class="">.*?<br>(?P<year>.*?)&nbsp.*?<span '  
             r'class="rating_num" property="v:average">(?P<score>.*?)</span>.*?'  
             r'<span>(?P<space>.*?)人评价</span>', re.S)  
result = obj.finditer(page_content)  
  
f = open("data.csv", mode="w")  
csvwriter =  csv.writer(f)  
for it in result:  
   dic = it.groupdict()  
   dic["year"] = dic['year'].strip()  
   csvwriter.writerow(dic.values())  
  
resp.close()

bs4实例

爬取优美图库前五张图片

import time  
import requests  
from bs4 import BeautifulSoup
 
url = "https://www.umeituku.com/bizhitupian/weimeibizhi/"  
resp = requests.get(url)  
resp.encoding = 'utf-8'  
  
main_page = BeautifulSoup(resp.text, "html.parser")  
alist = main_page.find("div", class_="TypeList").find_all("a")  
 
for a in alist:  
   href = a.get('href')  
   child_page_resp = requests.get(href)  
   child_page_resp.encoding = 'utf-8'  
   child_page_text = child_page_resp.text  
   child_page = BeautifulSoup(child_page_text, "html.parser")  
   p = child_page.find("p", align="center")  
   img = p.find("img")  
   src = img.get("src")  
   img_resp = requests.get(src)  
   img_name = src.split("/")[-1]  
   with open(img_name, mode="wb") as f:  
      f.write(img_resp.content)  
  
   print("over", i, img_name)  
   time.sleep(1)  
  
resp.close()

关系图谱

最近的笔记

反向链接