要求：指定一个网站，爬取这个网站中的所有的所有图片，例如中国气象网（http://www.weather.com.cn）。实现单线程和多线程的方式爬取。
代码如下：
import os
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from concurrent.futures import ThreadPoolExecutorBASE_URL = "http://www.weather.com.cn"
SAVE_DIR = "images"
os.makedirs(SAVE_DIR, exist_ok=True)HEADERS = {"User-Agent": "Mozilla/5.0"}def get_image_urls(url):try:html = requests.get(url, headers=HEADERS, timeout=10).textsoup = BeautifulSoup(html, "html.parser")return [urljoin(url, img.get("src")) for img in soup.find_all("img") if img.get("src")]except Exception as e:print(f"[获取失败] {e}")return []def download(img_url):try:name = os.path.join(SAVE_DIR, os.path.basename(img_url.split("?")[0]))r = requests.get(img_url, headers=HEADERS, timeout=10)if r.ok and "image" in r.headers.get("Content-Type", ""):with open(name, "wb") as f: f.write(r.content)print(f"[下载成功] {img_url}")except Exception as e:print(f"[失败] {img_url} -> {e}")if __name__ == "__main__":imgs = get_image_urls(BASE_URL)print(f"共发现 {len(imgs)} 张图片：")for i in imgs: print(i)mode = input("\n选择下载方式：1.单线程 2.多线程 > ").strip()if mode == "1":for i in imgs: download(i)else:with ThreadPoolExecutor(10) as ex: ex.map(download, imgs)
结果：
spider代码如下：
import scrapy
import json
from sina_stock.items import SinaStockItemclass StockSpider(scrapy.Spider):name = "stock_spider"allowed_domains = ["vip.stock.finance.sina.com.cn"]start_urls = ["https://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?node=hs_a&page=1"]def parse(self, response):data = json.loads(response.text)for stock in data:item = SinaStockItem()item["bStockNo"] = stock.get("code")item["bName"] = stock.get("name")item["bPrice"] = stock.get("trade")item["bChangeRate"] = stock.get("changepercent")item["bChange"] = stock.get("pricechange")item["bVolume"] = stock.get("volume")item["bAmount"] = stock.get("amount")item["bAmplitude"] = stock.get("amplitude")item["bHigh"] = stock.get("high")item["bLow"] = stock.get("low")item["bOpen"] = stock.get("open")item["bPrevClose"] = stock.get("settlement")yield item# 翻页逻辑（抓取前 5 页）current_page = int(response.url.split("=")[-1])if current_page < 5:next_page = current_page + 1next_url = f"https://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?node=hs_a&page={next_page}"yield scrapy.Request(next_url, callback=self.parse)
结果：
spider代码如下：
# boc_rate/spiders/boc_spider.py
import scrapy
from boc_rate.items import BocRateItemclass BocSpider(scrapy.Spider):name = "boc_rate"allowed_domains = ["boc.cn"]start_urls = ["https://www.boc.cn/sourcedb/whpj/"]custom_settings = {'ROBOTSTXT_OBEY': False}def parse(self, response):rows = response.xpath('//table//tr')[1:]for row in rows:item = BocRateItem()item['货币名称'] = row.xpath('./td[1]/text()').get(default='').strip()item['现汇买入价'] = row.xpath('./td[2]/text()').get(default='').strip()item['现钞买入价'] = row.xpath('./td[3]/text()').get(default='').strip()item['现汇卖出价'] = row.xpath('./td[4]/text()').get(default='').strip()item['现钞卖出价'] = row.xpath('./td[5]/text()').get(default='').strip()item['中行折算价'] = row.xpath('./td[6]/text()').get(default='').strip()item['发布日期'] = row.xpath('./td[7]/text()').get(default='').strip()item['发布时间'] = row.xpath('./td[8]/text()').get(default='').strip()yield item
结果：