Python 爬虫xpath多线程抓取豆瓣电影top250影片名

多线程抓取豆瓣top250，其实数据量不多，单线程完全够用，初学多线程抓取，就当练练手好了，下次换个数据量大的网页来抓取

import requests
from lxml import etree
import time
from concurrent.futures import ThreadPoolExecutor


def download_one_page(url, headers):
    # 拿到页面源代码

    resp = requests.get(url=url, headers=headers)
    html = etree.HTML(resp.text)
    divs = html.xpath('//*[@id="content"]/div/div[1]/ol/li')

    # time.sleep(0.5)

    for div in divs:
        movie_name = div.xpath('./div/div[2]/div[1]/a/span[1]/text()')[0]
 

        print(movie_name)


if __name__ == '__main__':

    start_time = time.time()
    headers = {"User-Agent":
                   "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36"}
	
	#开启线程池，10个线程
    with ThreadPoolExecutor(10) as t:

        for i in range(0, 251, 25):
            url = f"https://movie.douban.com/top250?start={i}&filter="
            t.submit(download_one_page, url, headers)

            # download_one_page(url,headers)

    final_time = time.time()
    cost_time = final_time - start_time
    print("---------" * 10)
    print("完成，一共耗时"   str(cost_time)   's')

可以看到多线程抓取网页数据速度确实很快，1s不到全部抓取完毕

这篇好文章是转载于：学新通技术网