scrapy 动态加载下载数据

配置就开启管道 中间件 之类的

中间件中改动

1.  
   from scrapy.http import HtmlResponse
2.  
    
3.  
    
4.  
   def process_response(self, request, response, spider):
5.  
   bro = spider.bro
6.  
   if request.url in spider.model_urls:
7.  
   bro.get(request.url)
8.  
   # 滚动加载更多数据
9.  
   # bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
10.  
     
11.  
    page_text = bro.page_source
12.  
    new_res = HtmlResponse(url=request.url, body=page_text, encoding='utf-8', request=request)
13.  
    return new_res
14.  
    else:
15.  
    return response

爬虫文件

1.  
   import scrapy
2.  
   from selenium import webdriver
3.  
   from selenium.webdriver.edge.service import Service
4.  
   from middlePro.items import MiddleproItem
5.  
    
6.  
    
7.  
   class MiddleSpider(scrapy.Spider):
8.  
   name = 'middle'
9.  
   # allowed_domains = ['www.百度.com']
10.  
    start_urls = ['https://news.163.com/']
11.  
    model_urls = []
12.  
     
13.  
    def __init__(self):
14.  
    self.ser = Service(r'/Users/xiaodunmeng/Desktop/xiaoproject/shixun/xinscrapy/middlePro/middlePro/msedgedriver')
15.  
    self.bro = webdriver.Edge(service=self.ser)
16.  
     
17.  
    def parse(self, response):
18.  
    li_list = response.xpath('//*[@id="index2016_wrap"]/div[2]/div[2]/div[2]/div[2]/div/ul/li')
19.  
    alist = [2, 3]
20.  
     
21.  
    for index in alist:
22.  
    model_url = li_list[index].xpath('./a/@href').extract_first()
23.  
    self.model_urls.append(model_url)
24.  
     
25.  
    for url in self.model_urls:
26.  
    yield scrapy.Request(url, callback=self.parse_model)
27.  
     
28.  
    def parse_model(self, response):
29.  
    div_lsit = response.xpath('//div[@class="ndi_main"]/div')
30.  
    for div in div_lsit:
31.  
    title = div.xpath('./div/div[1]/h3/a/text()').extract_first()
32.  
    new_detail_url = div.xpath('./div/div[1]/h3/a/@href').extract_first()
33.  
     
34.  
    item = MiddleproItem()
35.  
    item['title'] = title
36.  
     
37.  
    yield scrapy.Request(url=new_detail_url, callback=self.parse_detail, meta={'item': item})
38.  
     
39.  
    def parse_detail(self, response):
40.  
    content = response.xpath('//*[@id="content"]/div[2]//text()').extract()
41.  
    content = ''.join(content)
42.  
    item = response.meta['item']
43.  
    item['content'] = content
44.  
     
45.  
    yield item
46.  
     
47.  
    def closed(self, spider):
48.  
    self.bro.quit()
49.  
     

这篇好文章是转载于：学新通技术网