使用scrapy爬取平凡的世界

1.  
   import scrapy
2.  
   from ..items import PfdsjItem
3.  
    
4.  
    
5.  
   class PfsjSpider(scrapy.Spider):
6.  
   name = 'pfsj'
7.  
   #allowed_domains = ['xxx.com']
8.  
   start_urls = ['https://www.pingfandeshijie.net']
9.  
    
10.  
    #获取一级页面中的a链接地址
11.  
    def parse(self,response):
12.  
    #获取目录链接
13.  
    a_href=response.xpath("//center/table/tr/td/center/h2/a/@href").extract()
14.  
    # print(a_href)
15.  
    for v in a_href:
16.  
    # print(v)
17.  
    # 将返回的a链接交给调度器进行处理，将处理的结果传递给two_parse
18.  
    yield scrapy.Request(url=v,callback=self.two_parse)
19.  
     
20.  
    # 获取二级页面中的a链接地址
21.  
    def two_parse(self,respond):
22.  
    # print(respond)
23.  
    # 获取a链接
24.  
    a_href=respond.xpath('//div[@class="main"]/div[2]/ul/li/a/@href').extract()
25.  
    # print(a_href)
26.  
    for i in a_href:
27.  
    # 将返回的a链接交给调度器进行处理，将处理的结果传递给three_parse
28.  
    yield scrapy.Request(url=i,callback=self.three_parse)
29.  
     
30.  
    # 获取三级页面中的a链接地址
31.  
    def three_parse(self,respond):
32.  
    # print(type(book_name))
33.  
    page=respond.xpath('/html/body/div[3]/h1/text()').get().split()
34.  
    part=page[0]
35.  
    if len(page)>1:
36.  
    page_num=page[1]
37.  
    else:
38.  
    page_num = page[0]
39.  
    content=respond.xpath('//body/div[3]/div[2]/p/text()').extract()
40.  
    content='\n'.join(content)
41.  
    # print(content)
42.  
    item = PfdsjItem()
43.  
    # 给KugouItem对象属性赋值
44.  
    item['page_num'] = page_num
45.  
    item['part'] = part
46.  
    item['content'] = content.replace('\\u300', '')
47.  
    yield item

1.  
   # Define your item pipelines here
2.  
   #
3.  
   # Don't forget to add your pipeline to the ITEM_PIPELINES setting
4.  
   # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
5.  
    
6.  
   import os
7.  
   # useful for handling different item types with a single interface
8.  
   from itemadapter import ItemAdapter
9.  
    
10.  
     
11.  
    class PfdsjPipeline:
12.  
    # 第一次执行管道类文件的时候执行一次
13.  
    def open_spider(self,spider):
14.  
    dirname = './平凡的世界'
15.  
    if not os.path.exists(dirname):
16.  
    os.mkdir(dirname)
17.  
     
18.  
    def process_item(self, item, spider):
19.  
    dirname = './%s/'%('平凡的世界') item['part']
20.  
    if not os.path.exists(dirname):
21.  
    os.mkdir(dirname)
22.  
    # 章节名/章节数——标题
23.  
    filename = "./%s/%s/%s" % ('平凡的世界',item['part'],item['page_num'])
24.  
    with open(filename '.txt', 'a', encoding='utf-8') as f:
25.  
    f.write(item['content'])

1.  
   # Define here the models for your scraped items
2.  
   #
3.  
   # See documentation in:
4.  
   # https://docs.scrapy.org/en/latest/topics/items.html
5.  
    
6.  
   import scrapy
7.  
    
8.  
    
9.  
   class PfdsjItem(scrapy.Item):
10.  
    # define the fields for your item here like:
11.  
    # name = scrapy.Field()
12.  
    book_name = scrapy.Field()
13.  
    part = scrapy.Field()
14.  
    page_num = scrapy.Field()
15.  
    content=scrapy.Field()

所需第三方库：scrapy库

运行结果：

这篇好文章是转载于：学新通技术网