scrapy.Request深度爬取火影忍者人物详情并持久化存储到MySQL

1.创建项目

1.  
   scrapy startproject Naruto
2.  
   cd Naruto

2.创建爬虫文件

scrapy genspider naruto http://www.4399dmw.com/huoying/renwu/

3.项目结构

4.修改配置（settings）

1.  
   ROBOTSTXT_OBEY = False robots协议改为False
2.  
   LOG_LEVEL = 'ERROR' # 输出日志
3.  
   ITEM_PIPELINES = {
4.  
   # 'NaRuTo.pipelines.NarutoPipeline': 300,
5.  
   'NaRuTo.pipelines.MysqlPileLine': 300,
6.  
   } # 管道

5.爬虫文件（spiders下面的naruto）

1.  
   import scrapy
2.  
   from NaRuTo.items import NarutoItem
3.  
    
4.  
   class NarutoSpider(scrapy.Spider):
5.  
   name = 'naruto'
6.  
   # allowed_domains = ['www.xxx.com']
7.  
   start_urls = ['http://www.4399dmw.com/huoying/renwu/']
8.  
    
9.  
   def parse(self, response):
10.  
    # 解析出子页面的url
11.  
    href = response.xpath('//*[@id="iga"]/li/a/@href').extract()
12.  
    # 因为里面有重复的url，利用set方法去重
13.  
    new_href = list(set(href))
14.  
     
15.  
    for url in new_href:
16.  
    # 拼接成完整的url连接
17.  
    in_url = 'http://www.4399dmw.com' url
18.  
     
19.  
    try:
20.  
    # 请求传参，将request继续交给scrapy引擎自动爬取并通过回调函数返回结果
21.  
    yield scrapy.Request(url=in_url,
22.  
    callback=self.parse_content)
23.  
    except Exception as e:
24.  
    print('请求失败:', e)
25.  
     
26.  
    # 处理详情页数据
27.  
    def parse_content(self, response):
28.  
    # div_list = response.xpath('//*[@id="j-lazyimg"]/div[2]/div[1]/div[2]/div/div/div[2]')
29.  
    # for div in div_list:
30.  
    # 姓名
31.  
    name = response.xpath('//*[@id="j-lazyimg"]/div[2]/div[1]/div[2]/div/div/div[2]/div[1]/h1/text()').extract_first()
32.  
    # 详情
33.  
    detail = response.xpath('//*[@id="j-lazyimg"]/div[2]/div[1]/div[2]/div/div/div[2]/div[1]/p[1]/text()').extract_first()
34.  
    # 个人介绍
35.  
    introduce = response.xpath('//*[@id="j-lazyimg"]/div[2]/div[1]/div[2]/div/div/div[2]/div[2]/p//text()').extract()
36.  
    # 把爬取到的字符串里面的什么u3000替换为空（我也不知道是啥）
37.  
    new_introduce = ''.join(introduce).replace('\u3000', '').replace('\xa0', '')
38.  
     
39.  
    # 把爬取到的内容封装到字典里面
40.  
    all_data = {
41.  
    "name": name,
42.  
    "detail": detail,
43.  
    "introduce": new_introduce
44.  
    }
45.  
     
46.  
    # 实例化NarutoItem()
47.  
    item = NarutoItem()
48.  
     
49.  
    item['name'] = all_data['name']
50.  
    item['detail'] = all_data['detail']
51.  
    item['introduce'] = all_data['introduce']
52.  
     
53.  
    # 把item传入到管道（pipelines）
54.  
    yield item

6.item.py

1.  
   import scrapy
2.  
    
3.  
    
4.  
   class NarutoItem(scrapy.Item):
5.  
   # define the fields for your item here like:
6.  
   # name = scrapy.Field()
7.  
   name = scrapy.Field() # 忍者姓名
8.  
   detail = scrapy.Field() # 发布详情
9.  
   introduce = scrapy.Field() # 忍者介绍

7.管道（pipelines）

1.  
   import pymysql
2.  
    
3.  
    
4.  
   class MysqlPileLine(object):
5.  
   conn = None
6.  
   cursor = None
7.  
    
8.  
   def open_spider(self, spider):
9.  
   # 连接MySQL
10.  
    self.conn = pymysql.Connect(
11.  
    host='127.0.0.1',
12.  
    port=3306,
13.  
    user='root',
14.  
    password='***********',
15.  
    db='naruto',
16.  
    charset='utf8'
17.  
    )
18.  
     
19.  
    def process_item(self, item, spider):
20.  
    # 游标
21.  
    self.cursor = self.conn.cursor()
22.  
    insert_sql = 'insert into all_naruto_data values ("%s", "%s", "%s")' % (item['name'], item['detail'], item['introduce'])
23.  
    try:
24.  
    # 提交sql
25.  
    self.cursor.execute(insert_sql)
26.  
    self.conn.commit()
27.  
    except Exception as e:
28.  
    print('插入失败：', e)
29.  
    self.conn.rollback()
30.  
    return item
31.  
     
32.  
    # 关闭连接
33.  
    def close_spider(self, spider):
34.  
    self.cursor.close()
35.  
    self.conn.close()

7.忍者数据（一部分）

这篇好文章是转载于：学新通技术网