【scrapy框架】王者荣耀英雄信息爬取 python爬虫pythonqq25911173的博客-

【scrapy框架】王者荣耀英雄信息爬取 python爬虫

分析

入口页面地址

https://pvp.qq.com/web201605/herolist.shtml

第一步获取所有英雄的列表

可以看到英雄列表是在源码中可以被找到的

第二步 获取英雄的各种信息

英雄的基本信息放在一个class = “cover”的div中 我们主要采集英雄的名称 和 技能介绍

技能部分都在 class=" zk-con3 zk-con" 中 中的 ul中

爬取英雄列表

创建工程

scrapy startproject wzry cd wzry 

创建爬虫

scrapy genspider wzry_spider pvp.qq.com 

修改配置

# 不遵循robots协议 因为站点没有这个文件 爬虫会直接略过 ROBOTSTXT_OBEY = False # 添加请求头 DEFAULT_REQUEST_HEADERS = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en', 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36' } # 下载延迟 DOWNLOAD_DELAY = 1 

修改初始页面

start_urls = ['https://pvp.qq.com/web201605/herolist.shtml'] 

创建爬取列表方法(测试)

 def parse(self, response): print("=" * 50) print(response) print("=" * 50) 

运行爬虫

scrapy crawl wzry_spider 

2020-05-25 11:08:53 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023 2020-05-25 11:08:54 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://pvp.qq.com/web201605/herolist.shtml> (referer: None) ================================================== <200 https://pvp.qq.com/web201605/herolist.shtml> ================================================== 2020-05-25 11:08:54 [scrapy.core.engine] INFO: Closing spider (finished) 2020-05-25 11:08:54 [scrapy.statscollectors] INFO: Dumping Scrapy stats: {'downloader/request_bytes': 315,  

成功返回结果

爬取所有英雄链接方法

class WzrySpiderSpider(scrapy.Spider):     name = 'wzry_spider'     allowed_domains = ['pvp.qq.com']     start_urls = ['https://pvp.qq.com/web201605/herolist.shtml'] # 起始url     base_url = "https://pvp.qq.com/web201605/" # url 前缀 def parse(self, response): print("=" * 50) # print(response.body)         hero_list = response.xpath("//ul[@class='herolist clearfix']//li") for hero in hero_list:             url = self.base_url + hero.xpath("./a/@href").get() print(url) # yield scrapy.Request(url) print("=" * 50) 

https://pvp.qq.com/web201605/herodetail/114.shtml https://pvp.qq.com/web201605/herodetail/113.shtml https://pvp.qq.com/web201605/herodetail/112.shtml https://pvp.qq.com/web201605/herodetail/111.shtml https://pvp.qq.com/web201605/herodetail/110.shtml https://pvp.qq.com/web201605/herodetail/109.shtml https://pvp.qq.com/web201605/herodetail/108.shtml https://pvp.qq.com/web201605/herodetail/107.shtml https://pvp.qq.com/web201605/herodetail/106.shtml https://pvp.qq.com/web201605/herodetail/105.shtml ================================================== 

爬取英雄详情页面

• 获得基本信息

# 基本信息块 hero_info = response.xpath("//div[@class='cover']") hero_name = hero_info.xpath(".//h2[@class='cover-name']/text()").get() print(hero_name) # 分类 sort_num = hero_info.xpath(".//span[@class='herodetail-sort']/i/@class").get()[-1:] print(sort_num) # 生存能力 viability = hero_info.xpath(".//ul/li[1]/span/i/@style").get()[6:] print("生存能力:" + viability) # 伤害 aggressivity = hero_info.xpath(".//ul/li[2]/span/i/@style").get()[6:] print("攻击能力:" + aggressivity) effect = hero_info.xpath(".//ul/li[3]/span/i/@style").get()[6:] print("技能影响" + effect) difficulty = hero_info.xpath(".//ul/li[4]/span/i/@style").get()[6:] print("上手难度:" + difficulty) 

• 获取技能信息

 skill_list = response.xpath("//div[@class='skill-show']/div[@class='show-list']") for skill in skill_list:             skill_name = skill.xpath("./p[@class='skill-name']/b/text()").get() if not skill_name: continue # 冷却时间             cooling = skill.xpath("./p[@class='skill-name']/span[1]/text()").get().split("：")[1].strip().split('/') # 消耗             consume = skill.xpath("./p[@class='skill-name']/span[1]/text()").get().split("：")[1].strip().split('/') # "".strip() # 如果这个技能是空的就 continue # 技能介绍             skill_desc = skill.xpath("./p[@class='skill-desc']/text()").get()             new_skill = { "name": skill_name, "cooling": cooling, "consume": consume, "desc": skill_desc             }                  new_hero = HeroInfo(name=hero_name,                             sort_num=sort_num,                             viability=viability,                             aggressivity=aggressivity,                             effect=effect,                             difficulty=difficulty,                             skills_list=new_skill) yield new_hero           

• 英雄信息item类

class HeroInfo(scrapy.Item): # 存字符串     name = scrapy.Field()     sort_num = scrapy.Field()     viability = scrapy.Field()     aggressivity = scrapy.Field()     effect = scrapy.Field()     difficulty = scrapy.Field() # 存字典     skills_list = scrapy.Field() 

• 储存 pipelines.py文件

需要修改 ITEM_PIPELINES 配置 加入这个处理类

# -*- coding: utf-8 -*- from scrapy.exporters import JsonItemExporter   class WzryPipeline: def __init__(self): # 打开文件并实例化JsonItemExporter         self.fp = open('result.json', 'wb')         self.save_json = JsonItemExporter(self.fp, encoding="utf-8", ensure_ascii=False, indent=4) # 开始写入         self.save_json.start_exporting() def open_spider(self, spider): pass def close_spider(self, spider): # 结束写入         self.save_json.finish_exporting() # 关闭文件         self.fp.close() def process_item(self, item, spider): # 写入item         self.save_json.export_item(item) return item  

运行爬虫

scrapy crawl wzry_spider 

结果

全部代码

wzry_spider.py

# -*- coding: utf-8 -*- import scrapy from wzry.items import HeroInfo   class WzrySpiderSpider(scrapy.Spider):     name = 'wzry_spider'     allowed_domains = ['pvp.qq.com']     start_urls = ['https://pvp.qq.com/web201605/herolist.shtml'] # 起始url     base_url = "https://pvp.qq.com/web201605/" # url 前缀 def parse(self, response): # print(response.body)         hero_list = response.xpath("//ul[@class='herolist clearfix']//li") for hero in hero_list:             url = self.base_url + hero.xpath("./a/@href").get() yield scrapy.Request(url, callback=self.get_hero_info) def get_hero_info(self, response): # 基本信息块 global new_skill         hero_info = response.xpath("//div[@class='cover']")         hero_name = hero_info.xpath(".//h2[@class='cover-name']/text()").get() # 分类         sort_num = hero_info.xpath(".//span[@class='herodetail-sort']/i/@class").get()[-1:] # 生存能力         viability = hero_info.xpath(".//ul/li[1]/span/i/@style").get()[6:] # 伤害         aggressivity = hero_info.xpath(".//ul/li[2]/span/i/@style").get()[6:]         effect = hero_info.xpath(".//ul/li[3]/span/i/@style").get()[6:]         difficulty = hero_info.xpath(".//ul/li[4]/span/i/@style").get()[6:] # 技能列表         skill_list = response.xpath("//div[@class='skill-show']/div[@class='show-list']") for skill in skill_list:             skill_name = skill.xpath("./p[@class='skill-name']/b/text()").get() if not skill_name: continue # 冷却时间             cooling = skill.xpath("./p[@class='skill-name']/span[1]/text()").get().split("：" "")[1].strip().split('/') # 消耗             consume = skill.xpath("./p[@class='skill-name']/span[1]/text()").get().split("：" "")[1].strip().split('/') # "".strip() # 如果这个技能是空的就 continue # 技能介绍             skill_desc = skill.xpath("./p[@class='skill-desc']/text()").get()             new_skill = { "name": skill_name, "cooling": cooling, "consume": consume, "desc": skill_desc             }          new_hero = HeroInfo(name=hero_name,                             sort_num=sort_num,                             viability=viability,                             aggressivity=aggressivity,                             effect=effect,                             difficulty=difficulty,                             skills_list=new_skill) yield new_hero  

items.py

# -*- coding: utf-8 -*- # Define here the models for your scraped items # # See documentation in: # https://docs.scrapy.org/en/latest/topics/items.html import scrapy   class WzryItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() pass class HeroInfo(scrapy.Item): # 存字符串     name = scrapy.Field()     sort_num = scrapy.Field()     viability = scrapy.Field()     aggressivity = scrapy.Field()     effect = scrapy.Field()     difficulty = scrapy.Field() # 存字典     skills_list = scrapy.Field() 

pipelines.py

 # -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html from scrapy.exporters import JsonItemExporter   class WzryPipeline: def __init__(self):         self.fp = open('result.json', 'wb')         self.save_json = JsonItemExporter(self.fp, encoding="utf-8", ensure_ascii=False, indent=4)         self.save_json.start_exporting() def open_spider(self, spider): pass def close_spider(self, spider):         self.save_json.finish_exporting()         self.fp.close() def process_item(self, item, spider):         self.save_json.export_item(item) return item