只需要将img的src的属性值进行解析,提交到管道,管道就会对图片的src进行请求发送获取图片的二进制类型的数据,且还会帮我们进行持久化存储
- import scrapy
- from imgPro.items import ImgproItem
-
-
- class ImgSpider(scrapy.Spider):
- name = 'img'
- # allowed_domains = ['www.xxx.com']
- start_urls = ['http://pic.netbian.com/4kmeinv/']
-
- def parse(self, response):
- div_list = response.xpath('//*[@id="main"]/div[3]/ul/li')
- for div in div_list:
- #注意:使用伪属性
- src = 'http://pic.netbian.com' + div.xpath('./a/img/@src').extract_first()
- item = ImgproItem()
- item['src'] = src
-
- yield item
-
- from scrapy.pipelines.images import ImagesPipeline
- import scrapy
- class ImgproPipeline(ImagesPipeline):
-
- #就是可以根据图片地址进行图片数据的请求
- def get_media_requests(self, item, info):
-
- yield scrapy.Request(item['src'])
-
- #指定图片存储的路径
- def file_path(self, request, response=None, info=None):
- imgName = request.url.split('/')[-1]
- return imgName
-
- def item_completed(self, results, item, info):
- return item #返回给下一个即将被执行的管道类
-