Scrapy爬取视频

利用FilesPipeline 下载视频

1.setting.py

# 保存log信息的文件名
LOG_LEVEL = "INFO"
# LOG_STDOUT = True
# LOG_ENCODING = 'utf-8'
# # 路径  os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
# LOG_FILE = "info.log"

# 下载延迟
import random

DOWNLOAD_DELAY = random.random() + random.random()
RANDOMIZE_DOWNLOAD_DELAY = True

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.87 Safari/537.36'

# 视频下载路径
import os

BASE_DIR = os.path.dirname((os.path.abspath(__file__)))
MEDIA_ALLOW_REDIRECTS = True
FILES_STORE = os.path.join(BASE_DIR, "videos")  # 文件保存路径
FILES_URLS_FIELD = 'file_urls'  # 这里对应着item.py文件中的字段
FILES_RESULT_FIELD = 'files'  # 同样对应item.py文件中的字段
# 120 days of delay for files expiration
# FILES_EXPIRES = 120  # 设置文件失效时间


ITEM_PIPELINES = {
    'steam_video.pipelines.SteamVideoPipeline': 300,
    'steam_video.pipelines.SteamDownLoadPipeline': 100,   # 视频下载的管道
}

2.spider.py

# -*- coding: utf-8 -*-
import os
import pymysql
import re
import scrapy

from steam_video.settings import FILES_STORE as FILS
from steam_video.settings import MYSQL_HOST, MYSQL_POST, MYSQL_DATABASE, MYSQL_PASSWORD, MYSQL_USER


class VideosSpider(scrapy.Spider):
    name = 'videos'
    allowed_domains = ['.com']
    start_urls = ['https://www.baidu.com/']
    custom_settings = {
        # 'DOWNLOAD_TIMEOUT': 60000,
        'DOWNLOAD_MAXSIZE': 12406585060,   # 视频最大大小
        'DOWNLOAD_WARNSIZE': 12406585060   # 视频下载最大提醒尺寸
    }

    def parse(self, response):
        db = pymysql.connect(host=MYSQL_HOST, port=MYSQL_POST, database=MYSQL_DATABASE, user=MYSQL_USER,
                             password=MYSQL_PASSWORD,
                             charset='utf8', autocommit=True)
        cursor = db.cursor()
        cursor.execute(
            'SELECT id, appid, data_webm_source, data_webm_hd_source, data_mp4_source, data_mp4_hd_source, data_poster from yu_spider_app_video WHERE appid=730 and id=53')  # 获取图片url
        for appid in cursor.fetchall():
            for i in range(2, 7):
                item = {}
                item['id'] = appid[0]
                item['appid'] = appid[1]
                item['file_url'] = appid[i]  # 下载视频的url    前面要自己获取到 视频的url   可以自己爬
                item['video_name'] = str(int(item['appid']) % 100) + '/' + str(item['appid']) + '/' + str(
                    item['appid']) + '_' + re.findall(r'/(\d+)/', appid[i])[0] + '_' + \
                                     appid[i].split('/')[-1].split('?')[0]  # 后面图片要命名的名称
                print(item)
                print('*' * 100)
                file_path = FILS + '/' + item['video_name']
                if not os.path.exists(file_path):  # 判断视频是否存在
                    yield item

3.pipelines.py

# -*- coding: utf-8 -*-
from scrapy.pipelines.images import FilesPipeline  # 导入文件下载类
import scrapy, os, hashlib
from scrapy.http import Request
from scrapy.utils.python import to_bytes
from steam_video.settings import FILES_STORE as FILS


class SteamVideoPipeline(object):
    def process_item(self, item, spider):
        return item


# 下载图片管道
class SteamDownLoadPipeline(FilesPipeline):
    def get_media_requests(self, item, info):
        return scrapy.Request(item['file_url'],
                              meta={'video_name': item.get('video_name', None)})  # 下载视频  video_name为视频名称

    def file_path(self, request, response=None, info=None):

        def _warn():
            from scrapy.exceptions import ScrapyDeprecationWarning
            import warnings
            warnings.warn('FilesPipeline.file_key(url) method is deprecated, please use '
                          'file_path(request, response=None, info=None) instead',
                          category=ScrapyDeprecationWarning, stacklevel=1)

        # check if called from file_key with url as first argument
        if not isinstance(request, Request):
            _warn()
            url = request
        else:
            url = request.url

        # detect if file_key() method has been overridden
        if not hasattr(self.file_key, '_base'):
            _warn()
            return self.file_key(url)

        media_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
        media_ext = os.path.splitext(url)[1]  # change to request.url after deprecation
        # 这里我们使用自定义的文件名，如果meta中没有video_name,就使用url的hash值作为文件名
        file_path = FILS + '\\' + request.meta.get('video_name')
        file = file_path.replace('\\{}'.format(file_path.split('\\')[-1]), '')
        if not os.path.exists(file):
            os.makedirs(file)
        return '{}'.format(request.meta.get('video_name'))
        # return 'full/%s.mp4' % (''.join(request.meta.get('video_name', media_guid).split(' ')))

    def file_key(self, url):   # 服务器部署需要家里这个代码
        return self.file_path(url)
    file_key._base = True

    def item_completed(self, results, item, info):
        print('******the results is********:',
              results)  # resulte [(True, {'url': 'https://media.st.dl.bscstorage.net/steam/apps/904/movie480.mp4?t=1569623096', 'path': '220_904_movie480.mp4', 'checksum': '0f22435cdfe2d605480fc5396160d3a5'})]
        if results[0][0]:  # 判断视频是否下载成功
            print('下载成功:{}'.format(item))
        else:
            print('下载失败:{}'.format(item))