# -*- coding: utf-8 -*-
"""
Created on 2020/4/22 10:41 PM
---------
@summary:
---------
@author: Boris
@email: boris_liu@foxmail.com
"""

import feapder


class TestAirSpider(feapder.AirSpider):
    __custom_setting__ = dict(
        USE_SESSION=True,
        TASK_MAX_CACHED_SIZE=10,
    )

    def start_callback(self):
        print("爬虫开始")

    def end_callback(self):
        print("爬虫结束")

    def start_requests(self, *args, **kws):
        for i in range(1):
            print(i)
            yield feapder.Request("https://www.baidu.com")

    def download_midware(self, request):
        # request.headers = {'User-Agent': ""}
        # request.proxies = {"https":"https://12.12.12.12:6666"}
        # request.cookies = {}
        return request

    def validate(self, request, response):
        if response.status_code != 200:
            raise Exception("response code not 200")  # 重试

        # if "哈哈" not in response.text:
        #     return False # 抛弃当前请求

    def parse(self, request, response):
        print(response.bs4().title)
        print(response.xpath("//title").extract_first())


if __name__ == "__main__":
    TestAirSpider(thread_count=1).start()