25 lines
692 B
Python
25 lines
692 B
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
# @Time : 2024/11/24 下午6:26
|
||
|
|
# @Author : 河瞬
|
||
|
|
# @FileName: httpx_spider.py
|
||
|
|
# @Software: PyCharm
|
||
|
|
from httpx import AsyncClient, Proxy
|
||
|
|
|
||
|
|
from .base_spider import BaseSpider
|
||
|
|
|
||
|
|
|
||
|
|
class HttpxSpider(BaseSpider):
|
||
|
|
headers = {}
|
||
|
|
proxy = None
|
||
|
|
|
||
|
|
def __init__(self, proxy_url: str = None):
|
||
|
|
super().__init__(proxy_url)
|
||
|
|
self.proxy = Proxy(proxy_url) if self.use_proxy else None
|
||
|
|
|
||
|
|
def set_proxy(self, proxy_url: str):
|
||
|
|
super().set_proxy(proxy_url)
|
||
|
|
self.proxy = Proxy(proxy_url)
|
||
|
|
|
||
|
|
def get_session(self) -> AsyncClient:
|
||
|
|
return AsyncClient(headers=self.headers, proxy=self.proxy, timeout=self.timeout, follow_redirects=True)
|