Python爬虫技巧-异步请求装饰器

First Post:

Last Update:

常规

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
import httpx

# Decorator 
def request(func):

    def wrapper(*arg, **kwarg):
        req = func(*arg, **kwarg)
        client = httpx.Client()
        ret = client.send(req)
        return ret

    return wrapper

@request
def getUsers(method, url, params):
    return httpx.Request(method, url, params=params)

data = getUsers('GET', 'http://host:port/getUsers',{'user_query': 'username'})
print(data.text)

协程

通过协程来使用装饰器时,需要额外借助functools来对异步函数进行装饰

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
import httpx
import asyncio
import functools

def asyncRequest(client):

    def inner(func):

        @functools.wraps(func)
        async def wrapper(*arg,**kwarg):
            req = func(*arg, **kwarg)
            ret = await client.send(req)
            return ret
        
        return wrapper

    return inner


client = httpx.AsyncClient()

@asyncRequest(client)
def asyncGetUsers(method, url, params):
    return httpx.Request(method, url, params=params)


data = asyncio.run(getUsers('GET', 'http://host:port/getUsers',{'user_query': 'username'}))

print(data.text)
≡