python异步处理请求_python异步处理HTTP请求

我需要为django查询集的每个条目生成一个PDF报告。会有3万到4万个条目。在

PDF是通过外部API生成的。由于当前是按需生成的，因此这是通过HTTP请求/响应同步处理的。

对于这个任务，这是不同的，因为我认为我将使用django管理命令循环查询集并执行PDF生成。在

我应该遵循哪种方法来完成这项任务？我考虑了两种可能的解决方案，尽管我从未使用过这些技术：

1)Celery：将一个任务(具有不同负载的http请求)分配给一个worker，然后在任务完成后检索它。在

2)request-futures：以非阻塞方式使用请求。在

目标是并发使用API(例如，根据API可以处理的并发请求数，同时发送10或100个http请求)。在

有没有人在这里处理过类似的任务，并能就如何进行这项工作提出建议？在

下面是用multiprocessing进行的第一次尝试(注意：大多数代码都是重用的，而不是我自己编写的，因为我拥有了这个项目的所有权):

在class Checker(object):

def __init__(self, *args, **kwargs):

# ... various setup

# other methods

# .....

def run_single(self, uuid, verbose=False):

"""

run a single PDF generation and local download

"""

start = timer()

headers = self.headers

data, obj = self.get_review_data(uuid)

if verbose:

print("** Report: {} **".format(obj))

response = requests.post(

url=self.endpoint_url,

headers=headers,

data=json.dumps(data)

)

if verbose:

print('POST - Response: {} \n {} \n {} secs'.format(

response.status_code,

response.content,

response.elapsed.total_seconds())

)

run_url = self.check_progress(post_response=response, verbose=True)

if run_url:

self.get_file(run_url, obj, verbose=True)

print("*** Download {}in {} secs".format("(verbose) " if verbose else "", timer()-start))

def run_all(self, uuids, verbose=True):

start = timer()

for obj_uuid in review_uuids:

self.run_single(obj_uuid, verbose=verbose)

print("\n\n### Downloaded {}{} reviews in {} secs".format(

"(verbose) " if verbose else "",

len(uuids),

timer() - start)

)

def run_all_multi(self, uuids, workers=4, verbose=True):

pool = Pool(processes=workers)

pool.map(self.run_single, uuids)

def check_progress(self, post_response, attempts_limit=10000, verbose=False):

"""

check the progress of PDF generation querying periodically the API endpoint

"""

if post_response.status_code != 200:

if verbose: print("POST response status code != 200 - exit")

return None

url = 'https://apidomain.com/{path}'.format(

domain=self.domain,

path=post_response.json().get('links', {}).get('self', {}).get('href'),

headers = self.headers

)

job_id = post_response.json().get('jobId', '')

status = 'Running'

attempt_counter = 0

start = timer()

if verbose:

print("GET - url: {}".format(url))

while status == 'Running':

attempt_counter += 1

job_response = requests.get(

url=url,

headers=self.headers,

)

job_data = job_response.json()

status = job_data['status']

message = job_data['message']

progress = job_data['progress']

if status == 'Error':

if verbose:

end = timer()

print(

'{sc} - job_id: {job_id} - error_id: [{error_id}]: {message}'.format(

sc=job_response.status_code,

job_id=job_id,

error_id=job_data['errorId'],

message=message

), '{} secs'.format(end - start)

)

print('Attempts: {} \n {}% progress'.format(attempt_counter, progress))

return None

if status == 'Complete':

if verbose:

end = timer()

print('run_id: {run_id} - Complete - {secs} secs'.format(

run_id=run_id,

secs=end - start)

)

print('Attempts: {}'.format(attempt_counter))

print('{url}/files/'.format(url=url))

return '{url}/files/'.format(url=url)

if attempt_counter >= attempts_limit:

if verbose:

end = timer()

print('File failed to generate after {att_limit} retrieve attempts: ({progress}% progress)' \

' - {message}'.format(

att_limit=attempts_limit,

progress=int(progress * 100),

message=message

), '{} secs'.format(end-start))

return None

if verbose:

print('{}% progress - attempts: {}'.format(progress, attempt_counter), end='\r')

sys.stdout.flush()

time.sleep(1)

if verbose:

end = timer()

print(status, 'message: {} - attempts: {} - {} secs'.format(message, attempt_counter, end - start))

return None

def get_review_data(self, uuid, host=None, protocol=None):

review = get_object_or_404(MyModel, uuid)

internal_api_headers = {

'Authorization': 'Token {}'.format(

review.employee.csod_profile.csod_user_token

)

}

data = requests.get(

url=a_local_url,

params={'format': 'json', 'indirect': 'true'},

headers=internal_api_headers,

).json()

return (data, review)

def get_file(self, runs_url, obj, verbose=False):

runs_files_response = requests.get(

url=runs_url,

headers=self.headers,

stream=True,

)

runs_files_data = runs_files_response.json()

file_path = runs_files_data['files'][0]['links']['file']['href'] # remote generated file URI

file_response_url = 'https://apidomain.com/{path}'.format(path=file_path)

file_response = requests.get(

url=file_response_url,

headers=self.headers,

params={'userId': settings.CREDENTIALS['userId']},

stream=True,

)

if file_response.status_code != 200:

if verbose:

print('error in retrieving file for {r_id}\nurl: {url}\n'.format(

r_id=obj.uuid, url=file_response_url)

)

local_file_path = '{temp_dir}/{uuid}-{filename}-{employee}.pdf'.format(

temp_dir=self.local_temp_dir,

uuid=obj.uuid,

employee=slugify(obj.employee.get_full_name()),

filename=slugify(obj.task.name)

)

with open(local_file_path, 'wb') as f:

for block in file_response.iter_content(1024):

f.write(block)

if verbose:

print('\n --> {r} [{uuid}]'.format(r=review, uuid=obj.uuid))

print('\n --> File downloaded: {path}'.format(path=local_file_path))

@classmethod

def get_temp_directory(self):

"""

generate a local unique temporary directory

"""

return '{temp_dir}/'.format(

temp_dir=mkdtemp(dir=TEMP_DIR_PREFIX),

)

if __name__ == "__main__":

uuids = #list or generator of objs uuids

checker = Checker()

checker.run_all_multi(uuids=uuids)

不幸的是，运行checker.run_all_multi会产生以下效果Python的外壳冻结

不打印输出

不生成文件

我不得不从命令行杀死控制台，正常键盘中断停止工作

运行^{时，checker.run_all正常工作(一个接一个)。在

有什么特别的建议，关于为什么这段代码不起作用(不是关于我可以用什么来代替多处理)？在

谢谢大家。在