Browser queue wait time

本文深入探讨了浏览器队列等待时间的原因,包括JavaScript文件加载顺序和并发请求限制,并提供策略来优化这些因素,从而提高网页加载速度。通过将图像文件托管到单独的子域,可以有效避免并发请求限制的影响。

There can be multiple reasons for a request to wait in the browser queue before it is sent to the server. The two most common reasons are:


    • If there are multiple JavaScript files on a web page, they are loaded one after the other by the browser and not loaded simultaneously. They are loaded in the same sequence as they are present on the web page. A web developer should always consider if it is possible to convert multiple JavaScript files on a page into a single JavaScript file.


      • Each browser has a limit on the number of concurrent requests that can be made to a single domain by a web page. For example, this limit is six for Firefox 3. If there are eight images on a web page that have to be loaded from the same domain, six requests will be made simultaneously, and the request for two images has to wait before a response for any of the previous two requests is received from the server.

 


When optimizing the performance of a web application from the browser's perspective, browser queue wait time is a very important consideration.


So how to bypass the maximum concurrent requests limit by the browser ?


If, for whatever reason, a web page needs to make a lot of requests to the server to get images, CSS files, AJAX responses, and so on, then one common technique is to host the image files on a separate subdomain. For example, host all the images for the Packt site on images.packtpub.com instead of the www.packtpub.com subdomain. However, it is important to keep in mind that every subdomain that is added also requires a DNS lookup. Based on a study by Yahoo, having two to four subdomains for a site is a good compromise between parallel requests and DNS lookups.

from playwright.sync_api import sync_playwright import subprocess import time import os import threading import queue # 配置 debugging_ports = [9222, 9223, 9224, 9225, 9226] chrome_path = "C:\\Users\\Windows10\\Desktop\\chromedriver-win64\\chrome-win64\\chrome.exe" root_folder = "C:\\Users\\Windows10\\Desktop\\素材" # 获取所有 DZ+数字 文件夹 dz_numbered_folders = sorted([ folder for folder in os.listdir(root_folder) if folder.startswith("DZ") and folder[2:].isdigit() and os.path.isdir(os.path.join(root_folder, folder)) ]) print(f"找到 {len(dz_numbered_folders)} 个 DZ+数字 的文件夹:") # 创建任务队列 task_queue = queue.Queue() # 将所有任务放入队列 for folder in dz_numbered_folders: task_queue.put(folder) # 线程数(同时运行的最大浏览器实例数) THREAD_COUNT = len(debugging_ports) def worker(port): while not task_queue.empty(): folder_name = task_queue.get() try: upload_folder(folder_name, port) finally: task_queue.task_done() def upload_folder(folder_name, port): folder_path = os.path.join(root_folder, folder_name) process = subprocess.Popen([ chrome_path, f"--remote-debugging-port={port}", "--new-window", "https://www.dianxiaomi.com/web/popTemu/add" ]) time.sleep(2) try: with sync_playwright() as p: browser = p.chromium.connect_over_cdp(f"http://localhost:{port}") page = browser.contexts[0].pages[0] print(f"[端口 {port}] 正在上传文件夹:{folder_name}") # 悬停按钮并点击“本地图片” page.locator("button:has-text('选择图片')").hover() print(f"[{folder_name}] 鼠标已悬停在按钮上") time.sleep(1) page.wait_for_selector("li[data-menu-id='local']", timeout=1000) page.locator("li[data-menu-id='local']").click() print(f"[{folder_name}] 已点击“本地图片”选项") # 获取文件 files_in_folder = [ os.path.join(folder_path, f) for f in os.listdir(folder_path) if os.path.isfile(os.path.join(folder_path, f)) and f.lower().endswith(('.png', '.jpg', '.jpeg')) ] if files_in_folder: print(f"[{folder_name}] 准备上传 {len(files_in_folder)} 个文件...") page.set_input_files("input[type=file]", files_in_folder) print(f"[{folder_name}] 文件上传完成!") else: print(f"[{folder_name}] 没有图片可上传。") # 等待页面关闭 print(f"[{folder_name}] 请关闭浏览器窗口以继续...") start_time = time.time() timeout = 10 while True: try: if not browser.contexts or not browser.contexts[0].pages: print(f"[{folder_name}] 页面已关闭,继续下一个任务。") break except Exception as e: print(f"[{folder_name}] 检测页面状态异常:{e}") break if time.time() - start_time > timeout: print(f"[{folder_name}] 等待超时,强制继续下一个任务。") break time.sleep(1) browser.close() except Exception as e: print(f"[{folder_name}] 操作失败:{e}") finally: process.terminate() print(f"[{folder_name}] 浏览器进程已关闭。") # 启动线程池 threads = [] for i in range(THREAD_COUNT): thread = threading.Thread(target=worker, args=(debugging_ports[i % len(debugging_ports)],)) thread.start() threads.append(thread) # 等待所有任务完成 task_queue.join() # 停止线程 for thread in threads: thread.join() print("所有任务已完成!")
09-11
from playwright.sync_api import sync_playwright import subprocess import time import os import threading import queue import logging # 配置日志 logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S' ) logger = logging.getLogger() # 配置 debugging_ports = [9222, 9223, 9224, 9225, 9226, 9227, 9228, 9229] chrome_path = "C:\\Users\\Windows10\\Desktop\\chromedriver-win64\\chrome-win64\\chrome.exe" root_folder = "C:\\Users\\Windows10\\Desktop\\素材" # 获取所有 DZ+数字 文件夹 dz_numbered_folders = sorted([ folder for folder in os.listdir(root_folder) if folder.startswith("DZ") and folder[2:].isdigit() and os.path.isdir(os.path.join(root_folder, folder)) ]) logger.info(f"找到 {len(dz_numbered_folders)} 个 DZ+数字 的文件夹") # 创建任务队列和端口池 task_queue = queue.Queue() port_pool = queue.Queue() # 将所有任务放入队列 for folder in dz_numbered_folders: task_queue.put(folder) # 将所有端口放入端口池 for port in debugging_ports: port_pool.put(port) # 线程数(同时运行的最大浏览器实例数) THREAD_COUNT = min(4, len(debugging_ports)) # 确保不超过可用端口数 def worker(): """工作线程函数,从任务队列获取任务并从端口池获取端口""" while not task_queue.empty(): # 从端口池获取可用端口 port = port_pool.get() folder_name = task_queue.get() try: logger.info(f"线程 {threading.current_thread().name} 使用端口 {port} 处理任务 {folder_name}") upload_folder(folder_name, port) except Exception as e: logger.error(f"处理任务 {folder_name} 时出错: {e}") finally: # 任务完成后归还端口 port_pool.put(port) task_queue.task_done() logger.info(f"任务 {folder_name} 完成,端口 {port} 已归还") def upload_folder(folder_name, port): """上传文件夹的函数,独占使用一个端口""" folder_path = os.path.join(root_folder, folder_name) # 启动浏览器进程 process = subprocess.Popen([ chrome_path, f"--remote-debugging-port={port}", "--new-window", "https://www.dianxiaomi.com/web/popTemu/add" ]) # 等待浏览器启动 time.sleep(5) try: with sync_playwright() as p: # 连接到调试端口 browser = p.chromium.connect_over_cdp(f"http://localhost:{port}") context = browser.contexts[0] page = context.pages[0] # 获取文件夹中的图片文件 files_in_folder = [ os.path.join(folder_path, f) for f in os.listdir(folder_path) if (os.path.isfile(os.path.join(folder_path, f)) and f.lower().endswith(('.png', '.jpg', '.jpeg'))) ] logger.info(f"[端口 {port}] 正在上传文件夹: {folder_name}") time.sleep(5) # 悬停按钮并点击"本地图片" upload_button = page.locator("button:has-text('选择图片')") upload_button.hover() logger.info(f"[{folder_name}] 鼠标已悬停在按钮上") # 点击本地图片选项 local_option = page.locator("li[data-menu-id='local']") local_option.wait_for(timeout=1000) local_option.click() logger.info(f"[{folder_name}] 已点击'本地图片'选项") if files_in_folder: logger.info(f"[{folder_name}] 准备上传 {len(files_in_folder)} 个文件...") page.set_input_files("input[type=file]", files_in_folder) logger.info(f"[{folder_name}] 文件上传完成!") else: logger.warning(f"[{folder_name}] 没有图片可上传") time.sleep(3) # 等待页面关闭 logger.info(f"[{folder_name}] 请关闭浏览器窗口以继续...") try: # 使用事件监听而不是轮询 page.wait_for_event("close", timeout=30000) logger.info(f"[{folder_name}] 页面已关闭,继续下一个任务") except Exception as e: logger.warning(f"[{folder_name}] 等待页面关闭超时或出错: {e}") # 关闭浏览器连接 browser.close() except Exception as e: logger.error(f"[{folder_name}] 操作失败: {e}") finally: # 确保终止浏览器进程 try: process.terminate() logger.info(f"[{folder_name}] 浏览器进程已关闭") except: pass # 启动线程池 threads = [] for i in range(THREAD_COUNT): thread = threading.Thread(target=worker) thread.start() threads.append(thread) logger.info(f"启动线程 {thread.name}") # 等待所有任务完成 task_queue.join() logger.info("所有任务已完成!") # 停止线程 for thread in threads: thread.join() logger.info("所有线程已结束") 代码打开多个网页时,发现只有个别网页进行了上传操作,大部分网页未进行操作,如何优化
最新发布
09-11
评论
成就一亿技术人!
拼手气红包6.0元
还能输入1000个字符
 
红包 添加红包
表情包 插入表情
 条评论被折叠 查看
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值