-
Notifications
You must be signed in to change notification settings - Fork 1k
Closed
Labels
Description
Context:
- Playwright Version: playwright==1.30.0
- Operating System: Linux (Linux manjaro 5.15.94-1-MANJARO)
- Python version: 3.10.9
- Browser: Chromium
Code Snippet
import asyncio
from playwright.async_api import async_playwright, PlaywrightContextManager
from pympler import muppy
from pympler import summary
import gc
import tracemalloc
import logging
logger = logging.getLogger(__name__)
async def handle(route, request):
headers = {
}
await route.continue_(headers=headers)
def _make_request_logger():
async def _log_request(request) -> None:
referrer = await request.header_value("referer")
del referrer
return _log_request
def close_browser_call() -> None:
logger.debug("Browser closed")
async def main():
tracemalloc.start()
async with async_playwright() as p:
browser = await p.chromium.launch()
browser.on('disconnected', close_browser_call)
context = await browser.new_context(ignore_https_errors=True)
snapshot1 = tracemalloc.take_snapshot()
for i in range(100):
page = await context.new_page()
await page.route(
"**",
handle)
page.on("request",_make_request_logger())
try:
await page.goto("https://httpstat.us/200?sleep=3", wait_until="networkidle", timeout=100)
except Exception as e:
print(e)
finally:
await page.close()
del page
await context.close()
del context
context = await browser.new_context(ignore_https_errors=True)
gc.collect()
snapshot2 = tracemalloc.take_snapshot()
top_stats = snapshot2.compare_to(snapshot1, 'lineno')
print("[ Top 10 differences ]")
for stat in top_stats[:10]:
print(stat)
del snapshot2
asyncio.run(main())
Describe the bug
During mass crawling, the node may crash unexpectedly or be terminated by the oom-killer, and the program may get stuck.
I use kill -9
command to stop the node, then the following shows and the program hangs:
/home/test/source/shanhai/maliciousDownloader/malicious_downloader/venv/lib/python3.10/site-packages/playwright/driver/playwright.sh: line 6: 21110 Killed "$PLAYWRIGHT_NODEJS_PATH" "$SCRIPT_PATH/package/lib/cli/cli.js" "$@"