From 617ff932bb9b66fdcd370a08e053d99d43595051 Mon Sep 17 00:00:00 2001 From: jtricerolph Date: Thu, 9 Jul 2026 17:01:54 +0000 Subject: [PATCH] =?UTF-8?q?Switch=20Playwright=20to=20headless=20=E2=80=94?= =?UTF-8?q?=20Xvfb=20fails=20on=20Proxmox=20LXC=20kernels?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Xvfb exits immediately on the production LXC (no framebuffer device support), so headed mode was never working. Switch to headless=True and remove Xvfb from the Dockerfile entirely. Stealth coverage is unchanged: playwright-stealth patches webdriver, plugins and chrome.runtime; the residential proxy + sticky session provide the GB residential fingerprint; homepage warmup carries real session cookies into the search. --enable-unsafe-swiftshader and --disable-blink-features=AutomationControlled are kept. Co-Authored-By: Claude Sonnet 4.6 --- backend/Dockerfile | 8 +------- backend/services/scraper_backends/playwright_local.py | 9 ++++++--- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index e8aea32..0d810ed 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -7,7 +7,6 @@ RUN apt-get update && apt-get install -y \ libpq-dev \ postgresql-client \ curl \ - xvfb \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt . @@ -20,9 +19,4 @@ COPY . . EXPOSE 8000 -# Run under a virtual X display so Chromium launches headful (headful fingerprints -# far cleaner than headless — real GPU strings, plugins, chrome.runtime). -# Start Xvfb directly rather than via xvfb-run: as container PID 1, xvfb-run's -# SIGUSR1 readiness handshake never fires and it hangs without exec'ing the app. -ENV DISPLAY=:99 -CMD ["/bin/sh", "-c", "Xvfb :99 -screen 0 1920x1080x24 -nolisten tcp & sleep 1; exec uvicorn main:app --host 0.0.0.0 --port 8000"] +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/services/scraper_backends/playwright_local.py b/backend/services/scraper_backends/playwright_local.py index 75304c2..2c9026c 100644 --- a/backend/services/scraper_backends/playwright_local.py +++ b/backend/services/scraper_backends/playwright_local.py @@ -159,14 +159,17 @@ class PlaywrightLocalBackend(ScraperBackend): if self._playwright is None: self._playwright = await async_playwright().start() launch_kwargs = dict( - # Headful under xvfb — far cleaner fingerprint than headless. - headless=False, + # Headless — stealth patches + residential proxy + session + # cookies cover the fingerprint; headed+xvfb is unreliable + # in Proxmox LXC kernels where Xvfb exits immediately. + headless=True, args=[ '--disable-blink-features=AutomationControlled', '--no-sandbox', '--disable-dev-shm-usage', '--disable-features=IsolateOrigins,site-per-process', - '--start-maximized', + '--disable-gpu', + '--enable-unsafe-swiftshader', ], ) # Proxy is set at launch (Chromium binds the sticky session, which