mirror of
https://github.com/Dvorinka/facr-scraper.git
synced 2026-07-29 05:53:49 +00:00
feat(scraper): implement CloakBrowser support and enhance request stealth
Integrate CloakBrowser to improve success rates against Cloudflare challenges and implement more robust request handling in the Go backend. - Add CloakBrowser integration to Dockerfile and requirements - Implement domain-specific request semaphores in Go to prevent rate-limiting - Add shared HTTP client with cookie jar and header preservation for better session management - Enhance request headers in Go to include modern client hints (Sec-Ch-Ua) - Add benchmarking scripts to compare fetch methods (urllib vs Scrapling vs CloakBrowser) - Update docker-compose to support CloakBrowser environment variables - Optimize Docker image by pre-downloading patched Chromium binaries
This commit is contained in:
+13
-3
@@ -31,6 +31,10 @@ RUN pip install --no-cache-dir -r requirements-scrapling.txt
|
||||
# Install Playwright browsers with deps in one layer
|
||||
RUN playwright install chromium --with-deps
|
||||
|
||||
# Pre-download CloakBrowser patched Chromium binary so it doesn't
|
||||
# download at runtime (saves ~10-20s per cold-start request).
|
||||
RUN python -m cloakbrowser install
|
||||
|
||||
# Fix Python symlinks
|
||||
RUN ln -sf /usr/local/bin/python /opt/scrapling/bin/python \
|
||||
&& ln -sf /usr/local/bin/python /opt/scrapling/bin/python3
|
||||
@@ -50,6 +54,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libxkbcommon0 libatspi2.0-0 libx11-6 libxcomposite1 libxdamage1 \
|
||||
libxext6 libxfixes3 libxrandr2 libgbm1 libcairo2 libpango-1.0-0 \
|
||||
libasound2 \
|
||||
fonts-liberation fonts-noto-color-emoji fonts-noto-core \
|
||||
fontconfig locales \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Create non-root user
|
||||
@@ -64,10 +70,14 @@ COPY --from=python-builder /opt/scrapling /opt/scrapling
|
||||
# Copy Playwright browser cache
|
||||
COPY --from=python-builder /root/.cache/ms-playwright /home/scraper/.cache/ms-playwright
|
||||
|
||||
# Copy scrapling script
|
||||
COPY scripts/scrapling_fetch.py /opt/scrapling/scripts/scrapling_fetch.py
|
||||
# Copy CloakBrowser patched Chromium binary cache
|
||||
COPY --from=python-builder /root/.cloakbrowser /home/scraper/.cloakbrowser
|
||||
|
||||
# Create cache directory for Playwright
|
||||
# Copy scrapling and cloakbrowser scripts
|
||||
COPY scripts/scrapling_fetch.py /opt/scrapling/scripts/scrapling_fetch.py
|
||||
COPY scripts/cloakbrowser_fetch.py /opt/scrapling/scripts/cloakbrowser_fetch.py
|
||||
|
||||
# Create cache directory and fix permissions
|
||||
RUN mkdir -p /home/scraper/.cache && chown -R scraper:scraper /home/scraper /opt/scrapling
|
||||
|
||||
USER scraper
|
||||
|
||||
Reference in New Issue
Block a user