mirror of
https://github.com/Dvorinka/facr-scraper.git
synced 2026-07-28 21:43:49 +00:00
refactor: optimize docker image and implement lightweight fetching
This commit improves the overall efficiency and reliability of the scraper by: - Optimizing the Dockerfile by reducing layers, using `--no-install-recommends`, and consolidating Playwright installation. - Adding resource limits (CPU/Memory) to the docker-compose configuration. - Refactoring `main.go` to remove unused Cloudflare client structures and increasing cache TTL. - Implementing a `lightweight_fetch` mechanism in `scrapling_fetch.py` using `urllib` to attempt fast requests before falling back to the heavier Scrapling/Playwright engine. - Adding Cloudflare challenge detection to the lightweight fetcher.
This commit is contained in:
Tomas Dvorak
+20
-38
@@ -14,12 +14,12 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o facr-scraper .
|
||||
# Python stage for Scrapling
|
||||
FROM python:3.11-slim AS python-builder
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
# Install system dependencies for Playwright
|
||||
RUN apt-get update && apt-get install -y \
|
||||
wget \
|
||||
gnupg \
|
||||
ca-certificates \
|
||||
curl \
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
wget curl ca-certificates gnupg \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Create virtual environment and install Scrapling
|
||||
@@ -28,44 +28,27 @@ ENV PATH="/opt/scrapling/bin:$PATH"
|
||||
COPY requirements-scrapling.txt .
|
||||
RUN pip install --no-cache-dir -r requirements-scrapling.txt
|
||||
|
||||
# Install Playwright browsers
|
||||
RUN playwright install chromium
|
||||
RUN playwright install-deps
|
||||
# Install Playwright browsers with deps in one layer
|
||||
RUN playwright install chromium --with-deps
|
||||
|
||||
# Fix Python symlinks
|
||||
RUN ln -sf /usr/local/bin/python /opt/scrapling/bin/python
|
||||
RUN ln -sf /usr/local/bin/python /opt/scrapling/bin/python3
|
||||
RUN ln -sf /usr/local/bin/python /opt/scrapling/bin/python \
|
||||
&& ln -sf /usr/local/bin/python /opt/scrapling/bin/python3
|
||||
|
||||
# Final stage
|
||||
FROM python:3.11-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV PATH="/opt/scrapling/bin:$PATH"
|
||||
|
||||
# Install runtime dependencies for both Go and Playwright
|
||||
RUN apt-get update && apt-get install -y \
|
||||
ca-certificates \
|
||||
wget \
|
||||
curl \
|
||||
gnupg \
|
||||
libglib2.0-0 \
|
||||
libgobject-2.0-0 \
|
||||
libnspr4 \
|
||||
libnss3 \
|
||||
libdbus-1-3 \
|
||||
libatk1.0-0 \
|
||||
libatk-bridge2.0-0 \
|
||||
libcups2 \
|
||||
libexpat1 \
|
||||
libxcb1 \
|
||||
libxkbcommon0 \
|
||||
libatspi2.0-0 \
|
||||
libx11-6 \
|
||||
libxcomposite1 \
|
||||
libxdamage1 \
|
||||
libxext6 \
|
||||
libxfixes3 \
|
||||
libxrandr2 \
|
||||
libgbm1 \
|
||||
libcairo2 \
|
||||
libpango-1.0-0 \
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates wget curl \
|
||||
libglib2.0-0 libgobject-2.0-0 libnspr4 libnss3 libdbus-1-3 \
|
||||
libatk1.0-0 libatk-bridge2.0-0 libcups2 libexpat1 libxcb1 \
|
||||
libxkbcommon0 libatspi2.0-0 libx11-6 libxcomposite1 libxdamage1 \
|
||||
libxext6 libxfixes3 libxrandr2 libgbm1 libcairo2 libpango-1.0-0 \
|
||||
libasound2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
@@ -80,13 +63,12 @@ COPY --from=python-builder /opt/scrapling /opt/scrapling
|
||||
|
||||
# Copy Playwright browser cache
|
||||
COPY --from=python-builder /root/.cache/ms-playwright /home/scraper/.cache/ms-playwright
|
||||
ENV PATH="/opt/scrapling/bin:$PATH"
|
||||
|
||||
# Copy scrapling script
|
||||
COPY scripts/scrapling_fetch.py /opt/scrapling/scripts/scrapling_fetch.py
|
||||
|
||||
# Create cache directory for Playwright
|
||||
RUN mkdir -p /home/scraper/.cache && chown -R scraper:scraper /home/scraper
|
||||
RUN mkdir -p /home/scraper/.cache && chown -R scraper:scraper /home/scraper /opt/scrapling
|
||||
|
||||
USER scraper
|
||||
WORKDIR /home/scraper
|
||||
|
||||
@@ -13,6 +13,14 @@ services:
|
||||
- SCRAPLING_SCRIPT=/opt/scrapling/scripts/scrapling_fetch.py
|
||||
- DEBUG_SAVE_HTML=${DEBUG_SAVE_HTML:-}
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '2.0'
|
||||
memory: 4G
|
||||
reservations:
|
||||
cpus: '0.5'
|
||||
memory: 512M
|
||||
volumes:
|
||||
# Optional: Mount cache for Playwright browsers
|
||||
- playwright_cache:/home/scraper/.cache
|
||||
|
||||
@@ -16,6 +16,8 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
@@ -32,48 +34,6 @@ type Competition struct {
|
||||
Table *CompetitionTable `json:"table,omitempty"`
|
||||
}
|
||||
|
||||
// Cloudflare Browser Rendering API structures
|
||||
type CloudflareCrawlRequest struct {
|
||||
URL string `json:"url"`
|
||||
Limit int `json:"limit,omitempty"`
|
||||
Depth int `json:"depth,omitempty"`
|
||||
Formats []string `json:"formats,omitempty"`
|
||||
Render bool `json:"render,omitempty"`
|
||||
Source string `json:"source,omitempty"`
|
||||
Options map[string]interface{} `json:"options,omitempty"`
|
||||
}
|
||||
|
||||
type CloudflareCrawlResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Result string `json:"result"` // job ID
|
||||
}
|
||||
|
||||
type CloudflareCrawlJob struct {
|
||||
ID string `json:"id"`
|
||||
Status string `json:"status"`
|
||||
BrowserSecondsUsed float64 `json:"browserSecondsUsed"`
|
||||
Total int `json:"total"`
|
||||
Finished int `json:"finished"`
|
||||
Records []CloudflareCrawlRecord `json:"records"`
|
||||
Cursor string `json:"cursor,omitempty"`
|
||||
}
|
||||
|
||||
type CloudflareCrawlRecord struct {
|
||||
URL string `json:"url"`
|
||||
Status string `json:"status"`
|
||||
Markdown string `json:"markdown,omitempty"`
|
||||
HTML string `json:"html,omitempty"`
|
||||
JSON interface{} `json:"json,omitempty"`
|
||||
Metadata map[string]interface{} `json:"metadata"`
|
||||
}
|
||||
|
||||
type CloudflareClient struct {
|
||||
AccountID string
|
||||
APIToken string
|
||||
BaseURL string
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
type fetchOptions struct {
|
||||
Referer string
|
||||
}
|
||||
@@ -103,211 +63,95 @@ type cacheEntry struct {
|
||||
timestamp time.Time
|
||||
}
|
||||
|
||||
const cacheTTL = 5 * time.Minute
|
||||
const cacheTTL = 15 * time.Minute
|
||||
|
||||
// NewCloudflareClient creates a new Cloudflare Browser Rendering API client
|
||||
func NewCloudflareClient() *CloudflareClient {
|
||||
accountID := strings.TrimSpace(os.Getenv("CLOUDFLARE_ACCOUNT_ID"))
|
||||
apiToken := strings.TrimSpace(os.Getenv("CLOUDFLARE_API_TOKEN"))
|
||||
// domainBreakers is a per-domain circuit breaker map so failures on one site
|
||||
// don't block Scrapling for unrelated sites.
|
||||
var domainBreakers struct {
|
||||
mu sync.RWMutex
|
||||
breakers map[string]*circuitBreaker
|
||||
}
|
||||
|
||||
if accountID == "" || apiToken == "" {
|
||||
// scraplingSem limits concurrent Chromium launches to avoid zombie processes
|
||||
// and resource exhaustion.
|
||||
var scraplingSem = newSemaphore(2)
|
||||
|
||||
type circuitBreaker struct {
|
||||
failures int32
|
||||
lastFail time.Time
|
||||
threshold int
|
||||
timeout time.Duration
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func getDomainBreaker(domain string) *circuitBreaker {
|
||||
domainBreakers.mu.RLock()
|
||||
if cb, ok := domainBreakers.breakers[domain]; ok {
|
||||
domainBreakers.mu.RUnlock()
|
||||
return cb
|
||||
}
|
||||
domainBreakers.mu.RUnlock()
|
||||
|
||||
domainBreakers.mu.Lock()
|
||||
defer domainBreakers.mu.Unlock()
|
||||
if cb, ok := domainBreakers.breakers[domain]; ok {
|
||||
return cb
|
||||
}
|
||||
cb := &circuitBreaker{
|
||||
threshold: 15,
|
||||
timeout: 30 * time.Minute,
|
||||
}
|
||||
if domainBreakers.breakers == nil {
|
||||
domainBreakers.breakers = make(map[string]*circuitBreaker)
|
||||
}
|
||||
domainBreakers.breakers[domain] = cb
|
||||
return cb
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) RecordFailure() {
|
||||
atomic.AddInt32(&cb.failures, 1)
|
||||
cb.mu.Lock()
|
||||
cb.lastFail = time.Now()
|
||||
cb.mu.Unlock()
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) RecordSuccess() {
|
||||
atomic.StoreInt32(&cb.failures, 0)
|
||||
}
|
||||
|
||||
func (cb *circuitBreaker) IsOpen() bool {
|
||||
if atomic.LoadInt32(&cb.failures) < int32(cb.threshold) {
|
||||
return false
|
||||
}
|
||||
cb.mu.Lock()
|
||||
last := cb.lastFail
|
||||
cb.mu.Unlock()
|
||||
return time.Since(last) < cb.timeout
|
||||
}
|
||||
|
||||
// semaphore limits concurrent operations.
|
||||
type semaphore struct {
|
||||
ch chan struct{}
|
||||
}
|
||||
|
||||
func newSemaphore(n int) *semaphore {
|
||||
return &semaphore{ch: make(chan struct{}, n)}
|
||||
}
|
||||
|
||||
func (s *semaphore) Acquire(ctx context.Context) error {
|
||||
select {
|
||||
case s.ch <- struct{}{}:
|
||||
return nil
|
||||
}
|
||||
|
||||
return &CloudflareClient{
|
||||
AccountID: accountID,
|
||||
APIToken: apiToken,
|
||||
BaseURL: "https://api.cloudflare.com/client/v4",
|
||||
Client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// StartCrawl initiates a crawl job
|
||||
func (c *CloudflareClient) StartCrawl(ctx context.Context, req CloudflareCrawlRequest) (string, error) {
|
||||
if c == nil {
|
||||
return "", fmt.Errorf("Cloudflare client not initialized")
|
||||
func (s *semaphore) Release() {
|
||||
select {
|
||||
case <-s.ch:
|
||||
default:
|
||||
}
|
||||
|
||||
// Set defaults
|
||||
if req.Limit == 0 {
|
||||
req.Limit = 10
|
||||
}
|
||||
if req.Depth == 0 {
|
||||
req.Depth = 1
|
||||
}
|
||||
if len(req.Formats) == 0 {
|
||||
req.Formats = []string{"html", "markdown"}
|
||||
}
|
||||
if req.Source == "" {
|
||||
req.Source = "all"
|
||||
}
|
||||
|
||||
// Restrict to specific URL patterns for fotbal.cz to avoid crawling unrelated content
|
||||
if req.Options == nil {
|
||||
req.Options = make(map[string]interface{})
|
||||
}
|
||||
|
||||
// Only crawl URLs from the same domain and specific paths
|
||||
includePatterns := []string{
|
||||
"https://www.fotbal.cz/**",
|
||||
}
|
||||
excludePatterns := []string{
|
||||
"**/api/**",
|
||||
"**/static/**",
|
||||
"**/media/**",
|
||||
}
|
||||
|
||||
req.Options["includePatterns"] = includePatterns
|
||||
req.Options["excludePatterns"] = excludePatterns
|
||||
req.Options["includeExternalLinks"] = false
|
||||
req.Options["includeSubdomains"] = false
|
||||
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/accounts/%s/browser-rendering/crawl", c.BaseURL, c.AccountID)
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
httpReq.Header.Set("Authorization", "Bearer "+c.APIToken)
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := c.Client.Do(httpReq)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return "", fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var crawlResp CloudflareCrawlResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&crawlResp); err != nil {
|
||||
return "", fmt.Errorf("failed to decode response: %w", err)
|
||||
}
|
||||
|
||||
if !crawlResp.Success {
|
||||
return "", fmt.Errorf("API returned unsuccessful response")
|
||||
}
|
||||
|
||||
return crawlResp.Result, nil
|
||||
}
|
||||
|
||||
// GetCrawlResults retrieves the results of a crawl job
|
||||
func (c *CloudflareClient) GetCrawlResults(ctx context.Context, jobID string, limit int) (*CloudflareCrawlJob, error) {
|
||||
if c == nil {
|
||||
return nil, fmt.Errorf("Cloudflare client not initialized")
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/accounts/%s/browser-rendering/crawl/%s", c.BaseURL, c.AccountID, jobID)
|
||||
if limit > 0 {
|
||||
url += fmt.Sprintf("?limit=%d", limit)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
httpReq.Header.Set("Authorization", "Bearer "+c.APIToken)
|
||||
|
||||
resp, err := c.Client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("API request failed with status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var result struct {
|
||||
Success bool `json:"success"`
|
||||
Result CloudflareCrawlJob `json:"result"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode response: %w", err)
|
||||
}
|
||||
|
||||
if !result.Success {
|
||||
return nil, fmt.Errorf("API returned unsuccessful response")
|
||||
}
|
||||
|
||||
return &result.Result, nil
|
||||
}
|
||||
|
||||
// WaitForCrawlCompletion waits for a crawl job to complete and returns the results
|
||||
func (c *CloudflareClient) WaitForCrawlCompletion(ctx context.Context, jobID string, maxAttempts int, delay time.Duration) (*CloudflareCrawlJob, error) {
|
||||
if c == nil {
|
||||
return nil, fmt.Errorf("Cloudflare client not initialized")
|
||||
}
|
||||
|
||||
for i := 0; i < maxAttempts; i++ {
|
||||
job, err := c.GetCrawlResults(ctx, jobID, 1) // Use limit=1 for status checks
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if job.Status != "running" {
|
||||
// Get full results
|
||||
fullJob, err := c.GetCrawlResults(ctx, jobID, 0) // No limit for full results
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fullJob, nil
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(delay):
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("crawl job did not complete within timeout")
|
||||
}
|
||||
|
||||
// CrawlURL performs a complete crawl operation for a single URL
|
||||
func (c *CloudflareClient) CrawlURL(ctx context.Context, url string) (*CloudflareCrawlJob, error) {
|
||||
if c == nil {
|
||||
return nil, fmt.Errorf("Cloudflare client not initialized")
|
||||
}
|
||||
|
||||
req := CloudflareCrawlRequest{
|
||||
URL: url,
|
||||
Limit: 1, // Only crawl the specific URL
|
||||
Depth: 0, // Don't follow links
|
||||
Formats: []string{"html", "markdown"},
|
||||
Render: true,
|
||||
Source: "links", // Only crawl the specific URL, not sitemaps
|
||||
}
|
||||
|
||||
jobID, err := c.StartCrawl(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to start crawl: %w", err)
|
||||
}
|
||||
|
||||
// Wait for completion with reasonable timeout
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
job, err := c.WaitForCrawlCompletion(ctx, jobID, 24, 5*time.Second)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to wait for crawl completion: %w", err)
|
||||
}
|
||||
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func newBrowserRequest(url string, opts fetchOptions) (*http.Request, error) {
|
||||
@@ -364,11 +208,12 @@ func compactErrorText(s string) string {
|
||||
return s
|
||||
}
|
||||
|
||||
func fetchPageDirect(url string, opts fetchOptions) ([]byte, error) {
|
||||
func fetchPageDirectOnce(ctx context.Context, url string, opts fetchOptions) ([]byte, error) {
|
||||
req, err := newBrowserRequest(url, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req = req.WithContext(ctx)
|
||||
|
||||
client := &http.Client{Timeout: 15 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
@@ -392,12 +237,31 @@ func fetchPageDirect(url string, opts fetchOptions) ([]byte, error) {
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func fetchPageWithWget(url string, opts fetchOptions) ([]byte, error) {
|
||||
func fetchPageDirect(ctx context.Context, url string, opts fetchOptions) ([]byte, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if attempt > 0 {
|
||||
select {
|
||||
case <-time.After(time.Duration(attempt) * time.Second):
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
body, err := fetchPageDirectOnce(ctx, url, opts)
|
||||
if err == nil {
|
||||
return body, nil
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
return nil, fmt.Errorf("direct fetch failed after 3 attempts: %w", lastErr)
|
||||
}
|
||||
|
||||
func fetchPageWithWget(ctx context.Context, url string, opts fetchOptions) ([]byte, error) {
|
||||
if _, err := exec.LookPath("wget"); err != nil {
|
||||
return nil, fmt.Errorf("wget not available: %w", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
|
||||
args := []string{
|
||||
@@ -509,6 +373,52 @@ func ensureEmbeddedScraplingHelper() (string, error) {
|
||||
return embeddedScraplingHelperFile, nil
|
||||
}
|
||||
|
||||
func fetchPageWithCurl(ctx context.Context, url string, opts fetchOptions) ([]byte, error) {
|
||||
if _, err := exec.LookPath("curl"); err != nil {
|
||||
return nil, fmt.Errorf("curl not available: %w", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
|
||||
args := []string{
|
||||
"-sSL",
|
||||
"--max-time", "15",
|
||||
"-A", browserUserAgent,
|
||||
"-H", "Accept: " + browserAccept,
|
||||
"-H", "Accept-Language: " + browserAcceptLanguage,
|
||||
"-H", "Connection: keep-alive",
|
||||
}
|
||||
if opts.Referer != "" {
|
||||
args = append(args, "-H", "Referer: "+opts.Referer)
|
||||
}
|
||||
args = append(args, url)
|
||||
|
||||
cmd := exec.CommandContext(ctx, "curl", args...)
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
details := compactErrorText(stderr.String())
|
||||
if details == "" {
|
||||
details = compactErrorText(err.Error())
|
||||
}
|
||||
return nil, fmt.Errorf("curl request failed: %s", details)
|
||||
}
|
||||
|
||||
body := stdout.Bytes()
|
||||
if len(body) == 0 {
|
||||
return nil, fmt.Errorf("curl returned an empty body")
|
||||
}
|
||||
if looksLikeCloudflareBlock(body) {
|
||||
return nil, fmt.Errorf("curl returned a Cloudflare challenge page")
|
||||
}
|
||||
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func findScraplingHelperScript() (string, error) {
|
||||
cwd, _ := os.Getwd()
|
||||
|
||||
@@ -553,7 +463,17 @@ func findScraplingPython() string {
|
||||
)
|
||||
}
|
||||
|
||||
func fetchPageWithScrapling(url string, opts fetchOptions) ([]byte, error) {
|
||||
func fetchPageWithScrapling(ctx context.Context, url string, opts fetchOptions) ([]byte, error) {
|
||||
parsedURL, err := neturl.Parse(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Scrapling skipped: invalid URL: %w", err)
|
||||
}
|
||||
domain := parsedURL.Host
|
||||
|
||||
if getDomainBreaker(domain).IsOpen() {
|
||||
return nil, fmt.Errorf("Scrapling skipped: circuit breaker is open for %s", domain)
|
||||
}
|
||||
|
||||
pythonBin := findScraplingPython()
|
||||
if pythonBin == "" {
|
||||
return nil, fmt.Errorf("Scrapling skipped: no Python runtime found")
|
||||
@@ -564,10 +484,16 @@ func fetchPageWithScrapling(url string, opts fetchOptions) ([]byte, error) {
|
||||
return nil, fmt.Errorf("Scrapling skipped: %w", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
// Acquire global Scrapling semaphore to limit concurrent Chromium launches
|
||||
if err := scraplingSem.Acquire(ctx); err != nil {
|
||||
return nil, fmt.Errorf("Scrapling skipped: %w", err)
|
||||
}
|
||||
defer scraplingSem.Release()
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 90*time.Second)
|
||||
defer cancel()
|
||||
|
||||
args := []string{helperScript, "--url", url}
|
||||
args := []string{helperScript, "--url", url, "--timeout-ms", "60000", "--wait-ms", "500"}
|
||||
if opts.Referer != "" {
|
||||
args = append(args, "--referer", opts.Referer)
|
||||
}
|
||||
@@ -577,32 +503,51 @@ func fetchPageWithScrapling(url string, opts fetchOptions) ([]byte, error) {
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
// Run in a new process group so we can kill all Chromium children on timeout
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
// Kill the entire process group including Chromium children
|
||||
if cmd.Process != nil {
|
||||
syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
||||
}
|
||||
details := compactErrorText(stderr.String())
|
||||
if details == "" {
|
||||
details = compactErrorText(err.Error())
|
||||
}
|
||||
// Don't count context cancellations (client timeouts) or infrastructure failures
|
||||
if ctx.Err() == nil && !strings.Contains(details, "Executable doesn't exist") {
|
||||
getDomainBreaker(domain).RecordFailure()
|
||||
}
|
||||
return nil, fmt.Errorf("Scrapling request failed: %s", details)
|
||||
}
|
||||
|
||||
body := stdout.Bytes()
|
||||
if len(body) == 0 {
|
||||
getDomainBreaker(domain).RecordFailure()
|
||||
return nil, fmt.Errorf("Scrapling returned an empty body")
|
||||
}
|
||||
if looksLikeCloudflareBlock(body) {
|
||||
getDomainBreaker(domain).RecordFailure()
|
||||
return nil, fmt.Errorf("Scrapling returned a Cloudflare challenge page")
|
||||
}
|
||||
|
||||
getDomainBreaker(domain).RecordSuccess()
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func fetchPageWithFallback(url string) ([]byte, error) {
|
||||
return fetchPageWithFallbackOptions(url, fetchOptions{})
|
||||
func fetchPageWithFallback(ctx context.Context, url string) ([]byte, error) {
|
||||
return fetchPageWithFallbackOptions(ctx, url, fetchOptions{})
|
||||
}
|
||||
|
||||
// fetchPageWithFallback tries Go HTTP first, then wget, then Scrapling, then Cloudflare Browser Rendering.
|
||||
func fetchPageWithFallbackOptions(url string, opts fetchOptions) ([]byte, error) {
|
||||
// fetchPageWithFallback tries Go HTTP first, then curl/wget, then Scrapling.
|
||||
// When direct HTTP returns a Cloudflare block, curl/wget are skipped since they
|
||||
// will just return the same challenge page and waste ~20 seconds.
|
||||
func fetchPageWithFallbackOptions(ctx context.Context, url string, opts fetchOptions) ([]byte, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check cache first
|
||||
pageCacheMu.RLock()
|
||||
if entry, ok := pageCache[url]; ok {
|
||||
@@ -614,14 +559,29 @@ func fetchPageWithFallbackOptions(url string, opts fetchOptions) ([]byte, error)
|
||||
}
|
||||
pageCacheMu.RUnlock()
|
||||
|
||||
body, err := fetchPageDirect(url, opts)
|
||||
body, err := fetchPageDirect(ctx, url, opts)
|
||||
if err == nil {
|
||||
cachePage(url, body)
|
||||
return body, nil
|
||||
}
|
||||
log.Printf("Direct request failed for %s: %v", url, err)
|
||||
|
||||
body, err = fetchPageWithWget(url, opts)
|
||||
// If direct HTTP returned a Cloudflare block, skip curl/wget time-wasters
|
||||
// and go straight to Scrapling which can solve the challenge.
|
||||
if strings.Contains(err.Error(), "403") || strings.Contains(err.Error(), "Cloudflare") {
|
||||
log.Printf("Skipping curl/wget fallbacks for %s: direct HTTP hit Cloudflare wall", url)
|
||||
goto scraplingFallback
|
||||
}
|
||||
|
||||
body, err = fetchPageWithCurl(ctx, url, opts)
|
||||
if err == nil {
|
||||
log.Printf("Successfully retrieved content via curl for %s", url)
|
||||
cachePage(url, body)
|
||||
return body, nil
|
||||
}
|
||||
log.Printf("curl fallback failed for %s: %v", url, err)
|
||||
|
||||
body, err = fetchPageWithWget(ctx, url, opts)
|
||||
if err == nil {
|
||||
log.Printf("Successfully retrieved content via wget for %s", url)
|
||||
cachePage(url, body)
|
||||
@@ -629,7 +589,8 @@ func fetchPageWithFallbackOptions(url string, opts fetchOptions) ([]byte, error)
|
||||
}
|
||||
log.Printf("wget fallback failed for %s: %v", url, err)
|
||||
|
||||
body, err = fetchPageWithScrapling(url, opts)
|
||||
scraplingFallback:
|
||||
body, err = fetchPageWithScrapling(ctx, url, opts)
|
||||
if err == nil {
|
||||
log.Printf("Successfully retrieved content via Scrapling for %s", url)
|
||||
cachePage(url, body)
|
||||
@@ -637,32 +598,7 @@ func fetchPageWithFallbackOptions(url string, opts fetchOptions) ([]byte, error)
|
||||
}
|
||||
log.Printf("Scrapling fallback failed for %s: %v", url, err)
|
||||
|
||||
if cfClient := NewCloudflareClient(); cfClient != nil {
|
||||
log.Printf("Attempting Cloudflare crawl fallback for %s", url)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
job, err := cfClient.CrawlURL(ctx, url)
|
||||
if err != nil {
|
||||
log.Printf("Cloudflare crawl failed for %s: %v", url, err)
|
||||
return nil, fmt.Errorf("go scraping failed, wget failed, Scrapling failed, and Cloudflare crawl failed: %w", err)
|
||||
}
|
||||
|
||||
if len(job.Records) > 0 && job.Records[0].Status == "completed" {
|
||||
body := []byte(job.Records[0].HTML)
|
||||
if looksLikeCloudflareBlock(body) {
|
||||
return nil, fmt.Errorf("Cloudflare crawl returned a challenge page")
|
||||
}
|
||||
log.Printf("Successfully retrieved content via Cloudflare crawl for %s", url)
|
||||
cachePage(url, body)
|
||||
return body, nil
|
||||
}
|
||||
|
||||
log.Printf("Cloudflare crawl returned no completed records for %s", url)
|
||||
return nil, fmt.Errorf("Cloudflare crawl returned no completed records")
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("go scraping failed, wget failed, Scrapling failed, and Cloudflare client is not available")
|
||||
return nil, fmt.Errorf("all fetch methods failed for %s: %w", url, err)
|
||||
}
|
||||
|
||||
func cachePage(url string, body []byte) {
|
||||
@@ -674,13 +610,13 @@ func cachePage(url string, body []byte) {
|
||||
// parseCompetitionMatchesFromFotbal scrapes matches from the public fotbal.cz
|
||||
// competition page (e.g., https://www.fotbal.cz/souteze/turnaje/table/{id}).
|
||||
// It filters to only include matches involving the given clubName if provided.
|
||||
func parseCompetitionMatchesFromFotbal(pageURL, clubType, clubName, clubID string) []Match {
|
||||
func parseCompetitionMatchesFromFotbal(ctx context.Context, pageURL, clubType, clubName, clubID string) []Match {
|
||||
pageURL = strings.TrimSpace(pageURL)
|
||||
if pageURL == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
body, err := fetchPageWithFallback(pageURL)
|
||||
body, err := fetchPageWithFallback(ctx, pageURL)
|
||||
if err != nil {
|
||||
log.Printf("fotbal.cz matches fetch failed for %s: %v", pageURL, err)
|
||||
return nil
|
||||
@@ -846,8 +782,13 @@ func parseCompetitionMatchesFromFotbal(pageURL, clubType, clubName, clubID strin
|
||||
}
|
||||
|
||||
// parseCompetitionMatchesFromIS scrapes matches from the IS portal as fallback.
|
||||
func parseCompetitionMatchesFromIS(detailURL, clubType, clubName, clubID string) []Match {
|
||||
resp, err := http.Get(detailURL)
|
||||
func parseCompetitionMatchesFromIS(ctx context.Context, detailURL, clubType, clubName, clubID string) []Match {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", detailURL, nil)
|
||||
if err != nil {
|
||||
log.Printf("IS matches request error for %s: %v", detailURL, err)
|
||||
return nil
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("IS matches fetch error for %s: %v", detailURL, err)
|
||||
return nil
|
||||
@@ -1275,14 +1216,14 @@ func getClubSearch(w http.ResponseWriter, r *http.Request) {
|
||||
vals.Set("q", q)
|
||||
searchURL := "https://www.fotbal.cz/club/hledej?" + vals.Encode()
|
||||
|
||||
fetchSearchPage := func(url string) ([]byte, error) {
|
||||
return fetchPageWithFallbackOptions(url, fetchOptions{
|
||||
fetchSearchPage := func(ctx context.Context, url string) ([]byte, error) {
|
||||
return fetchPageWithFallbackOptions(ctx, url, fetchOptions{
|
||||
Referer: "https://www.fotbal.cz/club/hledej",
|
||||
})
|
||||
}
|
||||
|
||||
// Try direct HTTP request first
|
||||
body, err := fetchSearchPage(searchURL)
|
||||
body, err := fetchSearchPage(r.Context(), searchURL)
|
||||
if err != nil {
|
||||
log.Printf("Direct search request failed for %s: %v", searchURL, err)
|
||||
|
||||
@@ -1298,7 +1239,7 @@ func getClubSearch(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
body, err = fetchSearchPage(searchURL2)
|
||||
body, err = fetchSearchPage(r.Context(), searchURL2)
|
||||
if err != nil {
|
||||
log.Printf("Retried search request failed for %s: %v", searchURL2, err)
|
||||
// Return empty results instead of error
|
||||
@@ -1409,7 +1350,7 @@ func getClubTables(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/%s", baseURL, clubID)
|
||||
body, err := fetchPageWithFallback(url)
|
||||
body, err := fetchPageWithFallback(r.Context(), url)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("Error fetching club data: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
@@ -1459,81 +1400,102 @@ func getClubTables(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
})
|
||||
|
||||
// For each competition, fetch the standings tables from is.fotbal.cz
|
||||
// For each competition, fetch the standings tables from is.fotbal.cz concurrently
|
||||
sem := newSemaphore(4)
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
|
||||
for i := range competitions {
|
||||
comp := &competitions[i]
|
||||
tableURL := fmt.Sprintf("https://is.fotbal.cz/public/souteze/tabulky-souteze.aspx?req=%s&sport=%s", comp.ID, sportParam)
|
||||
resp, err := http.Get(tableURL)
|
||||
if err != nil {
|
||||
log.Printf("error fetching competition table for %s: %v", comp.ID, err)
|
||||
continue
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
if err := sem.Acquire(r.Context()); err != nil {
|
||||
return
|
||||
}
|
||||
defer sem.Release()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
log.Printf("non-200 response for %s: %d", comp.ID, resp.StatusCode)
|
||||
continue
|
||||
}
|
||||
comp := &competitions[idx]
|
||||
tableURL := fmt.Sprintf("https://is.fotbal.cz/public/souteze/tabulky-souteze.aspx?req=%s&sport=%s", comp.ID, sportParam)
|
||||
req, err := http.NewRequestWithContext(r.Context(), "GET", tableURL, nil)
|
||||
if err != nil {
|
||||
log.Printf("error creating request for competition table %s: %v", comp.ID, err)
|
||||
return
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("error fetching competition table for %s: %v", comp.ID, err)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
docTable, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
log.Printf("error parsing table HTML for %s: %v", comp.ID, err)
|
||||
continue
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
log.Printf("non-200 response for %s: %d", comp.ID, resp.StatusCode)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse section: Tabulka celková (only overall)
|
||||
var overall []TableRow
|
||||
docTable, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if err != nil {
|
||||
log.Printf("error parsing table HTML for %s: %v", comp.ID, err)
|
||||
return
|
||||
}
|
||||
|
||||
parseSection := func(headerText string) []TableRow {
|
||||
var rows []TableRow
|
||||
// Find the h3 with matching text, then the following .list.tabulky table
|
||||
docTable.Find("h3").EachWithBreak(func(_ int, h3 *goquery.Selection) bool {
|
||||
if strings.EqualFold(strings.TrimSpace(h3.Text()), headerText) {
|
||||
list := h3.NextAllFiltered("div.list.tabulky").First()
|
||||
if list.Length() == 0 {
|
||||
// Parse section: Tabulka celková (only overall)
|
||||
var overall []TableRow
|
||||
|
||||
parseSection := func(headerText string) []TableRow {
|
||||
var rows []TableRow
|
||||
// Find the h3 with matching text, then the following .list.tabulky table
|
||||
docTable.Find("h3").EachWithBreak(func(_ int, h3 *goquery.Selection) bool {
|
||||
if strings.EqualFold(strings.TrimSpace(h3.Text()), headerText) {
|
||||
list := h3.NextAllFiltered("div.list.tabulky").First()
|
||||
if list.Length() == 0 {
|
||||
return false
|
||||
}
|
||||
table := list.Find("table.vysledky-tabulky tbody")
|
||||
table.Find("tr").Each(func(_ int, tr *goquery.Selection) {
|
||||
// skip header rows containing th
|
||||
if tr.Find("th").Length() > 0 {
|
||||
return
|
||||
}
|
||||
tds := tr.Find("td")
|
||||
if tds.Length() < 8 {
|
||||
return
|
||||
}
|
||||
get := func(i int) string { return strings.TrimSpace(tds.Eq(i).Text()) }
|
||||
rank := get(0)
|
||||
team := get(1)
|
||||
teamID := extractUUIDFromHref(tds.Eq(1).Find("a").First().AttrOr("href", ""))
|
||||
played := get(2)
|
||||
wins := get(3)
|
||||
draws := get(4)
|
||||
losses := get(5)
|
||||
scoreRaw := get(6)
|
||||
// normalize score like "5 : 0" -> "5:0"
|
||||
score := scoreRaw
|
||||
if re := regexp.MustCompile(`\s*([0-9]+)\s*:\s*([0-9]+)\s*`); re != nil {
|
||||
if m := re.FindStringSubmatch(scoreRaw); len(m) == 3 {
|
||||
score = fmt.Sprintf("%s:%s", m[1], m[2])
|
||||
}
|
||||
}
|
||||
points := get(7)
|
||||
rows = append(rows, TableRow{
|
||||
Rank: rank, Team: team, TeamID: teamID, TeamLogoURL: getLogo(team, teamID), Played: played, Wins: wins, Draws: draws, Losses: losses, Score: score, Points: points,
|
||||
})
|
||||
})
|
||||
return false
|
||||
}
|
||||
table := list.Find("table.vysledky-tabulky tbody")
|
||||
table.Find("tr").Each(func(_ int, tr *goquery.Selection) {
|
||||
// skip header rows containing th
|
||||
if tr.Find("th").Length() > 0 {
|
||||
return
|
||||
}
|
||||
tds := tr.Find("td")
|
||||
if tds.Length() < 8 {
|
||||
return
|
||||
}
|
||||
get := func(i int) string { return strings.TrimSpace(tds.Eq(i).Text()) }
|
||||
rank := get(0)
|
||||
team := get(1)
|
||||
teamID := extractUUIDFromHref(tds.Eq(1).Find("a").First().AttrOr("href", ""))
|
||||
played := get(2)
|
||||
wins := get(3)
|
||||
draws := get(4)
|
||||
losses := get(5)
|
||||
scoreRaw := get(6)
|
||||
// normalize score like "5 : 0" -> "5:0"
|
||||
score := scoreRaw
|
||||
if re := regexp.MustCompile(`\s*([0-9]+)\s*:\s*([0-9]+)\s*`); re != nil {
|
||||
if m := re.FindStringSubmatch(scoreRaw); len(m) == 3 {
|
||||
score = fmt.Sprintf("%s:%s", m[1], m[2])
|
||||
}
|
||||
}
|
||||
points := get(7)
|
||||
rows = append(rows, TableRow{
|
||||
Rank: rank, Team: team, TeamID: teamID, TeamLogoURL: getLogo(team, teamID), Played: played, Wins: wins, Draws: draws, Losses: losses, Score: score, Points: points,
|
||||
})
|
||||
})
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
return rows
|
||||
}
|
||||
return true
|
||||
})
|
||||
return rows
|
||||
}
|
||||
|
||||
overall = parseSection("Tabulka celková")
|
||||
comp.Table = &CompetitionTable{Overall: overall}
|
||||
overall = parseSection("Tabulka celková")
|
||||
mu.Lock()
|
||||
comp.Table = &CompetitionTable{Overall: overall}
|
||||
mu.Unlock()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
clubName := strings.TrimSpace(doc.Find("h1.H4 span").First().Text())
|
||||
clubURL := strings.TrimSpace(doc.Find("h1.H4 a").First().AttrOr("href", ""))
|
||||
@@ -1583,7 +1545,7 @@ func getClubInfo(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("%s/%s", baseURL, clubID)
|
||||
body, err := fetchPageWithFallback(url)
|
||||
body, err := fetchPageWithFallback(r.Context(), url)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("Error fetching club data: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
@@ -1637,21 +1599,37 @@ func getClubInfo(w http.ResponseWriter, r *http.Request) {
|
||||
competitions = append(competitions, Competition{ID: compID, Code: code, Name: name, TeamCount: teamCount, MatchesLink: tableLink})
|
||||
})
|
||||
|
||||
// For each competition, fetch matches
|
||||
// For each competition, fetch matches concurrently with limits
|
||||
sem := newSemaphore(4)
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
|
||||
for i := range competitions {
|
||||
comp := &competitions[i]
|
||||
matchesLink := comp.MatchesLink
|
||||
// 1) Try parsing from the public fotbal.cz competition page (matches_link)
|
||||
matches := parseCompetitionMatchesFromFotbal(matchesLink, clubType, clubName, clubID)
|
||||
// Always try IS as well and prefer it if it provides at least as many matches
|
||||
detailURL := fmt.Sprintf("https://is.fotbal.cz/public/souteze/detail-souteze.aspx?req=%s&sport=%s", comp.ID, sportParam)
|
||||
isMatches := parseCompetitionMatchesFromIS(detailURL, clubType, clubName, clubID)
|
||||
// Prefer IS whenever it yields any results, as IS often contains alias team names
|
||||
if len(isMatches) > 0 {
|
||||
matches = isMatches
|
||||
}
|
||||
comp.Matches = matches
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
if err := sem.Acquire(r.Context()); err != nil {
|
||||
return
|
||||
}
|
||||
defer sem.Release()
|
||||
|
||||
comp := &competitions[idx]
|
||||
matchesLink := comp.MatchesLink
|
||||
// 1) Try parsing from the public fotbal.cz competition page (matches_link)
|
||||
matches := parseCompetitionMatchesFromFotbal(r.Context(), matchesLink, clubType, clubName, clubID)
|
||||
// Always try IS as well and prefer it if it provides at least as many matches
|
||||
detailURL := fmt.Sprintf("https://is.fotbal.cz/public/souteze/detail-souteze.aspx?req=%s&sport=%s", comp.ID, sportParam)
|
||||
isMatches := parseCompetitionMatchesFromIS(r.Context(), detailURL, clubType, clubName, clubID)
|
||||
// Prefer IS whenever it yields any results, as IS often contains alias team names
|
||||
if len(isMatches) > 0 {
|
||||
matches = isMatches
|
||||
}
|
||||
mu.Lock()
|
||||
comp.Matches = matches
|
||||
mu.Unlock()
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
clubInfo := ClubInfo{
|
||||
Name: clubName,
|
||||
@@ -1679,9 +1657,18 @@ func main() {
|
||||
http.Redirect(w, r, "/club/football/"+vars["id"], http.StatusMovedPermanently)
|
||||
}).Methods("GET")
|
||||
r.HandleFunc("/", docsHandler)
|
||||
port := "0.0.0.0:8686"
|
||||
fmt.Printf("Server running on http://%s\n", port)
|
||||
log.Fatal(http.ListenAndServe(port, r))
|
||||
|
||||
addr := "0.0.0.0:8686"
|
||||
srv := &http.Server{
|
||||
Addr: addr,
|
||||
Handler: r,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 5 * time.Minute,
|
||||
IdleTimeout: 120 * time.Second,
|
||||
MaxHeaderBytes: 1 << 20,
|
||||
}
|
||||
fmt.Printf("Server running on http://%s\n", addr)
|
||||
log.Fatal(srv.ListenAndServe())
|
||||
}
|
||||
|
||||
// docsHandler serves a simple HTML API documentation at the root endpoint.
|
||||
|
||||
+88
-31
@@ -3,7 +3,34 @@
|
||||
import argparse
|
||||
import contextlib
|
||||
import logging
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
|
||||
BROWSER_UA = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36"
|
||||
)
|
||||
|
||||
CF_SIGNS = [
|
||||
b"<title>just a moment...</title>",
|
||||
b"attention required!",
|
||||
b"enable javascript and cookies to continue",
|
||||
b"checking if the site connection is secure",
|
||||
b"cf-browser-verification",
|
||||
b"/cdn-cgi/challenge-platform/",
|
||||
]
|
||||
|
||||
|
||||
def looks_like_cloudflare_block(body: bytes) -> bool:
|
||||
if not body:
|
||||
return False
|
||||
low = body.lower()
|
||||
for sig in CF_SIGNS:
|
||||
if sig in low:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def response_body_bytes(response) -> bytes:
|
||||
@@ -20,60 +47,90 @@ def response_body_bytes(response) -> bytes:
|
||||
return str(response).encode("utf-8")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--url", required=True)
|
||||
parser.add_argument("--referer", default="")
|
||||
parser.add_argument("--timeout-ms", type=int, default=45000)
|
||||
parser.add_argument("--wait-ms", type=int, default=1000)
|
||||
args = parser.parse_args()
|
||||
def lightweight_fetch(url: str, referer: str = "", timeout: float = 15.0) -> bytes:
|
||||
"""Try a lightweight urllib fetch with browser headers first."""
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": BROWSER_UA,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
||||
"Accept-Language": "cs-CZ,cs;q=0.9,en;q=0.8",
|
||||
"Accept-Encoding": "identity",
|
||||
"Connection": "keep-alive",
|
||||
**({"Referer": referer} if referer else {}),
|
||||
},
|
||||
)
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
|
||||
body = resp.read()
|
||||
if looks_like_cloudflare_block(body):
|
||||
raise RuntimeError(" lightweight fetch returned Cloudflare challenge")
|
||||
return body
|
||||
|
||||
|
||||
def scrapling_fetch(url: str, referer: str = "", timeout_ms: int = 30000, wait_ms: int = 500) -> bytes:
|
||||
try:
|
||||
from scrapling.fetchers import StealthyFetcher
|
||||
except Exception as exc:
|
||||
print(f"Scrapling import failed: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
raise RuntimeError(f"Scrapling import failed: {exc}") from exc
|
||||
|
||||
logging.getLogger().setLevel(logging.ERROR)
|
||||
|
||||
extra_headers = {}
|
||||
if args.referer:
|
||||
extra_headers["Referer"] = args.referer
|
||||
if referer:
|
||||
extra_headers["Referer"] = referer
|
||||
|
||||
fetch_kwargs = {
|
||||
"headless": True,
|
||||
"network_idle": True,
|
||||
"google_search": False,
|
||||
"solve_cloudflare": True,
|
||||
"timeout": args.timeout_ms,
|
||||
"wait": args.wait_ms,
|
||||
"timeout": timeout_ms,
|
||||
"wait": wait_ms,
|
||||
}
|
||||
if extra_headers:
|
||||
fetch_kwargs["extra_headers"] = extra_headers
|
||||
|
||||
with contextlib.redirect_stdout(sys.stderr):
|
||||
response = StealthyFetcher.fetch(url, **fetch_kwargs)
|
||||
|
||||
status = getattr(response, "status", None)
|
||||
if isinstance(status, int) and status >= 400:
|
||||
raise RuntimeError(f"Scrapling returned HTTP {status}")
|
||||
|
||||
body = response_body_bytes(response)
|
||||
if not body:
|
||||
raise RuntimeError("Scrapling returned an empty body")
|
||||
return body
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--url", required=True)
|
||||
parser.add_argument("--referer", default="")
|
||||
parser.add_argument("--timeout-ms", type=int, default=30000)
|
||||
parser.add_argument("--wait-ms", type=int, default=500)
|
||||
args = parser.parse_args()
|
||||
|
||||
# 1) Try lightweight urllib fetch first (no browser, instant)
|
||||
try:
|
||||
with contextlib.redirect_stdout(sys.stderr):
|
||||
response = StealthyFetcher.fetch(args.url, **fetch_kwargs)
|
||||
body = lightweight_fetch(args.url, args.referer, timeout=min(args.timeout_ms / 1000.0, 15.0))
|
||||
sys.stdout.buffer.write(body)
|
||||
return 0
|
||||
except Exception as exc:
|
||||
print(f"Lightweight fetch failed: {exc}", file=sys.stderr)
|
||||
|
||||
# 2) Fall back to Scrapling / Playwright only if lightweight failed
|
||||
try:
|
||||
body = scrapling_fetch(args.url, args.referer, args.timeout_ms, args.wait_ms)
|
||||
sys.stdout.buffer.write(body)
|
||||
return 0
|
||||
except Exception as exc:
|
||||
print(f"Scrapling fetch failed: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
status = getattr(response, "status", None)
|
||||
if isinstance(status, int) and status >= 400:
|
||||
print(f"Scrapling returned HTTP {status}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
body = response_body_bytes(response)
|
||||
if not body:
|
||||
print("Scrapling returned an empty body", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
try:
|
||||
sys.stdout.buffer.write(body)
|
||||
except BrokenPipeError:
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
Reference in New Issue
Block a user