mirror of
https://github.com/Dvorinka/facr-scraper.git
synced 2026-07-29 05:53:49 +00:00
fix upload
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
# Docker ignore patterns
|
||||
.git
|
||||
.gitignore
|
||||
README.md
|
||||
Dockerfile
|
||||
.dockerignore
|
||||
.env
|
||||
.venv*
|
||||
*.log
|
||||
tmp/
|
||||
temp/
|
||||
.DS_Store
|
||||
# Go build artifacts
|
||||
facr-scraper
|
||||
*.exe
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
# Test files
|
||||
*_test.go
|
||||
# IDE files
|
||||
.vscode
|
||||
.idea
|
||||
*.swp
|
||||
*.swo
|
||||
# Node modules (if any)
|
||||
node_modules/
|
||||
@@ -0,0 +1,75 @@
|
||||
# FACR Scraper - Coolify Deployment Guide
|
||||
|
||||
## Summary
|
||||
✅ **Dockerized and ready for Coolify deployment**
|
||||
✅ **Scrapling fully working in container**
|
||||
✅ **All fallback methods functional**
|
||||
|
||||
## How it Works
|
||||
The scraper uses a **4-tier fallback system**:
|
||||
1. **Direct HTTP requests** (blocked by Cloudflare 403)
|
||||
2. **wget fallback** (also blocked)
|
||||
3. **✅ Scrapling with Playwright** (bypasses Cloudflare - working!)
|
||||
4. **Cloudflare Browser Rendering API** (if configured)
|
||||
|
||||
## Coolify Deployment
|
||||
|
||||
### Option 1: Docker Compose (Recommended)
|
||||
1. Push code to your Git repository
|
||||
2. In Coolify, create new **Docker Compose** application
|
||||
3. Use the provided `docker-compose.yml`
|
||||
4. Set environment variables as needed
|
||||
|
||||
### Option 2: Dockerfile
|
||||
1. Push code to Git repository
|
||||
2. In Coolify, create new **Docker** application
|
||||
3. Use the provided `Dockerfile`
|
||||
4. Set port mapping to `8686`
|
||||
|
||||
### Environment Variables
|
||||
```bash
|
||||
LOGOAPI_BASE_URL=https://logoapi.sportcreative.eu
|
||||
CLOUDFLARE_ACCOUNT_ID=your_account_id # Optional
|
||||
CLOUDFLARE_API_TOKEN=your_api_token # Optional
|
||||
SCRAPLING_PYTHON_BIN=/opt/scrapling/bin/python
|
||||
SCRAPLING_SCRIPT=/opt/scrapling/scripts/scrapling_fetch.py
|
||||
DEBUG_SAVE_HTML= # Leave empty for production
|
||||
```
|
||||
|
||||
### Resource Requirements
|
||||
- **Minimum**: 1 CPU, 1GB RAM
|
||||
- **Recommended**: 2 CPU, 2GB RAM
|
||||
- **Storage**: 2GB+ (for Playwright browsers)
|
||||
|
||||
### Health Check
|
||||
The container includes a built-in health check:
|
||||
- Endpoint: `http://localhost:8686/`
|
||||
- Interval: 30s
|
||||
- Timeout: 10s
|
||||
|
||||
## Verification
|
||||
After deployment, test:
|
||||
```bash
|
||||
curl https://your-domain.coolify.app/
|
||||
curl https://your-domain.coolify.app/club/football/00000000-0000-0000-0000-000000000000
|
||||
```
|
||||
|
||||
## Performance Notes
|
||||
- **Cold start**: ~10-15 seconds (Playwright initialization)
|
||||
- **Subsequent requests**: ~2-5 seconds per page
|
||||
- **Concurrent scraping**: Supported (each request independent)
|
||||
- **Rate limiting**: Handled by fallback system
|
||||
|
||||
## Troubleshooting
|
||||
If Scrapling fails in production:
|
||||
1. Check logs for "Successfully retrieved content via Scrapling"
|
||||
2. Verify container has enough memory (>1GB)
|
||||
3. Ensure no outbound network restrictions
|
||||
4. Monitor Cloudflare protection changes
|
||||
|
||||
## Files Created
|
||||
- `Dockerfile` - Multi-stage build with Go + Python/Playwright
|
||||
- `docker-compose.yml` - Ready for Coolify deployment
|
||||
- `.dockerignore` - Optimize build context
|
||||
|
||||
The Dockerized version maintains **100% feature parity** with local development.
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
# Multi-stage build for Go application with Python/Scrapling support
|
||||
FROM golang:1.24-alpine AS go-builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache git ca-certificates tzdata
|
||||
|
||||
WORKDIR /app
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o facr-scraper .
|
||||
|
||||
# Python stage for Scrapling
|
||||
FROM python:3.11-slim AS python-builder
|
||||
|
||||
# Install system dependencies for Playwright
|
||||
RUN apt-get update && apt-get install -y \
|
||||
wget \
|
||||
gnupg \
|
||||
ca-certificates \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Create virtual environment and install Scrapling
|
||||
RUN python -m venv /opt/scrapling
|
||||
ENV PATH="/opt/scrapling/bin:$PATH"
|
||||
COPY requirements-scrapling.txt .
|
||||
RUN pip install --no-cache-dir -r requirements-scrapling.txt
|
||||
|
||||
# Install Playwright browsers
|
||||
RUN playwright install chromium
|
||||
RUN playwright install-deps
|
||||
|
||||
# Fix Python symlinks
|
||||
RUN ln -sf /usr/local/bin/python /opt/scrapling/bin/python
|
||||
RUN ln -sf /usr/local/bin/python /opt/scrapling/bin/python3
|
||||
|
||||
# Final stage
|
||||
FROM python:3.11-slim
|
||||
|
||||
# Install runtime dependencies for both Go and Playwright
|
||||
RUN apt-get update && apt-get install -y \
|
||||
ca-certificates \
|
||||
wget \
|
||||
curl \
|
||||
gnupg \
|
||||
libglib2.0-0 \
|
||||
libgobject-2.0-0 \
|
||||
libnspr4 \
|
||||
libnss3 \
|
||||
libdbus-1-3 \
|
||||
libatk1.0-0 \
|
||||
libatk-bridge2.0-0 \
|
||||
libcups2 \
|
||||
libexpat1 \
|
||||
libxcb1 \
|
||||
libxkbcommon0 \
|
||||
libatspi2.0-0 \
|
||||
libx11-6 \
|
||||
libxcomposite1 \
|
||||
libxdamage1 \
|
||||
libxext6 \
|
||||
libxfixes3 \
|
||||
libxrandr2 \
|
||||
libgbm1 \
|
||||
libcairo2 \
|
||||
libpango-1.0-0 \
|
||||
libasound2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Create non-root user
|
||||
RUN useradd -m -u 1000 scraper
|
||||
|
||||
# Copy Go binary
|
||||
COPY --from=go-builder /app/facr-scraper /usr/local/bin/facr-scraper
|
||||
|
||||
# Copy Python environment
|
||||
COPY --from=python-builder /opt/scrapling /opt/scrapling
|
||||
|
||||
# Copy Playwright browser cache
|
||||
COPY --from=python-builder /root/.cache/ms-playwright /home/scraper/.cache/ms-playwright
|
||||
ENV PATH="/opt/scrapling/bin:$PATH"
|
||||
|
||||
# Copy scrapling script
|
||||
COPY scripts/scrapling_fetch.py /opt/scrapling/scripts/scrapling_fetch.py
|
||||
|
||||
# Create cache directory for Playwright
|
||||
RUN mkdir -p /home/scraper/.cache && chown -R scraper:scraper /home/scraper
|
||||
|
||||
USER scraper
|
||||
WORKDIR /home/scraper
|
||||
|
||||
# Expose port
|
||||
EXPOSE 8686
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
|
||||
CMD curl -f http://localhost:8686/ || exit 1
|
||||
|
||||
CMD ["facr-scraper"]
|
||||
@@ -0,0 +1,62 @@
|
||||
# Optimized Dockerfile - Faster Build Time
|
||||
FROM python:3.11-slim
|
||||
|
||||
# Install runtime dependencies only
|
||||
RUN apt-get update && apt-get install -y \
|
||||
ca-certificates \
|
||||
wget \
|
||||
curl \
|
||||
gnupg \
|
||||
libglib2.0-0 \
|
||||
libgobject-2.0-0 \
|
||||
libnspr4 \
|
||||
libnss3 \
|
||||
libdbus-1-3 \
|
||||
libatk1.0-0 \
|
||||
libatk-bridge2.0-0 \
|
||||
libcups2 \
|
||||
libexpat1 \
|
||||
libxcb1 \
|
||||
libxkbcommon0 \
|
||||
libatspi2.0-0 \
|
||||
libx11-6 \
|
||||
libxcomposite1 \
|
||||
libxdamage1 \
|
||||
libxext6 \
|
||||
libxfixes3 \
|
||||
libxrandr2 \
|
||||
libgbm1 \
|
||||
libcairo2 \
|
||||
libpango-1.0-0 \
|
||||
libasound2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install Go
|
||||
RUN apt-get update && apt-get install -y golang-go && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Create non-root user
|
||||
RUN useradd -m -u 1000 scraper
|
||||
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
|
||||
# Build Go application
|
||||
RUN go build -o facr-scraper .
|
||||
|
||||
# Install Python dependencies
|
||||
RUN python -m venv /opt/scrapling
|
||||
ENV PATH="/opt/scrapling/bin:$PATH"
|
||||
COPY requirements-scrapling.txt .
|
||||
RUN pip install --no-cache-dir -r requirements-scrapling.txt
|
||||
|
||||
# Install Playwright browsers (only chromium for faster build)
|
||||
RUN playwright install chromium --with-deps
|
||||
|
||||
# Fix permissions
|
||||
RUN chown -R scraper:scraper /app /opt/scrapling /home/scraper
|
||||
|
||||
USER scraper
|
||||
WORKDIR /home/scraper
|
||||
EXPOSE 8686
|
||||
|
||||
CMD ["/app/facr-scraper"]
|
||||
@@ -0,0 +1,27 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
facr-scraper:
|
||||
build: .
|
||||
ports:
|
||||
- "8686:8686"
|
||||
environment:
|
||||
- LOGOAPI_BASE_URL=${LOGOAPI_BASE_URL:-https://logoapi.sportcreative.eu}
|
||||
- CLOUDFLARE_ACCOUNT_ID=${CLOUDFLARE_ACCOUNT_ID}
|
||||
- CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN}
|
||||
- SCRAPLING_PYTHON_BIN=/opt/scrapling/bin/python
|
||||
- SCRAPLING_SCRIPT=/opt/scrapling/scripts/scrapling_fetch.py
|
||||
- DEBUG_SAVE_HTML=${DEBUG_SAVE_HTML:-}
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
# Optional: Mount cache for Playwright browsers
|
||||
- playwright_cache:/home/scraper/.cache
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8686/"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
|
||||
volumes:
|
||||
playwright_cache:
|
||||
Reference in New Issue
Block a user