mirror of
https://github.com/Dvorinka/Trackeep.git
synced 2026-06-03 20:12:58 +00:00
feat: major feature updates and cleanup
- Add Redis architecture implementation - Update browser extension functionality - Clean up deprecated files and documentation - Enhance backend handlers for auth, messages, search - Add new configuration options and settings - Update Docker and deployment configurations
This commit is contained in:
Tomas Dvorak
@@ -2,10 +2,15 @@ package handlers
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/smtp"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -771,3 +776,243 @@ func formatTimeAgo(t time.Time) string {
|
||||
return t.Format("Jan 2, 2006")
|
||||
}
|
||||
}
|
||||
|
||||
// GitHubRelease represents a GitHub release
|
||||
type GitHubRelease struct {
|
||||
TagName string `json:"tag_name"`
|
||||
Name string `json:"name"`
|
||||
Draft bool `json:"draft"`
|
||||
Prerelease bool `json:"prerelease"`
|
||||
PublishedAt string `json:"published_at"`
|
||||
Body string `json:"body"`
|
||||
}
|
||||
|
||||
// GetLatestVersion fetches the latest version from GitHub releases
|
||||
func GetLatestVersion() (string, error) {
|
||||
// GitHub API endpoint for releases
|
||||
url := "https://api.github.com/repos/dvorinka/trackeep/releases"
|
||||
|
||||
// Create HTTP request
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
// Set headers
|
||||
req.Header.Set("Accept", "application/vnd.github.v3+json")
|
||||
req.Header.Set("User-Agent", "Trackeep-Backend")
|
||||
|
||||
// Make request
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to fetch releases: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("GitHub API returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Read response
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
// Parse JSON
|
||||
var releases []GitHubRelease
|
||||
if err := json.Unmarshal(body, &releases); err != nil {
|
||||
return "", fmt.Errorf("failed to parse JSON: %w", err)
|
||||
}
|
||||
|
||||
// Find latest non-draft release
|
||||
for _, release := range releases {
|
||||
if !release.Draft && !release.Prerelease {
|
||||
return release.TagName, nil
|
||||
}
|
||||
}
|
||||
|
||||
// If no stable release found, return the latest release (including prerelease)
|
||||
if len(releases) > 0 {
|
||||
return releases[0].TagName, nil
|
||||
}
|
||||
|
||||
return "", errors.New("no releases found")
|
||||
}
|
||||
|
||||
// GetCurrentVersion detects the current running version
|
||||
func GetCurrentVersion() (string, error) {
|
||||
// Method 1: Check if running in Docker and get image info
|
||||
if isRunningInDocker() {
|
||||
if version, err := getDockerImageVersion(); err == nil && version != "" {
|
||||
return version, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Method 2: Check for version file or environment variable
|
||||
if version := os.Getenv("TRACKEEP_VERSION"); version != "" {
|
||||
return version, nil
|
||||
}
|
||||
|
||||
// Method 3: Try to read from version file
|
||||
if version, err := readVersionFile(); err == nil && version != "" {
|
||||
return version, nil
|
||||
}
|
||||
|
||||
// Method 4: Check git tag if running from source
|
||||
if version, err := getGitVersion(); err == nil && version != "" {
|
||||
return version, nil
|
||||
}
|
||||
|
||||
// Fallback: Return build time or unknown
|
||||
if buildTime := os.Getenv("BUILD_TIME"); buildTime != "" {
|
||||
return fmt.Sprintf("build-%s", buildTime), nil
|
||||
}
|
||||
|
||||
return "unknown", nil
|
||||
}
|
||||
|
||||
// isRunningInDocker checks if the application is running in a Docker container
|
||||
func isRunningInDocker() bool {
|
||||
// Check for .dockerenv file
|
||||
if _, err := os.Stat("/.dockerenv"); err == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check for Docker in cgroup
|
||||
data, err := os.ReadFile("/proc/1/cgroup")
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return strings.Contains(string(data), "docker")
|
||||
}
|
||||
|
||||
// getDockerImageVersion gets the Docker image tag
|
||||
func getDockerImageVersion() (string, error) {
|
||||
// Try to get container ID from cgroup
|
||||
containerID, err := getContainerID()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Try to inspect the container to get image info
|
||||
cmd := exec.Command("docker", "inspect", "--format='{{.Config.Image}}'", containerID)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
imageName := strings.TrimSpace(string(output))
|
||||
if strings.Contains(imageName, ":") {
|
||||
parts := strings.Split(imageName, ":")
|
||||
if len(parts) > 1 {
|
||||
tag := parts[len(parts)-1]
|
||||
// Remove quotes if present
|
||||
tag = strings.Trim(tag, "'")
|
||||
return tag, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "latest", nil
|
||||
}
|
||||
|
||||
// getContainerID attempts to get the current container ID
|
||||
func getContainerID() (string, error) {
|
||||
// Method 1: Read from /proc/self/cgroup
|
||||
data, err := os.ReadFile("/proc/self/cgroup")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
lines := strings.Split(string(data), "\n")
|
||||
for _, line := range lines {
|
||||
if strings.Contains(line, "docker") {
|
||||
parts := strings.Split(line, "/")
|
||||
if len(parts) > 0 {
|
||||
containerID := parts[len(parts)-1]
|
||||
// Remove any non-hex characters
|
||||
containerID = strings.Trim(containerID, " \t\r\n")
|
||||
if len(containerID) >= 12 {
|
||||
return containerID[:12], nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Method 2: Try to get from hostname
|
||||
hostname, err := os.Hostname()
|
||||
if err == nil && len(hostname) >= 12 {
|
||||
return hostname[:12], nil
|
||||
}
|
||||
|
||||
return "", errors.New("could not determine container ID")
|
||||
}
|
||||
|
||||
// readVersionFile tries to read version from a file
|
||||
func readVersionFile() (string, error) {
|
||||
// Try multiple possible version file locations
|
||||
versionFiles := []string{
|
||||
"/app/VERSION",
|
||||
"/app/version.txt",
|
||||
"./VERSION",
|
||||
"./version.txt",
|
||||
}
|
||||
|
||||
for _, file := range versionFiles {
|
||||
if data, err := os.ReadFile(file); err == nil {
|
||||
return strings.TrimSpace(string(data)), nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", errors.New("no version file found")
|
||||
}
|
||||
|
||||
// getGitVersion gets version from git tag
|
||||
func getGitVersion() (string, error) {
|
||||
if runtime.GOOS == "windows" {
|
||||
return "", errors.New("git version detection not supported on Windows")
|
||||
}
|
||||
|
||||
cmd := exec.Command("git", "describe", "--tags", "--abbrev=0")
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
version := strings.TrimSpace(string(output))
|
||||
return strings.TrimPrefix(version, "v"), nil
|
||||
}
|
||||
|
||||
// GetVersionHandler returns the current and latest version
|
||||
func GetVersionHandler(c *gin.Context) {
|
||||
latestVersion, err := GetLatestVersion()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{
|
||||
"error": "Failed to fetch latest version",
|
||||
"details": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Get current running version
|
||||
currentVersion, err := GetCurrentVersion()
|
||||
if err != nil {
|
||||
currentVersion = "unknown"
|
||||
}
|
||||
|
||||
// Clean the version tag (remove 'v' prefix if present)
|
||||
cleanLatestVersion := strings.TrimPrefix(latestVersion, "v")
|
||||
|
||||
response := gin.H{
|
||||
"current_version": currentVersion,
|
||||
"latest_version": cleanLatestVersion,
|
||||
"latest_tag": latestVersion, // Keep the original tag for reference
|
||||
"is_latest": currentVersion == cleanLatestVersion || currentVersion == "latest",
|
||||
"update_available": currentVersion != cleanLatestVersion && currentVersion != "latest",
|
||||
"running_in_docker": isRunningInDocker(),
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,437 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/trackeep/backend/config"
|
||||
"github.com/trackeep/backend/models"
|
||||
)
|
||||
|
||||
// CreateAPIKeyRequest represents a request to create an API key
|
||||
type CreateAPIKeyRequest struct {
|
||||
Name string `json:"name" binding:"required,min=1,max=100"`
|
||||
Permissions []string `json:"permissions" binding:"required"`
|
||||
ExpiresIn *int `json:"expires_in,omitempty"` // Days until expiration
|
||||
}
|
||||
|
||||
// APIKeyResponse represents API key response
|
||||
type APIKeyResponse struct {
|
||||
ID uint `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Key string `json:"key"`
|
||||
Permissions []string `json:"permissions"`
|
||||
ExpiresAt *time.Time `json:"expires_at,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// BrowserExtensionAuth represents browser extension authentication
|
||||
type BrowserExtensionAuth struct {
|
||||
ID uint `json:"id" gorm:"primaryKey"`
|
||||
UserID uint `json:"user_id" gorm:"not null"`
|
||||
ExtensionID string `json:"extension_id" gorm:"not null"`
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
IsActive bool `json:"is_active" gorm:"default:true"`
|
||||
LastSeen *time.Time `json:"last_seen,omitempty" gorm:"not null"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
}
|
||||
|
||||
// GenerateAPIKey creates a new API key for browser extension
|
||||
func GenerateAPIKey(c *gin.Context) {
|
||||
user, exists := c.Get("user")
|
||||
if !exists {
|
||||
c.JSON(401, gin.H{"error": "User not authenticated"})
|
||||
return
|
||||
}
|
||||
|
||||
currentUser := user.(models.User)
|
||||
|
||||
var req CreateAPIKeyRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
// Validate permissions
|
||||
validPermissions := map[string]bool{
|
||||
"bookmarks:read": true,
|
||||
"bookmarks:write": true,
|
||||
"files:read": true,
|
||||
"files:write": true,
|
||||
"notes:read": true,
|
||||
"notes:write": true,
|
||||
"tasks:read": true,
|
||||
"tasks:write": true,
|
||||
}
|
||||
|
||||
for _, perm := range req.Permissions {
|
||||
if !validPermissions[perm] {
|
||||
c.JSON(400, gin.H{"error": fmt.Sprintf("Invalid permission: %s", perm)})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Generate API key
|
||||
key := generateAPIKey()
|
||||
|
||||
// Set expiration if provided
|
||||
var expiresAt *time.Time
|
||||
if req.ExpiresIn != nil && *req.ExpiresIn > 0 {
|
||||
expiration := time.Now().AddDate(0, 0, *req.ExpiresIn)
|
||||
expiresAt = &expiration
|
||||
}
|
||||
|
||||
// Create API key record
|
||||
apiKey := models.APIKey{
|
||||
Name: req.Name,
|
||||
Key: key,
|
||||
UserID: currentUser.ID,
|
||||
Permissions: req.Permissions,
|
||||
IsActive: true,
|
||||
ExpiresAt: expiresAt,
|
||||
}
|
||||
|
||||
db := config.GetDB()
|
||||
if err := db.Create(&apiKey).Error; err != nil {
|
||||
c.JSON(500, gin.H{"error": "Failed to create API key"})
|
||||
return
|
||||
}
|
||||
|
||||
response := APIKeyResponse{
|
||||
ID: apiKey.ID,
|
||||
Name: apiKey.Name,
|
||||
Key: apiKey.Key,
|
||||
Permissions: apiKey.Permissions,
|
||||
ExpiresAt: apiKey.ExpiresAt,
|
||||
CreatedAt: apiKey.CreatedAt,
|
||||
}
|
||||
|
||||
c.JSON(201, response)
|
||||
}
|
||||
|
||||
// GetAPIKeys retrieves user's API keys
|
||||
func GetAPIKeys(c *gin.Context) {
|
||||
user, exists := c.Get("user")
|
||||
if !exists {
|
||||
c.JSON(401, gin.H{"error": "User not authenticated"})
|
||||
return
|
||||
}
|
||||
|
||||
currentUser := user.(models.User)
|
||||
|
||||
var apiKeys []models.APIKey
|
||||
db := config.GetDB()
|
||||
if err := db.Where("user_id = ? AND is_active = ?", currentUser.ID, true).Order("created_at desc").Find(&apiKeys).Error; err != nil {
|
||||
c.JSON(500, gin.H{"error": "Failed to retrieve API keys"})
|
||||
return
|
||||
}
|
||||
|
||||
// Don't return the actual keys in list view
|
||||
var response []map[string]interface{}
|
||||
for _, key := range apiKeys {
|
||||
response = append(response, map[string]interface{}{
|
||||
"id": key.ID,
|
||||
"name": key.Name,
|
||||
"permissions": key.Permissions,
|
||||
"is_active": key.IsActive,
|
||||
"last_used": key.LastUsed,
|
||||
"expires_at": key.ExpiresAt,
|
||||
"created_at": key.CreatedAt,
|
||||
"updated_at": key.UpdatedAt,
|
||||
})
|
||||
}
|
||||
|
||||
c.JSON(200, response)
|
||||
}
|
||||
|
||||
// RevokeAPIKey revokes an API key
|
||||
func RevokeAPIKey(c *gin.Context) {
|
||||
user, exists := c.Get("user")
|
||||
if !exists {
|
||||
c.JSON(401, gin.H{"error": "User not authenticated"})
|
||||
return
|
||||
}
|
||||
|
||||
currentUser := user.(models.User)
|
||||
keyID := c.Param("id")
|
||||
|
||||
db := config.GetDB()
|
||||
var apiKey models.APIKey
|
||||
if err := db.Where("id = ? AND user_id = ?", keyID, currentUser.ID).First(&apiKey).Error; err != nil {
|
||||
c.JSON(404, gin.H{"error": "API key not found"})
|
||||
return
|
||||
}
|
||||
|
||||
// Deactivate the key
|
||||
if err := db.Model(&apiKey).Update("is_active", false).Error; err != nil {
|
||||
c.JSON(500, gin.H{"error": "Failed to revoke API key"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(200, gin.H{"message": "API key revoked successfully"})
|
||||
}
|
||||
|
||||
// ValidateAPIKey validates an API key from browser extension
|
||||
func ValidateAPIKey(c *gin.Context) {
|
||||
authHeader := c.GetHeader("Authorization")
|
||||
if authHeader == "" {
|
||||
c.JSON(401, gin.H{"error": "Authorization header required"})
|
||||
return
|
||||
}
|
||||
|
||||
// Extract Bearer token
|
||||
parts := strings.SplitN(authHeader, " ", 2)
|
||||
if len(parts) != 2 || parts[0] != "Bearer" {
|
||||
c.JSON(401, gin.H{"error": "Invalid authorization format"})
|
||||
return
|
||||
}
|
||||
|
||||
apiKey := parts[1]
|
||||
|
||||
db := config.GetDB()
|
||||
var keyRecord models.APIKey
|
||||
if err := db.Where("key = ? AND is_active = ?", apiKey, true).Preload("User").First(&keyRecord).Error; err != nil {
|
||||
c.JSON(401, gin.H{"error": "Invalid API key"})
|
||||
return
|
||||
}
|
||||
|
||||
// Check expiration
|
||||
if keyRecord.ExpiresAt != nil && keyRecord.ExpiresAt.Before(time.Now()) {
|
||||
c.JSON(401, gin.H{"error": "API key expired"})
|
||||
return
|
||||
}
|
||||
|
||||
// Update last used timestamp
|
||||
now := time.Now()
|
||||
keyRecord.LastUsed = &now
|
||||
db.Model(&keyRecord).Update("last_used", now)
|
||||
|
||||
// Return user info for extension
|
||||
c.JSON(200, gin.H{
|
||||
"valid": true,
|
||||
"user_id": keyRecord.UserID,
|
||||
"permissions": keyRecord.Permissions,
|
||||
})
|
||||
}
|
||||
|
||||
// generateAPIKey generates a secure API key
|
||||
func generateAPIKey() string {
|
||||
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
keyLength := 32
|
||||
|
||||
bytes := make([]byte, keyLength)
|
||||
rand.Read(bytes)
|
||||
|
||||
for i, b := range bytes {
|
||||
bytes[i] = charset[b%byte(len(charset))]
|
||||
}
|
||||
|
||||
return "tk_" + string(bytes)
|
||||
}
|
||||
|
||||
// RegisterBrowserExtension registers a browser extension instance
|
||||
func RegisterBrowserExtension(c *gin.Context) {
|
||||
user, exists := c.Get("user")
|
||||
if !exists {
|
||||
c.JSON(401, gin.H{"error": "User not authenticated"})
|
||||
return
|
||||
}
|
||||
|
||||
currentUser := user.(models.User)
|
||||
|
||||
var req struct {
|
||||
ExtensionID string `json:"extension_id" binding:"required"`
|
||||
Name string `json:"name" binding:"required"`
|
||||
}
|
||||
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
// Check if extension already registered
|
||||
db := config.GetDB()
|
||||
var existingAuth BrowserExtensionAuth
|
||||
if err := db.Where("user_id = ? AND extension_id = ?", currentUser.ID, req.ExtensionID).First(&existingAuth).Error; err == nil {
|
||||
c.JSON(409, gin.H{"error": "Extension already registered"})
|
||||
return
|
||||
}
|
||||
|
||||
// Create new extension registration
|
||||
extAuth := BrowserExtensionAuth{
|
||||
UserID: currentUser.ID,
|
||||
ExtensionID: req.ExtensionID,
|
||||
Name: req.Name,
|
||||
IsActive: true,
|
||||
LastSeen: &time.Time{},
|
||||
}
|
||||
|
||||
if err := db.Create(&extAuth).Error; err != nil {
|
||||
c.JSON(500, gin.H{"error": "Failed to register extension"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(201, gin.H{
|
||||
"message": "Extension registered successfully",
|
||||
"extension_id": extAuth.ExtensionID,
|
||||
})
|
||||
}
|
||||
|
||||
// GetBrowserExtensions retrieves user's registered browser extensions
|
||||
func GetBrowserExtensions(c *gin.Context) {
|
||||
user, exists := c.Get("user")
|
||||
if !exists {
|
||||
c.JSON(401, gin.H{"error": "User not authenticated"})
|
||||
return
|
||||
}
|
||||
|
||||
currentUser := user.(models.User)
|
||||
|
||||
var extensions []BrowserExtensionAuth
|
||||
db := config.GetDB()
|
||||
if err := db.Where("user_id = ?", currentUser.ID).Order("created_at desc").Find(&extensions).Error; err != nil {
|
||||
c.JSON(500, gin.H{"error": "Failed to retrieve extensions"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(200, extensions)
|
||||
}
|
||||
|
||||
// RevokeBrowserExtension revokes a browser extension
|
||||
func RevokeBrowserExtension(c *gin.Context) {
|
||||
user, exists := c.Get("user")
|
||||
if !exists {
|
||||
c.JSON(401, gin.H{"error": "User not authenticated"})
|
||||
return
|
||||
}
|
||||
|
||||
currentUser := user.(models.User)
|
||||
extensionID := c.Param("id")
|
||||
|
||||
db := config.GetDB()
|
||||
var extAuth BrowserExtensionAuth
|
||||
if err := db.Where("extension_id = ? AND user_id = ?", extensionID, currentUser.ID).First(&extAuth).Error; err != nil {
|
||||
c.JSON(404, gin.H{"error": "Extension not found"})
|
||||
return
|
||||
}
|
||||
|
||||
// Deactivate the extension
|
||||
if err := db.Model(&extAuth).Update("is_active", false).Error; err != nil {
|
||||
c.JSON(500, gin.H{"error": "Failed to revoke extension"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(200, gin.H{"message": "Extension revoked successfully"})
|
||||
}
|
||||
|
||||
// DownloadBrowserExtension serves the browser extension as a downloadable zip file
|
||||
func DownloadBrowserExtension(c *gin.Context) {
|
||||
// Path to the browser extension directory
|
||||
extDir := "../browser-extension"
|
||||
|
||||
// Create a temporary zip file
|
||||
zipPath := "/tmp/browser-extension.zip"
|
||||
|
||||
// Create zip file
|
||||
err := createZip(extDir, zipPath)
|
||||
if err != nil {
|
||||
c.JSON(500, gin.H{"error": "Failed to create zip file"})
|
||||
return
|
||||
}
|
||||
|
||||
// Open the zip file
|
||||
zipFile, err := os.Open(zipPath)
|
||||
if err != nil {
|
||||
c.JSON(500, gin.H{"error": "Failed to open zip file"})
|
||||
return
|
||||
}
|
||||
defer zipFile.Close()
|
||||
|
||||
// Get file info
|
||||
fileInfo, err := zipFile.Stat()
|
||||
if err != nil {
|
||||
c.JSON(500, gin.H{"error": "Failed to get file info"})
|
||||
return
|
||||
}
|
||||
|
||||
// Set headers for download
|
||||
c.Header("Content-Type", "application/zip")
|
||||
c.Header("Content-Disposition", "attachment; filename=browser-extension.zip")
|
||||
c.Header("Content-Length", fmt.Sprintf("%d", fileInfo.Size()))
|
||||
|
||||
// Copy file to response
|
||||
io.Copy(c.Writer, zipFile)
|
||||
|
||||
// Clean up temporary file
|
||||
os.Remove(zipPath)
|
||||
}
|
||||
|
||||
// createZip creates a zip file from a directory
|
||||
func createZip(source, target string) error {
|
||||
zipfile, err := os.Create(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer zipfile.Close()
|
||||
|
||||
archive := zip.NewWriter(zipfile)
|
||||
defer archive.Close()
|
||||
|
||||
info, err := os.Stat(source)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var baseDir string
|
||||
if info.IsDir() {
|
||||
baseDir = filepath.Base(source)
|
||||
}
|
||||
|
||||
return filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
header, err := zip.FileInfoHeader(info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if baseDir != "" {
|
||||
header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
header.Name += "/"
|
||||
} else {
|
||||
header.Method = zip.Deflate
|
||||
}
|
||||
|
||||
writer, err := archive.CreateHeader(header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = io.Copy(writer, file)
|
||||
return err
|
||||
})
|
||||
}
|
||||
@@ -49,10 +49,17 @@ type AttachmentInput struct {
|
||||
Title string `json:"title"`
|
||||
}
|
||||
|
||||
type ReferenceInput struct {
|
||||
EntityType string `json:"entity_type"`
|
||||
EntityID uint `json:"entity_id"`
|
||||
DeepLink string `json:"deep_link"`
|
||||
}
|
||||
|
||||
type CreateMessageRequest struct {
|
||||
Body string `json:"body"`
|
||||
Attachments []AttachmentInput `json:"attachments"`
|
||||
Metadata map[string]interface{} `json:"metadata"`
|
||||
References []ReferenceInput `json:"references"`
|
||||
}
|
||||
|
||||
type UpdateMessageRequest struct {
|
||||
@@ -641,8 +648,8 @@ func CreateConversationMessage(c *gin.Context) {
|
||||
}
|
||||
|
||||
trimmedBody := strings.TrimSpace(req.Body)
|
||||
if trimmedBody == "" && len(req.Attachments) == 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Message body or attachments are required"})
|
||||
if trimmedBody == "" && len(req.Attachments) == 0 && len(req.References) == 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Message body, attachments, or references are required"})
|
||||
return
|
||||
}
|
||||
|
||||
@@ -656,6 +663,37 @@ func CreateConversationMessage(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
referenceRows := make([]models.MessageReference, 0, len(req.References))
|
||||
for _, ref := range req.References {
|
||||
entityType := normalizeReferenceType(ref.EntityType)
|
||||
if entityType == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid reference entity_type"})
|
||||
return
|
||||
}
|
||||
if ref.EntityID == 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid reference entity_id"})
|
||||
return
|
||||
}
|
||||
deepLink := strings.TrimSpace(ref.DeepLink)
|
||||
if deepLink == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid reference deep_link"})
|
||||
return
|
||||
}
|
||||
if !isReferenceDeepLinkAllowed(deepLink) {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Unsupported reference deep_link"})
|
||||
return
|
||||
}
|
||||
if !canReferenceEntity(models.DB, userID, entityType, ref.EntityID) {
|
||||
c.JSON(http.StatusForbidden, gin.H{"error": "Reference target is not accessible"})
|
||||
return
|
||||
}
|
||||
referenceRows = append(referenceRows, models.MessageReference{
|
||||
EntityType: entityType,
|
||||
EntityID: ref.EntityID,
|
||||
DeepLink: deepLink,
|
||||
})
|
||||
}
|
||||
|
||||
suggestions, inferredAttachments, isSensitive := services.DetectMessageContent(trimmedBody)
|
||||
for _, inferred := range inferredAttachments {
|
||||
if hasAttachment(attachmentRows, inferred.Kind, inferred.URL) {
|
||||
@@ -719,6 +757,13 @@ func CreateConversationMessage(c *gin.Context) {
|
||||
models.DB.Create(&attachmentRows)
|
||||
}
|
||||
|
||||
for i := range referenceRows {
|
||||
referenceRows[i].MessageID = message.ID
|
||||
}
|
||||
if len(referenceRows) > 0 {
|
||||
models.DB.Create(&referenceRows)
|
||||
}
|
||||
|
||||
if len(suggestions) > 0 {
|
||||
suggestionRows := make([]models.MessageSuggestion, 0, len(suggestions))
|
||||
for _, s := range suggestions {
|
||||
@@ -2159,6 +2204,33 @@ func normalizeAttachmentKind(kind string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeReferenceType(entityType string) string {
|
||||
t := strings.ToLower(strings.TrimSpace(entityType))
|
||||
switch t {
|
||||
case "task", "bookmark", "calendar_event", "youtube_video", "learning_path", "saved_search", "github", "password_vault_item", "ai_chat_session", "ai_chat_message":
|
||||
return t
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func isReferenceDeepLinkAllowed(deepLink string) bool {
|
||||
return strings.HasPrefix(deepLink, "/") || strings.HasPrefix(deepLink, "http://") || strings.HasPrefix(deepLink, "https://")
|
||||
}
|
||||
|
||||
func canReferenceEntity(db *gorm.DB, userID uint, entityType string, entityID uint) bool {
|
||||
switch entityType {
|
||||
case "ai_chat_session":
|
||||
var session models.ChatSession
|
||||
return db.Where("id = ? AND user_id = ?", entityID, userID).First(&session).Error == nil
|
||||
case "ai_chat_message":
|
||||
var message models.ChatMessage
|
||||
return db.Where("id = ? AND user_id = ?", entityID, userID).First(&message).Error == nil
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func compactMessageTitle(text string, limit int) string {
|
||||
trimmed := strings.TrimSpace(text)
|
||||
if len(trimmed) <= limit {
|
||||
|
||||
+351
-136
@@ -7,7 +7,6 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
@@ -63,78 +62,176 @@ func SearchWeb(c *gin.Context) {
|
||||
req.Count = 10
|
||||
}
|
||||
|
||||
apiKey := os.Getenv("BRAVE_API_KEY")
|
||||
if apiKey == "" {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Brave API key not configured"})
|
||||
// Get user ID from context (authentication is required)
|
||||
userID, exists := c.Get("user_id")
|
||||
if !exists {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "Authentication required for search functionality"})
|
||||
return
|
||||
}
|
||||
|
||||
// Build Brave Search API request
|
||||
baseURL := "https://api.search.brave.com/res/v1/web/search"
|
||||
q := url.Values{}
|
||||
q.Set("q", req.Query)
|
||||
q.Set("count", fmt.Sprint(req.Count))
|
||||
endpoint := fmt.Sprintf("%s?%s", baseURL, q.Encode())
|
||||
|
||||
reqHTTP, err := http.NewRequest(http.MethodGet, endpoint, nil)
|
||||
// Get user's search settings from database
|
||||
searchSettings, err := GetSearchSettingsForAPI(userID.(int))
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create Brave request"})
|
||||
return
|
||||
}
|
||||
reqHTTP.Header.Set("Accept", "application/json")
|
||||
reqHTTP.Header.Set("X-Subscription-Token", apiKey)
|
||||
|
||||
resp, err := http.DefaultClient.Do(reqHTTP)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": "Failed to contact Brave Search API"})
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": fmt.Sprintf("Brave API error: %d", resp.StatusCode)})
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get search settings"})
|
||||
return
|
||||
}
|
||||
|
||||
var braveResp BraveSearchResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&braveResp); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to decode Brave response"})
|
||||
return
|
||||
}
|
||||
// Check if user has search API key configured
|
||||
if searchSettings.SearchAPIProvider == "brave" {
|
||||
apiKey := searchSettings.BraveAPIKey
|
||||
if apiKey == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Brave Search API key not configured. Please configure your search API key in settings."})
|
||||
return
|
||||
}
|
||||
|
||||
// Prefer web.results, fall back to mixed.results
|
||||
resultsRaw := braveResp.Web.Results
|
||||
if len(resultsRaw) == 0 {
|
||||
resultsRaw = braveResp.Mixed.Results
|
||||
}
|
||||
// Build Brave Search API request
|
||||
baseURL := "https://api.search.brave.com/res/v1/web/search"
|
||||
q := url.Values{}
|
||||
q.Set("q", req.Query)
|
||||
q.Set("count", fmt.Sprint(req.Count))
|
||||
endpoint := fmt.Sprintf("%s?%s", baseURL, q.Encode())
|
||||
|
||||
results := make([]BraveSearchResult, 0, len(resultsRaw))
|
||||
for _, r := range resultsRaw {
|
||||
title, _ := r["title"].(string)
|
||||
urlStr, _ := r["url"].(string)
|
||||
desc, _ := r["description"].(string)
|
||||
lang, _ := r["language"].(string)
|
||||
pageAge, _ := r["page_age"].(string)
|
||||
reqHTTP, err := http.NewRequest(http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create Brave request"})
|
||||
return
|
||||
}
|
||||
reqHTTP.Header.Set("Accept", "application/json")
|
||||
reqHTTP.Header.Set("X-Subscription-Token", apiKey)
|
||||
|
||||
results = append(results, BraveSearchResult{
|
||||
Title: title,
|
||||
URL: urlStr,
|
||||
Description: desc,
|
||||
PublishedDate: pageAge,
|
||||
Language: lang,
|
||||
resp, err := http.DefaultClient.Do(reqHTTP)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": "Failed to contact Brave Search API"})
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": fmt.Sprintf("Brave API error: %d", resp.StatusCode)})
|
||||
return
|
||||
}
|
||||
|
||||
var braveResp BraveSearchResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&braveResp); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to decode Brave response"})
|
||||
return
|
||||
}
|
||||
|
||||
// Prefer web.results, fall back to mixed.results
|
||||
resultsRaw := braveResp.Web.Results
|
||||
if len(resultsRaw) == 0 {
|
||||
resultsRaw = braveResp.Mixed.Results
|
||||
}
|
||||
|
||||
results := make([]BraveSearchResult, 0, len(resultsRaw))
|
||||
for _, r := range resultsRaw {
|
||||
title, _ := r["title"].(string)
|
||||
urlStr, _ := r["url"].(string)
|
||||
desc, _ := r["description"].(string)
|
||||
lang, _ := r["language"].(string)
|
||||
pageAge, _ := r["page_age"].(string)
|
||||
|
||||
results = append(results, BraveSearchResult{
|
||||
Title: title,
|
||||
URL: urlStr,
|
||||
Description: desc,
|
||||
PublishedDate: pageAge,
|
||||
Language: lang,
|
||||
})
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"results": results,
|
||||
"query": gin.H{
|
||||
"original": braveResp.Query.Original,
|
||||
"display": braveResp.Query.Display,
|
||||
},
|
||||
"count": len(results),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"results": results,
|
||||
"query": gin.H{
|
||||
"original": braveResp.Query.Original,
|
||||
"display": braveResp.Query.Display,
|
||||
},
|
||||
"count": len(results),
|
||||
})
|
||||
// Use the configured provider
|
||||
if searchSettings.SearchAPIProvider == "brave" {
|
||||
apiKey := searchSettings.BraveAPIKey
|
||||
if apiKey == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Brave Search API key not configured. Please configure your search API key in settings."})
|
||||
return
|
||||
}
|
||||
|
||||
// Build Brave Search API request
|
||||
baseURL := searchSettings.BraveSearchBaseURL
|
||||
q := url.Values{}
|
||||
q.Set("q", req.Query)
|
||||
q.Set("count", fmt.Sprint(req.Count))
|
||||
endpoint := fmt.Sprintf("%s?%s", baseURL, q.Encode())
|
||||
|
||||
reqHTTP, err := http.NewRequest(http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create Brave request"})
|
||||
return
|
||||
}
|
||||
reqHTTP.Header.Set("Accept", "application/json")
|
||||
reqHTTP.Header.Set("X-Subscription-Token", apiKey)
|
||||
|
||||
resp, err := http.DefaultClient.Do(reqHTTP)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": "Failed to contact Brave Search API"})
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": fmt.Sprintf("Brave API error: %d", resp.StatusCode)})
|
||||
return
|
||||
}
|
||||
|
||||
var braveResp BraveSearchResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&braveResp); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to decode Brave response"})
|
||||
return
|
||||
}
|
||||
|
||||
// Prefer web.results, fall back to mixed.results
|
||||
resultsRaw := braveResp.Web.Results
|
||||
if len(resultsRaw) == 0 {
|
||||
resultsRaw = braveResp.Mixed.Results
|
||||
}
|
||||
|
||||
results := make([]BraveSearchResult, 0, len(resultsRaw))
|
||||
for _, r := range resultsRaw {
|
||||
title, _ := r["title"].(string)
|
||||
urlStr, _ := r["url"].(string)
|
||||
desc, _ := r["description"].(string)
|
||||
lang, _ := r["language"].(string)
|
||||
pageAge, _ := r["page_age"].(string)
|
||||
|
||||
results = append(results, BraveSearchResult{
|
||||
Title: title,
|
||||
URL: urlStr,
|
||||
Description: desc,
|
||||
PublishedDate: pageAge,
|
||||
Language: lang,
|
||||
})
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"results": results,
|
||||
"query": gin.H{
|
||||
"original": braveResp.Query.Original,
|
||||
"display": braveResp.Query.Display,
|
||||
},
|
||||
"count": len(results),
|
||||
})
|
||||
} else if searchSettings.SearchAPIProvider == "serper" {
|
||||
// TODO: Implement Serper API integration
|
||||
c.JSON(http.StatusNotImplemented, gin.H{"error": "Serper API integration not yet implemented"})
|
||||
} else {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "No valid search API provider configured. Please configure a search API provider in settings."})
|
||||
}
|
||||
}
|
||||
|
||||
// SearchNews handles POST /api/v1/search/news
|
||||
func SearchNews(c *gin.Context) {
|
||||
fmt.Printf("DEBUG: SearchNews function called\n")
|
||||
var req struct {
|
||||
@@ -151,97 +248,215 @@ func SearchNews(c *gin.Context) {
|
||||
req.Count = 10
|
||||
}
|
||||
|
||||
apiKey := os.Getenv("BRAVE_API_KEY")
|
||||
if apiKey == "" {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Brave API key not configured"})
|
||||
// Get user ID from context (authentication is required)
|
||||
userID, exists := c.Get("user_id")
|
||||
if !exists {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "Authentication required for search functionality"})
|
||||
return
|
||||
}
|
||||
|
||||
baseURL := "https://api.search.brave.com/res/v1/news/search"
|
||||
q := url.Values{}
|
||||
q.Set("q", req.Query)
|
||||
q.Set("count", fmt.Sprint(req.Count))
|
||||
endpoint := fmt.Sprintf("%s?%s", baseURL, q.Encode())
|
||||
|
||||
reqHTTP, err := http.NewRequest(http.MethodGet, endpoint, nil)
|
||||
// Get user's search settings from database
|
||||
searchSettings, err := GetSearchSettingsForAPI(userID.(int))
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create Brave request"})
|
||||
return
|
||||
}
|
||||
reqHTTP.Header.Set("Accept", "application/json")
|
||||
reqHTTP.Header.Set("X-Subscription-Token", apiKey)
|
||||
|
||||
resp, err := http.DefaultClient.Do(reqHTTP)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": "Failed to contact Brave News API"})
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get search settings"})
|
||||
return
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": fmt.Sprintf("Brave News API error: %d", resp.StatusCode)})
|
||||
return
|
||||
}
|
||||
|
||||
// Read the response body for debugging
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to read response body"})
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("DEBUG: Raw Brave News API response: %s\n", string(bodyBytes))
|
||||
|
||||
var braveResp BraveNewsResponse
|
||||
if err := json.NewDecoder(bytes.NewReader(bodyBytes)).Decode(&braveResp); err != nil {
|
||||
fmt.Printf("DEBUG: JSON decode error: %v\n", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to decode Brave news response"})
|
||||
return
|
||||
}
|
||||
|
||||
// Debug logging
|
||||
fmt.Printf("DEBUG: Parsed BraveNewsResponse: %+v\n", braveResp)
|
||||
fmt.Printf("DEBUG: Number of results: %d\n", len(braveResp.Results))
|
||||
|
||||
resultsRaw := braveResp.Results
|
||||
results := make([]BraveSearchResult, 0, len(resultsRaw))
|
||||
for _, r := range resultsRaw {
|
||||
title, _ := r["title"].(string)
|
||||
urlStr, _ := r["url"].(string)
|
||||
desc, _ := r["description"].(string)
|
||||
lang, _ := r["language"].(string)
|
||||
pubDate, _ := r["published_date"].(string)
|
||||
if pubDate == "" {
|
||||
pubDate, _ = r["page_age"].(string)
|
||||
// Check if user has search API key configured
|
||||
if searchSettings.SearchAPIProvider == "brave" {
|
||||
apiKey := searchSettings.BraveAPIKey
|
||||
if apiKey == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Brave Search API key not configured. Please configure your search API key in settings."})
|
||||
return
|
||||
}
|
||||
|
||||
results = append(results, BraveSearchResult{
|
||||
Title: title,
|
||||
URL: urlStr,
|
||||
Description: desc,
|
||||
PublishedDate: pubDate,
|
||||
Language: lang,
|
||||
baseURL := "https://api.search.brave.com/res/v1/news/search"
|
||||
q := url.Values{}
|
||||
q.Set("q", req.Query)
|
||||
q.Set("count", fmt.Sprint(req.Count))
|
||||
endpoint := fmt.Sprintf("%s?%s", baseURL, q.Encode())
|
||||
|
||||
reqHTTP, err := http.NewRequest(http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create Brave request"})
|
||||
return
|
||||
}
|
||||
reqHTTP.Header.Set("Accept", "application/json")
|
||||
reqHTTP.Header.Set("X-Subscription-Token", apiKey)
|
||||
|
||||
resp, err := http.DefaultClient.Do(reqHTTP)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": "Failed to contact Brave News API"})
|
||||
return
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": fmt.Sprintf("Brave News API error: %d", resp.StatusCode)})
|
||||
return
|
||||
}
|
||||
|
||||
// Read the response body for debugging
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to read response body"})
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("DEBUG: Raw Brave News API response: %s\n", string(bodyBytes))
|
||||
|
||||
var braveResp BraveNewsResponse
|
||||
if err := json.NewDecoder(bytes.NewReader(bodyBytes)).Decode(&braveResp); err != nil {
|
||||
fmt.Printf("DEBUG: JSON decode error: %v\n", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to decode Brave news response"})
|
||||
return
|
||||
}
|
||||
|
||||
// Debug logging
|
||||
fmt.Printf("DEBUG: Parsed BraveNewsResponse: %+v\n", braveResp)
|
||||
fmt.Printf("DEBUG: Number of results: %d\n", len(braveResp.Results))
|
||||
|
||||
resultsRaw := braveResp.Results
|
||||
results := make([]BraveSearchResult, 0, len(resultsRaw))
|
||||
for _, r := range resultsRaw {
|
||||
title, _ := r["title"].(string)
|
||||
urlStr, _ := r["url"].(string)
|
||||
desc, _ := r["description"].(string)
|
||||
lang, _ := r["language"].(string)
|
||||
pubDate, _ := r["published_date"].(string)
|
||||
if pubDate == "" {
|
||||
pubDate, _ = r["page_age"].(string)
|
||||
}
|
||||
|
||||
results = append(results, BraveSearchResult{
|
||||
Title: title,
|
||||
URL: urlStr,
|
||||
Description: desc,
|
||||
PublishedDate: pubDate,
|
||||
Language: lang,
|
||||
})
|
||||
}
|
||||
|
||||
original := braveResp.Query.Original
|
||||
display := braveResp.Query.Display
|
||||
if original == "" {
|
||||
original = req.Query
|
||||
}
|
||||
if display == "" {
|
||||
display = req.Query
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"results": results,
|
||||
"query": gin.H{
|
||||
"original": original,
|
||||
"display": display,
|
||||
},
|
||||
"count": len(results),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
original := braveResp.Query.Original
|
||||
display := braveResp.Query.Display
|
||||
if original == "" {
|
||||
original = req.Query
|
||||
}
|
||||
if display == "" {
|
||||
display = req.Query
|
||||
}
|
||||
// Use the configured provider
|
||||
if searchSettings.SearchAPIProvider == "brave" {
|
||||
apiKey := searchSettings.BraveAPIKey
|
||||
if apiKey == "" {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Brave API key not configured"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"results": results,
|
||||
"query": gin.H{
|
||||
"original": original,
|
||||
"display": display,
|
||||
},
|
||||
"count": len(results),
|
||||
})
|
||||
baseURL := "https://api.search.brave.com/res/v1/news/search"
|
||||
q := url.Values{}
|
||||
q.Set("q", req.Query)
|
||||
q.Set("count", fmt.Sprint(req.Count))
|
||||
endpoint := fmt.Sprintf("%s?%s", baseURL, q.Encode())
|
||||
|
||||
reqHTTP, err := http.NewRequest(http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create Brave request"})
|
||||
return
|
||||
}
|
||||
reqHTTP.Header.Set("Accept", "application/json")
|
||||
reqHTTP.Header.Set("X-Subscription-Token", apiKey)
|
||||
|
||||
resp, err := http.DefaultClient.Do(reqHTTP)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": "Failed to contact Brave News API"})
|
||||
return
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": fmt.Sprintf("Brave News API error: %d", resp.StatusCode)})
|
||||
return
|
||||
}
|
||||
|
||||
// Read the response body for debugging
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to read response body"})
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("DEBUG: Raw Brave News API response: %s\n", string(bodyBytes))
|
||||
|
||||
var braveResp BraveNewsResponse
|
||||
if err := json.NewDecoder(bytes.NewReader(bodyBytes)).Decode(&braveResp); err != nil {
|
||||
fmt.Printf("DEBUG: JSON decode error: %v\n", err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to decode Brave news response"})
|
||||
return
|
||||
}
|
||||
|
||||
// Debug logging
|
||||
fmt.Printf("DEBUG: Parsed BraveNewsResponse: %+v\n", braveResp)
|
||||
fmt.Printf("DEBUG: Number of results: %d\n", len(braveResp.Results))
|
||||
|
||||
resultsRaw := braveResp.Results
|
||||
results := make([]BraveSearchResult, 0, len(resultsRaw))
|
||||
for _, r := range resultsRaw {
|
||||
title, _ := r["title"].(string)
|
||||
urlStr, _ := r["url"].(string)
|
||||
desc, _ := r["description"].(string)
|
||||
lang, _ := r["language"].(string)
|
||||
pubDate, _ := r["published_date"].(string)
|
||||
if pubDate == "" {
|
||||
pubDate, _ = r["page_age"].(string)
|
||||
}
|
||||
|
||||
results = append(results, BraveSearchResult{
|
||||
Title: title,
|
||||
URL: urlStr,
|
||||
Description: desc,
|
||||
PublishedDate: pubDate,
|
||||
Language: lang,
|
||||
})
|
||||
}
|
||||
|
||||
original := braveResp.Query.Original
|
||||
display := braveResp.Query.Display
|
||||
if original == "" {
|
||||
original = req.Query
|
||||
}
|
||||
if display == "" {
|
||||
display = req.Query
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"results": results,
|
||||
"query": gin.H{
|
||||
"original": original,
|
||||
"display": display,
|
||||
},
|
||||
"count": len(results),
|
||||
})
|
||||
} else if searchSettings.SearchAPIProvider == "serper" {
|
||||
// TODO: Implement Serper API integration for news
|
||||
c.JSON(http.StatusNotImplemented, gin.H{"error": "Serper API integration not yet implemented"})
|
||||
} else {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "No valid search API provider configured. Please configure a search API provider in settings."})
|
||||
}
|
||||
}
|
||||
|
||||
// GetSearchSuggestions handles GET /api/v1/search/suggestions
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/trackeep/backend/models"
|
||||
)
|
||||
|
||||
// SearchSettings represents search API configuration
|
||||
type SearchSettings struct {
|
||||
BraveAPIKey string `json:"brave_api_key"`
|
||||
BraveSearchBaseURL string `json:"brave_search_base_url"`
|
||||
SerperAPIKey string `json:"serper_api_key"`
|
||||
SerperBaseURL string `json:"serper_base_url"`
|
||||
SearchAPIProvider string `json:"search_api_provider"`
|
||||
SearchResultsLimit int `json:"search_results_limit"`
|
||||
SearchCacheTTL int `json:"search_cache_ttl"`
|
||||
SearchRateLimit int `json:"search_rate_limit"`
|
||||
}
|
||||
|
||||
// GetSearchSettings handles GET /api/v1/auth/search/settings
|
||||
func GetSearchSettings(c *gin.Context) {
|
||||
userID := c.GetInt("user_id")
|
||||
|
||||
// Get settings from database
|
||||
settings, err := models.GetUserSearchSettings(uint(userID))
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get settings"})
|
||||
return
|
||||
}
|
||||
|
||||
// Convert to response format
|
||||
response := SearchSettings{
|
||||
BraveSearchBaseURL: settings.BraveSearchBaseURL,
|
||||
SerperBaseURL: settings.SerperBaseURL,
|
||||
SearchAPIProvider: settings.SearchAPIProvider,
|
||||
SearchResultsLimit: settings.SearchResultsLimit,
|
||||
SearchCacheTTL: settings.SearchCacheTTL,
|
||||
SearchRateLimit: settings.SearchRateLimit,
|
||||
}
|
||||
|
||||
// Mask API keys for security
|
||||
if settings.BraveAPIKey != "" && len(settings.BraveAPIKey) > 8 {
|
||||
response.BraveAPIKey = settings.BraveAPIKey[:4] + "********" + settings.BraveAPIKey[len(settings.BraveAPIKey)-4:]
|
||||
}
|
||||
if settings.SerperAPIKey != "" && len(settings.SerperAPIKey) > 8 {
|
||||
response.SerperAPIKey = settings.SerperAPIKey[:4] + "********" + settings.SerperAPIKey[len(settings.SerperAPIKey)-4:]
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
||||
// UpdateSearchSettings handles PUT /api/v1/auth/search/settings
|
||||
func UpdateSearchSettings(c *gin.Context) {
|
||||
userID := c.GetInt("user_id")
|
||||
|
||||
var newSettings SearchSettings
|
||||
if err := c.ShouldBindJSON(&newSettings); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
// Get existing settings to preserve API keys if they're masked
|
||||
existingSettings, err := models.GetUserSearchSettings(uint(userID))
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get existing settings"})
|
||||
return
|
||||
}
|
||||
|
||||
// Check if API keys are masked and preserve existing values
|
||||
if len(newSettings.BraveAPIKey) > 8 && newSettings.BraveAPIKey[4:12] == "********" {
|
||||
newSettings.BraveAPIKey = existingSettings.BraveAPIKey
|
||||
}
|
||||
if len(newSettings.SerperAPIKey) > 8 && newSettings.SerperAPIKey[4:12] == "********" {
|
||||
newSettings.SerperAPIKey = existingSettings.SerperAPIKey
|
||||
}
|
||||
|
||||
// Update model
|
||||
updatedSettings := &models.UserSearchSettings{
|
||||
BraveAPIKey: newSettings.BraveAPIKey,
|
||||
BraveSearchBaseURL: newSettings.BraveSearchBaseURL,
|
||||
SerperAPIKey: newSettings.SerperAPIKey,
|
||||
SerperBaseURL: newSettings.SerperBaseURL,
|
||||
SearchAPIProvider: newSettings.SearchAPIProvider,
|
||||
SearchResultsLimit: newSettings.SearchResultsLimit,
|
||||
SearchCacheTTL: newSettings.SearchCacheTTL,
|
||||
SearchRateLimit: newSettings.SearchRateLimit,
|
||||
}
|
||||
|
||||
// Save to database
|
||||
err = models.SaveUserSearchSettings(uint(userID), updatedSettings)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to save settings"})
|
||||
return
|
||||
}
|
||||
|
||||
// Return masked settings for consistency
|
||||
GetSearchSettings(c)
|
||||
}
|
||||
|
||||
// GetTestSearchSettings handles GET /api/v1/test-search-settings (for demo mode)
|
||||
func GetTestSearchSettings(c *gin.Context) {
|
||||
settings := getDefaultSearchSettings()
|
||||
|
||||
// Mask API keys for security
|
||||
if settings.BraveAPIKey != "" && len(settings.BraveAPIKey) > 8 {
|
||||
settings.BraveAPIKey = settings.BraveAPIKey[:4] + "********" + settings.BraveAPIKey[len(settings.BraveAPIKey)-4:]
|
||||
}
|
||||
if settings.SerperAPIKey != "" && len(settings.SerperAPIKey) > 8 {
|
||||
settings.SerperAPIKey = settings.SerperAPIKey[:4] + "********" + settings.SerperAPIKey[len(settings.SerperAPIKey)-4:]
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, settings)
|
||||
}
|
||||
|
||||
// GetSearchSettingsForAPI returns unmasked search settings for internal API use
|
||||
func GetSearchSettingsForAPI(userID int) (SearchSettings, error) {
|
||||
settings, err := models.GetUserSearchSettings(uint(userID))
|
||||
if err != nil {
|
||||
// Return default settings if error
|
||||
defaultSettings := getDefaultSearchSettings()
|
||||
return defaultSettings, nil
|
||||
}
|
||||
|
||||
return SearchSettings{
|
||||
BraveAPIKey: settings.BraveAPIKey,
|
||||
BraveSearchBaseURL: settings.BraveSearchBaseURL,
|
||||
SerperAPIKey: settings.SerperAPIKey,
|
||||
SerperBaseURL: settings.SerperBaseURL,
|
||||
SearchAPIProvider: settings.SearchAPIProvider,
|
||||
SearchResultsLimit: settings.SearchResultsLimit,
|
||||
SearchCacheTTL: settings.SearchCacheTTL,
|
||||
SearchRateLimit: settings.SearchRateLimit,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func getDefaultSearchSettings() SearchSettings {
|
||||
return SearchSettings{
|
||||
BraveAPIKey: getEnvWithDefault("BRAVE_API_KEY", "BSAw0HNI1v3rKmXlSTr0C_UfZDjw7fT"),
|
||||
BraveSearchBaseURL: getEnvWithDefault("BRAVE_SEARCH_BASE_URL", "https://api.search.brave.com/res/v1/web/search"),
|
||||
SerperAPIKey: getEnvWithDefault("SERPER_API_KEY", "6f1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2"),
|
||||
SerperBaseURL: getEnvWithDefault("SERPER_BASE_URL", "https://google.serper.dev/search"),
|
||||
SearchAPIProvider: getEnvWithDefault("SEARCH_API_PROVIDER", "brave"),
|
||||
SearchResultsLimit: getIntEnvWithDefault("SEARCH_RESULTS_LIMIT", 10),
|
||||
SearchCacheTTL: getIntEnvWithDefault("SEARCH_CACHE_TTL", 300),
|
||||
SearchRateLimit: getIntEnvWithDefault("SEARCH_RATE_LIMIT", 100),
|
||||
}
|
||||
}
|
||||
|
||||
func getEnvWithDefault(key, defaultValue string) string {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
return value
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
func getIntEnvWithDefault(key string, defaultValue int) int {
|
||||
value := os.Getenv(key)
|
||||
if value == "" {
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
if intValue, err := strconv.Atoi(value); err == nil {
|
||||
return intValue
|
||||
}
|
||||
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
func getBoolEnvWithDefault(key string, defaultValue bool) bool {
|
||||
value := os.Getenv(key)
|
||||
if value == "" {
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
if value == "true" || value == "1" {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"image/png"
|
||||
@@ -60,9 +61,13 @@ type TOTPLoginRequest struct {
|
||||
|
||||
// encrypt encrypts text using AES-GCM
|
||||
func encrypt(plaintext string) (string, error) {
|
||||
key := []byte(os.Getenv("ENCRYPTION_KEY"))
|
||||
keyHex := strings.TrimSpace(os.Getenv("JWT_SECRET"))
|
||||
key, err := hex.DecodeString(keyHex)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to decode JWT secret for encryption: %v", err)
|
||||
}
|
||||
if len(key) != 32 {
|
||||
return "", fmt.Errorf("encryption key must be 32 bytes")
|
||||
return "", fmt.Errorf("JWT secret must be 32 bytes when decoded, got %d", len(key))
|
||||
}
|
||||
|
||||
block, err := aes.NewCipher(key)
|
||||
@@ -86,9 +91,13 @@ func encrypt(plaintext string) (string, error) {
|
||||
|
||||
// decrypt decrypts text using AES-GCM
|
||||
func decrypt(ciphertext string) (string, error) {
|
||||
key := []byte(os.Getenv("ENCRYPTION_KEY"))
|
||||
keyHex := strings.TrimSpace(os.Getenv("JWT_SECRET"))
|
||||
key, err := hex.DecodeString(keyHex)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to decode JWT secret for encryption: %v", err)
|
||||
}
|
||||
if len(key) != 32 {
|
||||
return "", fmt.Errorf("encryption key must be 32 bytes")
|
||||
return "", fmt.Errorf("JWT secret must be 32 bytes when decoded, got %d", len(key))
|
||||
}
|
||||
|
||||
block, err := aes.NewCipher(key)
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/trackeep/backend/models"
|
||||
)
|
||||
|
||||
// UpdateSettings represents update and OAuth configuration
|
||||
type UpdateSettings struct {
|
||||
OAuthServiceURL string `json:"oauth_service_url"`
|
||||
AutoUpdateCheck bool `json:"auto_update_check"`
|
||||
UpdateCheckInterval string `json:"update_check_interval"`
|
||||
PrereleaseUpdates bool `json:"prerelease_updates"`
|
||||
}
|
||||
|
||||
// GetUpdateSettings handles GET /api/v1/auth/update/settings
|
||||
func GetUpdateSettings(c *gin.Context) {
|
||||
userID := c.GetInt("user_id")
|
||||
|
||||
// Get settings from database
|
||||
settings, err := models.GetUserUpdateSettings(uint(userID))
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get settings"})
|
||||
return
|
||||
}
|
||||
|
||||
// Convert to response format
|
||||
response := UpdateSettings{
|
||||
OAuthServiceURL: settings.OAuthServiceURL,
|
||||
AutoUpdateCheck: settings.AutoUpdateCheck,
|
||||
UpdateCheckInterval: settings.UpdateCheckInterval,
|
||||
PrereleaseUpdates: settings.PrereleaseUpdates,
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, response)
|
||||
}
|
||||
|
||||
// UpdateUpdateSettings handles PUT /api/v1/auth/update/settings
|
||||
func UpdateUpdateSettings(c *gin.Context) {
|
||||
userID := c.GetInt("user_id")
|
||||
|
||||
var newSettings UpdateSettings
|
||||
if err := c.ShouldBindJSON(&newSettings); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
// Update model
|
||||
updatedSettings := &models.UserUpdateSettings{
|
||||
OAuthServiceURL: newSettings.OAuthServiceURL,
|
||||
AutoUpdateCheck: newSettings.AutoUpdateCheck,
|
||||
UpdateCheckInterval: newSettings.UpdateCheckInterval,
|
||||
PrereleaseUpdates: newSettings.PrereleaseUpdates,
|
||||
}
|
||||
|
||||
// Save to database
|
||||
err := models.SaveUserUpdateSettings(uint(userID), updatedSettings)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to save settings"})
|
||||
return
|
||||
}
|
||||
|
||||
// Return updated settings
|
||||
GetUpdateSettings(c)
|
||||
}
|
||||
|
||||
// GetTestUpdateSettings handles GET /api/v1/test-update-settings (for demo mode)
|
||||
func GetTestUpdateSettings(c *gin.Context) {
|
||||
settings := getDefaultUpdateSettings()
|
||||
c.JSON(http.StatusOK, settings)
|
||||
}
|
||||
|
||||
// GetUpdateSettingsForAPI returns update settings for internal API use
|
||||
func GetUpdateSettingsForAPI(userID int) (UpdateSettings, error) {
|
||||
settings, err := models.GetUserUpdateSettings(uint(userID))
|
||||
if err != nil {
|
||||
// Return default settings if error
|
||||
defaultSettings := getDefaultUpdateSettings()
|
||||
return defaultSettings, nil
|
||||
}
|
||||
|
||||
return UpdateSettings{
|
||||
OAuthServiceURL: settings.OAuthServiceURL,
|
||||
AutoUpdateCheck: settings.AutoUpdateCheck,
|
||||
UpdateCheckInterval: settings.UpdateCheckInterval,
|
||||
PrereleaseUpdates: settings.PrereleaseUpdates,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func getDefaultUpdateSettings() UpdateSettings {
|
||||
return UpdateSettings{
|
||||
OAuthServiceURL: getEnvWithDefault("OAUTH_SERVICE_URL", "https://oauth.tdvorak.dev"),
|
||||
AutoUpdateCheck: getBoolEnvWithDefault("AUTO_UPDATE_CHECK", false),
|
||||
UpdateCheckInterval: getEnvWithDefault("UPDATE_CHECK_INTERVAL", "24h"),
|
||||
PrereleaseUpdates: getBoolEnvWithDefault("PRERELEASE_UPDATES", false),
|
||||
}
|
||||
}
|
||||
+181
-25
@@ -4,6 +4,7 @@ import (
|
||||
"archive/zip"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -66,33 +67,48 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
// getCurrentVersion reads the current version from frontend/package.json
|
||||
func getCurrentVersion() string {
|
||||
// Try to read from frontend/package.json first
|
||||
packageJsonPath := "frontend/package.json"
|
||||
if content, err := os.ReadFile(packageJsonPath); err == nil {
|
||||
var packageJson struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
if err := json.Unmarshal(content, &packageJson); err == nil && packageJson.Version != "" {
|
||||
log.Printf("Found version in frontend/package.json: %s", packageJson.Version)
|
||||
return packageJson.Version
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to backend/go.mod
|
||||
goModPath := "go.mod"
|
||||
if content, err := os.ReadFile(goModPath); err == nil {
|
||||
lines := strings.Split(string(content), "\n")
|
||||
for _, line := range lines {
|
||||
if strings.Contains(line, "module ") {
|
||||
// Extract version from module path or use a default
|
||||
// For now, return a default version
|
||||
log.Printf("Using fallback version from go.mod")
|
||||
return "1.2.5"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Final fallback
|
||||
log.Printf("Using default version - could not detect from source files")
|
||||
return "1.2.5"
|
||||
}
|
||||
|
||||
// CheckForUpdates checks if a new version is available using Docker registry
|
||||
func CheckForUpdates(c *gin.Context) {
|
||||
updateMutex.Lock()
|
||||
defer updateMutex.Unlock()
|
||||
|
||||
// Get current version from go.mod
|
||||
currentVersion := "1.2.5"
|
||||
// Get current version from frontend/package.json
|
||||
currentVersion := getCurrentVersion()
|
||||
|
||||
// Try to read from go.mod if running in development
|
||||
if _, err := os.Stat("go.mod"); err == nil {
|
||||
if content, err := os.ReadFile("go.mod"); err == nil {
|
||||
lines := strings.Split(string(content), "\n")
|
||||
for _, line := range lines {
|
||||
if strings.Contains(line, "go ") && strings.Contains(line, "1.2.5") {
|
||||
// Extract version from go.mod
|
||||
parts := strings.Fields(line)
|
||||
if len(parts) >= 2 {
|
||||
currentVersion = strings.TrimSpace(parts[1])
|
||||
log.Printf("Found version in go.mod: %s", currentVersion)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("Checking for updates using Docker registry (current version: %s)", currentVersion)
|
||||
log.Printf("Checking for updates using GitHub releases (current version: %s)", currentVersion)
|
||||
|
||||
// Check for updates using Docker registry
|
||||
updateInfo, updateAvailable, err := checkForUpdatesWithDocker(currentVersion)
|
||||
@@ -109,14 +125,20 @@ func CheckForUpdates(c *gin.Context) {
|
||||
currentUpdate = updateInfo
|
||||
updateProgress.Available = true
|
||||
} else {
|
||||
currentUpdate = nil
|
||||
// Still preserve updateInfo for displaying latest version, but mark as no update available
|
||||
currentUpdate = updateInfo
|
||||
updateProgress.Available = false
|
||||
}
|
||||
|
||||
latestVersion := ""
|
||||
if updateInfo != nil {
|
||||
latestVersion = updateInfo.Version
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"updateAvailable": updateAvailable,
|
||||
"currentVersion": currentVersion,
|
||||
"latestVersion": updateInfo.Version,
|
||||
"latestVersion": latestVersion,
|
||||
"updateInfo": currentUpdate,
|
||||
})
|
||||
}
|
||||
@@ -167,8 +189,142 @@ func UpdateProgressWebSocket(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// checkForUpdatesWithDocker checks for updates using Docker registry
|
||||
// checkForUpdatesWithDocker checks for updates using GitHub releases
|
||||
func checkForUpdatesWithDocker(currentVersion string) (*UpdateInfo, bool, error) {
|
||||
log.Printf("Checking for updates (current version: %s)", currentVersion)
|
||||
|
||||
// Get latest release from GitHub
|
||||
latestRelease, err := getLatestGitHubRelease()
|
||||
if err != nil {
|
||||
log.Printf("Failed to get latest release from GitHub: %v", err)
|
||||
// Fallback to Docker registry check
|
||||
return checkForUpdatesWithDockerRegistry(currentVersion)
|
||||
}
|
||||
|
||||
log.Printf("Latest release from GitHub: %s", latestRelease.Version)
|
||||
|
||||
// Compare versions
|
||||
if isNewerVersion(latestRelease.Version, currentVersion) {
|
||||
log.Printf("Update available: %s -> %s", currentVersion, latestRelease.Version)
|
||||
return latestRelease, true, nil
|
||||
}
|
||||
|
||||
log.Printf("No updates available - current version %s is latest", currentVersion)
|
||||
return latestRelease, false, nil
|
||||
}
|
||||
|
||||
// getLatestGitHubRelease fetches the latest release from GitHub API
|
||||
func getLatestGitHubRelease() (*UpdateInfo, error) {
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
url := "https://api.github.com/repos/Dvorinka/Trackeep/releases/latest"
|
||||
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch release: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("GitHub API returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Parse JSON response
|
||||
var release struct {
|
||||
TagName string `json:"tag_name"`
|
||||
Name string `json:"name"`
|
||||
Body string `json:"body"`
|
||||
PublishedAt string `json:"published_at"`
|
||||
Prerelease bool `json:"prerelease"`
|
||||
Draft bool `json:"draft"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&release); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode release JSON: %w", err)
|
||||
}
|
||||
|
||||
// Skip drafts and prereleases unless specifically allowed
|
||||
if release.Draft {
|
||||
return nil, fmt.Errorf("latest release is a draft")
|
||||
}
|
||||
|
||||
// Check if prereleases are allowed
|
||||
allowPrerelease := os.Getenv("PRERELEASE_UPDATES") == "true"
|
||||
if release.Prerelease && !allowPrerelease {
|
||||
// Try to get latest non-prerelease
|
||||
return getLatestStableRelease()
|
||||
}
|
||||
|
||||
// Clean version (remove 'v' prefix if present)
|
||||
version := strings.TrimPrefix(release.TagName, "v")
|
||||
|
||||
updateInfo := &UpdateInfo{
|
||||
Version: version,
|
||||
ReleaseNotes: release.Body,
|
||||
DownloadURL: "", // Docker images don't need download URL
|
||||
Mandatory: false,
|
||||
Size: "Docker images",
|
||||
Checksum: "",
|
||||
PublishedAt: release.PublishedAt,
|
||||
Prerelease: release.Prerelease,
|
||||
}
|
||||
|
||||
return updateInfo, nil
|
||||
}
|
||||
|
||||
// getLatestStableRelease gets the latest stable (non-prerelease) release
|
||||
func getLatestStableRelease() (*UpdateInfo, error) {
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
url := "https://api.github.com/repos/Dvorinka/Trackeep/releases"
|
||||
|
||||
resp, err := client.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch releases: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("GitHub API returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Parse JSON response
|
||||
var releases []struct {
|
||||
TagName string `json:"tag_name"`
|
||||
Name string `json:"name"`
|
||||
Body string `json:"body"`
|
||||
PublishedAt string `json:"published_at"`
|
||||
Prerelease bool `json:"prerelease"`
|
||||
Draft bool `json:"draft"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&releases); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode releases JSON: %w", err)
|
||||
}
|
||||
|
||||
// Find first stable (non-prerelease, non-draft) release
|
||||
for _, release := range releases {
|
||||
if !release.Draft && !release.Prerelease {
|
||||
version := strings.TrimPrefix(release.TagName, "v")
|
||||
|
||||
updateInfo := &UpdateInfo{
|
||||
Version: version,
|
||||
ReleaseNotes: release.Body,
|
||||
DownloadURL: "",
|
||||
Mandatory: false,
|
||||
Size: "Docker images",
|
||||
Checksum: "",
|
||||
PublishedAt: release.PublishedAt,
|
||||
Prerelease: false,
|
||||
}
|
||||
|
||||
return updateInfo, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("no stable releases found")
|
||||
}
|
||||
|
||||
// checkForUpdatesWithDockerRegistry fallback method using Docker registry
|
||||
func checkForUpdatesWithDockerRegistry(currentVersion string) (*UpdateInfo, bool, error) {
|
||||
// Define images to check (using latest)
|
||||
backendImage := "ghcr.io/dvorinka/trackeep/backend:latest"
|
||||
frontendImage := "ghcr.io/dvorinka/trackeep/frontend:latest"
|
||||
@@ -348,7 +504,7 @@ func updateWithDockerCompose() error {
|
||||
// Check if production docker-compose file exists
|
||||
composeFile := "docker-compose.prod.yml"
|
||||
if _, err := os.Stat(composeFile); err != nil {
|
||||
return fmt.Errorf("production docker-compose.yml not found")
|
||||
return fmt.Errorf("production docker-compose file not found")
|
||||
}
|
||||
|
||||
// Use docker compose command directly (assuming Docker is available on host)
|
||||
|
||||
+57
-10
@@ -22,19 +22,29 @@ func IsDemoMode() bool {
|
||||
}
|
||||
|
||||
func initializeSecuritySecrets() error {
|
||||
jwtSecret, err := utils.GetOrCreateJWTSecret()
|
||||
if err != nil {
|
||||
return err
|
||||
// Only set JWT_SECRET if not already provided in environment
|
||||
if os.Getenv("JWT_SECRET") == "" {
|
||||
jwtSecret, err := utils.GetOrCreateJWTSecret()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
os.Setenv("JWT_SECRET", jwtSecret)
|
||||
log.Println("JWT secret initialized from file")
|
||||
} else {
|
||||
log.Println("JWT secret found in environment variable")
|
||||
}
|
||||
os.Setenv("JWT_SECRET", jwtSecret)
|
||||
log.Println("JWT secret initialized successfully")
|
||||
|
||||
encryptionKey, err := utils.GetOrCreateEncryptionKey()
|
||||
if err != nil {
|
||||
return err
|
||||
// Only set ENCRYPTION_KEY if not already provided in environment
|
||||
if os.Getenv("ENCRYPTION_KEY") == "" {
|
||||
encryptionKey, err := utils.GetOrCreateEncryptionKey()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
os.Setenv("ENCRYPTION_KEY", encryptionKey)
|
||||
log.Println("Encryption key initialized from file")
|
||||
} else {
|
||||
log.Println("Encryption key found in environment variable")
|
||||
}
|
||||
os.Setenv("ENCRYPTION_KEY", encryptionKey)
|
||||
log.Println("Encryption key initialized successfully")
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -125,10 +135,17 @@ func main() {
|
||||
// Serve static files (frontend)
|
||||
r.Static("/assets", "../frontend/dist/assets")
|
||||
r.StaticFile("/", "../frontend/dist/index.html")
|
||||
|
||||
// Serve browser extension download
|
||||
r.GET("/browser-extension", handlers.DownloadBrowserExtension)
|
||||
|
||||
r.NoRoute(func(c *gin.Context) {
|
||||
c.File("../frontend/dist/index.html")
|
||||
})
|
||||
|
||||
// Version endpoint
|
||||
r.GET("/api/version", handlers.GetVersionHandler)
|
||||
|
||||
// Initialize handlers
|
||||
memberHandler := handlers.NewMemberHandler(config.GetDB())
|
||||
timeEntryHandler := handlers.NewTimeEntryHandler(config.GetDB())
|
||||
@@ -203,11 +220,23 @@ func main() {
|
||||
authProtected.GET("/ai/settings", handlers.GetAISettings)
|
||||
authProtected.PUT("/ai/settings", handlers.UpdateAISettings)
|
||||
authProtected.POST("/ai/test-connection", handlers.TestAIConnection)
|
||||
|
||||
// Search Settings routes
|
||||
authProtected.GET("/search/settings", handlers.GetSearchSettings)
|
||||
authProtected.PUT("/search/settings", handlers.UpdateSearchSettings)
|
||||
|
||||
// Update Settings routes
|
||||
authProtected.GET("/update/settings", handlers.GetUpdateSettings)
|
||||
authProtected.PUT("/update/settings", handlers.UpdateUpdateSettings)
|
||||
}
|
||||
|
||||
// Test AI settings without auth
|
||||
v1.GET("/test-ai-settings", handlers.GetAISettings)
|
||||
|
||||
// Test search and update settings without auth (for demo mode)
|
||||
v1.GET("/test-search-settings", handlers.GetTestSearchSettings)
|
||||
v1.GET("/test-update-settings", handlers.GetTestUpdateSettings)
|
||||
|
||||
// Dashboard routes (protected)
|
||||
dashboard := v1.Group("/dashboard")
|
||||
dashboard.Use(handlers.AuthMiddleware())
|
||||
@@ -721,6 +750,24 @@ func main() {
|
||||
performance.POST("/optimize", performanceHandler.OptimizeDatabase)
|
||||
performance.POST("/cleanup-audit-logs", performanceHandler.CleanupOldAuditLogs)
|
||||
}
|
||||
|
||||
// Browser Extension API routes
|
||||
browserExt := v1.Group("/browser-extension")
|
||||
browserExt.Use(handlers.AuthMiddleware())
|
||||
{
|
||||
// API Key management
|
||||
browserExt.POST("/api-keys/generate", handlers.GenerateAPIKey)
|
||||
browserExt.GET("/api-keys", handlers.GetAPIKeys)
|
||||
browserExt.DELETE("/api-keys/:id", handlers.RevokeAPIKey)
|
||||
|
||||
// Extension registration and validation
|
||||
browserExt.POST("/register", handlers.RegisterBrowserExtension)
|
||||
browserExt.GET("/extensions", handlers.GetBrowserExtensions)
|
||||
browserExt.DELETE("/extensions/:id", handlers.RevokeBrowserExtension)
|
||||
|
||||
// Public endpoints (for extension validation)
|
||||
browserExt.GET("/validate", handlers.ValidateAPIKey)
|
||||
}
|
||||
}
|
||||
|
||||
srv := &http.Server{
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// APIKey represents an API key for browser extension
|
||||
type APIKey struct {
|
||||
ID uint `json:"id" gorm:"primaryKey"`
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
Key string `json:"key" gorm:"not null;uniqueIndex"`
|
||||
UserID uint `json:"user_id" gorm:"not null"`
|
||||
Permissions []string `json:"permissions" gorm:"serializer:json"`
|
||||
IsActive bool `json:"is_active" gorm:"default:true"`
|
||||
LastUsed *time.Time `json:"last_used,omitempty" gorm:"not null"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
ExpiresAt *time.Time `json:"expires_at,omitempty" gorm:"not null"`
|
||||
User User `json:"user,omitempty" gorm:"foreignKey:UserID"`
|
||||
}
|
||||
|
||||
// BrowserExtension represents a browser extension registration
|
||||
type BrowserExtension struct {
|
||||
ID uint `json:"id" gorm:"primaryKey"`
|
||||
UserID uint `json:"user_id" gorm:"not null"`
|
||||
ExtensionID string `json:"extension_id" gorm:"not null"`
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
IsActive bool `json:"is_active" gorm:"default:true"`
|
||||
LastSeen *time.Time `json:"last_seen,omitempty" gorm:"not null"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
UpdatedAt time.Time `json:"updated_at" gorm:"autoUpdateTime"`
|
||||
User User `json:"user,omitempty" gorm:"foreignKey:UserID"`
|
||||
}
|
||||
@@ -49,6 +49,8 @@ func AutoMigrate() {
|
||||
&AISummary{},
|
||||
&AITaskSuggestion{},
|
||||
&UserAISettings{},
|
||||
&UserSearchSettings{},
|
||||
&UserUpdateSettings{},
|
||||
&AITagSuggestion{},
|
||||
&AIContentGeneration{},
|
||||
&AICodeReview{},
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// UserSearchSettings stores user-specific search API configurations
|
||||
type UserSearchSettings struct {
|
||||
ID uint `json:"id" gorm:"primaryKey"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
DeletedAt gorm.DeletedAt `json:"-" gorm:"index"`
|
||||
|
||||
UserID uint `json:"user_id" gorm:"not null;uniqueIndex"`
|
||||
User User `json:"user,omitempty" gorm:"foreignKey:UserID"`
|
||||
|
||||
// Brave Search Settings
|
||||
BraveAPIKey string `json:"-" gorm:"column:brave_api_key"` // Encrypted
|
||||
BraveSearchBaseURL string `json:"brave_search_base_url" gorm:"default:https://api.search.brave.com/res/v1/web/search"`
|
||||
|
||||
// Serper (Google) Search Settings
|
||||
SerperAPIKey string `json:"-" gorm:"column:serper_api_key"` // Encrypted
|
||||
SerperBaseURL string `json:"serper_base_url" gorm:"default:https://google.serper.dev/search"`
|
||||
|
||||
// Search Configuration
|
||||
SearchAPIProvider string `json:"search_api_provider" gorm:"default:brave"` // brave, serper
|
||||
SearchResultsLimit int `json:"search_results_limit" gorm:"default:10"`
|
||||
SearchCacheTTL int `json:"search_cache_ttl" gorm:"default:300"` // seconds
|
||||
SearchRateLimit int `json:"search_rate_limit" gorm:"default:100"` // requests per minute
|
||||
}
|
||||
|
||||
// GetUserSearchSettings retrieves search settings for a user
|
||||
func GetUserSearchSettings(userID uint) (*UserSearchSettings, error) {
|
||||
var settings UserSearchSettings
|
||||
err := DB.Where("user_id = ?", userID).First(&settings).Error
|
||||
if err != nil {
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
// Create default settings
|
||||
settings = UserSearchSettings{
|
||||
UserID: userID,
|
||||
BraveSearchBaseURL: "https://api.search.brave.com/res/v1/web/search",
|
||||
SerperBaseURL: "https://google.serper.dev/search",
|
||||
SearchAPIProvider: "brave",
|
||||
SearchResultsLimit: 10,
|
||||
SearchCacheTTL: 300,
|
||||
SearchRateLimit: 100,
|
||||
}
|
||||
// Save defaults
|
||||
if err := DB.Create(&settings).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &settings, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &settings, nil
|
||||
}
|
||||
|
||||
// SaveUserSearchSettings saves search settings for a user
|
||||
func SaveUserSearchSettings(userID uint, settings *UserSearchSettings) error {
|
||||
settings.UserID = userID
|
||||
return DB.Where("user_id = ?", userID).Assign(settings).FirstOrCreate(settings).Error
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// UserUpdateSettings stores user-specific update and OAuth configurations
|
||||
type UserUpdateSettings struct {
|
||||
ID uint `json:"id" gorm:"primaryKey"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
DeletedAt gorm.DeletedAt `json:"-" gorm:"index"`
|
||||
|
||||
UserID uint `json:"user_id" gorm:"not null;uniqueIndex"`
|
||||
User User `json:"user,omitempty" gorm:"foreignKey:UserID"`
|
||||
|
||||
// OAuth Service Configuration
|
||||
OAuthServiceURL string `json:"oauth_service_url" gorm:"default:https://oauth.trackeep.org"`
|
||||
|
||||
// Update Configuration
|
||||
AutoUpdateCheck bool `json:"auto_update_check" gorm:"default:false"`
|
||||
UpdateCheckInterval string `json:"update_check_interval" gorm:"default:24h"` // 1h, 6h, 12h, 24h, 168h
|
||||
PrereleaseUpdates bool `json:"prerelease_updates" gorm:"default:false"`
|
||||
}
|
||||
|
||||
// GetUserUpdateSettings retrieves update settings for a user
|
||||
func GetUserUpdateSettings(userID uint) (*UserUpdateSettings, error) {
|
||||
var settings UserUpdateSettings
|
||||
err := DB.Where("user_id = ?", userID).First(&settings).Error
|
||||
if err != nil {
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
// Create default settings
|
||||
settings = UserUpdateSettings{
|
||||
UserID: userID,
|
||||
OAuthServiceURL: "https://oauth.trackeep.org",
|
||||
AutoUpdateCheck: false,
|
||||
UpdateCheckInterval: "24h",
|
||||
PrereleaseUpdates: false,
|
||||
}
|
||||
// Save defaults
|
||||
if err := DB.Create(&settings).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &settings, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return &settings, nil
|
||||
}
|
||||
|
||||
// SaveUserUpdateSettings saves update settings for a user
|
||||
func SaveUserUpdateSettings(userID uint, settings *UserUpdateSettings) error {
|
||||
settings.UserID = userID
|
||||
return DB.Where("user_id = ?", userID).Assign(settings).FirstOrCreate(settings).Error
|
||||
}
|
||||
@@ -48,10 +48,14 @@ type User struct {
|
||||
LockedUntil *time.Time `json:"locked_until"`
|
||||
|
||||
// Privacy Settings
|
||||
ProfileVisibility string `json:"profile_visibility" gorm:"default:public"` // public, private, friends
|
||||
ShowEmail bool `json:"show_email" gorm:"default:false"`
|
||||
ShowActivity bool `json:"show_activity" gorm:"default:true"`
|
||||
AllowMessages bool `json:"allow_messages" gorm:"default:true"`
|
||||
ProfileVisibility string `json:"profile_visibility" gorm:"default:private"` // public, private, friends
|
||||
EmailNotifications bool `json:"email_notifications" gorm:"default:true"`
|
||||
PushNotifications bool `json:"push_notifications" gorm:"default:true"`
|
||||
|
||||
// Social Features
|
||||
ShowEmail bool `json:"show_email" gorm:"default:false"`
|
||||
ShowActivity bool `json:"show_activity" gorm:"default:true"`
|
||||
AllowMessages bool `json:"allow_messages" gorm:"default:true"`
|
||||
|
||||
// Social Stats
|
||||
FollowersCount int `json:"followers_count" gorm:"default:0"`
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
# YouTube Scraper Service
|
||||
|
||||
A standalone microservice for scraping YouTube video data. This service runs independently from the main Trackeep application.
|
||||
|
||||
## Features
|
||||
|
||||
- **Mock YouTube Data**: Provides mock YouTube video data for development and testing
|
||||
- **Channel Videos**: Fetch videos from specific YouTube channels
|
||||
- **Search**: Search through YouTube video metadata
|
||||
- **REST API**: Simple REST endpoints for integration
|
||||
|
||||
## API Endpoints
|
||||
|
||||
### Health Check
|
||||
```
|
||||
GET /
|
||||
```
|
||||
Returns service status and information.
|
||||
|
||||
### Get Channel Videos
|
||||
```
|
||||
GET /channel_videos?channel={channel_name}
|
||||
```
|
||||
Fetches videos for a specific YouTube channel.
|
||||
|
||||
**Parameters:**
|
||||
- `channel`: YouTube channel name (e.g., "@Fireship", "@NetworkChuck")
|
||||
|
||||
### Search Videos
|
||||
```
|
||||
GET /search?q={query}
|
||||
```
|
||||
Searches through video titles, descriptions, and channel names.
|
||||
|
||||
**Parameters:**
|
||||
- `q`: Search query
|
||||
|
||||
## Running the Service
|
||||
|
||||
### Development
|
||||
```bash
|
||||
cd youtube-scraper
|
||||
go run .
|
||||
```
|
||||
|
||||
### Production
|
||||
```bash
|
||||
cd youtube-scraper
|
||||
go build -o youtube-scraper .
|
||||
./youtube-scraper
|
||||
```
|
||||
|
||||
### Docker
|
||||
```bash
|
||||
docker build -f ../Dockerfile.youtube-scraper -t youtube-scraper ..
|
||||
docker run -p 7857:7857 youtube-scraper
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
- `PORT`: Service port (default: 7857)
|
||||
|
||||
## Mock Data
|
||||
|
||||
The service includes mock data for popular tech YouTube channels:
|
||||
- @Fireship
|
||||
- @NetworkChuck
|
||||
- @beyondfireship
|
||||
- @LinusTechTips
|
||||
- @Mrwhosetheboss
|
||||
- @JerryRigEverything
|
||||
- @JeffGeerling
|
||||
- @mkbhd
|
||||
|
||||
## Integration
|
||||
|
||||
This service is designed to be called by the main Trackeep application via HTTP requests. The main app can be configured to use this service for YouTube-related features.
|
||||
@@ -1,32 +0,0 @@
|
||||
module youtube-scraper
|
||||
|
||||
go 1.21
|
||||
|
||||
require github.com/gin-gonic/gin v1.9.1
|
||||
|
||||
require (
|
||||
github.com/bytedance/sonic v1.9.1 // indirect
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.14.0 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
|
||||
github.com/leodido/go-urn v1.2.4 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.11 // indirect
|
||||
golang.org/x/arch v0.3.0 // indirect
|
||||
golang.org/x/crypto v0.9.0 // indirect
|
||||
golang.org/x/net v0.10.0 // indirect
|
||||
golang.org/x/sys v0.8.0 // indirect
|
||||
golang.org/x/text v0.9.0 // indirect
|
||||
google.golang.org/protobuf v1.30.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
@@ -1,86 +0,0 @@
|
||||
github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
|
||||
github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s=
|
||||
github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams=
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
|
||||
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js=
|
||||
github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
|
||||
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
|
||||
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
|
||||
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
|
||||
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
|
||||
github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
|
||||
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
|
||||
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
|
||||
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
|
||||
golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
|
||||
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
|
||||
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
@@ -1,539 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type VideoResponse struct {
|
||||
VideoID string `json:"video_id"`
|
||||
ChannelName string `json:"channel_name"`
|
||||
}
|
||||
|
||||
var ctx = context.Background()
|
||||
|
||||
// ChannelVideosResponse represents the response for channel videos scraping
|
||||
type ChannelVideosResponse struct {
|
||||
Channel string `json:"channel"`
|
||||
ChannelURL string `json:"channel_url"`
|
||||
SubscribersText string `json:"subscribers_text"`
|
||||
Subscribers int64 `json:"subscribers"`
|
||||
Videos []VideoItem `json:"videos"`
|
||||
}
|
||||
|
||||
// VideoItem holds per-video metadata extracted from the /videos page
|
||||
type VideoItem struct {
|
||||
VideoID string `json:"video_id"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Length string `json:"length,omitempty"`
|
||||
ThumbnailURL string `json:"thumbnail_url,omitempty"`
|
||||
ViewsText string `json:"views_text,omitempty"`
|
||||
Views int64 `json:"views"`
|
||||
PublishedText string `json:"published_text,omitempty"`
|
||||
PublishedDate string `json:"published_date,omitempty"` // ISO 8601 date
|
||||
}
|
||||
|
||||
// normalizeChannelInput accepts a handle like "@FCBizoniUH" or "FCBizoniUH" or a full URL
|
||||
// and returns the canonical handle (with leading @) and the corresponding /videos URL.
|
||||
func normalizeChannelInput(input string) (handle string, url string) {
|
||||
in := strings.TrimSpace(input)
|
||||
lower := strings.ToLower(in)
|
||||
isURL := strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://") || strings.HasPrefix(lower, "www.") || strings.HasPrefix(lower, "youtube.com/")
|
||||
if isURL {
|
||||
// Ensure scheme
|
||||
if strings.HasPrefix(lower, "www.") || strings.HasPrefix(lower, "youtube.com/") {
|
||||
in = "https://" + strings.TrimPrefix(in, "www.")
|
||||
if !strings.HasPrefix(strings.ToLower(in), "https://youtube.com/") && !strings.HasPrefix(strings.ToLower(in), "https://www.youtube.com/") {
|
||||
in = "https://www." + strings.TrimPrefix(in, "https://")
|
||||
}
|
||||
}
|
||||
// Normalize m.youtube.com -> www.youtube.com
|
||||
in = strings.ReplaceAll(in, "m.youtube.com", "www.youtube.com")
|
||||
|
||||
// Extract handle if present
|
||||
reHandle := regexp.MustCompile(`https?://(www\.)?youtube\.com/(@[^/]+)`) // group with @
|
||||
if m := reHandle.FindStringSubmatch(in); len(m) >= 3 {
|
||||
handle = m[2]
|
||||
} else {
|
||||
// Try path segment after domain
|
||||
rePath := regexp.MustCompile(`https?://(www\.)?youtube\.com/([^/?#]+)`) // capture after domain
|
||||
if m2 := rePath.FindStringSubmatch(in); len(m2) >= 3 {
|
||||
seg := m2[2]
|
||||
if strings.HasPrefix(seg, "@") {
|
||||
handle = seg
|
||||
} else {
|
||||
handle = "@" + seg
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Respect provided tab if present: /videos, /shorts, /streams; default to /videos
|
||||
if strings.Contains(strings.ToLower(in), "/videos") || strings.Contains(strings.ToLower(in), "/shorts") || strings.Contains(strings.ToLower(in), "/streams") {
|
||||
url = in
|
||||
} else {
|
||||
// Build a /videos URL from detected handle
|
||||
if handle == "" {
|
||||
// If we couldn't find a handle, just use the original URL
|
||||
url = in
|
||||
} else {
|
||||
url = fmt.Sprintf("https://www.youtube.com/%s/videos", handle)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Not a URL; treat as handle or bare identifier
|
||||
if strings.HasPrefix(in, "@") {
|
||||
handle = in
|
||||
} else {
|
||||
handle = "@" + in
|
||||
}
|
||||
url = fmt.Sprintf("https://www.youtube.com/%s/videos", handle)
|
||||
}
|
||||
if handle == "" {
|
||||
// As a final fallback from given input
|
||||
handle = in
|
||||
if !strings.HasPrefix(handle, "@") {
|
||||
handle = "@" + handle
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// fetchChannelVideos scrapes the channel's /videos page and extracts video IDs present
|
||||
func fetchChannelVideos(channelInput string) (ChannelVideosResponse, error) {
|
||||
handle, channelURL := normalizeChannelInput(channelInput)
|
||||
log.Printf("Fetching channel videos: handle=%s url=%s", handle, channelURL)
|
||||
|
||||
// Craft request with a desktop UA to improve likelihood of getting full HTML payload
|
||||
req, err := http.NewRequest("GET", channelURL, nil)
|
||||
if err != nil {
|
||||
return ChannelVideosResponse{}, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return ChannelVideosResponse{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return ChannelVideosResponse{}, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return ChannelVideosResponse{}, err
|
||||
}
|
||||
html := string(body)
|
||||
|
||||
// Regex to capture all 11-char YouTube video IDs from initial data payload
|
||||
// Standard videos
|
||||
vidRe := regexp.MustCompile(`"videoRenderer":\{[^}]*?"videoId":"([a-zA-Z0-9_-]{11})"`)
|
||||
matches := vidRe.FindAllStringSubmatchIndex(html, -1)
|
||||
seen := make(map[string]struct{})
|
||||
var videos []VideoItem
|
||||
for _, idx := range matches {
|
||||
if len(idx) < 4 { // need at least match start/end and group start/end
|
||||
continue
|
||||
}
|
||||
// Extract ID
|
||||
id := html[idx[2]:idx[3]]
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
|
||||
// Build a local window around the match to parse related fields
|
||||
start := idx[0]
|
||||
if start-2000 > 0 {
|
||||
start = start - 2000
|
||||
}
|
||||
end := idx[1] + 8000
|
||||
if end > len(html) {
|
||||
end = len(html)
|
||||
}
|
||||
snippet := html[start:end]
|
||||
|
||||
vi := VideoItem{VideoID: id}
|
||||
// Prefer deterministic thumbnail URL derived from video ID
|
||||
vi.ThumbnailURL = fmt.Sprintf("https://img.youtube.com/vi/%s/maxresdefault.jpg", id)
|
||||
|
||||
// Title (may appear as simpleText or runs)
|
||||
if m := regexp.MustCompile(`"title":\{"runs":\[\{"text":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
|
||||
vi.Title = unescapeYT(m[1])
|
||||
} else if m := regexp.MustCompile(`"title":\{"simpleText":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
|
||||
vi.Title = unescapeYT(m[1])
|
||||
}
|
||||
|
||||
// Length
|
||||
if m := regexp.MustCompile(`"lengthText":\{[^}]*"simpleText":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
|
||||
// Generic lengthText.simpleText (with or without accessibility block)
|
||||
vi.Length = m[1]
|
||||
} else if m := regexp.MustCompile(`"lengthText":\{[^}]*"runs":\[\{"text":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
|
||||
// lengthText.runs[0].text
|
||||
vi.Length = m[1]
|
||||
} else if m := regexp.MustCompile(`"thumbnailOverlays":\[[^\]]*?"thumbnailOverlayTimeStatusRenderer":\{"text":\{"simpleText":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
|
||||
// Overlay badge duration
|
||||
vi.Length = m[1]
|
||||
} else if m := regexp.MustCompile(`yt-badge-shape__text">([^<]+)<`).FindStringSubmatch(snippet); len(m) >= 2 {
|
||||
// Fallback: raw HTML badge text seen in thumbnails
|
||||
vi.Length = strings.TrimSpace(m[1])
|
||||
}
|
||||
|
||||
// Extra fallback: search the global HTML near the video anchor for DOM-based duration
|
||||
if vi.Length == "" {
|
||||
anchorRe := regexp.MustCompile(fmt.Sprintf(`<a[^>]+href="/watch\?v=%s[^\"]*"`, regexp.QuoteMeta(id)))
|
||||
if loc := anchorRe.FindStringIndex(html); loc != nil {
|
||||
// Search a forward window after the anchor for duration elements
|
||||
start2 := loc[1]
|
||||
end2 := start2 + 4000
|
||||
if end2 > len(html) {
|
||||
end2 = len(html)
|
||||
}
|
||||
chunk := html[start2:end2]
|
||||
// Try yt-formatted-string id="length" inner text like 5:59
|
||||
if m := regexp.MustCompile(`yt-formatted-string[^>]*id="length"[^>]*>([0-9]{1,2}:[0-9]{2}(?::[0-9]{2})?)<`).FindStringSubmatch(chunk); len(m) >= 2 {
|
||||
vi.Length = strings.TrimSpace(m[1])
|
||||
} else if m := regexp.MustCompile(`yt-formatted-string[^>]*id="length"[^>]*aria-label="([^"]+)"`).FindStringSubmatch(chunk); len(m) >= 2 {
|
||||
if parsed := parseLocalizedDuration(unescapeYT(m[1])); parsed != "" {
|
||||
vi.Length = parsed
|
||||
}
|
||||
} else if m := regexp.MustCompile(`yt-badge-shape__text">([^<]+)<`).FindStringSubmatch(chunk); len(m) >= 2 {
|
||||
vi.Length = strings.TrimSpace(m[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Thumbnail URL (first in thumbnails array) as a fallback only if not set
|
||||
if vi.ThumbnailURL == "" {
|
||||
if m := regexp.MustCompile(`"thumbnail":\{"thumbnails":\[\{"url":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
|
||||
vi.ThumbnailURL = normalizeThumbURL(unescapeYT(m[1]))
|
||||
}
|
||||
}
|
||||
|
||||
// Published time text (e.g., "3 days ago")
|
||||
if m := regexp.MustCompile(`"publishedTimeText":\{"simpleText":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
|
||||
vi.PublishedText = m[1]
|
||||
vi.PublishedDate = parseRelativeToISO(m[1])
|
||||
}
|
||||
|
||||
// Views
|
||||
if m := regexp.MustCompile(`"viewCountText":\{"simpleText":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
|
||||
vi.ViewsText = m[1]
|
||||
vi.Views = parseCountText(m[1])
|
||||
} else if m := regexp.MustCompile(`"viewCountText":\{"runs":\[\{"text":"([^"]+)"`).FindStringSubmatch(snippet); len(m) >= 2 {
|
||||
vi.ViewsText = m[1] + " views"
|
||||
vi.Views = parseCountText(m[1])
|
||||
}
|
||||
|
||||
videos = append(videos, vi)
|
||||
}
|
||||
|
||||
// Attempt to derive a displayable channel handle/name
|
||||
channelDisplay := handle
|
||||
// Try to extract canonicalBaseUrl if present
|
||||
canRe := regexp.MustCompile(`"canonicalBaseUrl":"\\/(@[^\"]+)"`)
|
||||
if m := canRe.FindStringSubmatch(html); len(m) >= 2 {
|
||||
channelDisplay = m[1]
|
||||
}
|
||||
|
||||
// Extract subscribers (header section)
|
||||
subText := ""
|
||||
// Try simpleText first
|
||||
if m := regexp.MustCompile(`"subscriberCountText":\{"simpleText":"([^"]+)"`).FindStringSubmatch(html); len(m) >= 2 {
|
||||
subText = m[1]
|
||||
} else {
|
||||
// Try runs: join all text segments inside subscriberCountText.runs
|
||||
if loc := regexp.MustCompile(`"subscriberCountText":\{"runs":\[`).FindStringIndex(html); loc != nil {
|
||||
// Take a slice starting at runs and limited length
|
||||
slice := html[loc[1]:]
|
||||
// Find the closing ]
|
||||
if endIdx := strings.Index(slice, "]}"); endIdx != -1 {
|
||||
runsChunk := slice[:endIdx]
|
||||
// Collect all text fields inside runs
|
||||
texts := regexp.MustCompile(`"text":"([^"]+)"`).FindAllStringSubmatch(runsChunk, -1)
|
||||
var parts []string
|
||||
for _, t := range texts {
|
||||
if len(t) >= 2 {
|
||||
parts = append(parts, unescapeYT(t[1]))
|
||||
}
|
||||
}
|
||||
subText = strings.Join(parts, "")
|
||||
}
|
||||
}
|
||||
}
|
||||
// Fallbacks: approximateSubscriberCount or localized patterns like "131 odběratelů"
|
||||
if subText == "" {
|
||||
if m := regexp.MustCompile(`"approximateSubscriberCount":"([^"]+)"`).FindStringSubmatch(html); len(m) >= 2 {
|
||||
subText = m[1]
|
||||
}
|
||||
}
|
||||
if subText == "" {
|
||||
// Case-insensitive; match digits with optional spaces/commas/dots before localized label
|
||||
if m := regexp.MustCompile(`(?i)([0-9][0-9\s\.,]*)\s*(odběratel(?:é|ů)?|subscribers?)`).FindStringSubmatch(html); len(m) >= 2 {
|
||||
subText = strings.TrimSpace(m[0])
|
||||
}
|
||||
}
|
||||
subs := parseCountText(subText)
|
||||
|
||||
res := ChannelVideosResponse{
|
||||
Channel: channelDisplay,
|
||||
ChannelURL: channelURL,
|
||||
SubscribersText: subText,
|
||||
Subscribers: subs,
|
||||
Videos: videos,
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// unescapeYT fixes escaped sequences in YouTube HTML JSON strings
|
||||
func unescapeYT(s string) string {
|
||||
s = strings.ReplaceAll(s, `\/`, `/`)
|
||||
s = strings.ReplaceAll(s, `\u0026`, `&`)
|
||||
return s
|
||||
}
|
||||
|
||||
// normalizeThumbURL ensures thumbnails use https and removes query artifacts if needed
|
||||
func normalizeThumbURL(u string) string {
|
||||
u = unescapeYT(u)
|
||||
if strings.HasPrefix(u, "//") {
|
||||
u = "https:" + u
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// parseRelativeToISO converts strings like "3 days ago", "2 weeks ago", "1 year ago" to ISO date (yyyy-mm-dd)
|
||||
func parseRelativeToISO(rel string) string {
|
||||
now := time.Now()
|
||||
lower := strings.ToLower(rel)
|
||||
re := regexp.MustCompile(`(\d+)[\s-]*(second|minute|hour|day|week|month|year)s?\s+ago`)
|
||||
if m := re.FindStringSubmatch(lower); len(m) >= 3 {
|
||||
n, _ := strconv.Atoi(m[1])
|
||||
unit := m[2]
|
||||
dur := time.Duration(0)
|
||||
switch unit {
|
||||
case "second":
|
||||
dur = time.Duration(n) * time.Second
|
||||
return now.Add(-dur).Format("2006-01-02")
|
||||
case "minute":
|
||||
dur = time.Duration(n) * time.Minute
|
||||
return now.Add(-dur).Format("2006-01-02")
|
||||
case "hour":
|
||||
dur = time.Duration(n) * time.Hour
|
||||
return now.Add(-dur).Format("2006-01-02")
|
||||
case "day":
|
||||
return now.AddDate(0, 0, -n).Format("2006-01-02")
|
||||
case "week":
|
||||
return now.AddDate(0, 0, -7*n).Format("2006-01-02")
|
||||
case "month":
|
||||
return now.AddDate(0, -n, 0).Format("2006-01-02")
|
||||
case "year":
|
||||
return now.AddDate(-n, 0, 0).Format("2006-01-02")
|
||||
}
|
||||
}
|
||||
// Sometimes YouTube uses "Streamed X days ago" or "Premiered ..."
|
||||
re2 := regexp.MustCompile(`(streamed|premiered|started|live)\s+(\d+)\s+(second|minute|hour|day|week|month|year)s?\s+ago`)
|
||||
if m := re2.FindStringSubmatch(lower); len(m) >= 4 {
|
||||
n, _ := strconv.Atoi(m[2])
|
||||
unit := m[3]
|
||||
switch unit {
|
||||
case "second":
|
||||
return now.Add(-time.Duration(n) * time.Second).Format("2006-01-02")
|
||||
case "minute":
|
||||
return now.Add(-time.Duration(n) * time.Minute).Format("2006-01-02")
|
||||
case "hour":
|
||||
return now.Add(-time.Duration(n) * time.Hour).Format("2006-01-02")
|
||||
case "day":
|
||||
return now.AddDate(0, 0, -n).Format("2006-01-02")
|
||||
case "week":
|
||||
return now.AddDate(0, 0, -7*n).Format("2006-01-02")
|
||||
case "month":
|
||||
return now.AddDate(0, -n, 0).Format("2006-01-02")
|
||||
case "year":
|
||||
return now.AddDate(-n, 0, 0).Format("2006-01-02")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseLocalizedDuration converts localized duration phrases (e.g., "5 minut a 59 sekund")
|
||||
// into a mm:ss or hh:mm:ss string. Supports English and basic Czech variants.
|
||||
func parseLocalizedDuration(s string) string {
|
||||
t := strings.ToLower(strings.TrimSpace(s))
|
||||
// Replace HTML entities and non-breaking spaces
|
||||
t = strings.ReplaceAll(t, " ", " ")
|
||||
t = strings.ReplaceAll(t, "\u00a0", " ")
|
||||
t = strings.TrimSpace(t)
|
||||
|
||||
// If already in 00:00 or 0:00:00 form, return as-is trimmed
|
||||
if m := regexp.MustCompile(`^\d{1,2}:\d{2}(?::\d{2})?$`).FindString(t); m != "" {
|
||||
return m
|
||||
}
|
||||
|
||||
// Patterns like: 1 hour 2 minutes 3 seconds (EN)
|
||||
// or Czech: 1 hodina/hodiny/hodin, 2 minuty/minut, 3 sekundy/sekund
|
||||
// We'll extract numbers for h/m/s separately.
|
||||
var h, m, sec int
|
||||
|
||||
// English capture
|
||||
if mm := regexp.MustCompile(`(\d+)\s*hour`).FindStringSubmatch(t); len(mm) >= 2 {
|
||||
h, _ = strconv.Atoi(mm[1])
|
||||
}
|
||||
if mm := regexp.MustCompile(`(\d+)\s*minute`).FindStringSubmatch(t); len(mm) >= 2 {
|
||||
m, _ = strconv.Atoi(mm[1])
|
||||
}
|
||||
if mm := regexp.MustCompile(`(\d+)\s*second`).FindStringSubmatch(t); len(mm) >= 2 {
|
||||
sec, _ = strconv.Atoi(mm[1])
|
||||
}
|
||||
|
||||
// Czech capture
|
||||
if mm := regexp.MustCompile(`(\d+)\s*hodin(?:a|y)?`).FindStringSubmatch(t); len(mm) >= 2 {
|
||||
if h == 0 {
|
||||
h, _ = strconv.Atoi(mm[1])
|
||||
}
|
||||
}
|
||||
if mm := regexp.MustCompile(`(\d+)\s*minut(?:a|y)?`).FindStringSubmatch(t); len(mm) >= 2 {
|
||||
if m == 0 {
|
||||
m, _ = strconv.Atoi(mm[1])
|
||||
}
|
||||
}
|
||||
if mm := regexp.MustCompile(`(\d+)\s*sekund(?:a|y)?`).FindStringSubmatch(t); len(mm) >= 2 {
|
||||
if sec == 0 {
|
||||
sec, _ = strconv.Atoi(mm[1])
|
||||
}
|
||||
}
|
||||
|
||||
// If we still didn't parse anything but string contains a plain number like "5 minutes",
|
||||
// ensure we at least capture minutes.
|
||||
if h == 0 && m == 0 && sec == 0 {
|
||||
if mm := regexp.MustCompile(`^(\d+)$`).FindStringSubmatch(t); len(mm) >= 2 {
|
||||
m, _ = strconv.Atoi(mm[1])
|
||||
}
|
||||
}
|
||||
|
||||
// Build the time string
|
||||
if h > 0 {
|
||||
return fmt.Sprintf("%d:%02d:%02d", h, m, sec)
|
||||
}
|
||||
if m > 0 || sec > 0 {
|
||||
return fmt.Sprintf("%d:%02d", m, sec)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// parseCountText handles strings like "1,234 views", "12K subscribers", "3.4M"
|
||||
func parseCountText(s string) int64 {
|
||||
t := strings.ToLower(strings.TrimSpace(s))
|
||||
// keep only the first number token
|
||||
re := regexp.MustCompile(`([0-9]+(?:\.[0-9]+)?)([kmb])?`)
|
||||
if m := re.FindStringSubmatch(t); len(m) >= 2 {
|
||||
numStr := m[1]
|
||||
suf := ""
|
||||
if len(m) >= 3 {
|
||||
suf = m[2]
|
||||
}
|
||||
f, err := strconv.ParseFloat(numStr, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
switch suf {
|
||||
case "k":
|
||||
f *= 1_000
|
||||
case "m":
|
||||
f *= 1_000_000
|
||||
case "b":
|
||||
f *= 1_000_000_000
|
||||
}
|
||||
return int64(f)
|
||||
}
|
||||
// Fallback: strip non-digits and parse
|
||||
digits := regexp.MustCompile(`[^0-9]`).ReplaceAllString(t, "")
|
||||
if digits == "" {
|
||||
return 0
|
||||
}
|
||||
v, _ := strconv.ParseInt(digits, 10, 64)
|
||||
return v
|
||||
}
|
||||
|
||||
func channelVideosHandler(w http.ResponseWriter, r *http.Request) {
|
||||
channel := r.URL.Query().Get("channel")
|
||||
if channel == "" {
|
||||
log.Println("Missing channel parameter")
|
||||
http.Error(w, "Missing channel parameter. Provide a handle like @FCBizoniUH, FCBBizoniUH, or a full channel URL.", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
res, err := fetchChannelVideos(channel)
|
||||
if err != nil {
|
||||
log.Printf("Failed to fetch channel videos for %s: %v", channel, err)
|
||||
http.Error(w, "Failed to fetch channel videos", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(res)
|
||||
}
|
||||
|
||||
// CORS Middleware
|
||||
func corsMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Set CORS headers
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
|
||||
|
||||
// Handle preflight requests
|
||||
if r.Method == http.MethodOptions {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func rootHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
response := map[string]interface{}{
|
||||
"status": "ok",
|
||||
"service": "YouTube Scraper",
|
||||
"version": "1.0.0",
|
||||
"endpoints": map[string]string{
|
||||
"channel_videos": "/channel_videos?channel={handle_or_url}",
|
||||
},
|
||||
}
|
||||
json.NewEncoder(w).Encode(response)
|
||||
}
|
||||
|
||||
func main() {
|
||||
port := os.Getenv("PORT")
|
||||
if port == "" {
|
||||
port = "7857"
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
|
||||
// Create a new mux with CORS middleware
|
||||
handlerWithCORS := corsMiddleware(mux)
|
||||
|
||||
// Register routes on the original mux
|
||||
mux.HandleFunc("/", rootHandler)
|
||||
mux.HandleFunc("/channel_videos", channelVideosHandler)
|
||||
|
||||
log.Printf("YouTube Scraper starting on port %s", port)
|
||||
log.Fatal(http.ListenAndServe(":"+port, handlerWithCORS))
|
||||
}
|
||||
Reference in New Issue
Block a user